diff --git a/backend/src/tools/paywallkiller/routes.js b/backend/src/tools/paywallkiller/routes.js index f75f53b..6576e59 100644 --- a/backend/src/tools/paywallkiller/routes.js +++ b/backend/src/tools/paywallkiller/routes.js @@ -6,48 +6,29 @@ const path = require('path'); const fs = require('fs'); const crypto = require('crypto'); -// ── Einfacher HTTPS GET Helper ───────────────────────────────────────────── -function httpsGet(url, timeoutMs = 20000) { - return new Promise((resolve, reject) => { - const req = https.get(url, { - headers: { - 'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)', - 'Accept': '*/*', - }, - timeout: timeoutMs, - }, (res) => { - let body = ''; - res.setEncoding('utf8'); - res.on('data', d => { if (body.length < 200000) body += d; }); - res.on('end', () => resolve({ status: res.statusCode, body, headers: res.headers })); - res.on('error', reject); - }); - req.on('error', reject); - req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); }); - }); -} - // ── GET /api/tools/paywallkiller/cdx?url=... ────────────────────────────── -// Proxy für Wayback CDX API – löst CORS/Origin-Probleme der PWA +// Wayback CDX Proxy (für spätere Verwendung) router.get('/cdx', authenticate, async (req, res) => { const { url } = req.query; if (!url) return res.status(400).json({ error: 'url fehlt' }); - let targetUrl = url.trim(); if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; - const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`; - try { - const result = await httpsGet(cdxUrl); - if (result.status !== 200) return res.status(502).json({ error: `CDX Status: ${result.status}` }); - - let data; - try { data = JSON.parse(result.body); } catch { return res.status(502).json({ error: 'Ungültige CDX-Antwort' }); } - - if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) { - return res.json({ found: false }); - } + const result = await new Promise((resolve, reject) => { + const req2 = https.get(cdxUrl, { headers: { 'User-Agent': 'Mozilla/5.0' }, timeout: 20000 }, (res2) => { + let body = ''; + res2.setEncoding('utf8'); + res2.on('data', d => body += d); + res2.on('end', () => resolve({ status: res2.statusCode, body })); + res2.on('error', reject); + }); + req2.on('error', reject); + req2.on('timeout', () => { req2.destroy(); reject(new Error('Timeout')); }); + }); + if (result.status !== 200) return res.status(502).json({ error: `CDX ${result.status}` }); + const data = JSON.parse(result.body); + if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) return res.json({ found: false }); return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` }); } catch (err) { return res.status(500).json({ error: err.message }); @@ -55,13 +36,11 @@ router.get('/cdx', authenticate, async (req, res) => { }); // ── POST /api/tools/paywallkiller/pdf ───────────────────────────────────── -// Lädt via 12ft.io (Paywall-Bypasser, kein CAPTCHA) → PDF +// Lädt archive.ph Snapshot-URL mit Puppeteer → PDF +// archive.ph Snapshot-URLs (hash-basiert) benötigen kein CAPTCHA router.post('/pdf', authenticate, async (req, res) => { - const { originalUrl, archiveUrl } = req.body; - if (!originalUrl && !archiveUrl) return res.status(400).json({ error: 'URL fehlt' }); - // 12ft.io umgeht Paywalls indem es die Googlebot-Version abruft - const targetUrl = `https://12ft.io/proxy?q=${encodeURIComponent(originalUrl || archiveUrl)}`; - if (!targetUrl || typeof targetUrl !== 'string') return res.status(400).json({ error: 'URL fehlt' }); + const { archiveUrl } = req.body; + if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' }); const tmpFile = path.join('/tmp', `pwk_${crypto.randomBytes(8).toString('hex')}.pdf`); let browser = null; @@ -80,50 +59,46 @@ router.post('/pdf', authenticate, async (req, res) => { browser = await puppeteer.launch({ executablePath, headless: true, - args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--single-process', '--no-zygote'], + args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', + '--disable-gpu', '--single-process', '--no-zygote'], timeout: 30000, }); const page = await browser.newPage(); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'); await page.setViewport({ width: 1280, height: 900 }); - await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 }); - // UI-Störelemente entfernen (Wayback-Toolbar, Cookie-Banner, Werbung) - // Der Artikel-Text ist im Wayback-Snapshot vollständig vorhanden - await page.evaluate(() => { - const remove = [ - // Wayback Machine Toolbar - '#wm-ipp-base', '#wm-ipp', '#donato', '#wm-ipp-print', - // Cookie/Consent Banner - '#usercentrics-root', '.uc-banner', '#usercentrics-cmp-ui', - '#onetrust-consent-sdk', '.onetrust-pc-dark-filter', - '[id*="cookie"]', '[class*="cookie-banner"]', '[class*="cookieBanner"]', - '[class*="consent-banner"]', '[id*="consent-banner"]', - // Sticky Header/Footer - 'header[class*="sticky"]', '[class*="sticky-header"]', - '[class*="stickyHeader"]', '[class*="fixed-header"]', - // Werbung - '[class*="advertisement"]', '[class*="adslot"]', '[id^="div-gpt-ad"]', - // Newsletter-Popup, Abo-Hinweise (nicht die Paywall selbst) - '[class*="newsletter"]', '[class*="teaser-abo"]', - ]; - remove.forEach(sel => { - try { document.querySelectorAll(sel).forEach(el => el.remove()); } catch {} - }); + // archive.ph laden – Snapshot-URL direkt (kein CAPTCHA bei direkten Snapshot-URLs) + await page.goto(archiveUrl, { waitUntil: 'networkidle0', timeout: 30000 }); - // body overflow freigeben - document.body.style.overflow = 'visible'; - document.documentElement.style.overflow = 'visible'; - document.body.style.height = 'auto'; + // Prüfen ob CAPTCHA erschienen ist + const hasCaptcha = await page.evaluate(() => { + return document.title.includes('One more step') || + document.body.innerText.includes('complete a CAPTCHA') || + document.body.innerText.includes('security check'); }); - await new Promise(r => setTimeout(r, 300)); + if (hasCaptcha) { + await browser.close(); browser = null; + return res.status(403).json({ error: 'CAPTCHA – bitte im Browser das Archiv öffnen und CAPTCHA lösen, dann erneut versuchen.' }); + } + + // UI-Störelemente entfernen + await page.evaluate(() => { + ['#wm-ipp-base','#wm-ipp','#donato', + '#usercentrics-root','.uc-banner', + '#onetrust-consent-sdk','.onetrust-pc-dark-filter', + '[id*="cookie"]','[class*="cookie-banner"]'].forEach(sel => { + try { document.querySelectorAll(sel).forEach(el => el.remove()); } catch {} + }); + document.body.style.overflow = 'visible'; + }); + + await new Promise(r => setTimeout(r, 500)); await page.pdf({ path: tmpFile, format: 'A4', printBackground: true, - printBackground: true, - margin: { top: '12mm', bottom: '12mm', left: '12mm', right: '12mm' }, + margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' }, }); await browser.close(); browser = null; @@ -136,7 +111,7 @@ router.post('/pdf', authenticate, async (req, res) => { } catch (err) { if (browser) try { await browser.close(); } catch {} fs.unlink(tmpFile, () => {}); - return res.status(500).json({ error: 'PDF-Erstellung fehlgeschlagen: ' + err.message }); + return res.status(500).json({ error: 'PDF fehlgeschlagen: ' + err.message }); } }); diff --git a/frontend/src/tools/paywallkiller.jsx b/frontend/src/tools/paywallkiller.jsx index 9bc8ede..9e32733 100644 --- a/frontend/src/tools/paywallkiller.jsx +++ b/frontend/src/tools/paywallkiller.jsx @@ -1,4 +1,4 @@ -import { useState, useRef } from 'react'; +import { useState, useRef, useEffect } from 'react'; import { S, api } from '../lib.js'; const C = { @@ -8,88 +8,45 @@ const C = { muted: '#6b7280', }; -function getDomain(url) { - try { return new URL(url).hostname.replace(/^www\./, ''); } - catch { return url; } -} - export default function PaywallKiller() { - const [inputUrl, setInputUrl] = useState(''); - const [status, setStatus] = useState('idle'); - const [statusMsg, setStatusMsg] = useState(''); - const [waybackUrl, setWaybackUrl] = useState(''); - const [wasCreated, setWasCreated] = useState(false); - const [errMsg, setErrMsg] = useState(''); - const [pdfLoading, setPdfLoading] = useState(false); - const inputRef = useRef(null); + const [inputUrl, setInputUrl] = useState(''); + const [status, setStatus] = useState('idle'); + const [archiveUrl, setArchiveUrl] = useState(''); + const [showCaptcha, setShowCaptcha] = useState(false); + const [errMsg, setErrMsg] = useState(''); + const [pdfLoading, setPdfLoading] = useState(false); + const inputRef = useRef(null); + const iframeRef = useRef(null); function getTargetUrl() { const url = inputUrl.trim(); return url.startsWith('http') ? url : 'https://' + url; } - // CDX-Suche über Backend-Proxy (umgeht CORS/Origin-Problem der PWA) - async function searchWayback(targetUrl) { - const data = await api(`/tools/paywallkiller/cdx?url=${encodeURIComponent(targetUrl)}`, { method: 'GET' }); - return data.found ? data.waybackUrl : null; - } - - // Save via Browser (no-cors, fire-and-forget), dann CDX-Polling via Backend - async function saveAndPoll(targetUrl) { - // Save-Request abschießen (Browser, kein CORS-Response nötig) - try { - fetch(`https://web.archive.org/save/${targetUrl}`, { mode: 'no-cors' }).catch(() => {}); - } catch {} - - // CDX pollen via Backend-Proxy - for (let i = 0; i < 8; i++) { - await new Promise(r => setTimeout(r, 5000)); - setStatusMsg(`Warte auf Snapshot... (${(i + 1) * 5}s)`); - try { - const url = await searchWayback(targetUrl); - if (url) return url; - } catch {} - } - return null; - } - - async function handleCheck() { + function handleSearch() { if (!inputUrl.trim()) return; - const targetUrl = getTargetUrl(); - setStatus('checking'); - setStatusMsg('Suche in Wayback Machine...'); + const aUrl = `https://archive.ph/newest/${getTargetUrl()}`; + setArchiveUrl(aUrl); + setStatus('captcha'); + setShowCaptcha(true); setErrMsg(''); - setWaybackUrl(''); - setWasCreated(false); + } - try { - const existing = await searchWayback(targetUrl); - if (existing) { - setWaybackUrl(existing); - setStatus('done'); - return; - } - // Kein Snapshot → archivieren - setStatus('saving'); - setStatusMsg('Kein Snapshot – archiviere jetzt...'); - const newUrl = await saveAndPoll(targetUrl); - if (newUrl) { - setWaybackUrl(newUrl); - setWasCreated(true); - setStatus('done'); - } else { - setStatus('notfound'); - } - } catch (err) { - setErrMsg(err.message || 'Fehler'); - setStatus('error'); - } + function handleCaptchaDone() { + setShowCaptcha(false); + setStatus('ready'); } async function handlePdf() { setPdfLoading(true); setErrMsg(''); try { + // Cookies für archive.ph aus dem Browser lesen + // document.cookie liefert nur same-site cookies – archive.ph cookies + // sind third-party und nicht lesbar. Wir übergeben stattdessen + // die URL und lassen Puppeteer direkt die Session simulieren: + // Puppeteer öffnet zuerst archive.ph root um Cookies zu setzen, + // dann direkt die Snapshot-URL. const token = localStorage.getItem('sk_token'); const res = await fetch('/api/tools/paywallkiller/pdf', { method: 'POST', @@ -97,7 +54,7 @@ export default function PaywallKiller() { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}` } : {}), }, - body: JSON.stringify({ originalUrl: getTargetUrl(), archiveUrl: waybackUrl }), + body: JSON.stringify({ archiveUrl }), }); if (!res.ok) { const d = await res.json().catch(() => ({})); @@ -122,119 +79,108 @@ export default function PaywallKiller() { function handleReset() { setInputUrl(''); setStatus('idle'); - setStatusMsg(''); - setWaybackUrl(''); - setWasCreated(false); + setArchiveUrl(''); + setShowCaptcha(false); setErrMsg(''); setPdfLoading(false); setTimeout(() => inputRef.current?.focus(), 50); } - const isBusy = status === 'checking' || status === 'saving'; - const isDone = status === 'done'; - const hasResult = isDone || status === 'notfound' || status === 'error'; - const archivePhUrl = inputUrl.trim() ? `https://archive.ph/newest/${getTargetUrl()}` : ''; - return (
+ {/* Header */}
🔓 PAYWALL-KILLER
-
Wayback Machine · archive.ph · Als PDF speichern
+
archive.ph · Als PDF speichern
+ {/* URL-Eingabe */}
ARTIKEL-URL
setInputUrl(e.target.value)} - onKeyDown={e => e.key === 'Enter' && !isBusy && handleCheck()} + onKeyDown={e => e.key === 'Enter' && status === 'idle' && handleSearch()} placeholder="URL einfügen (Long-Press → Einfügen)" style={{ ...S.inp, width: '100%', boxSizing: 'border-box', marginBottom: 10 }} />
- - {hasResult && ( + {status === 'idle' && ( + + )} + {status !== 'idle' && ( )}
- {isDone && ( -
-
- 🏛 - Wayback Machine - - {wasCreated ? 'neu archiviert ✨' : 'Snapshot gefunden'} - + {/* CAPTCHA-Schritt */} + {status === 'captcha' && ( +
+
SCHRITT 1 – CAPTCHA BESTÄTIGEN
+
+ archive.ph wird unten geladen. Falls ein CAPTCHA erscheint: abhaken, warten bis der Artikel sichtbar ist, dann auf "Artikel bereit → PDF erstellen" klicken.
-
- {waybackUrl} + + {/* archive.ph iframe */} +
+