diff --git a/backend/src/tools/paywallkiller/routes.js b/backend/src/tools/paywallkiller/routes.js index ad21f0b..880a23d 100644 --- a/backend/src/tools/paywallkiller/routes.js +++ b/backend/src/tools/paywallkiller/routes.js @@ -1,142 +1,14 @@ const express = require('express'); const router = express.Router(); const { authenticate } = require('../../middleware/auth'); -const https = require('https'); -const http = require('http'); const path = require('path'); const fs = require('fs'); const crypto = require('crypto'); -// ── HTTP GET mit Redirect-Follow + Cookie-Jar ───────────────────────────── -function fetchPage(url, opts = {}) { - return new Promise((resolve, reject) => { - if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects')); - const mod = url.startsWith('https') ? https : http; - const cookieHeader = opts.cookieJar && Object.keys(opts.cookieJar).length - ? Object.entries(opts.cookieJar).map(([k,v]) => `${k}=${v}`).join('; ') - : undefined; - const headers = { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8', - 'Accept-Encoding': 'identity', - 'Connection': 'keep-alive', - ...(opts.referer ? { 'Referer': opts.referer } : {}), - ...(cookieHeader ? { 'Cookie': cookieHeader } : {}), - }; - const req = mod.get(url, { headers, timeout: 15000 }, (res) => { - const jar = opts.cookieJar || {}; - for (const c of (res.headers['set-cookie'] || [])) { - const m = c.match(/^([^=]+)=([^;]*)/); - if (m) jar[m[1].trim()] = m[2].trim(); - } - if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) { - let loc = res.headers.location; - if (!loc.startsWith('http')) { - try { const b = new URL(url); loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; } - catch { return reject(new Error('Ungültige Redirect-URL')); } - } - res.resume(); - return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1, cookieJar: jar, referer: url }).then(resolve).catch(reject); - } - let body = ''; - res.setEncoding('utf8'); - res.on('data', d => { if (body.length < 150000) body += d; }); - res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body, cookieJar: jar })); - res.on('error', reject); - }); - req.on('error', reject); - req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); }); - }); -} - -// ── GET /api/tools/paywallkiller/wayback?url=... ────────────────────────── -// Sucht vorhandenen Wayback-Snapshot via CDX API -router.get('/wayback', authenticate, async (req, res) => { - const { url } = req.query; - if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url Parameter fehlt' }); - let targetUrl = url.trim(); - if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; - - try { - const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`; - const r = await fetchPage(cdxUrl); - if (r.statusCode === 200 && r.body) { - let data; - try { data = JSON.parse(r.body); } catch { data = null; } - if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) { - return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` }); - } - } - return res.json({ found: false }); - } catch (err) { - return res.status(500).json({ error: 'Fehler: ' + err.message }); - } -}); - -// ── POST /api/tools/paywallkiller/wayback-save ──────────────────────────── -// Schießt Save-Request ab (fire-and-forget), pollt dann CDX auf Ergebnis -router.post('/wayback-save', authenticate, async (req, res) => { - const { url } = req.body; - if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url fehlt' }); - let targetUrl = url.trim(); - if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; - - async function checkCdx() { - const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`; - try { - const r = await fetchPage(cdxUrl); - if (r.statusCode === 200 && r.body) { - const data = JSON.parse(r.body); - if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) { - return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`; - } - } - } catch {} - return null; - } - - function fireSaveRequest() { - return new Promise((resolve) => { - const savePath = `/save/${targetUrl}`; - const req = https.get({ - hostname: 'web.archive.org', - path: savePath, - headers: { 'User-Agent': 'Mozilla/5.0 (compatible; WaybackSave/1.0)', 'Accept': 'text/html' }, - timeout: 8000, - }, (res) => { - const location = res.headers.location || ''; - res.resume(); - resolve(location); - }); - req.on('error', () => resolve('')); - req.on('timeout', () => { req.destroy(); resolve(''); }); - }); - } - - try { - const location = await fireSaveRequest(); - const directMatch = location.match(/\/web\/(\d{14})\//); - if (directMatch) { - const waybackUrl = location.startsWith('http') ? location : `https://web.archive.org${location}`; - return res.json({ success: true, waybackUrl }); - } - // Polling: bis zu 6x alle 5s - for (let i = 0; i < 6; i++) { - await new Promise(r => setTimeout(r, 5000)); - const found = await checkCdx(); - if (found) return res.json({ success: true, waybackUrl: found }); - } - return res.status(502).json({ error: 'Archivierung läuft noch oder wurde blockiert. Versuche es in einer Minute erneut.' }); - } catch (err) { - return res.status(500).json({ error: 'Save fehlgeschlagen: ' + err.message }); - } -}); - // ── POST /api/tools/paywallkiller/pdf ───────────────────────────────────── // Lädt Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF router.post('/pdf', authenticate, async (req, res) => { - const { archiveUrl, originalUrl } = req.body; + const { originalUrl, archiveUrl } = req.body; const targetUrl = originalUrl || archiveUrl; if (!targetUrl || typeof targetUrl !== 'string') return res.status(400).json({ error: 'URL fehlt' }); @@ -145,7 +17,12 @@ router.post('/pdf', authenticate, async (req, res) => { try { const puppeteer = require('puppeteer-core'); - const candidates = [process.env.CHROMIUM_PATH, '/usr/bin/chromium-browser', '/usr/bin/chromium', '/usr/bin/google-chrome'].filter(Boolean); + const candidates = [ + process.env.CHROMIUM_PATH, + '/usr/bin/chromium-browser', + '/usr/bin/chromium', + '/usr/bin/google-chrome', + ].filter(Boolean); const executablePath = candidates.find(p => { try { fs.accessSync(p); return true; } catch { return false; } }); if (!executablePath) return res.status(500).json({ error: 'Chromium nicht gefunden' }); @@ -162,6 +39,7 @@ router.post('/pdf', authenticate, async (req, res) => { await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 }); await new Promise(r => setTimeout(r, 1500)); + // Paywall-Overlays, Cookie-Banner, Blur-Filter entfernen await page.addStyleTag({ content: ` [class*="paywall"],[class*="Paywall"],[id*="paywall"], [class*="piano"],[class*="Piano"],[id*="piano"], diff --git a/frontend/src/tools/paywallkiller.jsx b/frontend/src/tools/paywallkiller.jsx index 896c543..945a0e8 100644 --- a/frontend/src/tools/paywallkiller.jsx +++ b/frontend/src/tools/paywallkiller.jsx @@ -6,7 +6,6 @@ const C = { success: '#4ade80', error: '#f87171', muted: '#6b7280', - info: '#60a5fa', }; function getDomain(url) { @@ -14,14 +13,42 @@ function getDomain(url) { catch { return url; } } +// Wayback CDX – direkt im Browser (CORS: * → kein Server-Proxy nötig) +async function searchWayback(targetUrl) { + const cdx = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`; + const res = await fetch(cdx); + if (!res.ok) throw new Error(`CDX Fehler: ${res.status}`); + const data = await res.json(); + if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) return null; + return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`; +} + +// Wayback Save – direkt im Browser via no-cors fetch (fire-and-forget) +// Dann CDX pollen bis Snapshot erscheint +async function saveAndPollWayback(targetUrl, onStatus) { + // Save abschießen (no-cors: kein Response lesbar, aber Request geht raus) + try { + await fetch(`https://web.archive.org/save/${targetUrl}`, { mode: 'no-cors' }); + } catch { /* ignorieren – no-cors wirft immer */ } + + // CDX pollen: 8x alle 5s = max 40s + for (let i = 0; i < 8; i++) { + onStatus(`Warte auf Snapshot... (${(i+1)*5}s)`); + await new Promise(r => setTimeout(r, 5000)); + const url = await searchWayback(targetUrl); + if (url) return url; + } + return null; +} + export default function PaywallKiller() { - const [inputUrl, setInputUrl] = useState(''); - // status: idle | checking | saving | done | error - const [status, setStatus] = useState('idle'); - const [waybackUrl, setWaybackUrl] = useState(''); - const [wasCreated, setWasCreated] = useState(false); - const [errMsg, setErrMsg] = useState(''); - const [pdfLoading, setPdfLoading] = useState(false); + const [inputUrl, setInputUrl] = useState(''); + const [status, setStatus] = useState('idle'); + const [statusMsg, setStatusMsg] = useState(''); + const [waybackUrl, setWaybackUrl] = useState(''); + const [wasCreated, setWasCreated] = useState(false); + const [errMsg, setErrMsg] = useState(''); + const [pdfLoading, setPdfLoading] = useState(false); const inputRef = useRef(null); function getTargetUrl() { @@ -30,47 +57,41 @@ export default function PaywallKiller() { } async function handleCheck() { - const url = inputUrl.trim(); - if (!url) return; + if (!inputUrl.trim()) return; + const targetUrl = getTargetUrl(); setStatus('checking'); + setStatusMsg('Suche in Wayback Machine...'); setErrMsg(''); setWaybackUrl(''); setWasCreated(false); - try { - const data = await api(`/tools/paywallkiller/wayback?url=${encodeURIComponent(getTargetUrl())}`, { method: 'GET' }); - if (data.found) { - setWaybackUrl(data.waybackUrl); - setStatus('done'); - } else { - // Kein Snapshot → direkt Save starten - await handleSave(); - } - } catch (err) { - setErrMsg(err.message || 'Fehler beim Suchen'); - setStatus('error'); - } - } - async function handleSave() { - setStatus('saving'); - setErrMsg(''); try { - const data = await api('/tools/paywallkiller/wayback-save', { body: { url: getTargetUrl() } }); - if (data.success) { - setWaybackUrl(data.waybackUrl); + // Schritt 1: vorhandenen Snapshot suchen + const existing = await searchWayback(targetUrl); + if (existing) { + setWaybackUrl(existing); + setStatus('done'); + return; + } + + // Schritt 2: Kein Snapshot → neu erstellen + pollen + setStatus('saving'); + setStatusMsg('Kein Snapshot gefunden – archiviere jetzt...'); + const newUrl = await saveAndPollWayback(targetUrl, msg => setStatusMsg(msg)); + if (newUrl) { + setWaybackUrl(newUrl); setWasCreated(true); setStatus('done'); } else { - throw new Error(data.error || 'Snapshot konnte nicht erstellt werden'); + setStatus('notfound'); } } catch (err) { - setErrMsg(err.message || 'Snapshot-Erstellung fehlgeschlagen'); + setErrMsg(err.message || 'Fehler'); setStatus('error'); } } async function handlePdf() { - if (!waybackUrl) return; setPdfLoading(true); setErrMsg(''); try { @@ -81,23 +102,23 @@ export default function PaywallKiller() { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}` } : {}), }, - body: JSON.stringify({ archiveUrl: waybackUrl, originalUrl: getTargetUrl() }), + body: JSON.stringify({ originalUrl: getTargetUrl(), archiveUrl: waybackUrl }), }); if (!res.ok) { - const data = await res.json().catch(() => ({})); - throw new Error(data.error || 'PDF-Fehler'); + const d = await res.json().catch(() => ({})); + throw new Error(d.error || 'PDF-Fehler'); } const blob = await res.blob(); const objUrl = URL.createObjectURL(blob); const a = document.createElement('a'); a.href = objUrl; - a.download = 'archiv_' + Date.now() + '.pdf'; + a.download = 'artikel_' + Date.now() + '.pdf'; document.body.appendChild(a); a.click(); document.body.removeChild(a); setTimeout(() => URL.revokeObjectURL(objUrl), 5000); } catch (err) { - setErrMsg(err.message || 'PDF konnte nicht erstellt werden'); + setErrMsg(err.message || 'PDF fehlgeschlagen'); } finally { setPdfLoading(false); } @@ -106,6 +127,7 @@ export default function PaywallKiller() { function handleReset() { setInputUrl(''); setStatus('idle'); + setStatusMsg(''); setWaybackUrl(''); setWasCreated(false); setErrMsg(''); @@ -113,29 +135,22 @@ export default function PaywallKiller() { setTimeout(() => inputRef.current?.focus(), 50); } - const isChecking = status === 'checking'; - const isSaving = status === 'saving'; - const isBusy = isChecking || isSaving; - const isDone = status === 'done'; - const archivePhUrl = inputUrl.trim() - ? `https://archive.ph/newest/${getTargetUrl()}` - : ''; + const isBusy = status === 'checking' || status === 'saving'; + const isDone = status === 'done'; + const hasResult = isDone || status === 'notfound' || status === 'error'; + const archivePhUrl = inputUrl.trim() ? `https://archive.ph/newest/${getTargetUrl()}` : ''; return (
- {/* Header */}
🔓 PAYWALL-KILLER
-
- Wayback Machine · archive.ph · Als PDF speichern -
+
Wayback Machine · archive.ph · Als PDF speichern
- {/* Eingabe */}
ARTIKEL-URL
- {(isDone || status === 'error') && ( + {hasResult && ( )}
- {isSaving && ( -
- Kein Snapshot gefunden – die Wayback Machine archiviert den Artikel jetzt. Das dauert 15–30 Sekunden... -
- )}
- {/* Ergebnis: Wayback Machine */} + {/* Wayback-Ergebnis */} {isDone && ( - <> -
-
- 🏛 - Wayback Machine - - {wasCreated ? 'neu archiviert ✨' : 'Snapshot gefunden'} - -
-
- {waybackUrl} -
-
- 🌐 Öffnen - -
- {pdfLoading && ( -
- Seite wird serverseitig gerendert (15–30s)... -
- )} +
+
+ 🏛 + Wayback Machine + + {wasCreated ? 'neu archiviert ✨' : 'Snapshot gefunden'} +
- - {/* archive.ph – immer anzeigen, nur Öffnen */} -
-
- 📦 - archive.ph -
-
- 🌐 Öffnen - Kein PDF – CAPTCHA-geschützt -
+
+ {waybackUrl}
- +
+ 🌐 Öffnen + +
+ {pdfLoading &&
Seite wird gerendert (15–30s)...
} +
+ )} + + {/* Kein Snapshot erstellt */} + {status === 'notfound' && ( +
+
+ 📭 + + Archivierung fehlgeschlagen – {getDomain(inputUrl)} blockiert möglicherweise die Wayback Machine. + +
+
+ )} + + {/* archive.ph – immer nach Suche anzeigen */} + {hasResult && archivePhUrl && ( +
+
+ 📦 + archive.ph +
+
+ 🌐 Öffnen + Kein PDF – CAPTCHA-geschützt +
+
)} - {/* Fehler */} {errMsg && (
-
+
⚠️ {errMsg}
)} - {/* Idle-Info */} {status === 'idle' && (
SO FUNKTIONIERT ES
{[ - ['1', 'URL des gesperrten Artikels einfügen (Long-Press → Einfügen)'], - ['2', 'Vorhandenen Snapshot in der Wayback Machine suchen'], - ['3', 'Falls keiner existiert: Artikel wird automatisch archiviert'], + ['1', 'URL einfügen (Long-Press → Einfügen)'], + ['2', 'Wayback Machine wird direkt im Browser durchsucht'], + ['3', 'Falls kein Snapshot: Artikel wird automatisch archiviert'], ['4', 'Snapshot öffnen oder als PDF speichern'], ].map(([num, text]) => (