diff --git a/backend/src/tools/paywallkiller/routes.js b/backend/src/tools/paywallkiller/routes.js index 880a23d..a00eabc 100644 --- a/backend/src/tools/paywallkiller/routes.js +++ b/backend/src/tools/paywallkiller/routes.js @@ -1,12 +1,60 @@ const express = require('express'); const router = express.Router(); const { authenticate } = require('../../middleware/auth'); +const https = require('https'); const path = require('path'); const fs = require('fs'); const crypto = require('crypto'); +// ── Einfacher HTTPS GET Helper ───────────────────────────────────────────── +function httpsGet(url, timeoutMs = 20000) { + return new Promise((resolve, reject) => { + const req = https.get(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)', + 'Accept': '*/*', + }, + timeout: timeoutMs, + }, (res) => { + let body = ''; + res.setEncoding('utf8'); + res.on('data', d => { if (body.length < 200000) body += d; }); + res.on('end', () => resolve({ status: res.statusCode, body, headers: res.headers })); + res.on('error', reject); + }); + req.on('error', reject); + req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); }); + }); +} + +// ── GET /api/tools/paywallkiller/cdx?url=... ────────────────────────────── +// Proxy für Wayback CDX API – löst CORS/Origin-Probleme der PWA +router.get('/cdx', authenticate, async (req, res) => { + const { url } = req.query; + if (!url) return res.status(400).json({ error: 'url fehlt' }); + + let targetUrl = url.trim(); + if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; + + const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`; + + try { + const result = await httpsGet(cdxUrl); + if (result.status !== 200) return res.status(502).json({ error: `CDX Status: ${result.status}` }); + + let data; + try { data = JSON.parse(result.body); } catch { return res.status(502).json({ error: 'Ungültige CDX-Antwort' }); } + + if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) { + return res.json({ found: false }); + } + return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` }); + } catch (err) { + return res.status(500).json({ error: err.message }); + } +}); + // ── POST /api/tools/paywallkiller/pdf ───────────────────────────────────── -// Lädt Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF router.post('/pdf', authenticate, async (req, res) => { const { originalUrl, archiveUrl } = req.body; const targetUrl = originalUrl || archiveUrl; @@ -39,7 +87,6 @@ router.post('/pdf', authenticate, async (req, res) => { await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 }); await new Promise(r => setTimeout(r, 1500)); - // Paywall-Overlays, Cookie-Banner, Blur-Filter entfernen await page.addStyleTag({ content: ` [class*="paywall"],[class*="Paywall"],[id*="paywall"], [class*="piano"],[class*="Piano"],[id*="piano"], diff --git a/frontend/src/tools/paywallkiller.jsx b/frontend/src/tools/paywallkiller.jsx index 1eb46c7..9b3fc04 100644 --- a/frontend/src/tools/paywallkiller.jsx +++ b/frontend/src/tools/paywallkiller.jsx @@ -13,52 +13,14 @@ function getDomain(url) { catch { return url; } } -// Wayback CDX – direkt im Browser -async function searchWayback(targetUrl) { - const cdx = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`; - let res; - try { - // Ohne custom headers – minimaler Request damit CORS-Preflight entfällt - res = await fetch(cdx); - } catch (e) { - throw new Error('Netzwerkfehler: ' + e.message); - } - if (!res.ok) throw new Error(`CDX ${res.status}`); - const data = await res.json(); - if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) return null; - return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`; -} - -// Wayback Save + CDX-Polling -async function saveAndPollWayback(targetUrl, onStatus) { - // Save via no-cors (kein lesbarer Response, aber Archivierung wird getriggert) - try { - fetch(`https://web.archive.org/save/${targetUrl}`, { - method: 'GET', - mode: 'no-cors', - }).catch(() => {}); - } catch {} - - // CDX pollen: 8x alle 5s = max 40s - for (let i = 0; i < 8; i++) { - await new Promise(r => setTimeout(r, 5000)); - onStatus(`Warte auf Snapshot... (${(i + 1) * 5}s)`); - try { - const url = await searchWayback(targetUrl); - if (url) return url; - } catch {} - } - return null; -} - export default function PaywallKiller() { - const [inputUrl, setInputUrl] = useState(''); - const [status, setStatus] = useState('idle'); - const [statusMsg, setStatusMsg] = useState(''); - const [waybackUrl, setWaybackUrl] = useState(''); - const [wasCreated, setWasCreated] = useState(false); - const [errMsg, setErrMsg] = useState(''); - const [pdfLoading, setPdfLoading] = useState(false); + const [inputUrl, setInputUrl] = useState(''); + const [status, setStatus] = useState('idle'); + const [statusMsg, setStatusMsg] = useState(''); + const [waybackUrl, setWaybackUrl] = useState(''); + const [wasCreated, setWasCreated] = useState(false); + const [errMsg, setErrMsg] = useState(''); + const [pdfLoading, setPdfLoading] = useState(false); const inputRef = useRef(null); function getTargetUrl() { @@ -66,6 +28,31 @@ export default function PaywallKiller() { return url.startsWith('http') ? url : 'https://' + url; } + // CDX-Suche über Backend-Proxy (umgeht CORS/Origin-Problem der PWA) + async function searchWayback(targetUrl) { + const data = await api(`/tools/paywallkiller/cdx?url=${encodeURIComponent(targetUrl)}`, { method: 'GET' }); + return data.found ? data.waybackUrl : null; + } + + // Save via Browser (no-cors, fire-and-forget), dann CDX-Polling via Backend + async function saveAndPoll(targetUrl) { + // Save-Request abschießen (Browser, kein CORS-Response nötig) + try { + fetch(`https://web.archive.org/save/${targetUrl}`, { mode: 'no-cors' }).catch(() => {}); + } catch {} + + // CDX pollen via Backend-Proxy + for (let i = 0; i < 8; i++) { + await new Promise(r => setTimeout(r, 5000)); + setStatusMsg(`Warte auf Snapshot... (${(i + 1) * 5}s)`); + try { + const url = await searchWayback(targetUrl); + if (url) return url; + } catch {} + } + return null; + } + async function handleCheck() { if (!inputUrl.trim()) return; const targetUrl = getTargetUrl(); @@ -76,18 +63,16 @@ export default function PaywallKiller() { setWasCreated(false); try { - // Schritt 1: vorhandenen Snapshot suchen const existing = await searchWayback(targetUrl); if (existing) { setWaybackUrl(existing); setStatus('done'); return; } - - // Schritt 2: Kein Snapshot → neu erstellen + pollen + // Kein Snapshot → archivieren setStatus('saving'); - setStatusMsg('Kein Snapshot gefunden – archiviere jetzt...'); - const newUrl = await saveAndPollWayback(targetUrl, msg => setStatusMsg(msg)); + setStatusMsg('Kein Snapshot – archiviere jetzt...'); + const newUrl = await saveAndPoll(targetUrl); if (newUrl) { setWaybackUrl(newUrl); setWasCreated(true); @@ -185,7 +170,6 @@ export default function PaywallKiller() { - {/* Wayback-Ergebnis */} {isDone && (
@@ -209,19 +193,17 @@ export default function PaywallKiller() {
)} - {/* Kein Snapshot erstellt */} {status === 'notfound' && (
📭 - Archivierung fehlgeschlagen – {getDomain(inputUrl)} blockiert möglicherweise die Wayback Machine. + Kein Snapshot gefunden – {getDomain(inputUrl)} blockiert möglicherweise die Wayback Machine.
)} - {/* archive.ph – immer nach Suche anzeigen */} {hasResult && archivePhUrl && (
@@ -249,7 +231,7 @@ export default function PaywallKiller() {
SO FUNKTIONIERT ES
{[ ['1', 'URL einfügen (Long-Press → Einfügen)'], - ['2', 'Wayback Machine wird direkt im Browser durchsucht'], + ['2', 'Wayback Machine wird serverseitig durchsucht'], ['3', 'Falls kein Snapshot: Artikel wird automatisch archiviert'], ['4', 'Snapshot öffnen oder als PDF speichern'], ].map(([num, text]) => (