From 6eb9e1cc723f56339270a8dfbd6c28feb8cf1ed1 Mon Sep 17 00:00:00 2001 From: Dicken Date: Fri, 12 Jun 2026 12:34:06 +0200 Subject: [PATCH] fix: alle Routes wiederhergestellt (wayback, wayback-save, pdf) --- backend/src/tools/paywallkiller/routes.js | 277 ++++++++-------------- 1 file changed, 98 insertions(+), 179 deletions(-) diff --git a/backend/src/tools/paywallkiller/routes.js b/backend/src/tools/paywallkiller/routes.js index 8b9147b..ad21f0b 100644 --- a/backend/src/tools/paywallkiller/routes.js +++ b/backend/src/tools/paywallkiller/routes.js @@ -12,35 +12,24 @@ function fetchPage(url, opts = {}) { return new Promise((resolve, reject) => { if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects')); const mod = url.startsWith('https') ? https : http; - - // Cookies aus jar zusammenbauen const cookieHeader = opts.cookieJar && Object.keys(opts.cookieJar).length ? Object.entries(opts.cookieJar).map(([k,v]) => `${k}=${v}`).join('; ') : undefined; - const headers = { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8', 'Accept-Encoding': 'identity', 'Connection': 'keep-alive', - 'Sec-Fetch-Dest': 'document', - 'Sec-Fetch-Mode': 'navigate', - 'Sec-Fetch-Site': opts.referer ? 'same-origin' : 'none', - 'Upgrade-Insecure-Requests': '1', ...(opts.referer ? { 'Referer': opts.referer } : {}), ...(cookieHeader ? { 'Cookie': cookieHeader } : {}), }; - const req = mod.get(url, { headers, timeout: 15000 }, (res) => { - // Cookies sammeln const jar = opts.cookieJar || {}; - const setCookies = res.headers['set-cookie'] || []; - for (const c of setCookies) { + for (const c of (res.headers['set-cookie'] || [])) { const m = c.match(/^([^=]+)=([^;]*)/); if (m) jar[m[1].trim()] = m[2].trim(); } - if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) { let loc = res.headers.location; if (!loc.startsWith('http')) { @@ -48,10 +37,8 @@ function fetchPage(url, opts = {}) { catch { return reject(new Error('Ungültige Redirect-URL')); } } res.resume(); - return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1, cookieJar: jar, referer: url }) - .then(resolve).catch(reject); + return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1, cookieJar: jar, referer: url }).then(resolve).catch(reject); } - let body = ''; res.setEncoding('utf8'); res.on('data', d => { if (body.length < 150000) body += d; }); @@ -63,137 +50,104 @@ function fetchPage(url, opts = {}) { }); } -// ── Session initialisieren (Startseite besuchen für Cookies) ────────────── -async function initSession() { - const result = await fetchPage('https://archive.ph/', { cookieJar: {} }); - return result.cookieJar; -} +// ── GET /api/tools/paywallkiller/wayback?url=... ────────────────────────── +// Sucht vorhandenen Wayback-Snapshot via CDX API +router.get('/wayback', authenticate, async (req, res) => { + const { url } = req.query; + if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url Parameter fehlt' }); + let targetUrl = url.trim(); + if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; -// ── Snapshot-URLs aus archive.is HTML extrahieren ───────────────────────── -function extractSnapshotUrls(html) { - const urls = []; - // Format: href="/HASH" oder href="https://archive.XX/HASH" - // Hash = 4-10 alphanumerische Zeichen, keine bekannten Pfade - const blacklist = new Set(['newest','oldest','submit','search','about','rss','api','timemap','http','www.']); - - const rel = /href="\/([a-zA-Z0-9]{4,10})(?:"|\/|\?)/g; - let m; - while ((m = rel.exec(html)) !== null) { - const h = m[1]; - if (!blacklist.has(h) && !/^\d+$/.test(h)) { - urls.push(`https://archive.ph/${h}`); + try { + const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`; + const r = await fetchPage(cdxUrl); + if (r.statusCode === 200 && r.body) { + let data; + try { data = JSON.parse(r.body); } catch { data = null; } + if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) { + return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` }); + } } + return res.json({ found: false }); + } catch (err) { + return res.status(500).json({ error: 'Fehler: ' + err.message }); } +}); - const abs = /href="(https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10})(?:"|\/|\?)/g; - while ((m = abs.exec(html)) !== null) { - const h = m[1].split('/').pop(); - if (!blacklist.has(h)) urls.push(m[1]); - } - - return [...new Set(urls)]; -} - -// ── Archivsuche mit Session-Cookie ──────────────────────────────────────── -async function findArchiveSnapshot(targetUrl) { - // Schritt 1: Session-Cookie holen - let cookieJar; - try { cookieJar = await initSession(); } - catch { cookieJar = {}; } - - // Schritt 2: Suche mit Cookie und Referer - const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`; - const result = await fetchPage(searchUrl, { - cookieJar, - referer: 'https://archive.ph/', - }); - - if (result.statusCode === 429) { - throw new Error('archive.is Rate-Limit erreicht – bitte kurz warten und nochmal versuchen'); - } - if (result.statusCode !== 200 || !result.body) return null; - - const snapshots = extractSnapshotUrls(result.body); - return snapshots.length > 0 ? snapshots[0] : null; -} - -// ── DEBUG endpoint ───────────────────────────────────────────────────────── -router.post('/debug', authenticate, async (req, res) => { +// ── POST /api/tools/paywallkiller/wayback-save ──────────────────────────── +// Schießt Save-Request ab (fire-and-forget), pollt dann CDX auf Ergebnis +router.post('/wayback-save', authenticate, async (req, res) => { const { url } = req.body; - if (!url) return res.status(400).json({ error: 'URL fehlt' }); + if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url fehlt' }); let targetUrl = url.trim(); if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; - try { - let cookieJar; - try { cookieJar = await initSession(); } catch(e) { cookieJar = {}; } + async function checkCdx() { + const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`; + try { + const r = await fetchPage(cdxUrl); + if (r.statusCode === 200 && r.body) { + const data = JSON.parse(r.body); + if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) { + return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`; + } + } + } catch {} + return null; + } - const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`; - const result = await fetchPage(searchUrl, { cookieJar, referer: 'https://archive.ph/' }); - const extracted = extractSnapshotUrls(result.body); - - res.json({ - statusCode: result.statusCode, - finalUrl: result.finalUrl, - cookiesObtained: Object.keys(cookieJar), - bodyLength: result.body.length, - bodyPreview: result.body.substring(0, 3000), - allHrefs: (result.body.match(/href="[^"]{1,80}"/g) || []).slice(0, 50), - extractedSnapshots: extracted, + function fireSaveRequest() { + return new Promise((resolve) => { + const savePath = `/save/${targetUrl}`; + const req = https.get({ + hostname: 'web.archive.org', + path: savePath, + headers: { 'User-Agent': 'Mozilla/5.0 (compatible; WaybackSave/1.0)', 'Accept': 'text/html' }, + timeout: 8000, + }, (res) => { + const location = res.headers.location || ''; + res.resume(); + resolve(location); + }); + req.on('error', () => resolve('')); + req.on('timeout', () => { req.destroy(); resolve(''); }); }); - } catch (err) { - res.status(500).json({ error: err.message }); } -}); - -// ── POST /check ──────────────────────────────────────────────────────────── -router.post('/check', authenticate, async (req, res) => { - const { url } = req.body; - if (!url || typeof url !== 'string') return res.status(400).json({ error: 'URL fehlt' }); - let targetUrl = url.trim(); - if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; try { - const snapshotUrl = await findArchiveSnapshot(targetUrl); - if (snapshotUrl) return res.json({ archiveUrl: snapshotUrl, type: 'newest' }); - return res.status(404).json({ error: 'Kein Archiv für diese URL gefunden' }); + const location = await fireSaveRequest(); + const directMatch = location.match(/\/web\/(\d{14})\//); + if (directMatch) { + const waybackUrl = location.startsWith('http') ? location : `https://web.archive.org${location}`; + return res.json({ success: true, waybackUrl }); + } + // Polling: bis zu 6x alle 5s + for (let i = 0; i < 6; i++) { + await new Promise(r => setTimeout(r, 5000)); + const found = await checkCdx(); + if (found) return res.json({ success: true, waybackUrl: found }); + } + return res.status(502).json({ error: 'Archivierung läuft noch oder wurde blockiert. Versuche es in einer Minute erneut.' }); } catch (err) { - return res.status(500).json({ error: err.message }); + return res.status(500).json({ error: 'Save fehlgeschlagen: ' + err.message }); } }); -// ── POST /pdf ────────────────────────────────────────────────────────────── -// Lädt die Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF +// ── POST /api/tools/paywallkiller/pdf ───────────────────────────────────── +// Lädt Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF router.post('/pdf', authenticate, async (req, res) => { const { archiveUrl, originalUrl } = req.body; - - // originalUrl bevorzugen (direkter Artikel ohne Wayback-Paywall) - // archiveUrl als Fallback (Wayback Machine) const targetUrl = originalUrl || archiveUrl; - if (!targetUrl || typeof targetUrl !== 'string') { - return res.status(400).json({ error: 'URL fehlt' }); - } + if (!targetUrl || typeof targetUrl !== 'string') return res.status(400).json({ error: 'URL fehlt' }); - const tmpFile = require('path').join('/tmp', `pwk_${require('crypto').randomBytes(8).toString('hex')}.pdf`); + const tmpFile = path.join('/tmp', `pwk_${crypto.randomBytes(8).toString('hex')}.pdf`); let browser = null; try { const puppeteer = require('puppeteer-core'); - - const chromiumPaths = [ - process.env.CHROMIUM_PATH, - '/usr/bin/chromium-browser', - '/usr/bin/chromium', - '/usr/bin/google-chrome', - ].filter(Boolean); - - let executablePath = null; - for (const p of chromiumPaths) { - if (require('fs').existsSync(p)) { executablePath = p; break; } - } - if (!executablePath) { - return res.status(500).json({ error: 'Chromium nicht gefunden' }); - } + const candidates = [process.env.CHROMIUM_PATH, '/usr/bin/chromium-browser', '/usr/bin/chromium', '/usr/bin/google-chrome'].filter(Boolean); + const executablePath = candidates.find(p => { try { fs.accessSync(p); return true; } catch { return false; } }); + if (!executablePath) return res.status(500).json({ error: 'Chromium nicht gefunden' }); browser = await puppeteer.launch({ executablePath, @@ -205,78 +159,43 @@ router.post('/pdf', authenticate, async (req, res) => { const page = await browser.newPage(); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'); await page.setViewport({ width: 1280, height: 900 }); - - // Seite laden – networkidle0 damit JS vollständig läuft (Artikel wird geladen) await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 }); - - // Kurz warten damit Artikel-Content gerendert ist await new Promise(r => setTimeout(r, 1500)); - // Paywall-Overlays, Cookie-Banner, Sticky-Header ausblenden await page.addStyleTag({ content: ` - /* Generische Paywall/Overlay-Selektoren */ - [class*="paywall"], [class*="Paywall"], - [class*="piano"], [class*="Piano"], - [id*="paywall"], [id*="piano"], - [class*="subscription"], [class*="subscribe"], - [class*="overlay"], [class*="modal"], - [class*="cookie"], [class*="Cookie"], - [class*="consent"], [class*="Consent"], - [class*="gdpr"], [class*="GDPR"], - [class*="sticky-header"], [class*="stickyHeader"], - [class*="fixed-header"], [class*="fixedHeader"], - /* WAZ/Funke spezifisch */ - .tp-modal, .tp-backdrop, #tp-container, - .article__paywall, .paywall-container, - .funke-piano, .piano-checkout, - [data-testid*="paywall"], [data-testid*="piano"], - /* Cookie-Banner */ - #usercentrics-root, .uc-banner, - #onetrust-consent-sdk, .onetrust-pc-dark-filter, - /* Sticky-Elemente */ - .sticky, [style*="position: sticky"], [style*="position:sticky"], - /* Allgemeine Werbung */ - [class*="ad-"], [class*="-ad-"], [id*="ad-"], - [class*="advertisement"] { - display: none !important; - visibility: hidden !important; - opacity: 0 !important; - } - /* Body-Overflow wiederherstellen falls Paywall ihn gesperrt hat */ - body, html { - overflow: visible !important; - height: auto !important; - } - /* Blur auf Artikel-Text entfernen */ - * { filter: none !important; } + [class*="paywall"],[class*="Paywall"],[id*="paywall"], + [class*="piano"],[class*="Piano"],[id*="piano"], + [class*="subscription"],[class*="subscribe"], + [class*="overlay"],[class*="modal"], + [class*="cookie"],[class*="Cookie"], + [class*="consent"],[class*="gdpr"], + .tp-modal,.tp-backdrop,#tp-container, + .article__paywall,.funke-piano,.piano-checkout, + [data-testid*="paywall"],[data-testid*="piano"], + #usercentrics-root,.uc-banner, + #onetrust-consent-sdk,.onetrust-pc-dark-filter, + #wm-ipp-base,#wm-ipp,#donato, + [class*="ad-"],[class*="-ad-"],[class*="advertisement"] + { display:none !important; visibility:hidden !important; } + body,html { overflow:visible !important; height:auto !important; } + * { filter:none !important; } ` }); - // Wayback-Toolbar auch ausblenden falls Wayback-URL - if (targetUrl.includes('web.archive.org')) { - await page.addStyleTag({ content: '#wm-ipp-base, #wm-ipp, #donato { display: none !important; }' }); - } - await page.pdf({ - path: tmpFile, - format: 'A4', - printBackground: true, + path: tmpFile, format: 'A4', printBackground: true, margin: { top: '12mm', bottom: '12mm', left: '12mm', right: '12mm' }, }); - - await browser.close(); - browser = null; + await browser.close(); browser = null; res.setHeader('Content-Type', 'application/pdf'); res.setHeader('Content-Disposition', `attachment; filename="artikel_${Date.now()}.pdf"`); - const stream = require('fs').createReadStream(tmpFile); + const stream = fs.createReadStream(tmpFile); stream.pipe(res); - stream.on('end', () => require('fs').unlink(tmpFile, () => {})); - stream.on('error', () => require('fs').unlink(tmpFile, () => {})); - + stream.on('end', () => fs.unlink(tmpFile, () => {})); + stream.on('error', () => fs.unlink(tmpFile, () => {})); } catch (err) { if (browser) try { await browser.close(); } catch {} - require('fs').unlink(tmpFile, () => {}); - console.error('[paywallkiller] PDF error:', err.message); + fs.unlink(tmpFile, () => {}); return res.status(500).json({ error: 'PDF-Erstellung fehlgeschlagen: ' + err.message }); } });