diff --git a/backend/src/tools/paywallkiller/routes.js b/backend/src/tools/paywallkiller/routes.js index e93be60..c09e32d 100644 --- a/backend/src/tools/paywallkiller/routes.js +++ b/backend/src/tools/paywallkiller/routes.js @@ -163,174 +163,122 @@ router.post('/check', authenticate, async (req, res) => { }); // ── POST /pdf ────────────────────────────────────────────────────────────── +// Lädt die Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF router.post('/pdf', authenticate, async (req, res) => { - const { archiveUrl } = req.body; - if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' }); - if (!archiveUrl.startsWith('https://web.archive.org/')) { - return res.status(400).json({ error: 'PDF nur für Wayback Machine URLs (web.archive.org) verfügbar' }); + const { archiveUrl, originalUrl } = req.body; + + // originalUrl bevorzugen (direkter Artikel ohne Wayback-Paywall) + // archiveUrl als Fallback (Wayback Machine) + const targetUrl = originalUrl || archiveUrl; + if (!targetUrl || typeof targetUrl !== 'string') { + return res.status(400).json({ error: 'URL fehlt' }); } - let browser; - const tmpFile = path.join('/tmp', `pwk_${crypto.randomBytes(8).toString('hex')}.pdf`); + const tmpFile = require('path').join('/tmp', `pwk_${require('crypto').randomBytes(8).toString('hex')}.pdf`); + let browser = null; try { - let puppeteer; - try { puppeteer = require('puppeteer-core'); } - catch { try { puppeteer = require('puppeteer'); } - catch { return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar' }); } } + const puppeteer = require('puppeteer-core'); - // System-Chromium aus Alpine nutzen (via CHROMIUM_PATH env oder bekannte Pfade) - const executablePath = process.env.CHROMIUM_PATH - || (() => { - const candidates = [ - '/usr/bin/chromium-browser', - '/usr/bin/chromium', - '/usr/bin/google-chrome', - '/usr/bin/google-chrome-stable', - ]; - const fs2 = require('fs'); - return candidates.find(p => { try { fs2.accessSync(p); return true; } catch { return false; } }) - || '/usr/bin/chromium-browser'; - })(); + const chromiumPaths = [ + process.env.CHROMIUM_PATH, + '/usr/bin/chromium-browser', + '/usr/bin/chromium', + '/usr/bin/google-chrome', + ].filter(Boolean); - const launchOptions = { + let executablePath = null; + for (const p of chromiumPaths) { + if (require('fs').existsSync(p)) { executablePath = p; break; } + } + if (!executablePath) { + return res.status(500).json({ error: 'Chromium nicht gefunden' }); + } + + browser = await puppeteer.launch({ executablePath, headless: true, - args: [ - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage', - '--disable-gpu', - '--disable-software-rasterizer', - '--single-process', - '--no-zygote', - ], - defaultViewport: { width: 1280, height: 900 }, - }; + args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--single-process', '--no-zygote'], + timeout: 30000, + }); - browser = await puppeteer.launch(launchOptions); const page = await browser.newPage(); - await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'); - await page.goto(archiveUrl, { waitUntil: 'networkidle2', timeout: 30000 }); - await page.pdf({ path: tmpFile, format: 'A4', printBackground: true, - margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' } }); - await browser.close(); browser = null; + await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'); + await page.setViewport({ width: 1280, height: 900 }); + + // Seite laden – networkidle0 damit JS vollständig läuft (Artikel wird geladen) + await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 }); + + // Kurz warten damit Artikel-Content gerendert ist + await new Promise(r => setTimeout(r, 1500)); + + // Paywall-Overlays, Cookie-Banner, Sticky-Header ausblenden + await page.addStyleTag({ content: ` + /* Generische Paywall/Overlay-Selektoren */ + [class*="paywall"], [class*="Paywall"], + [class*="piano"], [class*="Piano"], + [id*="paywall"], [id*="piano"], + [class*="subscription"], [class*="subscribe"], + [class*="overlay"], [class*="modal"], + [class*="cookie"], [class*="Cookie"], + [class*="consent"], [class*="Consent"], + [class*="gdpr"], [class*="GDPR"], + [class*="sticky-header"], [class*="stickyHeader"], + [class*="fixed-header"], [class*="fixedHeader"], + /* WAZ/Funke spezifisch */ + .tp-modal, .tp-backdrop, #tp-container, + .article__paywall, .paywall-container, + .funke-piano, .piano-checkout, + [data-testid*="paywall"], [data-testid*="piano"], + /* Cookie-Banner */ + #usercentrics-root, .uc-banner, + #onetrust-consent-sdk, .onetrust-pc-dark-filter, + /* Sticky-Elemente */ + .sticky, [style*="position: sticky"], [style*="position:sticky"], + /* Allgemeine Werbung */ + [class*="ad-"], [class*="-ad-"], [id*="ad-"], + [class*="advertisement"] { + display: none !important; + visibility: hidden !important; + opacity: 0 !important; + } + /* Body-Overflow wiederherstellen falls Paywall ihn gesperrt hat */ + body, html { + overflow: visible !important; + height: auto !important; + } + /* Blur auf Artikel-Text entfernen */ + * { filter: none !important; } + ` }); + + // Wayback-Toolbar auch ausblenden falls Wayback-URL + if (targetUrl.includes('web.archive.org')) { + await page.addStyleTag({ content: '#wm-ipp-base, #wm-ipp, #donato { display: none !important; }' }); + } + + await page.pdf({ + path: tmpFile, + format: 'A4', + printBackground: true, + margin: { top: '12mm', bottom: '12mm', left: '12mm', right: '12mm' }, + }); + + await browser.close(); + browser = null; res.setHeader('Content-Type', 'application/pdf'); - res.setHeader('Content-Disposition', `attachment; filename="archiv_${Date.now()}.pdf"`); - const stream = fs.createReadStream(tmpFile); + res.setHeader('Content-Disposition', `attachment; filename="artikel_${Date.now()}.pdf"`); + const stream = require('fs').createReadStream(tmpFile); stream.pipe(res); - stream.on('end', () => fs.unlink(tmpFile, () => {})); - stream.on('error', () => fs.unlink(tmpFile, () => {})); + stream.on('end', () => require('fs').unlink(tmpFile, () => {})); + stream.on('error', () => require('fs').unlink(tmpFile, () => {})); + } catch (err) { if (browser) try { await browser.close(); } catch {} - fs.unlink(tmpFile, () => {}); + require('fs').unlink(tmpFile, () => {}); + console.error('[paywallkiller] PDF error:', err.message); return res.status(500).json({ error: 'PDF-Erstellung fehlgeschlagen: ' + err.message }); } }); - -// ── GET /api/tools/paywallkiller/wayback?url=... ────────────────────────── -// 1. Sucht vorhandenen Snapshot via CDX API -// 2. Falls keiner → erstellt neuen Snapshot via Save API -router.get('/wayback', authenticate, async (req, res) => { - const { url } = req.query; - if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url Parameter fehlt' }); - - let targetUrl = url.trim(); - if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; - - // ── Schritt 1: Vorhandenen Snapshot suchen ────────────────────────────── - try { - const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`; - const cdxResult = await fetchPage(cdxUrl); - if (cdxResult.statusCode === 200 && cdxResult.body) { - let data; - try { data = JSON.parse(cdxResult.body); } catch { data = null; } - if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) { - const timestamp = data[1][0]; - return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${timestamp}/${targetUrl}`, created: false }); - } - } - } catch { /* weiter zu Save API */ } - - // ── Schritt 2: Keinen Snapshot gefunden → neu erstellen ───────────────── - return res.json({ found: false, canSave: true }); -}); - -// ── POST /api/tools/paywallkiller/wayback-save ──────────────────────────── -// Schießt Save-Request ab (fire-and-forget), pollt dann CDX auf Ergebnis -router.post('/wayback-save', authenticate, async (req, res) => { - const { url } = req.body; - if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url fehlt' }); - - let targetUrl = url.trim(); - if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; - - const https = require('https'); - - // ── Hilfsfunktion: CDX prüfen ────────────────────────────────────────── - async function checkCdx() { - const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`; - try { - const r = await fetchPage(cdxUrl); - if (r.statusCode === 200 && r.body) { - const data = JSON.parse(r.body); - if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) { - return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`; - } - } - } catch {} - return null; - } - - // ── Save-Request fire-and-forget (kein Warten auf Body) ─────────────── - function fireSaveRequest() { - return new Promise((resolve) => { - const saveUrl = new URL(`https://web.archive.org/save/${targetUrl}`); - const req = https.get({ - hostname: saveUrl.hostname, - path: saveUrl.pathname + saveUrl.search, - headers: { - 'User-Agent': 'Mozilla/5.0 (compatible; WaybackSave/1.0)', - 'Accept': 'text/html', - }, - timeout: 8000, - }, (res) => { - // Nur Headers lesen, Body sofort verwerfen - const location = res.headers.location || ''; - res.resume(); // Body ignorieren - resolve(location); - }); - req.on('error', () => resolve('')); - req.on('timeout', () => { req.destroy(); resolve(''); }); - }); - } - - try { - // Save abschießen - const location = await fireSaveRequest(); - - // Direkt aus Location-Header auslesen wenn vorhanden - const directMatch = location.match(/\/web\/(\d{14})\//); - if (directMatch) { - const waybackUrl = location.startsWith('http') ? location : `https://web.archive.org${location}`; - return res.json({ success: true, waybackUrl, timestamp: directMatch[1] }); - } - - // Polling: bis zu 6x alle 5s CDX abfragen (max 30s warten) - for (let i = 0; i < 6; i++) { - await new Promise(r => setTimeout(r, 5000)); - const found = await checkCdx(); - if (found) { - return res.json({ success: true, waybackUrl: found }); - } - } - - return res.status(502).json({ error: 'Archivierung läuft noch oder wurde blockiert. Versuche es in einer Minute erneut.' }); - } catch (err) { - return res.status(500).json({ error: 'Save fehlgeschlagen: ' + err.message }); - } -}); - -module.exports = router; +module.exports \ No newline at end of file diff --git a/frontend/src/tools/paywallkiller.jsx b/frontend/src/tools/paywallkiller.jsx index 7aa87fb..896c543 100644 --- a/frontend/src/tools/paywallkiller.jsx +++ b/frontend/src/tools/paywallkiller.jsx @@ -81,7 +81,7 @@ export default function PaywallKiller() { 'Content-Type': 'application/json', ...(token ? { Authorization: `Bearer ${token}` } : {}), }, - body: JSON.stringify({ archiveUrl: waybackUrl }), + body: JSON.stringify({ archiveUrl: waybackUrl, originalUrl: getTargetUrl() }), }); if (!res.ok) { const data = await res.json().catch(() => ({}));