feat: PDF lädt Original-URL + blendet Paywall-Overlays per CSS aus

backend/src/tools/paywallkiller/routes.js

@@ -163,174 +163,122 @@ router.post('/check', authenticate, async (req, res) => {
 });
 
 // ── POST /pdf ──────────────────────────────────────────────────────────────
+// Lädt die Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF
 router.post('/pdf', authenticate, async (req, res) => {
-  const { archiveUrl } = req.body;
+  const { archiveUrl, originalUrl } = req.body;
-  if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' });
+
-  if (!archiveUrl.startsWith('https://web.archive.org/')) {
+  // originalUrl bevorzugen (direkter Artikel ohne Wayback-Paywall)
-    return res.status(400).json({ error: 'PDF nur für Wayback Machine URLs (web.archive.org) verfügbar' });
+  // archiveUrl als Fallback (Wayback Machine)
+  const targetUrl = originalUrl || archiveUrl;
+  if (!targetUrl || typeof targetUrl !== 'string') {
+    return res.status(400).json({ error: 'URL fehlt' });
   }
 
-  let browser;
+  const tmpFile = require('path').join('/tmp', `pwk_${require('crypto').randomBytes(8).toString('hex')}.pdf`);
-  const tmpFile = path.join('/tmp', `pwk_${crypto.randomBytes(8).toString('hex')}.pdf`);
+  let browser = null;
 
   try {
-    let puppeteer;
+    const puppeteer = require('puppeteer-core');
-    try { puppeteer = require('puppeteer-core'); }
-    catch { try { puppeteer = require('puppeteer'); }
-    catch { return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar' }); } }
 
-    // System-Chromium aus Alpine nutzen (via CHROMIUM_PATH env oder bekannte Pfade)
+    const chromiumPaths = [
-    const executablePath = process.env.CHROMIUM_PATH
+      process.env.CHROMIUM_PATH,
-      || (() => {
-          const candidates = [
       '/usr/bin/chromium-browser',
       '/usr/bin/chromium',
       '/usr/bin/google-chrome',
-            '/usr/bin/google-chrome-stable',
+    ].filter(Boolean);
-          ];
-          const fs2 = require('fs');
-          return candidates.find(p => { try { fs2.accessSync(p); return true; } catch { return false; } })
-            || '/usr/bin/chromium-browser';
-        })();
 
-    const launchOptions = {
+    let executablePath = null;
+    for (const p of chromiumPaths) {
+      if (require('fs').existsSync(p)) { executablePath = p; break; }
+    }
+    if (!executablePath) {
+      return res.status(500).json({ error: 'Chromium nicht gefunden' });
+    }
+
+    browser = await puppeteer.launch({
       executablePath,
       headless: true,
-      args: [
+      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--single-process', '--no-zygote'],
-        '--no-sandbox',
+      timeout: 30000,
-        '--disable-setuid-sandbox',
+    });
-        '--disable-dev-shm-usage',
-        '--disable-gpu',
-        '--disable-software-rasterizer',
-        '--single-process',
-        '--no-zygote',
-      ],
-      defaultViewport: { width: 1280, height: 900 },
-    };
 
-    browser = await puppeteer.launch(launchOptions);
     const page = await browser.newPage();
-    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
+    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36');
-    await page.goto(archiveUrl, { waitUntil: 'networkidle2', timeout: 30000 });
+    await page.setViewport({ width: 1280, height: 900 });
-    await page.pdf({ path: tmpFile, format: 'A4', printBackground: true,
+
-      margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' } });
+    // Seite laden – networkidle0 damit JS vollständig läuft (Artikel wird geladen)
-    await browser.close(); browser = null;
+    await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 });
+
+    // Kurz warten damit Artikel-Content gerendert ist
+    await new Promise(r => setTimeout(r, 1500));
+
+    // Paywall-Overlays, Cookie-Banner, Sticky-Header ausblenden
+    await page.addStyleTag({ content: `
+      /* Generische Paywall/Overlay-Selektoren */
+      [class*="paywall"], [class*="Paywall"],
+      [class*="piano"], [class*="Piano"],
+      [id*="paywall"], [id*="piano"],
+      [class*="subscription"], [class*="subscribe"],
+      [class*="overlay"], [class*="modal"],
+      [class*="cookie"], [class*="Cookie"],
+      [class*="consent"], [class*="Consent"],
+      [class*="gdpr"], [class*="GDPR"],
+      [class*="sticky-header"], [class*="stickyHeader"],
+      [class*="fixed-header"], [class*="fixedHeader"],
+      /* WAZ/Funke spezifisch */
+      .tp-modal, .tp-backdrop, #tp-container,
+      .article__paywall, .paywall-container,
+      .funke-piano, .piano-checkout,
+      [data-testid*="paywall"], [data-testid*="piano"],
+      /* Cookie-Banner */
+      #usercentrics-root, .uc-banner,
+      #onetrust-consent-sdk, .onetrust-pc-dark-filter,
+      /* Sticky-Elemente */
+      .sticky, [style*="position: sticky"], [style*="position:sticky"],
+      /* Allgemeine Werbung */
+      [class*="ad-"], [class*="-ad-"], [id*="ad-"],
+      [class*="advertisement"] { 
+        display: none !important; 
+        visibility: hidden !important;
+        opacity: 0 !important;
+      }
+      /* Body-Overflow wiederherstellen falls Paywall ihn gesperrt hat */
+      body, html {
+        overflow: visible !important;
+        height: auto !important;
+      }
+      /* Blur auf Artikel-Text entfernen */
+      * { filter: none !important; }
+    ` });
+
+    // Wayback-Toolbar auch ausblenden falls Wayback-URL
+    if (targetUrl.includes('web.archive.org')) {
+      await page.addStyleTag({ content: '#wm-ipp-base, #wm-ipp, #donato { display: none !important; }' });
+    }
+
+    await page.pdf({
+      path: tmpFile,
+      format: 'A4',
+      printBackground: true,
+      margin: { top: '12mm', bottom: '12mm', left: '12mm', right: '12mm' },
+    });
+
+    await browser.close();
+    browser = null;
 
     res.setHeader('Content-Type', 'application/pdf');
-    res.setHeader('Content-Disposition', `attachment; filename="archiv_${Date.now()}.pdf"`);
+    res.setHeader('Content-Disposition', `attachment; filename="artikel_${Date.now()}.pdf"`);
-    const stream = fs.createReadStream(tmpFile);
+    const stream = require('fs').createReadStream(tmpFile);
     stream.pipe(res);
-    stream.on('end', () => fs.unlink(tmpFile, () => {}));
+    stream.on('end', () => require('fs').unlink(tmpFile, () => {}));
-    stream.on('error', () => fs.unlink(tmpFile, () => {}));
+    stream.on('error', () => require('fs').unlink(tmpFile, () => {}));
+
   } catch (err) {
     if (browser) try { await browser.close(); } catch {}
-    fs.unlink(tmpFile, () => {});
+    require('fs').unlink(tmpFile, () => {});
+    console.error('[paywallkiller] PDF error:', err.message);
     return res.status(500).json({ error: 'PDF-Erstellung fehlgeschlagen: ' + err.message });
   }
 });
 
-
+module.exports
-// ── GET /api/tools/paywallkiller/wayback?url=... ──────────────────────────
-// 1. Sucht vorhandenen Snapshot via CDX API
-// 2. Falls keiner → erstellt neuen Snapshot via Save API
-router.get('/wayback', authenticate, async (req, res) => {
-  const { url } = req.query;
-  if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url Parameter fehlt' });
-
-  let targetUrl = url.trim();
-  if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
-
-  // ── Schritt 1: Vorhandenen Snapshot suchen ──────────────────────────────
-  try {
-    const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
-    const cdxResult = await fetchPage(cdxUrl);
-    if (cdxResult.statusCode === 200 && cdxResult.body) {
-      let data;
-      try { data = JSON.parse(cdxResult.body); } catch { data = null; }
-      if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
-        const timestamp = data[1][0];
-        return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${timestamp}/${targetUrl}`, created: false });
-      }
-    }
-  } catch { /* weiter zu Save API */ }
-
-  // ── Schritt 2: Keinen Snapshot gefunden → neu erstellen ─────────────────
-  return res.json({ found: false, canSave: true });
-});
-
-// ── POST /api/tools/paywallkiller/wayback-save ────────────────────────────
-// Schießt Save-Request ab (fire-and-forget), pollt dann CDX auf Ergebnis
-router.post('/wayback-save', authenticate, async (req, res) => {
-  const { url } = req.body;
-  if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url fehlt' });
-
-  let targetUrl = url.trim();
-  if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
-
-  const https = require('https');
-
-  // ── Hilfsfunktion: CDX prüfen ──────────────────────────────────────────
-  async function checkCdx() {
-    const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
-    try {
-      const r = await fetchPage(cdxUrl);
-      if (r.statusCode === 200 && r.body) {
-        const data = JSON.parse(r.body);
-        if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
-          return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`;
-        }
-      }
-    } catch {}
-    return null;
-  }
-
-  // ── Save-Request fire-and-forget (kein Warten auf Body) ───────────────
-  function fireSaveRequest() {
-    return new Promise((resolve) => {
-      const saveUrl = new URL(`https://web.archive.org/save/${targetUrl}`);
-      const req = https.get({
-        hostname: saveUrl.hostname,
-        path: saveUrl.pathname + saveUrl.search,
-        headers: {
-          'User-Agent': 'Mozilla/5.0 (compatible; WaybackSave/1.0)',
-          'Accept': 'text/html',
-        },
-        timeout: 8000,
-      }, (res) => {
-        // Nur Headers lesen, Body sofort verwerfen
-        const location = res.headers.location || '';
-        res.resume(); // Body ignorieren
-        resolve(location);
-      });
-      req.on('error', () => resolve(''));
-      req.on('timeout', () => { req.destroy(); resolve(''); });
-    });
-  }
-
-  try {
-    // Save abschießen
-    const location = await fireSaveRequest();
-
-    // Direkt aus Location-Header auslesen wenn vorhanden
-    const directMatch = location.match(/\/web\/(\d{14})\//);
-    if (directMatch) {
-      const waybackUrl = location.startsWith('http') ? location : `https://web.archive.org${location}`;
-      return res.json({ success: true, waybackUrl, timestamp: directMatch[1] });
-    }
-
-    // Polling: bis zu 6x alle 5s CDX abfragen (max 30s warten)
-    for (let i = 0; i < 6; i++) {
-      await new Promise(r => setTimeout(r, 5000));
-      const found = await checkCdx();
-      if (found) {
-        return res.json({ success: true, waybackUrl: found });
-      }
-    }
-
-    return res.status(502).json({ error: 'Archivierung läuft noch oder wurde blockiert. Versuche es in einer Minute erneut.' });
-  } catch (err) {
-    return res.status(500).json({ error: 'Save fehlgeschlagen: ' + err.message });
-  }
-});
-
-module.exports = router;

frontend/src/tools/paywallkiller.jsx

@@ -81,7 +81,7 @@ export default function PaywallKiller() {
           'Content-Type': 'application/json',
           ...(token ? { Authorization: `Bearer ${token}` } : {}),
         },
-        body: JSON.stringify({ archiveUrl: waybackUrl }),
+        body: JSON.stringify({ archiveUrl: waybackUrl, originalUrl: getTargetUrl() }),
       });
       if (!res.ok) {
         const data = await res.json().catch(() => ({}));