feat: iframe CAPTCHA-Flow, PDF von archive.ph Snapshot-URL

backend/src/tools/paywallkiller/routes.js

@@ -6,48 +6,29 @@ const path = require('path');
 const fs = require('fs');
 const crypto = require('crypto');
 
-// ── Einfacher HTTPS GET Helper ─────────────────────────────────────────────
-function httpsGet(url, timeoutMs = 20000) {
-  return new Promise((resolve, reject) => {
-    const req = https.get(url, {
-      headers: {
-        'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)',
-        'Accept': '*/*',
-      },
-      timeout: timeoutMs,
-    }, (res) => {
-      let body = '';
-      res.setEncoding('utf8');
-      res.on('data', d => { if (body.length < 200000) body += d; });
-      res.on('end', () => resolve({ status: res.statusCode, body, headers: res.headers }));
-      res.on('error', reject);
-    });
-    req.on('error', reject);
-    req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
-  });
-}
-
 // ── GET /api/tools/paywallkiller/cdx?url=... ──────────────────────────────
-// Proxy für Wayback CDX API – löst CORS/Origin-Probleme der PWA
+// Wayback CDX Proxy (für spätere Verwendung)
 router.get('/cdx', authenticate, async (req, res) => {
   const { url } = req.query;
   if (!url) return res.status(400).json({ error: 'url fehlt' });
-
   let targetUrl = url.trim();
   if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
-
   const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
-
   try {
-    const result = await httpsGet(cdxUrl);
-    if (result.status !== 200) return res.status(502).json({ error: `CDX Status: ${result.status}` });
-
-    let data;
-    try { data = JSON.parse(result.body); } catch { return res.status(502).json({ error: 'Ungültige CDX-Antwort' }); }
-
-    if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) {
-      return res.json({ found: false });
-    }
+    const result = await new Promise((resolve, reject) => {
+      const req2 = https.get(cdxUrl, { headers: { 'User-Agent': 'Mozilla/5.0' }, timeout: 20000 }, (res2) => {
+        let body = '';
+        res2.setEncoding('utf8');
+        res2.on('data', d => body += d);
+        res2.on('end', () => resolve({ status: res2.statusCode, body }));
+        res2.on('error', reject);
+      });
+      req2.on('error', reject);
+      req2.on('timeout', () => { req2.destroy(); reject(new Error('Timeout')); });
+    });
+    if (result.status !== 200) return res.status(502).json({ error: `CDX ${result.status}` });
+    const data = JSON.parse(result.body);
+    if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) return res.json({ found: false });
     return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` });
   } catch (err) {
     return res.status(500).json({ error: err.message });
@@ -55,13 +36,11 @@ router.get('/cdx', authenticate, async (req, res) => {
 });
 
 // ── POST /api/tools/paywallkiller/pdf ─────────────────────────────────────
-// Lädt via 12ft.io (Paywall-Bypasser, kein CAPTCHA) → PDF
+// Lädt archive.ph Snapshot-URL mit Puppeteer → PDF
+// archive.ph Snapshot-URLs (hash-basiert) benötigen kein CAPTCHA
 router.post('/pdf', authenticate, async (req, res) => {
-  const { originalUrl, archiveUrl } = req.body;
-  if (!originalUrl && !archiveUrl) return res.status(400).json({ error: 'URL fehlt' });
-  // 12ft.io umgeht Paywalls indem es die Googlebot-Version abruft
-  const targetUrl = `https://12ft.io/proxy?q=${encodeURIComponent(originalUrl || archiveUrl)}`;
-  if (!targetUrl || typeof targetUrl !== 'string') return res.status(400).json({ error: 'URL fehlt' });
+  const { archiveUrl } = req.body;
+  if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' });
 
   const tmpFile = path.join('/tmp', `pwk_${crypto.randomBytes(8).toString('hex')}.pdf`);
   let browser = null;
@@ -80,50 +59,46 @@ router.post('/pdf', authenticate, async (req, res) => {
     browser = await puppeteer.launch({
       executablePath,
       headless: true,
-      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--single-process', '--no-zygote'],
+      args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage',
+             '--disable-gpu', '--single-process', '--no-zygote'],
       timeout: 30000,
     });
 
     const page = await browser.newPage();
     await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36');
     await page.setViewport({ width: 1280, height: 900 });
-    await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 });
 
-    // UI-Störelemente entfernen (Wayback-Toolbar, Cookie-Banner, Werbung)
-    // Der Artikel-Text ist im Wayback-Snapshot vollständig vorhanden
-    await page.evaluate(() => {
-      const remove = [
-        // Wayback Machine Toolbar
-        '#wm-ipp-base', '#wm-ipp', '#donato', '#wm-ipp-print',
-        // Cookie/Consent Banner
-        '#usercentrics-root', '.uc-banner', '#usercentrics-cmp-ui',
-        '#onetrust-consent-sdk', '.onetrust-pc-dark-filter',
-        '[id*="cookie"]', '[class*="cookie-banner"]', '[class*="cookieBanner"]',
-        '[class*="consent-banner"]', '[id*="consent-banner"]',
-        // Sticky Header/Footer
-        'header[class*="sticky"]', '[class*="sticky-header"]',
-        '[class*="stickyHeader"]', '[class*="fixed-header"]',
-        // Werbung
-        '[class*="advertisement"]', '[class*="adslot"]', '[id^="div-gpt-ad"]',
-        // Newsletter-Popup, Abo-Hinweise (nicht die Paywall selbst)
-        '[class*="newsletter"]', '[class*="teaser-abo"]',
-      ];
-      remove.forEach(sel => {
-        try { document.querySelectorAll(sel).forEach(el => el.remove()); } catch {}
-      });
+    // archive.ph laden – Snapshot-URL direkt (kein CAPTCHA bei direkten Snapshot-URLs)
+    await page.goto(archiveUrl, { waitUntil: 'networkidle0', timeout: 30000 });
 
-      // body overflow freigeben
-      document.body.style.overflow = 'visible';
-      document.documentElement.style.overflow = 'visible';
-      document.body.style.height = 'auto';
+    // Prüfen ob CAPTCHA erschienen ist
+    const hasCaptcha = await page.evaluate(() => {
+      return document.title.includes('One more step') ||
+             document.body.innerText.includes('complete a CAPTCHA') ||
+             document.body.innerText.includes('security check');
     });
 
-    await new Promise(r => setTimeout(r, 300));
+    if (hasCaptcha) {
+      await browser.close(); browser = null;
+      return res.status(403).json({ error: 'CAPTCHA – bitte im Browser das Archiv öffnen und CAPTCHA lösen, dann erneut versuchen.' });
+    }
+
+    // UI-Störelemente entfernen
+    await page.evaluate(() => {
+      ['#wm-ipp-base','#wm-ipp','#donato',
+       '#usercentrics-root','.uc-banner',
+       '#onetrust-consent-sdk','.onetrust-pc-dark-filter',
+       '[id*="cookie"]','[class*="cookie-banner"]'].forEach(sel => {
+        try { document.querySelectorAll(sel).forEach(el => el.remove()); } catch {}
+      });
+      document.body.style.overflow = 'visible';
+    });
+
+    await new Promise(r => setTimeout(r, 500));
 
     await page.pdf({
       path: tmpFile, format: 'A4', printBackground: true,
-      printBackground: true,
-      margin: { top: '12mm', bottom: '12mm', left: '12mm', right: '12mm' },
+      margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' },
     });
     await browser.close(); browser = null;
 
@@ -136,7 +111,7 @@ router.post('/pdf', authenticate, async (req, res) => {
   } catch (err) {
     if (browser) try { await browser.close(); } catch {}
     fs.unlink(tmpFile, () => {});
-    return res.status(500).json({ error: 'PDF-Erstellung fehlgeschlagen: ' + err.message });
+    return res.status(500).json({ error: 'PDF fehlgeschlagen: ' + err.message });
   }
 });