fix: CDX via Backend-Proxy /cdx statt Browser-fetch

backend/src/tools/paywallkiller/routes.js

@@ -1,12 +1,60 @@
 const express = require('express');
 const router = express.Router();
 const { authenticate } = require('../../middleware/auth');
+const https = require('https');
 const path = require('path');
 const fs = require('fs');
 const crypto = require('crypto');
 
+// ── Einfacher HTTPS GET Helper ─────────────────────────────────────────────
+function httpsGet(url, timeoutMs = 20000) {
+  return new Promise((resolve, reject) => {
+    const req = https.get(url, {
+      headers: {
+        'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)',
+        'Accept': '*/*',
+      },
+      timeout: timeoutMs,
+    }, (res) => {
+      let body = '';
+      res.setEncoding('utf8');
+      res.on('data', d => { if (body.length < 200000) body += d; });
+      res.on('end', () => resolve({ status: res.statusCode, body, headers: res.headers }));
+      res.on('error', reject);
+    });
+    req.on('error', reject);
+    req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
+  });
+}
+
+// ── GET /api/tools/paywallkiller/cdx?url=... ──────────────────────────────
+// Proxy für Wayback CDX API – löst CORS/Origin-Probleme der PWA
+router.get('/cdx', authenticate, async (req, res) => {
+  const { url } = req.query;
+  if (!url) return res.status(400).json({ error: 'url fehlt' });
+
+  let targetUrl = url.trim();
+  if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
+
+  const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
+
+  try {
+    const result = await httpsGet(cdxUrl);
+    if (result.status !== 200) return res.status(502).json({ error: `CDX Status: ${result.status}` });
+
+    let data;
+    try { data = JSON.parse(result.body); } catch { return res.status(502).json({ error: 'Ungültige CDX-Antwort' }); }
+
+    if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) {
+      return res.json({ found: false });
+    }
+    return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` });
+  } catch (err) {
+    return res.status(500).json({ error: err.message });
+  }
+});
+
 // ── POST /api/tools/paywallkiller/pdf ─────────────────────────────────────
-// Lädt Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF
 router.post('/pdf', authenticate, async (req, res) => {
   const { originalUrl, archiveUrl } = req.body;
   const targetUrl = originalUrl || archiveUrl;
@@ -39,7 +87,6 @@ router.post('/pdf', authenticate, async (req, res) => {
     await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 });
     await new Promise(r => setTimeout(r, 1500));
 
-    // Paywall-Overlays, Cookie-Banner, Blur-Filter entfernen
     await page.addStyleTag({ content: `
       [class*="paywall"],[class*="Paywall"],[id*="paywall"],
       [class*="piano"],[class*="Piano"],[id*="piano"],

frontend/src/tools/paywallkiller.jsx

@@ -13,52 +13,14 @@ function getDomain(url) {
   catch { return url; }
 }
 
-// Wayback CDX – direkt im Browser
-async function searchWayback(targetUrl) {
-  const cdx = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
-  let res;
-  try {
-    // Ohne custom headers – minimaler Request damit CORS-Preflight entfällt
-    res = await fetch(cdx);
-  } catch (e) {
-    throw new Error('Netzwerkfehler: ' + e.message);
-  }
-  if (!res.ok) throw new Error(`CDX ${res.status}`);
-  const data = await res.json();
-  if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) return null;
-  return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`;
-}
-
-// Wayback Save + CDX-Polling
-async function saveAndPollWayback(targetUrl, onStatus) {
-  // Save via no-cors (kein lesbarer Response, aber Archivierung wird getriggert)
-  try {
-    fetch(`https://web.archive.org/save/${targetUrl}`, {
-      method: 'GET',
-      mode: 'no-cors',
-    }).catch(() => {});
-  } catch {}
-
-  // CDX pollen: 8x alle 5s = max 40s
-  for (let i = 0; i < 8; i++) {
-    await new Promise(r => setTimeout(r, 5000));
-    onStatus(`Warte auf Snapshot... (${(i + 1) * 5}s)`);
-    try {
-      const url = await searchWayback(targetUrl);
-      if (url) return url;
-    } catch {}
-  }
-  return null;
-}
-
 export default function PaywallKiller() {
-  const [inputUrl,    setInputUrl]    = useState('');
+  const [inputUrl,   setInputUrl]   = useState('');
-  const [status,      setStatus]      = useState('idle');
+  const [status,     setStatus]     = useState('idle');
-  const [statusMsg,   setStatusMsg]   = useState('');
+  const [statusMsg,  setStatusMsg]  = useState('');
-  const [waybackUrl,  setWaybackUrl]  = useState('');
+  const [waybackUrl, setWaybackUrl] = useState('');
-  const [wasCreated,  setWasCreated]  = useState(false);
+  const [wasCreated, setWasCreated] = useState(false);
-  const [errMsg,      setErrMsg]      = useState('');
+  const [errMsg,     setErrMsg]     = useState('');
-  const [pdfLoading,  setPdfLoading]  = useState(false);
+  const [pdfLoading, setPdfLoading] = useState(false);
   const inputRef = useRef(null);
 
   function getTargetUrl() {
@@ -66,6 +28,31 @@ export default function PaywallKiller() {
     return url.startsWith('http') ? url : 'https://' + url;
   }
 
+  // CDX-Suche über Backend-Proxy (umgeht CORS/Origin-Problem der PWA)
+  async function searchWayback(targetUrl) {
+    const data = await api(`/tools/paywallkiller/cdx?url=${encodeURIComponent(targetUrl)}`, { method: 'GET' });
+    return data.found ? data.waybackUrl : null;
+  }
+
+  // Save via Browser (no-cors, fire-and-forget), dann CDX-Polling via Backend
+  async function saveAndPoll(targetUrl) {
+    // Save-Request abschießen (Browser, kein CORS-Response nötig)
+    try {
+      fetch(`https://web.archive.org/save/${targetUrl}`, { mode: 'no-cors' }).catch(() => {});
+    } catch {}
+
+    // CDX pollen via Backend-Proxy
+    for (let i = 0; i < 8; i++) {
+      await new Promise(r => setTimeout(r, 5000));
+      setStatusMsg(`Warte auf Snapshot... (${(i + 1) * 5}s)`);
+      try {
+        const url = await searchWayback(targetUrl);
+        if (url) return url;
+      } catch {}
+    }
+    return null;
+  }
+
   async function handleCheck() {
     if (!inputUrl.trim()) return;
     const targetUrl = getTargetUrl();
@@ -76,18 +63,16 @@ export default function PaywallKiller() {
     setWasCreated(false);
 
     try {
-      // Schritt 1: vorhandenen Snapshot suchen
       const existing = await searchWayback(targetUrl);
       if (existing) {
         setWaybackUrl(existing);
         setStatus('done');
         return;
       }
-
+      // Kein Snapshot → archivieren
-      // Schritt 2: Kein Snapshot → neu erstellen + pollen
       setStatus('saving');
-      setStatusMsg('Kein Snapshot gefunden – archiviere jetzt...');
+      setStatusMsg('Kein Snapshot – archiviere jetzt...');
-      const newUrl = await saveAndPollWayback(targetUrl, msg => setStatusMsg(msg));
+      const newUrl = await saveAndPoll(targetUrl);
       if (newUrl) {
         setWaybackUrl(newUrl);
         setWasCreated(true);
@@ -185,7 +170,6 @@ export default function PaywallKiller() {
         </div>
       </div>
 
-      {/* Wayback-Ergebnis */}
       {isDone && (
         <div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 12 }}>
           <div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
@@ -209,19 +193,17 @@ export default function PaywallKiller() {
         </div>
       )}
 
-      {/* Kein Snapshot erstellt */}
       {status === 'notfound' && (
         <div style={{ ...S.card, borderColor: `${C.error}44`, marginBottom: 12 }}>
           <div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
             <span>📭</span>
             <span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>
-              Archivierung fehlgeschlagen – {getDomain(inputUrl)} blockiert möglicherweise die Wayback Machine.
+              Kein Snapshot gefunden – {getDomain(inputUrl)} blockiert möglicherweise die Wayback Machine.
             </span>
           </div>
         </div>
       )}
 
-      {/* archive.ph – immer nach Suche anzeigen */}
       {hasResult && archivePhUrl && (
         <div style={{ ...S.card, borderColor: 'rgba(255,255,255,0.1)', marginBottom: 12 }}>
           <div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
@@ -249,7 +231,7 @@ export default function PaywallKiller() {
           <div style={{ ...S.head, marginBottom: 10 }}>SO FUNKTIONIERT ES</div>
           {[
             ['1', 'URL einfügen (Long-Press → Einfügen)'],
-            ['2', 'Wayback Machine wird direkt im Browser durchsucht'],
+            ['2', 'Wayback Machine wird serverseitig durchsucht'],
             ['3', 'Falls kein Snapshot: Artikel wird automatisch archiviert'],
             ['4', 'Snapshot öffnen oder als PDF speichern'],
           ].map(([num, text]) => (