fix: Wayback CDX API im Frontend + archive.ph immer anzeigen

2026-06-12 08:53:58 +02:00
parent 373bb3b6a0
commit c83f78595f
2 changed files with 104 additions and 163 deletions
--- a/backend/src/tools/paywallkiller/routes.js
+++ b/backend/src/tools/paywallkiller/routes.js
@@ -166,8 +166,8 @@ router.post('/check', authenticate, async (req, res) => {
 router.post('/pdf', authenticate, async (req, res) => {
  const { archiveUrl } = req.body;
  if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' });
-  if (!/^https?:\/\/archive\.(is|ph|today|fo|li|vn|md|gg)\//.test(archiveUrl)) {
-    return res.status(400).json({ error: 'Ungültige archive.is URL' });
+  if (!/^https?:\/\/(archive\.(is|ph|today|fo|li|vn|md|gg)|web\.archive\.org)\//.test(archiveUrl)) {
+    return res.status(400).json({ error: 'Ungültige Archiv-URL' });
  }

  let browser;
--- a/frontend/src/tools/paywallkiller.jsx
+++ b/frontend/src/tools/paywallkiller.jsx
@@ -1,4 +1,4 @@
-import { useState, useRef, useEffect } from 'react';
+import { useState, useRef } from 'react';
 import { S } from '../lib.js';

 const C = {
@@ -13,131 +13,56 @@ function getDomain(url) {
  catch { return url; }
 }

-// ── Iframe-basierte Archivsuche (umgeht Server-IP-Block) ──────────────────
-// Öffnet archive.ph/?url=... in einem versteckten iframe und liest
-// nach dem Load die finale URL aus (Snapshot-Hash-URL).
-function useArchiveSearch() {
-  const iframeRef = useRef(null);
-  const resolveRef = useRef(null);
-  const timeoutRef = useRef(null);
-
-  useEffect(() => {
-    return () => {
-      if (timeoutRef.current) clearTimeout(timeoutRef.current);
-    };
-  }, []);
-
-  function search(targetUrl) {
-    return new Promise((resolve, reject) => {
-      resolveRef.current = { resolve, reject };
-
-      // Altes iframe entfernen
-      if (iframeRef.current) {
-        document.body.removeChild(iframeRef.current);
-        iframeRef.current = null;
-      }
-
-      const iframe = document.createElement('iframe');
-      iframe.style.cssText = 'position:fixed;top:-9999px;left:-9999px;width:1px;height:1px;opacity:0;pointer-events:none;border:none;';
-      iframe.sandbox = 'allow-scripts allow-same-origin allow-forms';
-
-      iframe.onload = () => {
-        if (timeoutRef.current) clearTimeout(timeoutRef.current);
-        try {
-          // Nach dem Load: finale URL des iframes prüfen
-          const finalUrl = iframe.contentWindow?.location?.href || '';
-          cleanup();
-
-          if (!finalUrl || finalUrl === 'about:blank') {
-            return resolve({ found: false });
-          }
-
-          // Snapshot-URL hat Form https://archive.ph/HASH (4-10 Zeichen)
-          if (/archive\.(is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10}$/.test(finalUrl)) {
-            return resolve({ found: true, url: finalUrl });
-          }
-
-          // Suchergebnis-Seite geladen: Links nach Snapshots durchsuchen
-          try {
-            const doc = iframe.contentDocument || iframe.contentWindow?.document;
-            if (doc) {
-              // Alle Links durchsuchen
-              const links = Array.from(doc.querySelectorAll('a[href]'));
-              for (const a of links) {
-                const href = a.getAttribute('href') || '';
-                const abs = href.startsWith('http') ? href : `https://archive.ph${href}`;
-                if (/archive\.(is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10}$/.test(abs)) {
-                  const h = abs.split('/').pop();
-                  const blacklist = ['newest','oldest','submit','search','about','rss','api','timemap'];
-                  if (!blacklist.includes(h) && !/^\d+$/.test(h)) {
-                    return resolve({ found: true, url: abs });
-                  }
-                }
-              }
-            }
-          } catch {
-            // Cross-Origin Zugriff blockiert – trotzdem finalUrl prüfen
-          }
-
-          resolve({ found: false });
-        } catch (e) {
-          cleanup();
-          resolve({ found: false });
-        }
-      };
-
-      iframe.onerror = () => {
-        cleanup();
-        resolve({ found: false });
-      };
-
-      // Timeout nach 15s
-      timeoutRef.current = setTimeout(() => {
-        cleanup();
-        reject(new Error('Timeout beim Laden von archive.ph'));
-      }, 15000);
-
-      const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`;
-      iframe.src = searchUrl;
-      document.body.appendChild(iframe);
-      iframeRef.current = iframe;
-
-      function cleanup() {
-        if (timeoutRef.current) clearTimeout(timeoutRef.current);
-        if (iframeRef.current) {
-          try { document.body.removeChild(iframeRef.current); } catch {}
-          iframeRef.current = null;
-        }
-      }
-    });
-  }
-
-  return { search };
+// ── Wayback Machine CDX API – direkt im Browser (CORS: *) ─────────────────
+async function checkWayback(targetUrl) {
+  // Neuesten Snapshot suchen (limit=1, sortiert newest-first via fl+limit)
+  const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
+  const res = await fetch(cdxUrl, { signal: AbortSignal.timeout(10000) });
+  if (!res.ok) return null;
+  const data = await res.json();
+  // data = [["timestamp"],["20260608201434"]] oder [] wenn nichts gefunden
+  if (!Array.isArray(data) || data.length < 2) return null;
+  const timestamp = data[1]?.[0];
+  if (!timestamp) return null;
+  return `https://web.archive.org/web/${timestamp}/${targetUrl}`;
 }

 export default function PaywallKiller() {
-  const [inputUrl,    setInputUrl]    = useState('');
-  const [status,      setStatus]      = useState('idle');
-  const [archiveUrl,  setArchiveUrl]  = useState('');
-  const [errMsg,      setErrMsg]      = useState('');
-  const [pdfLoading,  setPdfLoading]  = useState(false);
+  const [inputUrl,   setInputUrl]   = useState('');
+  const [status,     setStatus]     = useState('idle');
+  const [waybackUrl, setWaybackUrl] = useState('');
+  const [archiveUrl, setArchiveUrl] = useState('');
+  const [errMsg,     setErrMsg]     = useState('');
+  const [pdfLoading, setPdfLoading] = useState(false);
+  const [pdfTarget,  setPdfTarget]  = useState('');
  const inputRef = useRef(null);
-  const { search } = useArchiveSearch();

  async function handleCheck() {
    const url = inputUrl.trim();
    if (!url) return;
    let targetUrl = url.startsWith('http') ? url : 'https://' + url;
+
    setStatus('checking');
    setErrMsg('');
+    setWaybackUrl('');
    setArchiveUrl('');
+
    try {
-      const result = await search(targetUrl);
-      if (result.found) {
-        setArchiveUrl(result.url);
-        setStatus('found');
+      // archive.ph URL immer konstruieren (ohne Prüfung – direkt nutzbar)
+      const archivePh = `https://archive.ph/newest/${targetUrl}`;
+
+      // Wayback CDX parallel prüfen
+      let wb = null;
+      try { wb = await checkWayback(targetUrl); } catch { /* ignorieren */ }
+
+      setWaybackUrl(wb || '');
+      setArchiveUrl(archivePh);
+
+      if (wb) {
+        setStatus('found_both');
      } else {
-        setStatus('notfound');
+        // Auch ohne Wayback-Treffer archive.ph anbieten
+        setStatus('found_archive');
      }
    } catch (err) {
      setErrMsg(err.message || 'Fehler beim Suchen');
@@ -145,7 +70,8 @@ export default function PaywallKiller() {
    }
  }

-  async function handlePdf() {
+  async function handlePdf(url) {
+    setPdfTarget(url);
    setPdfLoading(true);
    try {
      const token = localStorage.getItem('sk_token');
@@ -155,7 +81,7 @@ export default function PaywallKiller() {
          'Content-Type': 'application/json',
          ...(token ? { Authorization: `Bearer ${token}` } : {}),
        },
-        body: JSON.stringify({ archiveUrl }),
+        body: JSON.stringify({ archiveUrl: url }),
      });
      if (!res.ok) {
        const data = await res.json().catch(() => ({}));
@@ -172,15 +98,16 @@ export default function PaywallKiller() {
      setTimeout(() => URL.revokeObjectURL(objUrl), 5000);
    } catch (err) {
      setErrMsg(err.message || 'PDF konnte nicht erstellt werden');
-      setStatus('error');
    } finally {
      setPdfLoading(false);
+      setPdfTarget('');
    }
  }

  function handleReset() {
    setInputUrl('');
    setStatus('idle');
+    setWaybackUrl('');
    setArchiveUrl('');
    setErrMsg('');
    setPdfLoading(false);
@@ -195,10 +122,20 @@ export default function PaywallKiller() {
  }

  const isChecking = status === 'checking';
-  const isFound    = status === 'found';
-  const isNotFound = status === 'notfound';
-  const isError    = status === 'error';
-  const hasResult  = isFound || isNotFound || isError;
+  const hasResult  = status.startsWith('found') || status === 'error';
+  const hasWayback = status === 'found_both' && waybackUrl;
+  const hasArchive = (status === 'found_both' || status === 'found_archive') && archiveUrl;
+
+  // PDF-Button für eine bestimmte URL
+  function PdfBtn({ url }) {
+    const loading = pdfLoading && pdfTarget === url;
+    return (
+      <button onClick={() => handlePdf(url)} disabled={pdfLoading}
+        style={{ ...S.btn(C.accent), opacity: pdfLoading ? 0.5 : 1, cursor: pdfLoading ? 'default' : 'pointer', fontSize: 12 }}>
+        {loading ? '⏳ PDF...' : '⬇ PDF'}
+      </button>
+    );
+  }

  return (
    <div style={{ padding: '20px 16px', maxWidth: 640, margin: '0 auto' }}>
@@ -209,10 +146,11 @@ export default function PaywallKiller() {
          <span style={{ color: C.accent, fontFamily: 'monospace', fontSize: 13, letterSpacing: 1 }}>PAYWALL-KILLER</span>
        </div>
        <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace' }}>
-          Artikel über archive.ph abrufen · Als PDF speichern
+          Wayback Machine · archive.ph · Als PDF speichern
        </div>
      </div>

+      {/* Eingabe */}
      <div style={{ ...S.card, marginBottom: 16 }}>
        <div style={{ ...S.head, marginBottom: 10 }}>ARTIKEL-URL</div>
        <div style={{ display: 'flex', gap: 8, marginBottom: 10 }}>
@@ -232,7 +170,7 @@ export default function PaywallKiller() {
        <div style={{ display: 'flex', gap: 8 }}>
          <button onClick={handleCheck} disabled={isChecking || !inputUrl.trim()}
            style={{ ...S.btn(C.accent), flex: 1, opacity: (isChecking || !inputUrl.trim()) ? 0.45 : 1, cursor: (isChecking || !inputUrl.trim()) ? 'default' : 'pointer' }}>
-            {isChecking ? '⏳ Lade archive.ph...' : '🔍 Archiv suchen'}
+            {isChecking ? '⏳ Suche Archiv...' : '🔍 Archiv suchen'}
          </button>
          {hasResult && (
            <button onClick={handleReset}
@@ -241,59 +179,61 @@ export default function PaywallKiller() {
            </button>
          )}
        </div>
-        {isChecking && (
-          <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 10 }}>
-            archive.ph wird direkt im Browser abgefragt...
-          </div>
-        )}
      </div>

-      {isFound && (
-        <div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 16 }}>
-          <div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 12 }}>
-            <span>✅</span>
-            <span style={{ color: C.success, fontFamily: 'monospace', fontSize: 12 }}>Archiv gefunden</span>
+      {/* Wayback Machine Treffer */}
+      {hasWayback && (
+        <div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 12 }}>
+          <div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
+            <span>🏛</span>
+            <span style={{ color: C.success, fontFamily: 'monospace', fontSize: 12 }}>Wayback Machine</span>
+            <span style={{ background: `${C.success}20`, border: `1px solid ${C.success}44`, borderRadius: 4, padding: '1px 6px', fontSize: 10, color: C.success, fontFamily: 'monospace' }}>gefunden</span>
          </div>
-          <div style={{ background: 'rgba(255,255,255,0.03)', border: '1px solid rgba(255,255,255,0.08)', borderRadius: 6, padding: '8px 10px', marginBottom: 14, wordBreak: 'break-all' }}>
-            <a href={archiveUrl} target="_blank" rel="noreferrer" style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>
-              {archiveUrl}
-            </a>
+          <div style={{ background: 'rgba(255,255,255,0.03)', border: '1px solid rgba(255,255,255,0.08)', borderRadius: 6, padding: '7px 10px', marginBottom: 10, wordBreak: 'break-all' }}>
+            <a href={waybackUrl} target="_blank" rel="noreferrer" style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>{waybackUrl}</a>
          </div>
-          <div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}>
-            <a href={archiveUrl} target="_blank" rel="noreferrer" style={{ ...S.btn(C.success), textDecoration: 'none' }}>
-              🌐 Im Browser öffnen
-            </a>
-            <button onClick={handlePdf} disabled={pdfLoading}
-              style={{ ...S.btn(C.accent), opacity: pdfLoading ? 0.5 : 1, cursor: pdfLoading ? 'default' : 'pointer' }}>
-              {pdfLoading ? '⏳ Erstelle PDF...' : '⬇ Als PDF speichern'}
-            </button>
+          <div style={{ display: 'flex', gap: 8 }}>
+            <a href={waybackUrl} target="_blank" rel="noreferrer" style={{ ...S.btn(C.success), textDecoration: 'none', fontSize: 12 }}>🌐 Öffnen</a>
+            <PdfBtn url={waybackUrl} />
          </div>
-          {pdfLoading && (
-            <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 10 }}>
-              Seite wird serverseitig gerendert, dauert 15–30 Sekunden...
+        </div>
+      )}
+
+      {/* archive.ph Link */}
+      {hasArchive && (
+        <div style={{ ...S.card, borderColor: hasWayback ? 'rgba(255,255,255,0.1)' : `${C.success}55`, marginBottom: 12 }}>
+          <div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
+            <span>📦</span>
+            <span style={{ color: hasWayback ? 'rgba(255,255,255,0.6)' : C.success, fontFamily: 'monospace', fontSize: 12 }}>archive.ph</span>
+            {!hasWayback && <span style={{ background: `${C.success}20`, border: `1px solid ${C.success}44`, borderRadius: 4, padding: '1px 6px', fontSize: 10, color: C.success, fontFamily: 'monospace' }}>neuester Snapshot</span>}
+          </div>
+          <div style={{ background: 'rgba(255,255,255,0.03)', border: '1px solid rgba(255,255,255,0.08)', borderRadius: 6, padding: '7px 10px', marginBottom: 10, wordBreak: 'break-all' }}>
+            <a href={archiveUrl} target="_blank" rel="noreferrer" style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>{archiveUrl}</a>
+          </div>
+          <div style={{ display: 'flex', gap: 8 }}>
+            <a href={archiveUrl} target="_blank" rel="noreferrer" style={{ ...S.btn(C.success), textDecoration: 'none', fontSize: 12 }}>🌐 Öffnen</a>
+            <PdfBtn url={archiveUrl} />
+          </div>
+          {!hasWayback && (
+            <div style={{ color: C.muted, fontSize: 10, fontFamily: 'monospace', marginTop: 8 }}>
+              Kein Wayback-Snapshot gefunden. archive.ph-Link öffnet direkt den neuesten Snapshot (falls vorhanden).
            </div>
          )}
        </div>
      )}

-      {isNotFound && (
-        <div style={{ ...S.card, borderColor: `${C.error}44` }}>
-          <div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 8 }}>
-            <span>📭</span>
-            <span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>Kein Archiv gefunden</span>
+      {/* PDF-Ladehinweis */}
+      {pdfLoading && (
+        <div style={{ ...S.card, borderColor: `${C.accent}44`, marginBottom: 12 }}>
+          <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace' }}>
+            ⏳ Seite wird serverseitig gerendert, bitte warten (15–30s)...
          </div>
-          <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', lineHeight: 1.6, marginBottom: 12 }}>
-            Für <strong style={{ color: 'rgba(255,255,255,0.55)' }}>{getDomain(inputUrl)}</strong> wurde kein Snapshot auf archive.ph gefunden.
-          </div>
-          <a href={`https://archive.ph/${inputUrl.trim()}`} target="_blank" rel="noreferrer"
-            style={{ ...S.btn(C.muted), textDecoration: 'none', display: 'inline-block', fontSize: 11 }}>
-            🔗 Manuell auf archive.ph prüfen
-          </a>
        </div>
      )}

-      {isError && (
-        <div style={{ ...S.card, borderColor: `${C.error}44` }}>
+      {/* Fehler */}
+      {errMsg && (
+        <div style={{ ...S.card, borderColor: `${C.error}44`, marginBottom: 12 }}>
          <div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
            <span>⚠️</span>
            <span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>{errMsg}</span>
@@ -301,12 +241,13 @@ export default function PaywallKiller() {
        </div>
      )}

+      {/* Idle-Info */}
      {status === 'idle' && (
        <div style={{ ...S.card }}>
          <div style={{ ...S.head, marginBottom: 10 }}>SO FUNKTIONIERT ES</div>
          {[
            ['1', 'URL des gesperrten Artikels einfügen'],
-            ['2', 'Archiv auf archive.ph suchen (direkt im Browser)'],
+            ['2', 'Wayback Machine + archive.ph werden geprüft'],
            ['3', 'Im Browser öffnen oder als PDF speichern'],
          ].map(([num, text]) => (
            <div key={num} style={{ display: 'flex', gap: 10, alignItems: 'center', marginBottom: 8 }}>