fix: archive.is Suche via HTML-Parse, Button-Farben gefixt

2026-06-12 01:42:42 +02:00
parent 38c27ac5bb
commit 902d5fd99e
2 changed files with 146 additions and 137 deletions
--- a/backend/src/tools/paywallkiller/routes.js
+++ b/backend/src/tools/paywallkiller/routes.js
@@ -7,60 +7,71 @@ const path = require('path');
 const fs = require('fs');
 const crypto = require('crypto');

-// ── Hilfsfunktion: folgt Redirects, gibt finale URL zurück ───────────────
-function fetchFollowRedirects(url, redirectCount = 0) {
+// ── HTTP GET, gibt finalUrl + statusCode + body (optional) zurück ─────────
+function fetchPage(url, opts = {}) {
  return new Promise((resolve, reject) => {
-    if (redirectCount > 8) return reject(new Error('Zu viele Redirects'));
+    if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects'));
    const mod = url.startsWith('https') ? https : http;
    const req = mod.get(url, {
      headers: {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
+        'Accept': 'text/html,application/xhtml+xml',
        'Accept-Language': 'de,en;q=0.5',
      },
      timeout: 12000,
    }, (res) => {
-      if ([301, 302, 303, 307, 308].includes(res.statusCode) && res.headers.location) {
-        let location = res.headers.location;
-        if (!location.startsWith('http')) {
-          try {
-            const base = new URL(url);
-            location = location.startsWith('/')
-              ? `${base.protocol}//${base.host}${location}`
-              : `${base.protocol}//${base.host}/${location}`;
-          } catch { return reject(new Error('Ungültige Redirect-URL')); }
+      if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) {
+        let loc = res.headers.location;
+        if (!loc.startsWith('http')) {
+          try { const b = new URL(url); loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; }
+          catch { return reject(new Error('Ungültige Redirect-URL')); }
        }
        res.resume();
-        return fetchFollowRedirects(location, redirectCount + 1).then(resolve).catch(reject);
+        return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1 }).then(resolve).catch(reject);
      }
+
+      if (opts.bodyNeeded) {
+        let body = '';
+        res.setEncoding('utf8');
+        res.on('data', d => { if (body.length < 50000) body += d; });
+        res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body }));
+      } else {
        res.resume();
        resolve({ statusCode: res.statusCode, finalUrl: url });
+      }
    });
    req.on('error', reject);
    req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
  });
 }

-// ── Prüfe ob archive.is einen Snapshot hat ────────────────────────────────
-// archive.is/newest/URL leitet auf archive.is/HASH weiter wenn ein Snapshot existiert.
-// Wenn kein Snapshot → Redirect zurück auf archive.is/ (Root) oder /newest/... bleibt stehen.
-async function checkArchive(archiveUrl) {
+// ── Archivsuche über archive.is ───────────────────────────────────────────
+// archive.is/?url=ZIEL gibt HTML zurück mit Links zu archivierten Snapshots.
+// Wir parsen den ersten Snapshot-Link daraus.
+async function findArchiveSnapshot(targetUrl) {
+  const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`;
  try {
-    const result = await fetchFollowRedirects(archiveUrl);
-    const final = result.finalUrl;
-    // Kein Redirect = nichts gefunden (blieb auf gleicher URL)
-    if (final === archiveUrl) return { found: false };
-    // Snapshot-URL hat die Form https://archive.is/ABCDE (alphanumeric hash, 4-8 Zeichen)
-    if (/https?:\/\/archive\.(is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10}$/.test(final)) {
-      return { found: true, resolvedUrl: final };
+    const result = await fetchPage(searchUrl, { bodyNeeded: true });
+    if (result.statusCode !== 200 || !result.body) return null;
+
+    // Snapshots stehen als Links in der Form href="/ABCDE" oder href="https://archive.is/ABCDE"
+    // Wir suchen den neuesten (ersten in der Liste – archive.is sortiert newest first)
+    const hashPattern = /href="(?:https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg))?\/([a-zA-Z0-9]{4,10})"/g;
+    let match;
+    const hashes = new Set();
+    while ((match = hashPattern.exec(result.body)) !== null) {
+      const h = match[1];
+      // Blacklist bekannter Nicht-Snapshot-Pfade
+      if (['newest','oldest','submit','search','about','faq','rss'].includes(h)) continue;
+      hashes.add(h);
    }
-    // archive.is Root oder Submit-Seite = kein Treffer
-    if (/archive\.[a-z]+\/?$/.test(final)) return { found: false };
-    if (final.includes('/submit')) return { found: false };
-    // Sonst: irgendwohin weitergeleitet → als Treffer werten
-    return { found: true, resolvedUrl: final };
+    if (hashes.size === 0) return null;
+
+    // Erster Hash = neuester Snapshot
+    const newestHash = [...hashes][0];
+    return `https://archive.is/${newestHash}`;
  } catch {
-    return { found: false };
+    return null;
  }
 }

@@ -72,17 +83,10 @@ router.post('/check', authenticate, async (req, res) => {
  let targetUrl = url.trim();
  if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;

-  const newestUrl = `https://archive.is/newest/${targetUrl}`;
-  const oldestUrl = `https://archive.is/oldest/${targetUrl}`;
-
  try {
-    const newest = await checkArchive(newestUrl);
-    if (newest.found) {
-      return res.json({ archiveUrl: newest.resolvedUrl || newestUrl, type: 'newest' });
-    }
-    const oldest = await checkArchive(oldestUrl);
-    if (oldest.found) {
-      return res.json({ archiveUrl: oldest.resolvedUrl || oldestUrl, type: 'oldest' });
+    const snapshotUrl = await findArchiveSnapshot(targetUrl);
+    if (snapshotUrl) {
+      return res.json({ archiveUrl: snapshotUrl, type: 'newest' });
    }
    return res.status(404).json({ error: 'Kein Archiv für diese URL gefunden' });
  } catch (err) {
@@ -103,17 +107,12 @@ router.post('/pdf', authenticate, async (req, res) => {

  try {
    let puppeteer, chromium;
-
    try {
      puppeteer = require('puppeteer-core');
      chromium  = require('@sparticuz/chromium');
    } catch {
-      try {
-        puppeteer = require('puppeteer');
-        chromium  = null;
-      } catch {
-        return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar – puppeteer nicht installiert' });
-      }
+      try { puppeteer = require('puppeteer'); chromium = null; }
+      catch { return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar – puppeteer nicht installiert' }); }
    }

    const launchOptions = chromium
@@ -130,9 +129,7 @@ router.post('/pdf', authenticate, async (req, res) => {
    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
    await page.goto(archiveUrl, { waitUntil: 'networkidle2', timeout: 30000 });
    await page.pdf({
-      path: tmpFile,
-      format: 'A4',
-      printBackground: true,
+      path: tmpFile, format: 'A4', printBackground: true,
      margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' },
    });
    await browser.close();
@@ -144,7 +141,6 @@ router.post('/pdf', authenticate, async (req, res) => {
    stream.pipe(res);
    stream.on('end', () => fs.unlink(tmpFile, () => {}));
    stream.on('error', () => fs.unlink(tmpFile, () => {}));
-
  } catch (err) {
    if (browser) try { await browser.close(); } catch {}
    fs.unlink(tmpFile, () => {});
--- a/frontend/src/tools/paywallkiller.jsx
+++ b/frontend/src/tools/paywallkiller.jsx
@@ -1,16 +1,14 @@
 import { useState, useRef } from 'react';
 import { S, api } from '../lib.js';

-// ── Farben ─────────────────────────────────────────────────────────────────
 const C = {
  accent:  '#f59e0b',
  success: '#4ade80',
  error:   '#f87171',
-  muted:    'rgba(255,255,255,0.4)',
-  dim:      'rgba(255,255,255,0.07)',
+  muted:   '#6b7280',
+  dim:     '#1f2937',
 };

-// ── Hilfsfunktion: Domain aus URL extrahieren ──────────────────────────────
 function getDomain(url) {
  try { return new URL(url).hostname.replace(/^www\./, ''); }
  catch { return url; }
@@ -18,14 +16,13 @@ function getDomain(url) {

 export default function PaywallKiller() {
  const [inputUrl,    setInputUrl]    = useState('');
-  const [status,      setStatus]      = useState('idle'); // idle | checking | found | notfound | pdf | error
+  const [status,      setStatus]      = useState('idle');
  const [archiveUrl,  setArchiveUrl]  = useState('');
  const [archiveType, setArchiveType] = useState('');
  const [errMsg,      setErrMsg]      = useState('');
  const [pdfLoading,  setPdfLoading]  = useState(false);
  const inputRef = useRef(null);

-  // ── URL prüfen ────────────────────────────────────────────────────────────
  async function handleCheck() {
    const url = inputUrl.trim();
    if (!url) return;
@@ -48,7 +45,6 @@ export default function PaywallKiller() {
    }
  }

-  // ── PDF herunterladen ─────────────────────────────────────────────────────
  async function handlePdf() {
    setPdfLoading(true);
    try {
@@ -61,12 +57,10 @@ export default function PaywallKiller() {
        },
        body: JSON.stringify({ archiveUrl }),
      });
-
      if (!res.ok) {
        const data = await res.json().catch(() => ({}));
        throw new Error(data.error || 'PDF-Fehler');
      }
-
      const blob = await res.blob();
      const objUrl = URL.createObjectURL(blob);
      const a = document.createElement('a');
@@ -84,7 +78,6 @@ export default function PaywallKiller() {
    }
  }

-  // ── Reset ─────────────────────────────────────────────────────────────────
  function handleReset() {
    setInputUrl('');
    setStatus('idle');
@@ -95,7 +88,6 @@ export default function PaywallKiller() {
    setTimeout(() => inputRef.current?.focus(), 50);
  }

-  // ── Paste aus Clipboard ───────────────────────────────────────────────────
  async function handlePaste() {
    try {
      const text = await navigator.clipboard.readText();
@@ -109,14 +101,15 @@ export default function PaywallKiller() {
  const isFound    = status === 'found';
  const isNotFound = status === 'notfound';
  const isError    = status === 'error';
+  const hasResult  = isFound || isNotFound || isError;

  return (
-    <div style={{ padding: '20px 16px', maxWidth: 660, margin: '0 auto' }}>
+    <div style={{ padding: '20px 16px', maxWidth: 640, margin: '0 auto' }}>

      {/* Header */}
      <div style={{ marginBottom: 24 }}>
        <div style={{ display: 'flex', alignItems: 'center', gap: 10, marginBottom: 4 }}>
-          <span style={{ fontSize: 22 }}>🔓</span>
+          <span style={{ fontSize: 20 }}>🔓</span>
          <span style={{ color: C.accent, fontFamily: 'monospace', fontSize: 13, letterSpacing: 1 }}>
            PAYWALL-KILLER
          </span>
@@ -126,9 +119,11 @@ export default function PaywallKiller() {
        </div>
      </div>

-      {/* URL-Eingabe */}
+      {/* Eingabe */}
      <div style={{ ...S.card, marginBottom: 16 }}>
        <div style={{ ...S.head, marginBottom: 10 }}>ARTIKEL-URL</div>
+
+        {/* Input + Paste-Button */}
        <div style={{ display: 'flex', gap: 8, marginBottom: 10 }}>
          <input
            ref={inputRef}
@@ -138,10 +133,25 @@ export default function PaywallKiller() {
            placeholder="https://www.zeitung.de/artikel/..."
            style={{ ...S.inp, flex: 1 }}
          />
-          <button onClick={handlePaste} style={{ ...S.btn(C.muted, true), padding: '5px 12px' }} title="Aus Zwischenablage einfügen">
+          <button
+            onClick={handlePaste}
+            title="Aus Zwischenablage einfügen"
+            style={{
+              background: 'rgba(255,255,255,0.06)',
+              border: '1px solid rgba(255,255,255,0.12)',
+              borderRadius: 6,
+              padding: '0 12px',
+              color: '#fff',
+              cursor: 'pointer',
+              fontSize: 14,
+              flexShrink: 0,
+            }}
+          >
            📋
          </button>
        </div>
+
+        {/* Action-Buttons */}
        <div style={{ display: 'flex', gap: 8 }}>
          <button
            onClick={handleCheck}
@@ -149,15 +159,28 @@ export default function PaywallKiller() {
            style={{
              ...S.btn(C.accent),
              flex: 1,
-              opacity: (isChecking || !inputUrl.trim()) ? 0.5 : 1,
+              opacity: (isChecking || !inputUrl.trim()) ? 0.45 : 1,
              cursor: (isChecking || !inputUrl.trim()) ? 'default' : 'pointer',
            }}
          >
            {isChecking ? '⏳ Suche Archiv...' : '🔍 Archiv suchen'}
          </button>
-          {(isFound || isNotFound || isError) && (
-            <button onClick={handleReset} style={S.btn(C.muted, false)}>
-              ✕ Zurücksetzen
+          {hasResult && (
+            <button
+              onClick={handleReset}
+              style={{
+                background: 'rgba(255,255,255,0.06)',
+                border: '1px solid rgba(255,255,255,0.14)',
+                borderRadius: 6,
+                padding: '8px 14px',
+                color: 'rgba(255,255,255,0.6)',
+                cursor: 'pointer',
+                fontFamily: 'monospace',
+                fontSize: 12,
+                whiteSpace: 'nowrap',
+              }}
+            >
+              ✕ Reset
            </button>
          )}
        </div>
@@ -165,26 +188,25 @@ export default function PaywallKiller() {

      {/* Gefunden */}
      {isFound && (
-        <div style={{ ...S.card, borderColor: `${C.success}44`, marginBottom: 16 }}>
+        <div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 16 }}>
          <div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 12 }}>
-            <span style={{ fontSize: 16 }}>✅</span>
+            <span>✅</span>
            <span style={{ color: C.success, fontFamily: 'monospace', fontSize: 12 }}>
              Archiv gefunden
+            </span>
            <span style={{
-                marginLeft: 8,
-                background: `${C.accent}22`,
-                border: `1px solid ${C.accent}44`,
+              background: `${C.accent}20`,
+              border: `1px solid ${C.accent}55`,
              borderRadius: 4,
-                padding: '1px 6px',
+              padding: '1px 7px',
              fontSize: 10,
              color: C.accent,
+              fontFamily: 'monospace',
            }}>
-                {archiveType === 'newest' ? 'neuestes' : 'ältestes'} Archiv
-              </span>
+              {archiveType === 'newest' ? 'neuester Snapshot' : 'ältester Snapshot'}
            </span>
          </div>

-          {/* Archive-URL anzeigen */}
          <div style={{
            background: 'rgba(255,255,255,0.03)',
            border: '1px solid rgba(255,255,255,0.08)',
@@ -193,18 +215,18 @@ export default function PaywallKiller() {
            marginBottom: 14,
            wordBreak: 'break-all',
          }}>
-            <a href={archiveUrl} target="_blank" rel="noreferrer" style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>
+            <a href={archiveUrl} target="_blank" rel="noreferrer"
+              style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>
              {archiveUrl}
            </a>
          </div>

-          {/* Aktions-Buttons */}
          <div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}>
            <a
              href={archiveUrl}
              target="_blank"
              rel="noreferrer"
-              style={{ ...S.btn(C.success), textDecoration: 'none', display: 'inline-block' }}
+              style={{ ...S.btn(C.success), textDecoration: 'none' }}
            >
              🌐 Im Browser öffnen
            </a>
@@ -221,10 +243,9 @@ export default function PaywallKiller() {
            </button>
          </div>

-          {/* PDF-Hinweis */}
          {pdfLoading && (
            <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 10 }}>
-              Seite wird geladen und als PDF gerendert. Das kann 15–30 Sekunden dauern...
+              Seite wird gerendert, das dauert 15–30 Sekunden...
            </div>
          )}
        </div>
@@ -234,16 +255,15 @@ export default function PaywallKiller() {
      {isNotFound && (
        <div style={{ ...S.card, borderColor: `${C.error}44` }}>
          <div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 8 }}>
-            <span style={{ fontSize: 16 }}>📭</span>
+            <span>📭</span>
            <span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>
              Kein Archiv gefunden
            </span>
          </div>
-          <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', lineHeight: 1.6 }}>
-            Für <strong style={{ color: 'rgba(255,255,255,0.6)' }}>{getDomain(inputUrl)}</strong> existiert
-            kein Eintrag auf archive.is. Der Artikel wurde möglicherweise noch nie archiviert.
+          <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', lineHeight: 1.6, marginBottom: 12 }}>
+            Für <strong style={{ color: 'rgba(255,255,255,0.55)' }}>{getDomain(inputUrl)}</strong> wurde
+            kein Snapshot auf archive.is gefunden.
          </div>
-          <div style={{ marginTop: 12 }}>
          <a
            href={`https://archive.is/${inputUrl.trim()}`}
            target="_blank"
@@ -253,14 +273,13 @@ export default function PaywallKiller() {
            🔗 Manuell auf archive.is prüfen
          </a>
        </div>
-        </div>
      )}

      {/* Fehler */}
      {isError && (
        <div style={{ ...S.card, borderColor: `${C.error}44` }}>
          <div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
-            <span style={{ fontSize: 16 }}>⚠️</span>
+            <span>⚠️</span>
            <span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>
              {errMsg}
            </span>
@@ -268,34 +287,28 @@ export default function PaywallKiller() {
        </div>
      )}

-      {/* Info-Sektion */}
+      {/* Idle-Info */}
      {status === 'idle' && (
-        <div style={{ ...S.card, marginTop: 8 }}>
-          <div style={{ ...S.head, marginBottom: 10 }}>WIE ES FUNKTIONIERT</div>
-          <div style={{ display: 'flex', flexDirection: 'column', gap: 8 }}>
+        <div style={{ ...S.card }}>
+          <div style={{ ...S.head, marginBottom: 10 }}>SO FUNKTIONIERT ES</div>
          {[
            ['1', 'URL des gesperrten Artikels einfügen'],
-              ['2', 'Archiv auf archive.is suchen (newest → oldest)'],
-              ['3', 'Archivierte Version im Browser öffnen oder als PDF speichern'],
+            ['2', 'Archiv auf archive.is suchen'],
+            ['3', 'Im Browser öffnen oder als PDF speichern'],
          ].map(([num, text]) => (
-              <div key={num} style={{ display: 'flex', gap: 10, alignItems: 'flex-start' }}>
+            <div key={num} style={{ display: 'flex', gap: 10, alignItems: 'center', marginBottom: 8 }}>
              <span style={{
-                  background: `${C.accent}22`,
+                background: `${C.accent}20`,
                border: `1px solid ${C.accent}44`,
                borderRadius: '50%',
-                  width: 20, height: 20,
+                width: 20, height: 20, flexShrink: 0,
                display: 'flex', alignItems: 'center', justifyContent: 'center',
-                  fontSize: 10, color: C.accent, fontFamily: 'monospace', flexShrink: 0,
-                }}>
-                  {num}
-                </span>
-                <span style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', lineHeight: 1.6 }}>
-                  {text}
-                </span>
+                fontSize: 10, color: C.accent, fontFamily: 'monospace',
+              }}>{num}</span>
+              <span style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace' }}>{text}</span>
            </div>
          ))}
        </div>
-        </div>
      )}
    </div>
  );