fix: archive.ph Session-Cookie + Browser-Headers gegen 429

backend/src/tools/paywallkiller/routes.js

@@ -7,37 +7,55 @@ const path = require('path');
 const fs = require('fs');
 const crypto = require('crypto');
 
-// ── HTTP GET mit Redirect-Follow, gibt Body zurück ────────────────────────
+// ── HTTP GET mit Redirect-Follow + Cookie-Jar ─────────────────────────────
 function fetchPage(url, opts = {}) {
   return new Promise((resolve, reject) => {
     if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects'));
     const mod = url.startsWith('https') ? https : http;
-    const req = mod.get(url, {
+
-      headers: {
+    // Cookies aus jar zusammenbauen
+    const cookieHeader = opts.cookieJar && Object.keys(opts.cookieJar).length
+      ? Object.entries(opts.cookieJar).map(([k,v]) => `${k}=${v}`).join('; ')
+      : undefined;
+
+    const headers = {
       'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
-        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+      'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
-        'Accept-Language': 'de,en;q=0.5',
+      'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
       'Accept-Encoding': 'identity',
       'Connection': 'keep-alive',
-      },
+      'Sec-Fetch-Dest': 'document',
-      timeout: 15000,
+      'Sec-Fetch-Mode': 'navigate',
-    }, (res) => {
+      'Sec-Fetch-Site': opts.referer ? 'same-origin' : 'none',
+      'Upgrade-Insecure-Requests': '1',
+      ...(opts.referer ? { 'Referer': opts.referer } : {}),
+      ...(cookieHeader ? { 'Cookie': cookieHeader } : {}),
+    };
+
+    const req = mod.get(url, { headers, timeout: 15000 }, (res) => {
+      // Cookies sammeln
+      const jar = opts.cookieJar || {};
+      const setCookies = res.headers['set-cookie'] || [];
+      for (const c of setCookies) {
+        const m = c.match(/^([^=]+)=([^;]*)/);
+        if (m) jar[m[1].trim()] = m[2].trim();
+      }
+
       if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) {
         let loc = res.headers.location;
         if (!loc.startsWith('http')) {
-          try {
+          try { const b = new URL(url); loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; }
-            const b = new URL(url);
+          catch { return reject(new Error('Ungültige Redirect-URL')); }
-            loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`;
-          } catch { return reject(new Error('Ungültige Redirect-URL')); }
         }
         res.resume();
-        return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1 }).then(resolve).catch(reject);
+        return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1, cookieJar: jar, referer: url })
+          .then(resolve).catch(reject);
       }
 
       let body = '';
       res.setEncoding('utf8');
-      res.on('data', d => { if (body.length < 100000) body += d; });
+      res.on('data', d => { if (body.length < 150000) body += d; });
-      res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body, headers: res.headers }));
+      res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body, cookieJar: jar }));
       res.on('error', reject);
     });
     req.on('error', reject);
@@ -45,44 +63,82 @@ function fetchPage(url, opts = {}) {
   });
 }
 
+// ── Session initialisieren (Startseite besuchen für Cookies) ──────────────
+async function initSession() {
+  const result = await fetchPage('https://archive.ph/', { cookieJar: {} });
+  return result.cookieJar;
+}
+
 // ── Snapshot-URLs aus archive.is HTML extrahieren ─────────────────────────
 function extractSnapshotUrls(html) {
   const urls = [];
-  // Format 1: href="/HASH" - relativer Pfad (4-8 alphanumerische Zeichen)
+  // Format: href="/HASH" oder href="https://archive.XX/HASH"
-  const rel = /href="\/([a-zA-Z0-9]{4,8})(?:"|\?|\/)/g;
+  // Hash = 4-10 alphanumerische Zeichen, keine bekannten Pfade
+  const blacklist = new Set(['newest','oldest','submit','search','about','rss','api','timemap','http','www.']);
+
+  const rel = /href="\/([a-zA-Z0-9]{4,10})(?:"|\/|\?)/g;
   let m;
   while ((m = rel.exec(html)) !== null) {
     const h = m[1];
-    if (!['http','www.','subm','sear','abou','time','rss_','feed','api_'].some(x => h.startsWith(x))) {
+    if (!blacklist.has(h) && !/^\d+$/.test(h)) {
       urls.push(`https://archive.ph/${h}`);
     }
   }
-  // Format 2: href="https://archive.XX/HASH"
+
-  const abs = /href="(https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,8})(?:"|\?)/g;
+  const abs = /href="(https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10})(?:"|\/|\?)/g;
   while ((m = abs.exec(html)) !== null) {
-    urls.push(m[1]);
+    const h = m[1].split('/').pop();
+    if (!blacklist.has(h)) urls.push(m[1]);
   }
+
   return [...new Set(urls)];
 }
 
-// ── DEBUG: Roher Response von archive.is ──────────────────────────────────
+// ── Archivsuche mit Session-Cookie ────────────────────────────────────────
+async function findArchiveSnapshot(targetUrl) {
+  // Schritt 1: Session-Cookie holen
+  let cookieJar;
+  try { cookieJar = await initSession(); }
+  catch { cookieJar = {}; }
+
+  // Schritt 2: Suche mit Cookie und Referer
+  const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`;
+  const result = await fetchPage(searchUrl, {
+    cookieJar,
+    referer: 'https://archive.ph/',
+  });
+
+  if (result.statusCode === 429) {
+    throw new Error('archive.is Rate-Limit erreicht – bitte kurz warten und nochmal versuchen');
+  }
+  if (result.statusCode !== 200 || !result.body) return null;
+
+  const snapshots = extractSnapshotUrls(result.body);
+  return snapshots.length > 0 ? snapshots[0] : null;
+}
+
+// ── DEBUG endpoint ─────────────────────────────────────────────────────────
 router.post('/debug', authenticate, async (req, res) => {
   const { url } = req.body;
   if (!url) return res.status(400).json({ error: 'URL fehlt' });
   let targetUrl = url.trim();
   if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
 
-  const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`;
   try {
-    const result = await fetchPage(searchUrl);
+    let cookieJar;
+    try { cookieJar = await initSession(); } catch(e) { cookieJar = {}; }
+
+    const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`;
+    const result = await fetchPage(searchUrl, { cookieJar, referer: 'https://archive.ph/' });
     const extracted = extractSnapshotUrls(result.body);
+
     res.json({
       statusCode: result.statusCode,
       finalUrl: result.finalUrl,
+      cookiesObtained: Object.keys(cookieJar),
       bodyLength: result.body.length,
-      bodyPreview: result.body.substring(0, 2000),
+      bodyPreview: result.body.substring(0, 3000),
-      // Alle href-Werte zur Analyse
+      allHrefs: (result.body.match(/href="[^"]{1,80}"/g) || []).slice(0, 50),
-      allHrefs: (result.body.match(/href="[^"]{1,60}"/g) || []).slice(0, 40),
       extractedSnapshots: extracted,
     });
   } catch (err) {
@@ -90,15 +146,6 @@ router.post('/debug', authenticate, async (req, res) => {
   }
 });
 
-// ── Archivsuche ────────────────────────────────────────────────────────────
-async function findArchiveSnapshot(targetUrl) {
-  const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`;
-  const result = await fetchPage(searchUrl);
-  if (result.statusCode !== 200 || !result.body) return null;
-  const snapshots = extractSnapshotUrls(result.body);
-  return snapshots.length > 0 ? snapshots[0] : null;
-}
-
 // ── POST /check ────────────────────────────────────────────────────────────
 router.post('/check', authenticate, async (req, res) => {
   const { url } = req.body;
@@ -111,7 +158,7 @@ router.post('/check', authenticate, async (req, res) => {
     if (snapshotUrl) return res.json({ archiveUrl: snapshotUrl, type: 'newest' });
     return res.status(404).json({ error: 'Kein Archiv für diese URL gefunden' });
   } catch (err) {
-    return res.status(500).json({ error: 'Fehler beim Prüfen: ' + err.message });
+    return res.status(500).json({ error: err.message });
   }
 });