debug: archive.is raw response endpoint

backend/src/tools/paywallkiller/routes.js

@@ -7,7 +7,7 @@ const path = require('path');
 const fs = require('fs');
 const crypto = require('crypto');
 
-// ── HTTP GET, gibt finalUrl + statusCode + body (optional) zurück ─────────
+// ── HTTP GET mit Redirect-Follow, gibt Body zurück ────────────────────────
 function fetchPage(url, opts = {}) {
   return new Promise((resolve, reject) => {
     if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects'));
@@ -15,86 +15,107 @@ function fetchPage(url, opts = {}) {
     const req = mod.get(url, {
       headers: {
         'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
-        'Accept': 'text/html,application/xhtml+xml',
+        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
         'Accept-Language': 'de,en;q=0.5',
+        'Accept-Encoding': 'identity',
+        'Connection': 'keep-alive',
       },
-      timeout: 12000,
+      timeout: 15000,
     }, (res) => {
       if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) {
         let loc = res.headers.location;
         if (!loc.startsWith('http')) {
-          try { const b = new URL(url); loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; }
-          catch { return reject(new Error('Ungültige Redirect-URL')); }
+          try {
+            const b = new URL(url);
+            loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`;
+          } catch { return reject(new Error('Ungültige Redirect-URL')); }
         }
         res.resume();
         return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1 }).then(resolve).catch(reject);
       }
 
-      if (opts.bodyNeeded) {
-        let body = '';
-        res.setEncoding('utf8');
-        res.on('data', d => { if (body.length < 50000) body += d; });
-        res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body }));
-      } else {
-        res.resume();
-        resolve({ statusCode: res.statusCode, finalUrl: url });
-      }
+      let body = '';
+      res.setEncoding('utf8');
+      res.on('data', d => { if (body.length < 100000) body += d; });
+      res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body, headers: res.headers }));
+      res.on('error', reject);
     });
     req.on('error', reject);
     req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
   });
 }
 
-// ── Archivsuche über archive.is ───────────────────────────────────────────
-// archive.is/?url=ZIEL gibt HTML zurück mit Links zu archivierten Snapshots.
-// Wir parsen den ersten Snapshot-Link daraus.
-async function findArchiveSnapshot(targetUrl) {
-  const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`;
-  try {
-    const result = await fetchPage(searchUrl, { bodyNeeded: true });
-    if (result.statusCode !== 200 || !result.body) return null;
-
-    // Snapshots stehen als Links in der Form href="/ABCDE" oder href="https://archive.is/ABCDE"
-    // Wir suchen den neuesten (ersten in der Liste – archive.is sortiert newest first)
-    const hashPattern = /href="(?:https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg))?\/([a-zA-Z0-9]{4,10})"/g;
-    let match;
-    const hashes = new Set();
-    while ((match = hashPattern.exec(result.body)) !== null) {
-      const h = match[1];
-      // Blacklist bekannter Nicht-Snapshot-Pfade
-      if (['newest','oldest','submit','search','about','faq','rss'].includes(h)) continue;
-      hashes.add(h);
+// ── Snapshot-URLs aus archive.is HTML extrahieren ─────────────────────────
+function extractSnapshotUrls(html) {
+  const urls = [];
+  // Format 1: href="/HASH" - relativer Pfad (4-8 alphanumerische Zeichen)
+  const rel = /href="\/([a-zA-Z0-9]{4,8})(?:"|\?|\/)/g;
+  let m;
+  while ((m = rel.exec(html)) !== null) {
+    const h = m[1];
+    if (!['http','www.','subm','sear','abou','time','rss_','feed','api_'].some(x => h.startsWith(x))) {
+      urls.push(`https://archive.ph/${h}`);
     }
-    if (hashes.size === 0) return null;
-
-    // Erster Hash = neuester Snapshot
-    const newestHash = [...hashes][0];
-    return `https://archive.is/${newestHash}`;
-  } catch {
-    return null;
   }
+  // Format 2: href="https://archive.XX/HASH"
+  const abs = /href="(https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,8})(?:"|\?)/g;
+  while ((m = abs.exec(html)) !== null) {
+    urls.push(m[1]);
+  }
+  return [...new Set(urls)];
 }
 
-// ── POST /api/tools/paywallkiller/check ───────────────────────────────────
+// ── DEBUG: Roher Response von archive.is ──────────────────────────────────
+router.post('/debug', authenticate, async (req, res) => {
+  const { url } = req.body;
+  if (!url) return res.status(400).json({ error: 'URL fehlt' });
+  let targetUrl = url.trim();
+  if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
+
+  const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`;
+  try {
+    const result = await fetchPage(searchUrl);
+    const extracted = extractSnapshotUrls(result.body);
+    res.json({
+      statusCode: result.statusCode,
+      finalUrl: result.finalUrl,
+      bodyLength: result.body.length,
+      bodyPreview: result.body.substring(0, 2000),
+      // Alle href-Werte zur Analyse
+      allHrefs: (result.body.match(/href="[^"]{1,60}"/g) || []).slice(0, 40),
+      extractedSnapshots: extracted,
+    });
+  } catch (err) {
+    res.status(500).json({ error: err.message });
+  }
+});
+
+// ── Archivsuche ────────────────────────────────────────────────────────────
+async function findArchiveSnapshot(targetUrl) {
+  const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`;
+  const result = await fetchPage(searchUrl);
+  if (result.statusCode !== 200 || !result.body) return null;
+  const snapshots = extractSnapshotUrls(result.body);
+  return snapshots.length > 0 ? snapshots[0] : null;
+}
+
+// ── POST /check ────────────────────────────────────────────────────────────
 router.post('/check', authenticate, async (req, res) => {
   const { url } = req.body;
   if (!url || typeof url !== 'string') return res.status(400).json({ error: 'URL fehlt' });
-
   let targetUrl = url.trim();
   if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
 
   try {
     const snapshotUrl = await findArchiveSnapshot(targetUrl);
-    if (snapshotUrl) {
-      return res.json({ archiveUrl: snapshotUrl, type: 'newest' });
-    }
+    if (snapshotUrl) return res.json({ archiveUrl: snapshotUrl, type: 'newest' });
     return res.status(404).json({ error: 'Kein Archiv für diese URL gefunden' });
   } catch (err) {
     return res.status(500).json({ error: 'Fehler beim Prüfen: ' + err.message });
   }
 });
 
-// ── POST /api/tools/paywallkiller/pdf ─────────────────────────────────────
+// ── POST /pdf ──────────────────────────────────────────────────────────────
 router.post('/pdf', authenticate, async (req, res) => {
   const { archiveUrl } = req.body;
   if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' });
@@ -107,33 +128,24 @@ router.post('/pdf', authenticate, async (req, res) => {
 
   try {
     let puppeteer, chromium;
-    try {
-      puppeteer = require('puppeteer-core');
-      chromium  = require('@sparticuz/chromium');
-    } catch {
-      try { puppeteer = require('puppeteer'); chromium = null; }
-      catch { return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar – puppeteer nicht installiert' }); }
-    }
+    try { puppeteer = require('puppeteer-core'); chromium = require('@sparticuz/chromium'); }
+    catch { try { puppeteer = require('puppeteer'); chromium = null; }
+    catch { return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar' }); } }
 
     const launchOptions = chromium
-      ? {
-          args: [...chromium.args, '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'],
+      ? { args: [...chromium.args, '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'],
           defaultViewport: { width: 1280, height: 900 },
           executablePath: process.env.CHROMIUM_PATH || await chromium.executablePath(),
-          headless: true,
-        }
+          headless: true }
       : { headless: 'new', args: ['--no-sandbox', '--disable-setuid-sandbox'] };
 
     browser = await puppeteer.launch(launchOptions);
     const page = await browser.newPage();
     await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
     await page.goto(archiveUrl, { waitUntil: 'networkidle2', timeout: 30000 });
-    await page.pdf({
-      path: tmpFile, format: 'A4', printBackground: true,
-      margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' },
-    });
-    await browser.close();
-    browser = null;
+    await page.pdf({ path: tmpFile, format: 'A4', printBackground: true,
+      margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' } });
+    await browser.close(); browser = null;
 
     res.setHeader('Content-Type', 'application/pdf');
     res.setHeader('Content-Disposition', `attachment; filename="archiv_${Date.now()}.pdf"`);