diff --git a/backend/src/tools/paywallkiller/routes.js b/backend/src/tools/paywallkiller/routes.js index fa0c535..9b9d28a 100644 --- a/backend/src/tools/paywallkiller/routes.js +++ b/backend/src/tools/paywallkiller/routes.js @@ -7,37 +7,55 @@ const path = require('path'); const fs = require('fs'); const crypto = require('crypto'); -// ── HTTP GET mit Redirect-Follow, gibt Body zurück ──────────────────────── +// ── HTTP GET mit Redirect-Follow + Cookie-Jar ───────────────────────────── function fetchPage(url, opts = {}) { return new Promise((resolve, reject) => { if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects')); const mod = url.startsWith('https') ? https : http; - const req = mod.get(url, { - headers: { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', - 'Accept-Language': 'de,en;q=0.5', - 'Accept-Encoding': 'identity', - 'Connection': 'keep-alive', - }, - timeout: 15000, - }, (res) => { + + // Cookies aus jar zusammenbauen + const cookieHeader = opts.cookieJar && Object.keys(opts.cookieJar).length + ? Object.entries(opts.cookieJar).map(([k,v]) => `${k}=${v}`).join('; ') + : undefined; + + const headers = { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8', + 'Accept-Encoding': 'identity', + 'Connection': 'keep-alive', + 'Sec-Fetch-Dest': 'document', + 'Sec-Fetch-Mode': 'navigate', + 'Sec-Fetch-Site': opts.referer ? 'same-origin' : 'none', + 'Upgrade-Insecure-Requests': '1', + ...(opts.referer ? { 'Referer': opts.referer } : {}), + ...(cookieHeader ? { 'Cookie': cookieHeader } : {}), + }; + + const req = mod.get(url, { headers, timeout: 15000 }, (res) => { + // Cookies sammeln + const jar = opts.cookieJar || {}; + const setCookies = res.headers['set-cookie'] || []; + for (const c of setCookies) { + const m = c.match(/^([^=]+)=([^;]*)/); + if (m) jar[m[1].trim()] = m[2].trim(); + } + if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) { let loc = res.headers.location; if (!loc.startsWith('http')) { - try { - const b = new URL(url); - loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; - } catch { return reject(new Error('Ungültige Redirect-URL')); } + try { const b = new URL(url); loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; } + catch { return reject(new Error('Ungültige Redirect-URL')); } } res.resume(); - return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1 }).then(resolve).catch(reject); + return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1, cookieJar: jar, referer: url }) + .then(resolve).catch(reject); } let body = ''; res.setEncoding('utf8'); - res.on('data', d => { if (body.length < 100000) body += d; }); - res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body, headers: res.headers })); + res.on('data', d => { if (body.length < 150000) body += d; }); + res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body, cookieJar: jar })); res.on('error', reject); }); req.on('error', reject); @@ -45,44 +63,82 @@ function fetchPage(url, opts = {}) { }); } +// ── Session initialisieren (Startseite besuchen für Cookies) ────────────── +async function initSession() { + const result = await fetchPage('https://archive.ph/', { cookieJar: {} }); + return result.cookieJar; +} + // ── Snapshot-URLs aus archive.is HTML extrahieren ───────────────────────── function extractSnapshotUrls(html) { const urls = []; - // Format 1: href="/HASH" - relativer Pfad (4-8 alphanumerische Zeichen) - const rel = /href="\/([a-zA-Z0-9]{4,8})(?:"|\?|\/)/g; + // Format: href="/HASH" oder href="https://archive.XX/HASH" + // Hash = 4-10 alphanumerische Zeichen, keine bekannten Pfade + const blacklist = new Set(['newest','oldest','submit','search','about','rss','api','timemap','http','www.']); + + const rel = /href="\/([a-zA-Z0-9]{4,10})(?:"|\/|\?)/g; let m; while ((m = rel.exec(html)) !== null) { const h = m[1]; - if (!['http','www.','subm','sear','abou','time','rss_','feed','api_'].some(x => h.startsWith(x))) { + if (!blacklist.has(h) && !/^\d+$/.test(h)) { urls.push(`https://archive.ph/${h}`); } } - // Format 2: href="https://archive.XX/HASH" - const abs = /href="(https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,8})(?:"|\?)/g; + + const abs = /href="(https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10})(?:"|\/|\?)/g; while ((m = abs.exec(html)) !== null) { - urls.push(m[1]); + const h = m[1].split('/').pop(); + if (!blacklist.has(h)) urls.push(m[1]); } + return [...new Set(urls)]; } -// ── DEBUG: Roher Response von archive.is ────────────────────────────────── +// ── Archivsuche mit Session-Cookie ──────────────────────────────────────── +async function findArchiveSnapshot(targetUrl) { + // Schritt 1: Session-Cookie holen + let cookieJar; + try { cookieJar = await initSession(); } + catch { cookieJar = {}; } + + // Schritt 2: Suche mit Cookie und Referer + const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`; + const result = await fetchPage(searchUrl, { + cookieJar, + referer: 'https://archive.ph/', + }); + + if (result.statusCode === 429) { + throw new Error('archive.is Rate-Limit erreicht – bitte kurz warten und nochmal versuchen'); + } + if (result.statusCode !== 200 || !result.body) return null; + + const snapshots = extractSnapshotUrls(result.body); + return snapshots.length > 0 ? snapshots[0] : null; +} + +// ── DEBUG endpoint ───────────────────────────────────────────────────────── router.post('/debug', authenticate, async (req, res) => { const { url } = req.body; if (!url) return res.status(400).json({ error: 'URL fehlt' }); let targetUrl = url.trim(); if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; - const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`; try { - const result = await fetchPage(searchUrl); + let cookieJar; + try { cookieJar = await initSession(); } catch(e) { cookieJar = {}; } + + const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`; + const result = await fetchPage(searchUrl, { cookieJar, referer: 'https://archive.ph/' }); const extracted = extractSnapshotUrls(result.body); + res.json({ statusCode: result.statusCode, finalUrl: result.finalUrl, + cookiesObtained: Object.keys(cookieJar), bodyLength: result.body.length, - bodyPreview: result.body.substring(0, 2000), - // Alle href-Werte zur Analyse - allHrefs: (result.body.match(/href="[^"]{1,60}"/g) || []).slice(0, 40), + bodyPreview: result.body.substring(0, 3000), + allHrefs: (result.body.match(/href="[^"]{1,80}"/g) || []).slice(0, 50), extractedSnapshots: extracted, }); } catch (err) { @@ -90,15 +146,6 @@ router.post('/debug', authenticate, async (req, res) => { } }); -// ── Archivsuche ──────────────────────────────────────────────────────────── -async function findArchiveSnapshot(targetUrl) { - const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`; - const result = await fetchPage(searchUrl); - if (result.statusCode !== 200 || !result.body) return null; - const snapshots = extractSnapshotUrls(result.body); - return snapshots.length > 0 ? snapshots[0] : null; -} - // ── POST /check ──────────────────────────────────────────────────────────── router.post('/check', authenticate, async (req, res) => { const { url } = req.body; @@ -111,7 +158,7 @@ router.post('/check', authenticate, async (req, res) => { if (snapshotUrl) return res.json({ archiveUrl: snapshotUrl, type: 'newest' }); return res.status(404).json({ error: 'Kein Archiv für diese URL gefunden' }); } catch (err) { - return res.status(500).json({ error: 'Fehler beim Prüfen: ' + err.message }); + return res.status(500).json({ error: err.message }); } });