diff --git a/backend/src/tools/paywallkiller/routes.js b/backend/src/tools/paywallkiller/routes.js index d83e5d4..fa0c535 100644 --- a/backend/src/tools/paywallkiller/routes.js +++ b/backend/src/tools/paywallkiller/routes.js @@ -7,7 +7,7 @@ const path = require('path'); const fs = require('fs'); const crypto = require('crypto'); -// ── HTTP GET, gibt finalUrl + statusCode + body (optional) zurück ───────── +// ── HTTP GET mit Redirect-Follow, gibt Body zurück ──────────────────────── function fetchPage(url, opts = {}) { return new Promise((resolve, reject) => { if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects')); @@ -15,86 +15,107 @@ function fetchPage(url, opts = {}) { const req = mod.get(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', 'Accept-Language': 'de,en;q=0.5', + 'Accept-Encoding': 'identity', + 'Connection': 'keep-alive', }, - timeout: 12000, + timeout: 15000, }, (res) => { if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) { let loc = res.headers.location; if (!loc.startsWith('http')) { - try { const b = new URL(url); loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; } - catch { return reject(new Error('Ungültige Redirect-URL')); } + try { + const b = new URL(url); + loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; + } catch { return reject(new Error('Ungültige Redirect-URL')); } } res.resume(); return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1 }).then(resolve).catch(reject); } - if (opts.bodyNeeded) { - let body = ''; - res.setEncoding('utf8'); - res.on('data', d => { if (body.length < 50000) body += d; }); - res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body })); - } else { - res.resume(); - resolve({ statusCode: res.statusCode, finalUrl: url }); - } + let body = ''; + res.setEncoding('utf8'); + res.on('data', d => { if (body.length < 100000) body += d; }); + res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body, headers: res.headers })); + res.on('error', reject); }); req.on('error', reject); req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); }); }); } -// ── Archivsuche über archive.is ─────────────────────────────────────────── -// archive.is/?url=ZIEL gibt HTML zurück mit Links zu archivierten Snapshots. -// Wir parsen den ersten Snapshot-Link daraus. -async function findArchiveSnapshot(targetUrl) { - const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`; - try { - const result = await fetchPage(searchUrl, { bodyNeeded: true }); - if (result.statusCode !== 200 || !result.body) return null; - - // Snapshots stehen als Links in der Form href="/ABCDE" oder href="https://archive.is/ABCDE" - // Wir suchen den neuesten (ersten in der Liste – archive.is sortiert newest first) - const hashPattern = /href="(?:https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg))?\/([a-zA-Z0-9]{4,10})"/g; - let match; - const hashes = new Set(); - while ((match = hashPattern.exec(result.body)) !== null) { - const h = match[1]; - // Blacklist bekannter Nicht-Snapshot-Pfade - if (['newest','oldest','submit','search','about','faq','rss'].includes(h)) continue; - hashes.add(h); +// ── Snapshot-URLs aus archive.is HTML extrahieren ───────────────────────── +function extractSnapshotUrls(html) { + const urls = []; + // Format 1: href="/HASH" - relativer Pfad (4-8 alphanumerische Zeichen) + const rel = /href="\/([a-zA-Z0-9]{4,8})(?:"|\?|\/)/g; + let m; + while ((m = rel.exec(html)) !== null) { + const h = m[1]; + if (!['http','www.','subm','sear','abou','time','rss_','feed','api_'].some(x => h.startsWith(x))) { + urls.push(`https://archive.ph/${h}`); } - if (hashes.size === 0) return null; - - // Erster Hash = neuester Snapshot - const newestHash = [...hashes][0]; - return `https://archive.is/${newestHash}`; - } catch { - return null; } + // Format 2: href="https://archive.XX/HASH" + const abs = /href="(https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,8})(?:"|\?)/g; + while ((m = abs.exec(html)) !== null) { + urls.push(m[1]); + } + return [...new Set(urls)]; } -// ── POST /api/tools/paywallkiller/check ─────────────────────────────────── +// ── DEBUG: Roher Response von archive.is ────────────────────────────────── +router.post('/debug', authenticate, async (req, res) => { + const { url } = req.body; + if (!url) return res.status(400).json({ error: 'URL fehlt' }); + let targetUrl = url.trim(); + if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; + + const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`; + try { + const result = await fetchPage(searchUrl); + const extracted = extractSnapshotUrls(result.body); + res.json({ + statusCode: result.statusCode, + finalUrl: result.finalUrl, + bodyLength: result.body.length, + bodyPreview: result.body.substring(0, 2000), + // Alle href-Werte zur Analyse + allHrefs: (result.body.match(/href="[^"]{1,60}"/g) || []).slice(0, 40), + extractedSnapshots: extracted, + }); + } catch (err) { + res.status(500).json({ error: err.message }); + } +}); + +// ── Archivsuche ──────────────────────────────────────────────────────────── +async function findArchiveSnapshot(targetUrl) { + const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`; + const result = await fetchPage(searchUrl); + if (result.statusCode !== 200 || !result.body) return null; + const snapshots = extractSnapshotUrls(result.body); + return snapshots.length > 0 ? snapshots[0] : null; +} + +// ── POST /check ──────────────────────────────────────────────────────────── router.post('/check', authenticate, async (req, res) => { const { url } = req.body; if (!url || typeof url !== 'string') return res.status(400).json({ error: 'URL fehlt' }); - let targetUrl = url.trim(); if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; try { const snapshotUrl = await findArchiveSnapshot(targetUrl); - if (snapshotUrl) { - return res.json({ archiveUrl: snapshotUrl, type: 'newest' }); - } + if (snapshotUrl) return res.json({ archiveUrl: snapshotUrl, type: 'newest' }); return res.status(404).json({ error: 'Kein Archiv für diese URL gefunden' }); } catch (err) { return res.status(500).json({ error: 'Fehler beim Prüfen: ' + err.message }); } }); -// ── POST /api/tools/paywallkiller/pdf ───────────────────────────────────── +// ── POST /pdf ────────────────────────────────────────────────────────────── router.post('/pdf', authenticate, async (req, res) => { const { archiveUrl } = req.body; if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' }); @@ -107,33 +128,24 @@ router.post('/pdf', authenticate, async (req, res) => { try { let puppeteer, chromium; - try { - puppeteer = require('puppeteer-core'); - chromium = require('@sparticuz/chromium'); - } catch { - try { puppeteer = require('puppeteer'); chromium = null; } - catch { return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar – puppeteer nicht installiert' }); } - } + try { puppeteer = require('puppeteer-core'); chromium = require('@sparticuz/chromium'); } + catch { try { puppeteer = require('puppeteer'); chromium = null; } + catch { return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar' }); } } const launchOptions = chromium - ? { - args: [...chromium.args, '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], + ? { args: [...chromium.args, '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], defaultViewport: { width: 1280, height: 900 }, executablePath: process.env.CHROMIUM_PATH || await chromium.executablePath(), - headless: true, - } + headless: true } : { headless: 'new', args: ['--no-sandbox', '--disable-setuid-sandbox'] }; browser = await puppeteer.launch(launchOptions); const page = await browser.newPage(); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'); await page.goto(archiveUrl, { waitUntil: 'networkidle2', timeout: 30000 }); - await page.pdf({ - path: tmpFile, format: 'A4', printBackground: true, - margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' }, - }); - await browser.close(); - browser = null; + await page.pdf({ path: tmpFile, format: 'A4', printBackground: true, + margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' } }); + await browser.close(); browser = null; res.setHeader('Content-Type', 'application/pdf'); res.setHeader('Content-Disposition', `attachment; filename="archiv_${Date.now()}.pdf"`);