diff --git a/backend/src/tools/paywallkiller/routes.js b/backend/src/tools/paywallkiller/routes.js index 0a93fcf..cd8aa46 100644 --- a/backend/src/tools/paywallkiller/routes.js +++ b/backend/src/tools/paywallkiller/routes.js @@ -7,49 +7,64 @@ const path = require('path'); const fs = require('fs'); const crypto = require('crypto'); -// ── Hilfsfunktion: HTTP GET mit Follow-Redirect ──────────────────────────── -function fetchUrl(url, options = {}) { +// ── Hilfsfunktion: folgt Redirects, gibt finale URL zurück ─────────────── +function fetchFollowRedirects(url, redirectCount = 0) { return new Promise((resolve, reject) => { + if (redirectCount > 8) return reject(new Error('Zu viele Redirects')); const mod = url.startsWith('https') ? https : http; const req = mod.get(url, { headers: { - 'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)', - ...options.headers, + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', + 'Accept-Language': 'de,en;q=0.5', }, - timeout: 10000, + timeout: 12000, }, (res) => { - // Redirects manuell folgen (max 5) - if ([301, 302, 303, 307, 308].includes(res.statusCode) && res.headers.location && (options.redirects || 0) < 5) { - const location = res.headers.location.startsWith('http') - ? res.headers.location - : new URL(res.headers.location, url).href; + if ([301, 302, 303, 307, 308].includes(res.statusCode) && res.headers.location) { + let location = res.headers.location; + if (!location.startsWith('http')) { + try { + const base = new URL(url); + location = location.startsWith('/') + ? `${base.protocol}//${base.host}${location}` + : `${base.protocol}//${base.host}/${location}`; + } catch { return reject(new Error('Ungültige Redirect-URL')); } + } res.resume(); - return fetchUrl(location, { ...options, redirects: (options.redirects || 0) + 1 }) - .then(resolve).catch(reject); + return fetchFollowRedirects(location, redirectCount + 1).then(resolve).catch(reject); } - resolve({ statusCode: res.statusCode, url, finalUrl: url, headers: res.headers, res }); + res.resume(); + resolve({ statusCode: res.statusCode, finalUrl: url }); }); req.on('error', reject); req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); }); }); } -// ── Prüfe ob archive.is-URL existiert (HEAD-ähnlich via GET + sofortiges Abbrechen) ── +// ── Prüfe ob archive.is einen Snapshot hat ──────────────────────────────── +// archive.is/newest/URL leitet auf archive.is/HASH weiter wenn ein Snapshot existiert. +// Wenn kein Snapshot → Redirect zurück auf archive.is/ (Root) oder /newest/... bleibt stehen. async function checkArchive(archiveUrl) { try { - const result = await fetchUrl(archiveUrl); - result.res.destroy(); - // archive.is gibt 200 zurück wenn Archiv existiert, 404 oder Redirect zu / wenn nicht - if (result.statusCode === 200) return true; - return false; + const result = await fetchFollowRedirects(archiveUrl); + const final = result.finalUrl; + // Kein Redirect = nichts gefunden (blieb auf gleicher URL) + if (final === archiveUrl) return { found: false }; + // Snapshot-URL hat die Form https://archive.is/ABCDE (alphanumeric hash, 4-8 Zeichen) + if (/https?:\/\/archive\.(is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10}$/.test(final)) { + return { found: true, resolvedUrl: final }; + } + // archive.is Root oder Submit-Seite = kein Treffer + if (/archive\.[a-z]+\/?$/.test(final)) return { found: false }; + if (final.includes('/submit')) return { found: false }; + // Sonst: irgendwohin weitergeleitet → als Treffer werten + return { found: true, resolvedUrl: final }; } catch { - return false; + return { found: false }; } } // ── POST /api/tools/paywallkiller/check ─────────────────────────────────── -// Body: { url: "https://..." } -// Response: { archiveUrl: "https://archive.is/...", type: "newest"|"oldest" } router.post('/check', authenticate, async (req, res) => { const { url } = req.body; if (!url || typeof url !== 'string') return res.status(400).json({ error: 'URL fehlt' }); @@ -61,13 +76,13 @@ router.post('/check', authenticate, async (req, res) => { const oldestUrl = `https://archive.is/oldest/${targetUrl}`; try { - const newestOk = await checkArchive(newestUrl); - if (newestOk) { - return res.json({ archiveUrl: newestUrl, type: 'newest' }); + const newest = await checkArchive(newestUrl); + if (newest.found) { + return res.json({ archiveUrl: newest.resolvedUrl || newestUrl, type: 'newest' }); } - const oldestOk = await checkArchive(oldestUrl); - if (oldestOk) { - return res.json({ archiveUrl: oldestUrl, type: 'oldest' }); + const oldest = await checkArchive(oldestUrl); + if (oldest.found) { + return res.json({ archiveUrl: oldest.resolvedUrl || oldestUrl, type: 'oldest' }); } return res.status(404).json({ error: 'Kein Archiv für diese URL gefunden' }); } catch (err) { @@ -76,12 +91,12 @@ router.post('/check', authenticate, async (req, res) => { }); // ── POST /api/tools/paywallkiller/pdf ───────────────────────────────────── -// Body: { archiveUrl: "https://archive.is/..." } -// Response: PDF als Download-Stream, Datei danach gelöscht router.post('/pdf', authenticate, async (req, res) => { const { archiveUrl } = req.body; if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' }); - if (!archiveUrl.startsWith('https://archive.is/')) return res.status(400).json({ error: 'Ungültige archive.is URL' }); + if (!/^https?:\/\/archive\.(is|ph|today|fo|li|vn|md|gg)\//.test(archiveUrl)) { + return res.status(400).json({ error: 'Ungültige archive.is URL' }); + } let browser; const tmpFile = path.join('/tmp', `pwk_${crypto.randomBytes(8).toString('hex')}.pdf`); @@ -89,12 +104,10 @@ router.post('/pdf', authenticate, async (req, res) => { try { let puppeteer, chromium; - // Versuche puppeteer-core + @sparticuz/chromium (Alpine-kompatibel) try { puppeteer = require('puppeteer-core'); chromium = require('@sparticuz/chromium'); } catch { - // Fallback: normales puppeteer (falls installiert) try { puppeteer = require('puppeteer'); chromium = null; @@ -105,12 +118,7 @@ router.post('/pdf', authenticate, async (req, res) => { const launchOptions = chromium ? { - args: [ - ...chromium.args, - '--no-sandbox', - '--disable-setuid-sandbox', - '--disable-dev-shm-usage', - ], + args: [...chromium.args, '--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage'], defaultViewport: { width: 1280, height: 900 }, executablePath: process.env.CHROMIUM_PATH || await chromium.executablePath(), headless: true, @@ -119,32 +127,23 @@ router.post('/pdf', authenticate, async (req, res) => { browser = await puppeteer.launch(launchOptions); const page = await browser.newPage(); - await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'); await page.goto(archiveUrl, { waitUntil: 'networkidle2', timeout: 30000 }); - await page.pdf({ path: tmpFile, format: 'A4', printBackground: true, margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' }, }); - await browser.close(); browser = null; - const filename = 'archiv_' + Date.now() + '.pdf'; res.setHeader('Content-Type', 'application/pdf'); - res.setHeader('Content-Disposition', `attachment; filename="${filename}"`); - + res.setHeader('Content-Disposition', `attachment; filename="archiv_${Date.now()}.pdf"`); const stream = fs.createReadStream(tmpFile); stream.pipe(res); - stream.on('end', () => { - fs.unlink(tmpFile, () => {}); - }); - stream.on('error', () => { - fs.unlink(tmpFile, () => {}); - }); + stream.on('end', () => fs.unlink(tmpFile, () => {})); + stream.on('error', () => fs.unlink(tmpFile, () => {})); } catch (err) { if (browser) try { await browser.close(); } catch {}