Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: alle Routes wiederhergestellt (wayback, wayback-save, pdf)

Browse Source
This commit is contained in:
Dicken
2026-06-12 12:34:06 +02:00
parent 1ab3d2494f
commit 6eb9e1cc72
1 changed files with 98 additions and 179 deletions
Download Patch File Download Diff File Expand all files Collapse all files

277
backend/src/tools/paywallkiller/routes.js
View File

@@ -12,35 +12,24 @@ function fetchPage(url, opts = {}) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects')); if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects'));
const mod = url.startsWith('https') ? https : http; const mod = url.startsWith('https') ? https : http;
// Cookies aus jar zusammenbauen
const cookieHeader = opts.cookieJar && Object.keys(opts.cookieJar).length const cookieHeader = opts.cookieJar && Object.keys(opts.cookieJar).length
? Object.entries(opts.cookieJar).map(([k,v]) => `${k}=${v}`).join('; ') ? Object.entries(opts.cookieJar).map(([k,v]) => `${k}=${v}`).join('; ')
: undefined; : undefined;
const headers = { const headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8', 'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
'Accept-Encoding': 'identity', 'Accept-Encoding': 'identity',
'Connection': 'keep-alive', 'Connection': 'keep-alive',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': opts.referer ? 'same-origin' : 'none',
'Upgrade-Insecure-Requests': '1',
...(opts.referer ? { 'Referer': opts.referer } : {}), ...(opts.referer ? { 'Referer': opts.referer } : {}),
...(cookieHeader ? { 'Cookie': cookieHeader } : {}), ...(cookieHeader ? { 'Cookie': cookieHeader } : {}),
}; };
const req = mod.get(url, { headers, timeout: 15000 }, (res) => { const req = mod.get(url, { headers, timeout: 15000 }, (res) => {
// Cookies sammeln
const jar = opts.cookieJar || {}; const jar = opts.cookieJar || {};
const setCookies = res.headers['set-cookie'] || []; for (const c of (res.headers['set-cookie'] || [])) {
for (const c of setCookies) {
const m = c.match(/^([^=]+)=([^;]*)/); const m = c.match(/^([^=]+)=([^;]*)/);
if (m) jar[m[1].trim()] = m[2].trim(); if (m) jar[m[1].trim()] = m[2].trim();
} }
if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) { if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) {
let loc = res.headers.location; let loc = res.headers.location;
if (!loc.startsWith('http')) { if (!loc.startsWith('http')) {
@@ -48,10 +37,8 @@ function fetchPage(url, opts = {}) {
catch { return reject(new Error('Ungültige Redirect-URL')); } catch { return reject(new Error('Ungültige Redirect-URL')); }
} }
res.resume(); res.resume();
return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1, cookieJar: jar, referer: url }) return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1, cookieJar: jar, referer: url }).then(resolve).catch(reject);
.then(resolve).catch(reject);
} }
let body = ''; let body = '';
res.setEncoding('utf8'); res.setEncoding('utf8');
res.on('data', d => { if (body.length < 150000) body += d; }); res.on('data', d => { if (body.length < 150000) body += d; });
@@ -63,137 +50,104 @@ function fetchPage(url, opts = {}) {
}); });
} }
// ── Session initialisieren (Startseite besuchen für Cookies) ────────────── // ── GET /api/tools/paywallkiller/wayback?url=... ──────────────────────────
async function initSession() { // Sucht vorhandenen Wayback-Snapshot via CDX API
const result = await fetchPage('https://archive.ph/', { cookieJar: {} }); router.get('/wayback', authenticate, async (req, res) => {
return result.cookieJar; const { url } = req.query;
} if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url Parameter fehlt' });
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
// ── Snapshot-URLs aus archive.is HTML extrahieren ───────────────────────── try {
function extractSnapshotUrls(html) { const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
const urls = []; const r = await fetchPage(cdxUrl);
// Format: href="/HASH" oder href="https://archive.XX/HASH" if (r.statusCode === 200 && r.body) {
// Hash = 4-10 alphanumerische Zeichen, keine bekannten Pfade let data;
const blacklist = new Set(['newest','oldest','submit','search','about','rss','api','timemap','http','www.']); try { data = JSON.parse(r.body); } catch { data = null; }
if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
const rel = /href="\/([a-zA-Z0-9]{4,10})(?:"|\/|\?)/g; return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` });
let m; }
while ((m = rel.exec(html)) !== null) {
const h = m[1];
if (!blacklist.has(h) && !/^\d+$/.test(h)) {
urls.push(`https://archive.ph/${h}`);
} }
return res.json({ found: false });
} catch (err) {
return res.status(500).json({ error: 'Fehler: ' + err.message });
} }
});
const abs = /href="(https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10})(?:"|\/|\?)/g; // ── POST /api/tools/paywallkiller/wayback-save ────────────────────────────
while ((m = abs.exec(html)) !== null) { // Schießt Save-Request ab (fire-and-forget), pollt dann CDX auf Ergebnis
const h = m[1].split('/').pop(); router.post('/wayback-save', authenticate, async (req, res) => {
if (!blacklist.has(h)) urls.push(m[1]);
}
return [...new Set(urls)];
}
// ── Archivsuche mit Session-Cookie ────────────────────────────────────────
async function findArchiveSnapshot(targetUrl) {
// Schritt 1: Session-Cookie holen
let cookieJar;
try { cookieJar = await initSession(); }
catch { cookieJar = {}; }
// Schritt 2: Suche mit Cookie und Referer
const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`;
const result = await fetchPage(searchUrl, {
cookieJar,
referer: 'https://archive.ph/',
});
if (result.statusCode === 429) {
throw new Error('archive.is Rate-Limit erreicht – bitte kurz warten und nochmal versuchen');
}
if (result.statusCode !== 200 || !result.body) return null;
const snapshots = extractSnapshotUrls(result.body);
return snapshots.length > 0 ? snapshots[0] : null;
}
// ── DEBUG endpoint ─────────────────────────────────────────────────────────
router.post('/debug', authenticate, async (req, res) => {
const { url } = req.body; const { url } = req.body;
if (!url) return res.status(400).json({ error: 'URL fehlt' }); if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url fehlt' });
let targetUrl = url.trim(); let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl; if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
try { async function checkCdx() {
let cookieJar; const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
try { cookieJar = await initSession(); } catch(e) { cookieJar = {}; } try {
const r = await fetchPage(cdxUrl);
if (r.statusCode === 200 && r.body) {
const data = JSON.parse(r.body);
if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`;
}
}
} catch {}
return null;
}
const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`; function fireSaveRequest() {
const result = await fetchPage(searchUrl, { cookieJar, referer: 'https://archive.ph/' }); return new Promise((resolve) => {
const extracted = extractSnapshotUrls(result.body); const savePath = `/save/${targetUrl}`;
const req = https.get({
res.json({ hostname: 'web.archive.org',
statusCode: result.statusCode, path: savePath,
finalUrl: result.finalUrl, headers: { 'User-Agent': 'Mozilla/5.0 (compatible; WaybackSave/1.0)', 'Accept': 'text/html' },
cookiesObtained: Object.keys(cookieJar), timeout: 8000,
bodyLength: result.body.length, }, (res) => {
bodyPreview: result.body.substring(0, 3000), const location = res.headers.location || '';
allHrefs: (result.body.match(/href="[^"]{1,80}"/g) || []).slice(0, 50), res.resume();
extractedSnapshots: extracted, resolve(location);
});
req.on('error', () => resolve(''));
req.on('timeout', () => { req.destroy(); resolve(''); });
}); });
} catch (err) {
res.status(500).json({ error: err.message });
} }
});
// ── POST /check ────────────────────────────────────────────────────────────
router.post('/check', authenticate, async (req, res) => {
const { url } = req.body;
if (!url || typeof url !== 'string') return res.status(400).json({ error: 'URL fehlt' });
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
try { try {
const snapshotUrl = await findArchiveSnapshot(targetUrl); const location = await fireSaveRequest();
if (snapshotUrl) return res.json({ archiveUrl: snapshotUrl, type: 'newest' }); const directMatch = location.match(/\/web\/(\d{14})\//);
return res.status(404).json({ error: 'Kein Archiv für diese URL gefunden' }); if (directMatch) {
const waybackUrl = location.startsWith('http') ? location : `https://web.archive.org${location}`;
return res.json({ success: true, waybackUrl });
}
// Polling: bis zu 6x alle 5s
for (let i = 0; i < 6; i++) {
await new Promise(r => setTimeout(r, 5000));
const found = await checkCdx();
if (found) return res.json({ success: true, waybackUrl: found });
}
return res.status(502).json({ error: 'Archivierung läuft noch oder wurde blockiert. Versuche es in einer Minute erneut.' });
} catch (err) { } catch (err) {
return res.status(500).json({ error: err.message }); return res.status(500).json({ error: 'Save fehlgeschlagen: ' + err.message });
} }
}); });
// ── POST /pdf ────────────────────────────────────────────────────────────── // ── POST /api/tools/paywallkiller/pdf ─────────────────────────────────────
// Lädt die Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF // Lädt Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF
router.post('/pdf', authenticate, async (req, res) => { router.post('/pdf', authenticate, async (req, res) => {
const { archiveUrl, originalUrl } = req.body; const { archiveUrl, originalUrl } = req.body;
// originalUrl bevorzugen (direkter Artikel ohne Wayback-Paywall)
// archiveUrl als Fallback (Wayback Machine)
const targetUrl = originalUrl || archiveUrl; const targetUrl = originalUrl || archiveUrl;
if (!targetUrl || typeof targetUrl !== 'string') { if (!targetUrl || typeof targetUrl !== 'string') return res.status(400).json({ error: 'URL fehlt' });
return res.status(400).json({ error: 'URL fehlt' });
}
const tmpFile = require('path').join('/tmp', `pwk_${require('crypto').randomBytes(8).toString('hex')}.pdf`); const tmpFile = path.join('/tmp', `pwk_${crypto.randomBytes(8).toString('hex')}.pdf`);
let browser = null; let browser = null;
try { try {
const puppeteer = require('puppeteer-core'); const puppeteer = require('puppeteer-core');
const candidates = [process.env.CHROMIUM_PATH, '/usr/bin/chromium-browser', '/usr/bin/chromium', '/usr/bin/google-chrome'].filter(Boolean);
const chromiumPaths = [ const executablePath = candidates.find(p => { try { fs.accessSync(p); return true; } catch { return false; } });
process.env.CHROMIUM_PATH, if (!executablePath) return res.status(500).json({ error: 'Chromium nicht gefunden' });
'/usr/bin/chromium-browser',
'/usr/bin/chromium',
'/usr/bin/google-chrome',
].filter(Boolean);
let executablePath = null;
for (const p of chromiumPaths) {
if (require('fs').existsSync(p)) { executablePath = p; break; }
}
if (!executablePath) {
return res.status(500).json({ error: 'Chromium nicht gefunden' });
}
browser = await puppeteer.launch({ browser = await puppeteer.launch({
executablePath, executablePath,
@@ -205,78 +159,43 @@ router.post('/pdf', authenticate, async (req, res) => {
const page = await browser.newPage(); const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36'); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36');
await page.setViewport({ width: 1280, height: 900 }); await page.setViewport({ width: 1280, height: 900 });
// Seite laden – networkidle0 damit JS vollständig läuft (Artikel wird geladen)
await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 }); await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 });
// Kurz warten damit Artikel-Content gerendert ist
await new Promise(r => setTimeout(r, 1500)); await new Promise(r => setTimeout(r, 1500));
// Paywall-Overlays, Cookie-Banner, Sticky-Header ausblenden
await page.addStyleTag({ content: ` await page.addStyleTag({ content: `
/* Generische Paywall/Overlay-Selektoren */ [class*="paywall"],[class*="Paywall"],[id*="paywall"],
[class*="paywall"], [class*="Paywall"], [class*="piano"],[class*="Piano"],[id*="piano"],
[class*="piano"], [class*="Piano"], [class*="subscription"],[class*="subscribe"],
[id*="paywall"], [id*="piano"], [class*="overlay"],[class*="modal"],
[class*="subscription"], [class*="subscribe"], [class*="cookie"],[class*="Cookie"],
[class*="overlay"], [class*="modal"], [class*="consent"],[class*="gdpr"],
[class*="cookie"], [class*="Cookie"], .tp-modal,.tp-backdrop,#tp-container,
[class*="consent"], [class*="Consent"], .article__paywall,.funke-piano,.piano-checkout,
[class*="gdpr"], [class*="GDPR"], [data-testid*="paywall"],[data-testid*="piano"],
[class*="sticky-header"], [class*="stickyHeader"], #usercentrics-root,.uc-banner,
[class*="fixed-header"], [class*="fixedHeader"], #onetrust-consent-sdk,.onetrust-pc-dark-filter,
/* WAZ/Funke spezifisch */ #wm-ipp-base,#wm-ipp,#donato,
.tp-modal, .tp-backdrop, #tp-container, [class*="ad-"],[class*="-ad-"],[class*="advertisement"]
.article__paywall, .paywall-container, { display:none !important; visibility:hidden !important; }
.funke-piano, .piano-checkout, body,html { overflow:visible !important; height:auto !important; }
[data-testid*="paywall"], [data-testid*="piano"], * { filter:none !important; }
/* Cookie-Banner */
#usercentrics-root, .uc-banner,
#onetrust-consent-sdk, .onetrust-pc-dark-filter,
/* Sticky-Elemente */
.sticky, [style*="position: sticky"], [style*="position:sticky"],
/* Allgemeine Werbung */
[class*="ad-"], [class*="-ad-"], [id*="ad-"],
[class*="advertisement"] {
display: none !important;
visibility: hidden !important;
opacity: 0 !important;
}
/* Body-Overflow wiederherstellen falls Paywall ihn gesperrt hat */
body, html {
overflow: visible !important;
height: auto !important;
}
/* Blur auf Artikel-Text entfernen */
* { filter: none !important; }
` }); ` });
// Wayback-Toolbar auch ausblenden falls Wayback-URL
if (targetUrl.includes('web.archive.org')) {
await page.addStyleTag({ content: '#wm-ipp-base, #wm-ipp, #donato { display: none !important; }' });
}
await page.pdf({ await page.pdf({
path: tmpFile, path: tmpFile, format: 'A4', printBackground: true,
format: 'A4',
printBackground: true,
margin: { top: '12mm', bottom: '12mm', left: '12mm', right: '12mm' }, margin: { top: '12mm', bottom: '12mm', left: '12mm', right: '12mm' },
}); });
await browser.close(); browser = null;
await browser.close();
browser = null;
res.setHeader('Content-Type', 'application/pdf'); res.setHeader('Content-Type', 'application/pdf');
res.setHeader('Content-Disposition', `attachment; filename="artikel_${Date.now()}.pdf"`); res.setHeader('Content-Disposition', `attachment; filename="artikel_${Date.now()}.pdf"`);
const stream = require('fs').createReadStream(tmpFile); const stream = fs.createReadStream(tmpFile);
stream.pipe(res); stream.pipe(res);
stream.on('end', () => require('fs').unlink(tmpFile, () => {})); stream.on('end', () => fs.unlink(tmpFile, () => {}));
stream.on('error', () => require('fs').unlink(tmpFile, () => {})); stream.on('error', () => fs.unlink(tmpFile, () => {}));
} catch (err) { } catch (err) {
if (browser) try { await browser.close(); } catch {} if (browser) try { await browser.close(); } catch {}
require('fs').unlink(tmpFile, () => {}); fs.unlink(tmpFile, () => {});
console.error('[paywallkiller] PDF error:', err.message);
return res.status(500).json({ error: 'PDF-Erstellung fehlgeschlagen: ' + err.message }); return res.status(500).json({ error: 'PDF-Erstellung fehlgeschlagen: ' + err.message });
} }
}); });