Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: PDF lädt Original-URL + blendet Paywall-Overlays per CSS aus

Browse Source
This commit is contained in:
Dicken
2026-06-12 12:24:06 +02:00
parent c75c488d3b
commit 8877587309
2 changed files with 100 additions and 152 deletions
Download Patch File Download Diff File Expand all files Collapse all files

250
backend/src/tools/paywallkiller/routes.js
View File

@@ -163,174 +163,122 @@ router.post('/check', authenticate, async (req, res) => {
});
// ── POST /pdf ──────────────────────────────────────────────────────────────
// Lädt die Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF
router.post('/pdf', authenticate, async (req, res) => {
const { archiveUrl } = req.body;
if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' });
if (!archiveUrl.startsWith('https://web.archive.org/')) {
return res.status(400).json({ error: 'PDF nur für Wayback Machine URLs (web.archive.org) verfügbar' });
const { archiveUrl, originalUrl } = req.body;
// originalUrl bevorzugen (direkter Artikel ohne Wayback-Paywall)
// archiveUrl als Fallback (Wayback Machine)
const targetUrl = originalUrl || archiveUrl;
if (!targetUrl || typeof targetUrl !== 'string') {
return res.status(400).json({ error: 'URL fehlt' });
}
let browser;
const tmpFile = path.join('/tmp', `pwk_${crypto.randomBytes(8).toString('hex')}.pdf`);
const tmpFile = require('path').join('/tmp', `pwk_${require('crypto').randomBytes(8).toString('hex')}.pdf`);
let browser = null;
try {
let puppeteer;
try { puppeteer = require('puppeteer-core'); }
catch { try { puppeteer = require('puppeteer'); }
catch { return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar' }); } }
const puppeteer = require('puppeteer-core');
// System-Chromium aus Alpine nutzen (via CHROMIUM_PATH env oder bekannte Pfade)
const executablePath = process.env.CHROMIUM_PATH
|| (() => {
const candidates = [
'/usr/bin/chromium-browser',
'/usr/bin/chromium',
'/usr/bin/google-chrome',
'/usr/bin/google-chrome-stable',
];
const fs2 = require('fs');
return candidates.find(p => { try { fs2.accessSync(p); return true; } catch { return false; } })
|| '/usr/bin/chromium-browser';
})();
const chromiumPaths = [
process.env.CHROMIUM_PATH,
'/usr/bin/chromium-browser',
'/usr/bin/chromium',
'/usr/bin/google-chrome',
].filter(Boolean);
const launchOptions = {
let executablePath = null;
for (const p of chromiumPaths) {
if (require('fs').existsSync(p)) { executablePath = p; break; }
}
if (!executablePath) {
return res.status(500).json({ error: 'Chromium nicht gefunden' });
}
browser = await puppeteer.launch({
executablePath,
headless: true,
args: [
'--no-sandbox',
'--disable-setuid-sandbox',
'--disable-dev-shm-usage',
'--disable-gpu',
'--disable-software-rasterizer',
'--single-process',
'--no-zygote',
],
defaultViewport: { width: 1280, height: 900 },
};
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--single-process', '--no-zygote'],
timeout: 30000,
});
browser = await puppeteer.launch(launchOptions);
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
await page.goto(archiveUrl, { waitUntil: 'networkidle2', timeout: 30000 });
await page.pdf({ path: tmpFile, format: 'A4', printBackground: true,
margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' } });
await browser.close(); browser = null;
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36');
await page.setViewport({ width: 1280, height: 900 });
// Seite laden – networkidle0 damit JS vollständig läuft (Artikel wird geladen)
await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 });
// Kurz warten damit Artikel-Content gerendert ist
await new Promise(r => setTimeout(r, 1500));
// Paywall-Overlays, Cookie-Banner, Sticky-Header ausblenden
await page.addStyleTag({ content: `
/* Generische Paywall/Overlay-Selektoren */
[class*="paywall"], [class*="Paywall"],
[class*="piano"], [class*="Piano"],
[id*="paywall"], [id*="piano"],
[class*="subscription"], [class*="subscribe"],
[class*="overlay"], [class*="modal"],
[class*="cookie"], [class*="Cookie"],
[class*="consent"], [class*="Consent"],
[class*="gdpr"], [class*="GDPR"],
[class*="sticky-header"], [class*="stickyHeader"],
[class*="fixed-header"], [class*="fixedHeader"],
/* WAZ/Funke spezifisch */
.tp-modal, .tp-backdrop, #tp-container,
.article__paywall, .paywall-container,
.funke-piano, .piano-checkout,
[data-testid*="paywall"], [data-testid*="piano"],
/* Cookie-Banner */
#usercentrics-root, .uc-banner,
#onetrust-consent-sdk, .onetrust-pc-dark-filter,
/* Sticky-Elemente */
.sticky, [style*="position: sticky"], [style*="position:sticky"],
/* Allgemeine Werbung */
[class*="ad-"], [class*="-ad-"], [id*="ad-"],
[class*="advertisement"] {
display: none !important;
visibility: hidden !important;
opacity: 0 !important;
}
/* Body-Overflow wiederherstellen falls Paywall ihn gesperrt hat */
body, html {
overflow: visible !important;
height: auto !important;
}
/* Blur auf Artikel-Text entfernen */
* { filter: none !important; }
` });
// Wayback-Toolbar auch ausblenden falls Wayback-URL
if (targetUrl.includes('web.archive.org')) {
await page.addStyleTag({ content: '#wm-ipp-base, #wm-ipp, #donato { display: none !important; }' });
}
await page.pdf({
path: tmpFile,
format: 'A4',
printBackground: true,
margin: { top: '12mm', bottom: '12mm', left: '12mm', right: '12mm' },
});
await browser.close();
browser = null;
res.setHeader('Content-Type', 'application/pdf');
res.setHeader('Content-Disposition', `attachment; filename="archiv_${Date.now()}.pdf"`);
const stream = fs.createReadStream(tmpFile);
res.setHeader('Content-Disposition', `attachment; filename="artikel_${Date.now()}.pdf"`);
const stream = require('fs').createReadStream(tmpFile);
stream.pipe(res);
stream.on('end', () => fs.unlink(tmpFile, () => {}));
stream.on('error', () => fs.unlink(tmpFile, () => {}));
stream.on('end', () => require('fs').unlink(tmpFile, () => {}));
stream.on('error', () => require('fs').unlink(tmpFile, () => {}));
} catch (err) {
if (browser) try { await browser.close(); } catch {}
fs.unlink(tmpFile, () => {});
require('fs').unlink(tmpFile, () => {});
console.error('[paywallkiller] PDF error:', err.message);
return res.status(500).json({ error: 'PDF-Erstellung fehlgeschlagen: ' + err.message });
}
});
// ── GET /api/tools/paywallkiller/wayback?url=... ──────────────────────────
// 1. Sucht vorhandenen Snapshot via CDX API
// 2. Falls keiner → erstellt neuen Snapshot via Save API
router.get('/wayback', authenticate, async (req, res) => {
const { url } = req.query;
if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url Parameter fehlt' });
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
// ── Schritt 1: Vorhandenen Snapshot suchen ──────────────────────────────
try {
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
const cdxResult = await fetchPage(cdxUrl);
if (cdxResult.statusCode === 200 && cdxResult.body) {
let data;
try { data = JSON.parse(cdxResult.body); } catch { data = null; }
if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
const timestamp = data[1][0];
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${timestamp}/${targetUrl}`, created: false });
}
}
} catch { /* weiter zu Save API */ }
// ── Schritt 2: Keinen Snapshot gefunden → neu erstellen ─────────────────
return res.json({ found: false, canSave: true });
});
// ── POST /api/tools/paywallkiller/wayback-save ────────────────────────────
// Schießt Save-Request ab (fire-and-forget), pollt dann CDX auf Ergebnis
router.post('/wayback-save', authenticate, async (req, res) => {
const { url } = req.body;
if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url fehlt' });
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
const https = require('https');
// ── Hilfsfunktion: CDX prüfen ──────────────────────────────────────────
async function checkCdx() {
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
try {
const r = await fetchPage(cdxUrl);
if (r.statusCode === 200 && r.body) {
const data = JSON.parse(r.body);
if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`;
}
}
} catch {}
return null;
}
// ── Save-Request fire-and-forget (kein Warten auf Body) ───────────────
function fireSaveRequest() {
return new Promise((resolve) => {
const saveUrl = new URL(`https://web.archive.org/save/${targetUrl}`);
const req = https.get({
hostname: saveUrl.hostname,
path: saveUrl.pathname + saveUrl.search,
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; WaybackSave/1.0)',
'Accept': 'text/html',
},
timeout: 8000,
}, (res) => {
// Nur Headers lesen, Body sofort verwerfen
const location = res.headers.location || '';
res.resume(); // Body ignorieren
resolve(location);
});
req.on('error', () => resolve(''));
req.on('timeout', () => { req.destroy(); resolve(''); });
});
}
try {
// Save abschießen
const location = await fireSaveRequest();
// Direkt aus Location-Header auslesen wenn vorhanden
const directMatch = location.match(/\/web\/(\d{14})\//);
if (directMatch) {
const waybackUrl = location.startsWith('http') ? location : `https://web.archive.org${location}`;
return res.json({ success: true, waybackUrl, timestamp: directMatch[1] });
}
// Polling: bis zu 6x alle 5s CDX abfragen (max 30s warten)
for (let i = 0; i < 6; i++) {
await new Promise(r => setTimeout(r, 5000));
const found = await checkCdx();
if (found) {
return res.json({ success: true, waybackUrl: found });
}
}
return res.status(502).json({ error: 'Archivierung läuft noch oder wurde blockiert. Versuche es in einer Minute erneut.' });
} catch (err) {
return res.status(500).json({ error: 'Save fehlgeschlagen: ' + err.message });
}
});
module.exports = router;
module.exports