Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: /render endpoint bereinigt archive.ph HTML (Banner weg, Links original)

Browse Source
This commit is contained in:
Dicken
2026-06-12 16:04:41 +02:00
parent 2e89669c9b
commit 5469a17a3f
2 changed files with 230 additions and 162 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
backend/src/tools/paywallkiller/routes.js
View File

@@ -2,12 +2,144 @@ const express = require('express');
const router = express.Router();
const { authenticate } = require('../../middleware/auth');
const https = require('https');
const http = require('http');
const crypto = require('crypto');
const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
// ── HTTPS GET Helper ───────────────────────────────────────────────────────
function fetchHtml(url, redirects = 0) {
return new Promise((resolve, reject) => {
if (redirects > 5) return reject(new Error('Zu viele Redirects'));
const mod = url.startsWith('https') ? https : http;
const req = mod.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml',
'Accept-Language': 'de,en;q=0.9',
'Accept-Encoding': 'identity',
},
timeout: 20000,
}, (res) => {
if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) {
let loc = res.headers.location;
if (!loc.startsWith('http')) {
const b = new URL(url);
loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`;
}
res.resume();
return fetchHtml(loc, redirects + 1).then(resolve).catch(reject);
}
let body = '';
res.setEncoding('utf8');
res.on('data', d => { if (body.length < 5000000) body += d; });
res.on('end', () => resolve({ status: res.statusCode, body, finalUrl: url }));
res.on('error', reject);
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
});
}
// ── GET /api/tools/paywallkiller/render?url=... ───────────────────────────
// Holt archive.ph Seite, entfernt Banner + schreibt Links um → sauberes HTML
router.get('/render', authenticate, async (req, res) => {
const { url } = req.query;
if (!url) return res.status(400).send('url fehlt');
// Nur archive.ph URLs erlauben
if (!/^https?:\/\/archive\.(ph|is|today|fo|li|vn|md|gg)\//.test(url)) {
return res.status(400).send('Nur archive.ph URLs erlaubt');
}
try {
const result = await fetchHtml(url);
if (result.status !== 200) {
return res.status(502).send(`archive.ph antwortete mit ${result.status}`);
}
let html = result.body;
// ── Banner entfernen ──────────────────────────────────────────────────
// Der archive.ph Banner hat die ID "HEADER" oder ist ein <div> ganz oben
html = html
// Kompletten Wayback/archive Header-Block entfernen
.replace(/<div[^>]*id=["']HEADER["'][^>]*>[\s\S]*?<\/div>/gi, '')
.replace(/<div[^>]*id=["']wm-ipp[^"']*["'][^>]*>[\s\S]*?<\/div>/gi, '')
// archive.ph spezifische Toolbar
.replace(/<div[^>]*class=["'][^"']*THUMBS-BLOCK[^"']*["'][^>]*>[\s\S]*?<\/div>/gi, '')
.replace(/<div[^>]*id=["']tools["'][^>]*>[\s\S]*?<\/div>/gi, '');
// ── Links zurück auf Originale umschreiben ────────────────────────────
// archive.ph schreibt Links um zu: https://archive.ph/TIMESTAMP/https://original.url
// Wir extrahieren die Original-URL
html = html.replace(
/https?:\/\/archive\.(?:ph|is|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]+\/(https?:\/\/[^"'\s>]+)/g,
'$1'
);
// Auch relative Archive-Links
html = html.replace(
/href="\/[a-zA-Z0-9]{4,10}\/(https?:\/\/[^"]+)"/g,
'href="$1"'
);
// ── CSS für sauberes Drucken injizieren ───────────────────────────────
const printCss = `
<style id="pwk-print-style">
/* Archive.ph Banner + Toolbar ausblenden */
#HEADER, #tools, #THUMBS-BLOCK,
div[id^="HEADER"], div[id^="tools"],
.TEXT-BLOCK > div:first-child,
[class*="THUMBS"], [class*="tools-block"] {
display: none !important;
}
/* Druck-Optimierung */
@media print {
#HEADER, #tools, [id^="wm-"], [class*="archive-toolbar"],
div:first-child > div:first-child { display: none !important; }
body { margin: 0 !important; }
a[href] { color: inherit !important; text-decoration: none !important; }
a[href]::after { content: "" !important; }
}
</style>
<script>
// Banner sofort nach Load entfernen
document.addEventListener('DOMContentLoaded', function() {
['HEADER','tools','THUMBS-BLOCK'].forEach(function(id) {
var el = document.getElementById(id);
if (el) el.remove();
});
// Ersten Child-Div entfernen wenn er archive-Toolbar ist
var first = document.body && document.body.firstElementChild;
if (first && (first.id === 'HEADER' || first.className.includes('HEADER'))) {
first.remove();
}
});
</script>`;
// CSS vor </head> einfügen
if (html.includes('</head>')) {
html = html.replace('</head>', printCss + '</head>');
} else {
html = printCss + html;
}
// Base-Tag setzen damit relative URLs funktionieren
const baseTag = `<base href="${url}">`;
if (html.includes('<head>')) {
html = html.replace('<head>', '<head>' + baseTag);
}
res.setHeader('Content-Type', 'text/html; charset=utf-8');
// X-Frame-Options entfernen damit wir es anzeigen können
res.removeHeader('X-Frame-Options');
res.send(html);
} catch (err) {
res.status(500).send(`Fehler: ${err.message}`);
}
});
// ── GET /api/tools/paywallkiller/cdx?url=... ──────────────────────────────
// Wayback CDX Proxy (für spätere Verwendung)
router.get('/cdx', authenticate, async (req, res) => {
const { url } = req.query;
if (!url) return res.status(400).json({ error: 'url fehlt' });
@@ -15,19 +147,9 @@ router.get('/cdx', authenticate, async (req, res) => {
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
try {
const result = await new Promise((resolve, reject) => {
const req2 = https.get(cdxUrl, { headers: { 'User-Agent': 'Mozilla/5.0' }, timeout: 20000 }, (res2) => {
let body = '';
res2.setEncoding('utf8');
res2.on('data', d => body += d);
res2.on('end', () => resolve({ status: res2.statusCode, body }));
res2.on('error', reject);
});
req2.on('error', reject);
req2.on('timeout', () => { req2.destroy(); reject(new Error('Timeout')); });
});
if (result.status !== 200) return res.status(502).json({ error: `CDX ${result.status}` });
const data = JSON.parse(result.body);
const r = await fetchHtml(cdxUrl);
if (r.status !== 200) return res.status(502).json({ error: `CDX ${r.status}` });
const data = JSON.parse(r.body);
if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) return res.json({ found: false });
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` });
} catch (err) {
@@ -35,84 +157,4 @@ router.get('/cdx', authenticate, async (req, res) => {
}
});
// ── POST /api/tools/paywallkiller/pdf ─────────────────────────────────────
// Lädt archive.ph Snapshot-URL mit Puppeteer → PDF
// archive.ph Snapshot-URLs (hash-basiert) benötigen kein CAPTCHA
router.post('/pdf', authenticate, async (req, res) => {
const { archiveUrl } = req.body;
if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' });
const tmpFile = path.join('/tmp', `pwk_${crypto.randomBytes(8).toString('hex')}.pdf`);
let browser = null;
try {
const puppeteer = require('puppeteer-core');
const candidates = [
process.env.CHROMIUM_PATH,
'/usr/bin/chromium-browser',
'/usr/bin/chromium',
'/usr/bin/google-chrome',
].filter(Boolean);
const executablePath = candidates.find(p => { try { fs.accessSync(p); return true; } catch { return false; } });
if (!executablePath) return res.status(500).json({ error: 'Chromium nicht gefunden' });
browser = await puppeteer.launch({
executablePath,
headless: true,
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage',
'--disable-gpu', '--single-process', '--no-zygote'],
timeout: 30000,
});
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36');
await page.setViewport({ width: 1280, height: 900 });
// archive.ph laden – Snapshot-URL direkt (kein CAPTCHA bei direkten Snapshot-URLs)
await page.goto(archiveUrl, { waitUntil: 'networkidle0', timeout: 30000 });
// Prüfen ob CAPTCHA erschienen ist
const hasCaptcha = await page.evaluate(() => {
return document.title.includes('One more step') ||
document.body.innerText.includes('complete a CAPTCHA') ||
document.body.innerText.includes('security check');
});
if (hasCaptcha) {
await browser.close(); browser = null;
return res.status(403).json({ error: 'CAPTCHA – bitte im Browser das Archiv öffnen und CAPTCHA lösen, dann erneut versuchen.' });
}
// UI-Störelemente entfernen
await page.evaluate(() => {
['#wm-ipp-base','#wm-ipp','#donato',
'#usercentrics-root','.uc-banner',
'#onetrust-consent-sdk','.onetrust-pc-dark-filter',
'[id*="cookie"]','[class*="cookie-banner"]'].forEach(sel => {
try { document.querySelectorAll(sel).forEach(el => el.remove()); } catch {}
});
document.body.style.overflow = 'visible';
});
await new Promise(r => setTimeout(r, 500));
await page.pdf({
path: tmpFile, format: 'A4', printBackground: true,
margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' },
});
await browser.close(); browser = null;
res.setHeader('Content-Type', 'application/pdf');
res.setHeader('Content-Disposition', `attachment; filename="artikel_${Date.now()}.pdf"`);
const stream = fs.createReadStream(tmpFile);
stream.pipe(res);
stream.on('end', () => fs.unlink(tmpFile, () => {}));
stream.on('error', () => fs.unlink(tmpFile, () => {}));
} catch (err) {
if (browser) try { await browser.close(); } catch {}
fs.unlink(tmpFile, () => {});
return res.status(500).json({ error: 'PDF fehlgeschlagen: ' + err.message });
}
});
module.exports = router;