refactor: Paywall-Killer nur Browser-Flow, kein Server mehr
This commit is contained in:
@@ -1,168 +1,8 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { authenticate } = require('../../middleware/auth');
|
||||
const https = require('https');
|
||||
const http = require('http');
|
||||
const crypto = require('crypto');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
// ── HTTPS GET Helper ───────────────────────────────────────────────────────
|
||||
function fetchHtml(url, redirects = 0) {
|
||||
return new Promise((resolve, reject) => {
|
||||
if (redirects > 5) return reject(new Error('Zu viele Redirects'));
|
||||
const mod = url.startsWith('https') ? https : http;
|
||||
const req = mod.get(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml',
|
||||
'Accept-Language': 'de,en;q=0.9',
|
||||
'Accept-Encoding': 'identity',
|
||||
},
|
||||
timeout: 20000,
|
||||
}, (res) => {
|
||||
if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) {
|
||||
let loc = res.headers.location;
|
||||
if (!loc.startsWith('http')) {
|
||||
const b = new URL(url);
|
||||
loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`;
|
||||
}
|
||||
res.resume();
|
||||
return fetchHtml(loc, redirects + 1).then(resolve).catch(reject);
|
||||
}
|
||||
let body = '';
|
||||
res.setEncoding('utf8');
|
||||
res.on('data', d => { if (body.length < 5000000) body += d; });
|
||||
res.on('end', () => resolve({ status: res.statusCode, body, finalUrl: url }));
|
||||
res.on('error', reject);
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
|
||||
});
|
||||
}
|
||||
|
||||
// ── GET /api/tools/paywallkiller/render?url=...&token=... ────────────────
|
||||
// Holt archive.ph Seite, entfernt Banner + schreibt Links um → sauberes HTML
|
||||
// Token als Query-Parameter da der Endpoint direkt im Browser geöffnet wird
|
||||
router.get('/render', (req, res, next) => {
|
||||
const jwt = require('jsonwebtoken');
|
||||
const SECRET = process.env.JWT_SECRET || 'dev-secret';
|
||||
const token = req.query.token || (req.headers.authorization || '').replace('Bearer ', '');
|
||||
if (!token) return res.status(401).send('Nicht eingeloggt');
|
||||
try { req.user = jwt.verify(token, SECRET); next(); }
|
||||
catch { return res.status(401).send('Token ungültig'); }
|
||||
}, async (req, res) => {
|
||||
const { url } = req.query;
|
||||
if (!url) return res.status(400).send('url fehlt');
|
||||
|
||||
// Nur archive.ph URLs erlauben
|
||||
if (!/^https?:\/\/archive\.(ph|is|today|fo|li|vn|md|gg)\//.test(url)) {
|
||||
return res.status(400).send('Nur archive.ph URLs erlaubt');
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await fetchHtml(url);
|
||||
if (result.status !== 200) {
|
||||
return res.status(502).send(`archive.ph antwortete mit ${result.status}`);
|
||||
}
|
||||
|
||||
let html = result.body;
|
||||
|
||||
// ── Banner entfernen ──────────────────────────────────────────────────
|
||||
// Der archive.ph Banner hat die ID "HEADER" oder ist ein <div> ganz oben
|
||||
html = html
|
||||
// Kompletten Wayback/archive Header-Block entfernen
|
||||
.replace(/<div[^>]*id=["']HEADER["'][^>]*>[\s\S]*?<\/div>/gi, '')
|
||||
.replace(/<div[^>]*id=["']wm-ipp[^"']*["'][^>]*>[\s\S]*?<\/div>/gi, '')
|
||||
// archive.ph spezifische Toolbar
|
||||
.replace(/<div[^>]*class=["'][^"']*THUMBS-BLOCK[^"']*["'][^>]*>[\s\S]*?<\/div>/gi, '')
|
||||
.replace(/<div[^>]*id=["']tools["'][^>]*>[\s\S]*?<\/div>/gi, '');
|
||||
|
||||
// ── Links zurück auf Originale umschreiben ────────────────────────────
|
||||
// archive.ph schreibt Links um zu: https://archive.ph/TIMESTAMP/https://original.url
|
||||
// Wir extrahieren die Original-URL
|
||||
html = html.replace(
|
||||
/https?:\/\/archive\.(?:ph|is|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]+\/(https?:\/\/[^"'\s>]+)/g,
|
||||
'$1'
|
||||
);
|
||||
// Auch relative Archive-Links
|
||||
html = html.replace(
|
||||
/href="\/[a-zA-Z0-9]{4,10}\/(https?:\/\/[^"]+)"/g,
|
||||
'href="$1"'
|
||||
);
|
||||
|
||||
// ── CSS für sauberes Drucken injizieren ───────────────────────────────
|
||||
const printCss = `
|
||||
<style id="pwk-print-style">
|
||||
/* Archive.ph Banner + Toolbar ausblenden */
|
||||
#HEADER, #tools, #THUMBS-BLOCK,
|
||||
div[id^="HEADER"], div[id^="tools"],
|
||||
.TEXT-BLOCK > div:first-child,
|
||||
[class*="THUMBS"], [class*="tools-block"] {
|
||||
display: none !important;
|
||||
}
|
||||
/* Druck-Optimierung */
|
||||
@media print {
|
||||
#HEADER, #tools, [id^="wm-"], [class*="archive-toolbar"],
|
||||
div:first-child > div:first-child { display: none !important; }
|
||||
body { margin: 0 !important; }
|
||||
a[href] { color: inherit !important; text-decoration: none !important; }
|
||||
a[href]::after { content: "" !important; }
|
||||
}
|
||||
</style>
|
||||
<script>
|
||||
// Banner sofort nach Load entfernen
|
||||
document.addEventListener('DOMContentLoaded', function() {
|
||||
['HEADER','tools','THUMBS-BLOCK'].forEach(function(id) {
|
||||
var el = document.getElementById(id);
|
||||
if (el) el.remove();
|
||||
});
|
||||
// Ersten Child-Div entfernen wenn er archive-Toolbar ist
|
||||
var first = document.body && document.body.firstElementChild;
|
||||
if (first && (first.id === 'HEADER' || first.className.includes('HEADER'))) {
|
||||
first.remove();
|
||||
}
|
||||
});
|
||||
</script>`;
|
||||
|
||||
// CSS vor </head> einfügen
|
||||
if (html.includes('</head>')) {
|
||||
html = html.replace('</head>', printCss + '</head>');
|
||||
} else {
|
||||
html = printCss + html;
|
||||
}
|
||||
|
||||
// Base-Tag setzen damit relative URLs funktionieren
|
||||
const baseTag = `<base href="${url}">`;
|
||||
if (html.includes('<head>')) {
|
||||
html = html.replace('<head>', '<head>' + baseTag);
|
||||
}
|
||||
|
||||
res.setHeader('Content-Type', 'text/html; charset=utf-8');
|
||||
// X-Frame-Options entfernen damit wir es anzeigen können
|
||||
res.removeHeader('X-Frame-Options');
|
||||
res.send(html);
|
||||
} catch (err) {
|
||||
res.status(500).send(`Fehler: ${err.message}`);
|
||||
}
|
||||
});
|
||||
|
||||
// ── GET /api/tools/paywallkiller/cdx?url=... ──────────────────────────────
|
||||
router.get('/cdx', authenticate, async (req, res) => {
|
||||
const { url } = req.query;
|
||||
if (!url) return res.status(400).json({ error: 'url fehlt' });
|
||||
let targetUrl = url.trim();
|
||||
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
|
||||
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
|
||||
try {
|
||||
const r = await fetchHtml(cdxUrl);
|
||||
if (r.status !== 200) return res.status(502).json({ error: `CDX ${r.status}` });
|
||||
const data = JSON.parse(r.body);
|
||||
if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) return res.json({ found: false });
|
||||
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
// Paywall-Killer hat keinen aktiven Backend-Endpoint mehr.
|
||||
// archive.ph blockt Server-seitige Requests (429).
|
||||
// Der komplette Flow läuft im Browser des Users.
|
||||
|
||||
module.exports = router;
|
||||
|
||||
Reference in New Issue
Block a user