Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: Wayback komplett im Browser, Server nur noch für PDF

Browse Source
This commit is contained in:
Dicken
2026-06-12 12:47:58 +02:00
parent 6eb9e1cc72
commit 4c909392e5
2 changed files with 124 additions and 233 deletions
Download Patch File Download Diff File Expand all files Collapse all files

138
backend/src/tools/paywallkiller/routes.js
View File

@@ -1,142 +1,14 @@
const express = require('express');
const router = express.Router();
const { authenticate } = require('../../middleware/auth');
const https = require('https');
const http = require('http');
const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
// ── HTTP GET mit Redirect-Follow + Cookie-Jar ─────────────────────────────
function fetchPage(url, opts = {}) {
return new Promise((resolve, reject) => {
if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects'));
const mod = url.startsWith('https') ? https : http;
const cookieHeader = opts.cookieJar && Object.keys(opts.cookieJar).length
? Object.entries(opts.cookieJar).map(([k,v]) => `${k}=${v}`).join('; ')
: undefined;
const headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
'Accept-Encoding': 'identity',
'Connection': 'keep-alive',
...(opts.referer ? { 'Referer': opts.referer } : {}),
...(cookieHeader ? { 'Cookie': cookieHeader } : {}),
};
const req = mod.get(url, { headers, timeout: 15000 }, (res) => {
const jar = opts.cookieJar || {};
for (const c of (res.headers['set-cookie'] || [])) {
const m = c.match(/^([^=]+)=([^;]*)/);
if (m) jar[m[1].trim()] = m[2].trim();
}
if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) {
let loc = res.headers.location;
if (!loc.startsWith('http')) {
try { const b = new URL(url); loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; }
catch { return reject(new Error('Ungültige Redirect-URL')); }
}
res.resume();
return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1, cookieJar: jar, referer: url }).then(resolve).catch(reject);
}
let body = '';
res.setEncoding('utf8');
res.on('data', d => { if (body.length < 150000) body += d; });
res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body, cookieJar: jar }));
res.on('error', reject);
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
});
}
// ── GET /api/tools/paywallkiller/wayback?url=... ──────────────────────────
// Sucht vorhandenen Wayback-Snapshot via CDX API
router.get('/wayback', authenticate, async (req, res) => {
const { url } = req.query;
if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url Parameter fehlt' });
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
try {
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
const r = await fetchPage(cdxUrl);
if (r.statusCode === 200 && r.body) {
let data;
try { data = JSON.parse(r.body); } catch { data = null; }
if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` });
}
}
return res.json({ found: false });
} catch (err) {
return res.status(500).json({ error: 'Fehler: ' + err.message });
}
});
// ── POST /api/tools/paywallkiller/wayback-save ────────────────────────────
// Schießt Save-Request ab (fire-and-forget), pollt dann CDX auf Ergebnis
router.post('/wayback-save', authenticate, async (req, res) => {
const { url } = req.body;
if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url fehlt' });
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
async function checkCdx() {
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
try {
const r = await fetchPage(cdxUrl);
if (r.statusCode === 200 && r.body) {
const data = JSON.parse(r.body);
if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`;
}
}
} catch {}
return null;
}
function fireSaveRequest() {
return new Promise((resolve) => {
const savePath = `/save/${targetUrl}`;
const req = https.get({
hostname: 'web.archive.org',
path: savePath,
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; WaybackSave/1.0)', 'Accept': 'text/html' },
timeout: 8000,
}, (res) => {
const location = res.headers.location || '';
res.resume();
resolve(location);
});
req.on('error', () => resolve(''));
req.on('timeout', () => { req.destroy(); resolve(''); });
});
}
try {
const location = await fireSaveRequest();
const directMatch = location.match(/\/web\/(\d{14})\//);
if (directMatch) {
const waybackUrl = location.startsWith('http') ? location : `https://web.archive.org${location}`;
return res.json({ success: true, waybackUrl });
}
// Polling: bis zu 6x alle 5s
for (let i = 0; i < 6; i++) {
await new Promise(r => setTimeout(r, 5000));
const found = await checkCdx();
if (found) return res.json({ success: true, waybackUrl: found });
}
return res.status(502).json({ error: 'Archivierung läuft noch oder wurde blockiert. Versuche es in einer Minute erneut.' });
} catch (err) {
return res.status(500).json({ error: 'Save fehlgeschlagen: ' + err.message });
}
});
// ── POST /api/tools/paywallkiller/pdf ─────────────────────────────────────
// Lädt Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF
router.post('/pdf', authenticate, async (req, res) => {
const { archiveUrl, originalUrl } = req.body;
const { originalUrl, archiveUrl } = req.body;
const targetUrl = originalUrl || archiveUrl;
if (!targetUrl || typeof targetUrl !== 'string') return res.status(400).json({ error: 'URL fehlt' });
@@ -145,7 +17,12 @@ router.post('/pdf', authenticate, async (req, res) => {
try {
const puppeteer = require('puppeteer-core');
const candidates = [process.env.CHROMIUM_PATH, '/usr/bin/chromium-browser', '/usr/bin/chromium', '/usr/bin/google-chrome'].filter(Boolean);
const candidates = [
process.env.CHROMIUM_PATH,
'/usr/bin/chromium-browser',
'/usr/bin/chromium',
'/usr/bin/google-chrome',
].filter(Boolean);
const executablePath = candidates.find(p => { try { fs.accessSync(p); return true; } catch { return false; } });
if (!executablePath) return res.status(500).json({ error: 'Chromium nicht gefunden' });
@@ -162,6 +39,7 @@ router.post('/pdf', authenticate, async (req, res) => {
await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 });
await new Promise(r => setTimeout(r, 1500));
// Paywall-Overlays, Cookie-Banner, Blur-Filter entfernen
await page.addStyleTag({ content: `
[class*="paywall"],[class*="Paywall"],[id*="paywall"],
[class*="piano"],[class*="Piano"],[id*="piano"],

149
frontend/src/tools/paywallkiller.jsx
View File

@@ -6,7 +6,6 @@ const C = {
success: '#4ade80',
error: '#f87171',
muted: '#6b7280',
info: '#60a5fa',
};
function getDomain(url) {
@@ -14,10 +13,38 @@ function getDomain(url) {
catch { return url; }
}
// Wayback CDX – direkt im Browser (CORS: * → kein Server-Proxy nötig)
async function searchWayback(targetUrl) {
const cdx = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
const res = await fetch(cdx);
if (!res.ok) throw new Error(`CDX Fehler: ${res.status}`);
const data = await res.json();
if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) return null;
return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`;
}
// Wayback Save – direkt im Browser via no-cors fetch (fire-and-forget)
// Dann CDX pollen bis Snapshot erscheint
async function saveAndPollWayback(targetUrl, onStatus) {
// Save abschießen (no-cors: kein Response lesbar, aber Request geht raus)
try {
await fetch(`https://web.archive.org/save/${targetUrl}`, { mode: 'no-cors' });
} catch { /* ignorieren – no-cors wirft immer */ }
// CDX pollen: 8x alle 5s = max 40s
for (let i = 0; i < 8; i++) {
onStatus(`Warte auf Snapshot... (${(i+1)*5}s)`);
await new Promise(r => setTimeout(r, 5000));
const url = await searchWayback(targetUrl);
if (url) return url;
}
return null;
}
export default function PaywallKiller() {
const [inputUrl, setInputUrl] = useState('');
// status: idle | checking | saving | done | error
const [status, setStatus] = useState('idle');
const [statusMsg, setStatusMsg] = useState('');
const [waybackUrl, setWaybackUrl] = useState('');
const [wasCreated, setWasCreated] = useState(false);
const [errMsg, setErrMsg] = useState('');
@@ -30,47 +57,41 @@ export default function PaywallKiller() {
}
async function handleCheck() {
const url = inputUrl.trim();
if (!url) return;
if (!inputUrl.trim()) return;
const targetUrl = getTargetUrl();
setStatus('checking');
setStatusMsg('Suche in Wayback Machine...');
setErrMsg('');
setWaybackUrl('');
setWasCreated(false);
try {
const data = await api(`/tools/paywallkiller/wayback?url=${encodeURIComponent(getTargetUrl())}`, { method: 'GET' });
if (data.found) {
setWaybackUrl(data.waybackUrl);
// Schritt 1: vorhandenen Snapshot suchen
const existing = await searchWayback(targetUrl);
if (existing) {
setWaybackUrl(existing);
setStatus('done');
} else {
// Kein Snapshot → direkt Save starten
await handleSave();
}
} catch (err) {
setErrMsg(err.message || 'Fehler beim Suchen');
setStatus('error');
}
return;
}
async function handleSave() {
// Schritt 2: Kein Snapshot → neu erstellen + pollen
setStatus('saving');
setErrMsg('');
try {
const data = await api('/tools/paywallkiller/wayback-save', { body: { url: getTargetUrl() } });
if (data.success) {
setWaybackUrl(data.waybackUrl);
setStatusMsg('Kein Snapshot gefunden – archiviere jetzt...');
const newUrl = await saveAndPollWayback(targetUrl, msg => setStatusMsg(msg));
if (newUrl) {
setWaybackUrl(newUrl);
setWasCreated(true);
setStatus('done');
} else {
throw new Error(data.error || 'Snapshot konnte nicht erstellt werden');
setStatus('notfound');
}
} catch (err) {
setErrMsg(err.message || 'Snapshot-Erstellung fehlgeschlagen');
setErrMsg(err.message || 'Fehler');
setStatus('error');
}
}
async function handlePdf() {
if (!waybackUrl) return;
setPdfLoading(true);
setErrMsg('');
try {
@@ -81,23 +102,23 @@ export default function PaywallKiller() {
'Content-Type': 'application/json',
...(token ? { Authorization: `Bearer ${token}` } : {}),
},
body: JSON.stringify({ archiveUrl: waybackUrl, originalUrl: getTargetUrl() }),
body: JSON.stringify({ originalUrl: getTargetUrl(), archiveUrl: waybackUrl }),
});
if (!res.ok) {
const data = await res.json().catch(() => ({}));
throw new Error(data.error || 'PDF-Fehler');
const d = await res.json().catch(() => ({}));
throw new Error(d.error || 'PDF-Fehler');
}
const blob = await res.blob();
const objUrl = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = objUrl;
a.download = 'archiv_' + Date.now() + '.pdf';
a.download = 'artikel_' + Date.now() + '.pdf';
document.body.appendChild(a);
a.click();
document.body.removeChild(a);
setTimeout(() => URL.revokeObjectURL(objUrl), 5000);
} catch (err) {
setErrMsg(err.message || 'PDF konnte nicht erstellt werden');
setErrMsg(err.message || 'PDF fehlgeschlagen');
} finally {
setPdfLoading(false);
}
@@ -106,6 +127,7 @@ export default function PaywallKiller() {
function handleReset() {
setInputUrl('');
setStatus('idle');
setStatusMsg('');
setWaybackUrl('');
setWasCreated(false);
setErrMsg('');
@@ -113,29 +135,22 @@ export default function PaywallKiller() {
setTimeout(() => inputRef.current?.focus(), 50);
}
const isChecking = status === 'checking';
const isSaving = status === 'saving';
const isBusy = isChecking || isSaving;
const isBusy = status === 'checking' || status === 'saving';
const isDone = status === 'done';
const archivePhUrl = inputUrl.trim()
? `https://archive.ph/newest/${getTargetUrl()}`
: '';
const hasResult = isDone || status === 'notfound' || status === 'error';
const archivePhUrl = inputUrl.trim() ? `https://archive.ph/newest/${getTargetUrl()}` : '';
return (
<div style={{ padding: '20px 16px', maxWidth: 640, margin: '0 auto' }}>
{/* Header */}
<div style={{ marginBottom: 24 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 10, marginBottom: 4 }}>
<span style={{ fontSize: 20 }}>🔓</span>
<span style={{ color: C.accent, fontFamily: 'monospace', fontSize: 13, letterSpacing: 1 }}>PAYWALL-KILLER</span>
</div>
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace' }}>
Wayback Machine · archive.ph · Als PDF speichern
</div>
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace' }}>Wayback Machine · archive.ph · Als PDF speichern</div>
</div>
{/* Eingabe */}
<div style={{ ...S.card, marginBottom: 16 }}>
<div style={{ ...S.head, marginBottom: 10 }}>ARTIKEL-URL</div>
<input
@@ -149,25 +164,19 @@ export default function PaywallKiller() {
<div style={{ display: 'flex', gap: 8 }}>
<button onClick={handleCheck} disabled={isBusy || !inputUrl.trim()}
style={{ ...S.btn(C.accent), flex: 1, opacity: (isBusy || !inputUrl.trim()) ? 0.45 : 1, cursor: (isBusy || !inputUrl.trim()) ? 'default' : 'pointer' }}>
{isChecking ? '⏳ Suche Archiv...' : isSaving ? '⏳ Erstelle Snapshot...' : '🔍 Archiv suchen'}
{isBusy ? `⏳ ${statusMsg}` : '🔍 Archiv suchen'}
</button>
{(isDone || status === 'error') && (
{hasResult && (
<button onClick={handleReset}
style={{ background: 'rgba(255,255,255,0.06)', border: '1px solid rgba(255,255,255,0.14)', borderRadius: 6, padding: '8px 14px', color: 'rgba(255,255,255,0.6)', cursor: 'pointer', fontFamily: 'monospace', fontSize: 12, whiteSpace: 'nowrap' }}>
✕ Reset
</button>
)}
</div>
{isSaving && (
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 10, lineHeight: 1.5 }}>
Kein Snapshot gefunden – die Wayback Machine archiviert den Artikel jetzt. Das dauert 15–30 Sekunden...
</div>
)}
</div>
{/* Ergebnis: Wayback Machine */}
{/* Wayback-Ergebnis */}
{isDone && (
<>
<div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 12 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
<span>🏛</span>
@@ -177,57 +186,61 @@ export default function PaywallKiller() {
</span>
</div>
<div style={{ background: 'rgba(255,255,255,0.03)', border: '1px solid rgba(255,255,255,0.08)', borderRadius: 6, padding: '7px 10px', marginBottom: 10, wordBreak: 'break-all' }}>
<a href={waybackUrl} target="_blank" rel="noreferrer"
style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>{waybackUrl}</a>
<a href={waybackUrl} target="_blank" rel="noreferrer" style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>{waybackUrl}</a>
</div>
<div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}>
<a href={waybackUrl} target="_blank" rel="noreferrer"
style={{ ...S.btn(C.success), textDecoration: 'none', fontSize: 12 }}>🌐 Öffnen</a>
<a href={waybackUrl} target="_blank" rel="noreferrer" style={{ ...S.btn(C.success), textDecoration: 'none', fontSize: 12 }}>🌐 Öffnen</a>
<button onClick={handlePdf} disabled={pdfLoading}
style={{ ...S.btn(C.accent), opacity: pdfLoading ? 0.5 : 1, cursor: pdfLoading ? 'default' : 'pointer', fontSize: 12 }}>
{pdfLoading ? '⏳ Erstelle PDF...' : '⬇ Als PDF'}
{pdfLoading ? '⏳ PDF...' : '⬇ Als PDF'}
</button>
</div>
{pdfLoading && (
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 8 }}>
Seite wird serverseitig gerendert (15–30s)...
{pdfLoading && <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 8 }}>Seite wird gerendert (15–30s)...</div>}
</div>
)}
</div>
{/* archive.ph – immer anzeigen, nur Öffnen */}
{/* Kein Snapshot erstellt */}
{status === 'notfound' && (
<div style={{ ...S.card, borderColor: `${C.error}44`, marginBottom: 12 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
<span>📭</span>
<span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>
Archivierung fehlgeschlagen – {getDomain(inputUrl)} blockiert möglicherweise die Wayback Machine.
</span>
</div>
</div>
)}
{/* archive.ph – immer nach Suche anzeigen */}
{hasResult && archivePhUrl && (
<div style={{ ...S.card, borderColor: 'rgba(255,255,255,0.1)', marginBottom: 12 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
<span>📦</span>
<span style={{ color: 'rgba(255,255,255,0.55)', fontFamily: 'monospace', fontSize: 12 }}>archive.ph</span>
</div>
<div style={{ display: 'flex', gap: 8, alignItems: 'center' }}>
<a href={archivePhUrl} target="_blank" rel="noreferrer"
style={{ ...S.btn(C.muted), textDecoration: 'none', fontSize: 12 }}>🌐 Öffnen</a>
<a href={archivePhUrl} target="_blank" rel="noreferrer" style={{ ...S.btn(C.muted), textDecoration: 'none', fontSize: 12 }}>🌐 Öffnen</a>
<span style={{ color: C.muted, fontSize: 10, fontFamily: 'monospace' }}>Kein PDF – CAPTCHA-geschützt</span>
</div>
</div>
</>
)}
{/* Fehler */}
{errMsg && (
<div style={{ ...S.card, borderColor: `${C.error}44`, marginTop: 8 }}>
<div style={{ display: 'flex', alignItems: 'flex-start', gap: 8 }}>
<div style={{ display: 'flex', gap: 8 }}>
<span>⚠️</span>
<span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>{errMsg}</span>
</div>
</div>
)}
{/* Idle-Info */}
{status === 'idle' && (
<div style={{ ...S.card }}>
<div style={{ ...S.head, marginBottom: 10 }}>SO FUNKTIONIERT ES</div>
{[
['1', 'URL des gesperrten Artikels einfügen (Long-Press → Einfügen)'],
['2', 'Vorhandenen Snapshot in der Wayback Machine suchen'],
['3', 'Falls keiner existiert: Artikel wird automatisch archiviert'],
['1', 'URL einfügen (Long-Press → Einfügen)'],
['2', 'Wayback Machine wird direkt im Browser durchsucht'],
['3', 'Falls kein Snapshot: Artikel wird automatisch archiviert'],
['4', 'Snapshot öffnen oder als PDF speichern'],
].map(([num, text]) => (
<div key={num} style={{ display: 'flex', gap: 10, alignItems: 'flex-start', marginBottom: 8 }}>