feat: iframe CAPTCHA-Flow, PDF von archive.ph Snapshot-URL

This commit is contained in:
2026-06-12 14:33:39 +02:00
parent 762783b8ab
commit 25b906bee6
2 changed files with 137 additions and 216 deletions

View File

@@ -6,48 +6,29 @@ const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
// ── Einfacher HTTPS GET Helper ─────────────────────────────────────────────
function httpsGet(url, timeoutMs = 20000) {
return new Promise((resolve, reject) => {
const req = https.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)',
'Accept': '*/*',
},
timeout: timeoutMs,
}, (res) => {
let body = '';
res.setEncoding('utf8');
res.on('data', d => { if (body.length < 200000) body += d; });
res.on('end', () => resolve({ status: res.statusCode, body, headers: res.headers }));
res.on('error', reject);
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
});
}
// ── GET /api/tools/paywallkiller/cdx?url=... ──────────────────────────────
// Proxy für Wayback CDX API löst CORS/Origin-Probleme der PWA
// Wayback CDX Proxy (für spätere Verwendung)
router.get('/cdx', authenticate, async (req, res) => {
const { url } = req.query;
if (!url) return res.status(400).json({ error: 'url fehlt' });
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
try {
const result = await httpsGet(cdxUrl);
if (result.status !== 200) return res.status(502).json({ error: `CDX Status: ${result.status}` });
let data;
try { data = JSON.parse(result.body); } catch { return res.status(502).json({ error: 'Ungültige CDX-Antwort' }); }
if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) {
return res.json({ found: false });
}
const result = await new Promise((resolve, reject) => {
const req2 = https.get(cdxUrl, { headers: { 'User-Agent': 'Mozilla/5.0' }, timeout: 20000 }, (res2) => {
let body = '';
res2.setEncoding('utf8');
res2.on('data', d => body += d);
res2.on('end', () => resolve({ status: res2.statusCode, body }));
res2.on('error', reject);
});
req2.on('error', reject);
req2.on('timeout', () => { req2.destroy(); reject(new Error('Timeout')); });
});
if (result.status !== 200) return res.status(502).json({ error: `CDX ${result.status}` });
const data = JSON.parse(result.body);
if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) return res.json({ found: false });
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` });
} catch (err) {
return res.status(500).json({ error: err.message });
@@ -55,13 +36,11 @@ router.get('/cdx', authenticate, async (req, res) => {
});
// ── POST /api/tools/paywallkiller/pdf ─────────────────────────────────────
// Lädt via 12ft.io (Paywall-Bypasser, kein CAPTCHA) → PDF
// Lädt archive.ph Snapshot-URL mit Puppeteer → PDF
// archive.ph Snapshot-URLs (hash-basiert) benötigen kein CAPTCHA
router.post('/pdf', authenticate, async (req, res) => {
const { originalUrl, archiveUrl } = req.body;
if (!originalUrl && !archiveUrl) return res.status(400).json({ error: 'URL fehlt' });
// 12ft.io umgeht Paywalls indem es die Googlebot-Version abruft
const targetUrl = `https://12ft.io/proxy?q=${encodeURIComponent(originalUrl || archiveUrl)}`;
if (!targetUrl || typeof targetUrl !== 'string') return res.status(400).json({ error: 'URL fehlt' });
const { archiveUrl } = req.body;
if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' });
const tmpFile = path.join('/tmp', `pwk_${crypto.randomBytes(8).toString('hex')}.pdf`);
let browser = null;
@@ -80,50 +59,46 @@ router.post('/pdf', authenticate, async (req, res) => {
browser = await puppeteer.launch({
executablePath,
headless: true,
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu', '--single-process', '--no-zygote'],
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage',
'--disable-gpu', '--single-process', '--no-zygote'],
timeout: 30000,
});
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36');
await page.setViewport({ width: 1280, height: 900 });
await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 });
// UI-Störelemente entfernen (Wayback-Toolbar, Cookie-Banner, Werbung)
// Der Artikel-Text ist im Wayback-Snapshot vollständig vorhanden
// archive.ph laden Snapshot-URL direkt (kein CAPTCHA bei direkten Snapshot-URLs)
await page.goto(archiveUrl, { waitUntil: 'networkidle0', timeout: 30000 });
// Prüfen ob CAPTCHA erschienen ist
const hasCaptcha = await page.evaluate(() => {
return document.title.includes('One more step') ||
document.body.innerText.includes('complete a CAPTCHA') ||
document.body.innerText.includes('security check');
});
if (hasCaptcha) {
await browser.close(); browser = null;
return res.status(403).json({ error: 'CAPTCHA bitte im Browser das Archiv öffnen und CAPTCHA lösen, dann erneut versuchen.' });
}
// UI-Störelemente entfernen
await page.evaluate(() => {
const remove = [
// Wayback Machine Toolbar
'#wm-ipp-base', '#wm-ipp', '#donato', '#wm-ipp-print',
// Cookie/Consent Banner
'#usercentrics-root', '.uc-banner', '#usercentrics-cmp-ui',
['#wm-ipp-base','#wm-ipp','#donato',
'#usercentrics-root','.uc-banner',
'#onetrust-consent-sdk','.onetrust-pc-dark-filter',
'[id*="cookie"]', '[class*="cookie-banner"]', '[class*="cookieBanner"]',
'[class*="consent-banner"]', '[id*="consent-banner"]',
// Sticky Header/Footer
'header[class*="sticky"]', '[class*="sticky-header"]',
'[class*="stickyHeader"]', '[class*="fixed-header"]',
// Werbung
'[class*="advertisement"]', '[class*="adslot"]', '[id^="div-gpt-ad"]',
// Newsletter-Popup, Abo-Hinweise (nicht die Paywall selbst)
'[class*="newsletter"]', '[class*="teaser-abo"]',
];
remove.forEach(sel => {
'[id*="cookie"]','[class*="cookie-banner"]'].forEach(sel => {
try { document.querySelectorAll(sel).forEach(el => el.remove()); } catch {}
});
// body overflow freigeben
document.body.style.overflow = 'visible';
document.documentElement.style.overflow = 'visible';
document.body.style.height = 'auto';
});
await new Promise(r => setTimeout(r, 300));
await new Promise(r => setTimeout(r, 500));
await page.pdf({
path: tmpFile, format: 'A4', printBackground: true,
printBackground: true,
margin: { top: '12mm', bottom: '12mm', left: '12mm', right: '12mm' },
margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' },
});
await browser.close(); browser = null;
@@ -136,7 +111,7 @@ router.post('/pdf', authenticate, async (req, res) => {
} catch (err) {
if (browser) try { await browser.close(); } catch {}
fs.unlink(tmpFile, () => {});
return res.status(500).json({ error: 'PDF-Erstellung fehlgeschlagen: ' + err.message });
return res.status(500).json({ error: 'PDF fehlgeschlagen: ' + err.message });
}
});

View File

@@ -1,4 +1,4 @@
import { useState, useRef } from 'react';
import { useState, useRef, useEffect } from 'react';
import { S, api } from '../lib.js';
const C = {
@@ -8,88 +8,45 @@ const C = {
muted: '#6b7280',
};
function getDomain(url) {
try { return new URL(url).hostname.replace(/^www\./, ''); }
catch { return url; }
}
export default function PaywallKiller() {
const [inputUrl, setInputUrl] = useState('');
const [status, setStatus] = useState('idle');
const [statusMsg, setStatusMsg] = useState('');
const [waybackUrl, setWaybackUrl] = useState('');
const [wasCreated, setWasCreated] = useState(false);
const [archiveUrl, setArchiveUrl] = useState('');
const [showCaptcha, setShowCaptcha] = useState(false);
const [errMsg, setErrMsg] = useState('');
const [pdfLoading, setPdfLoading] = useState(false);
const inputRef = useRef(null);
const iframeRef = useRef(null);
function getTargetUrl() {
const url = inputUrl.trim();
return url.startsWith('http') ? url : 'https://' + url;
}
// CDX-Suche über Backend-Proxy (umgeht CORS/Origin-Problem der PWA)
async function searchWayback(targetUrl) {
const data = await api(`/tools/paywallkiller/cdx?url=${encodeURIComponent(targetUrl)}`, { method: 'GET' });
return data.found ? data.waybackUrl : null;
}
// Save via Browser (no-cors, fire-and-forget), dann CDX-Polling via Backend
async function saveAndPoll(targetUrl) {
// Save-Request abschießen (Browser, kein CORS-Response nötig)
try {
fetch(`https://web.archive.org/save/${targetUrl}`, { mode: 'no-cors' }).catch(() => {});
} catch {}
// CDX pollen via Backend-Proxy
for (let i = 0; i < 8; i++) {
await new Promise(r => setTimeout(r, 5000));
setStatusMsg(`Warte auf Snapshot... (${(i + 1) * 5}s)`);
try {
const url = await searchWayback(targetUrl);
if (url) return url;
} catch {}
}
return null;
}
async function handleCheck() {
function handleSearch() {
if (!inputUrl.trim()) return;
const targetUrl = getTargetUrl();
setStatus('checking');
setStatusMsg('Suche in Wayback Machine...');
const aUrl = `https://archive.ph/newest/${getTargetUrl()}`;
setArchiveUrl(aUrl);
setStatus('captcha');
setShowCaptcha(true);
setErrMsg('');
setWaybackUrl('');
setWasCreated(false);
}
try {
const existing = await searchWayback(targetUrl);
if (existing) {
setWaybackUrl(existing);
setStatus('done');
return;
}
// Kein Snapshot → archivieren
setStatus('saving');
setStatusMsg('Kein Snapshot archiviere jetzt...');
const newUrl = await saveAndPoll(targetUrl);
if (newUrl) {
setWaybackUrl(newUrl);
setWasCreated(true);
setStatus('done');
} else {
setStatus('notfound');
}
} catch (err) {
setErrMsg(err.message || 'Fehler');
setStatus('error');
}
function handleCaptchaDone() {
setShowCaptcha(false);
setStatus('ready');
}
async function handlePdf() {
setPdfLoading(true);
setErrMsg('');
try {
// Cookies für archive.ph aus dem Browser lesen
// document.cookie liefert nur same-site cookies archive.ph cookies
// sind third-party und nicht lesbar. Wir übergeben stattdessen
// die URL und lassen Puppeteer direkt die Session simulieren:
// Puppeteer öffnet zuerst archive.ph root um Cookies zu setzen,
// dann direkt die Snapshot-URL.
const token = localStorage.getItem('sk_token');
const res = await fetch('/api/tools/paywallkiller/pdf', {
method: 'POST',
@@ -97,7 +54,7 @@ export default function PaywallKiller() {
'Content-Type': 'application/json',
...(token ? { Authorization: `Bearer ${token}` } : {}),
},
body: JSON.stringify({ originalUrl: getTargetUrl(), archiveUrl: waybackUrl }),
body: JSON.stringify({ archiveUrl }),
});
if (!res.ok) {
const d = await res.json().catch(() => ({}));
@@ -122,119 +79,108 @@ export default function PaywallKiller() {
function handleReset() {
setInputUrl('');
setStatus('idle');
setStatusMsg('');
setWaybackUrl('');
setWasCreated(false);
setArchiveUrl('');
setShowCaptcha(false);
setErrMsg('');
setPdfLoading(false);
setTimeout(() => inputRef.current?.focus(), 50);
}
const isBusy = status === 'checking' || status === 'saving';
const isDone = status === 'done';
const hasResult = isDone || status === 'notfound' || status === 'error';
const archivePhUrl = inputUrl.trim() ? `https://archive.ph/newest/${getTargetUrl()}` : '';
return (
<div style={{ padding: '20px 16px', maxWidth: 640, margin: '0 auto' }}>
{/* Header */}
<div style={{ marginBottom: 24 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 10, marginBottom: 4 }}>
<span style={{ fontSize: 20 }}>🔓</span>
<span style={{ color: C.accent, fontFamily: 'monospace', fontSize: 13, letterSpacing: 1 }}>PAYWALL-KILLER</span>
</div>
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace' }}>Wayback Machine · archive.ph · Als PDF speichern</div>
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace' }}>archive.ph · Als PDF speichern</div>
</div>
{/* URL-Eingabe */}
<div style={{ ...S.card, marginBottom: 16 }}>
<div style={{ ...S.head, marginBottom: 10 }}>ARTIKEL-URL</div>
<input
ref={inputRef}
value={inputUrl}
onChange={e => setInputUrl(e.target.value)}
onKeyDown={e => e.key === 'Enter' && !isBusy && handleCheck()}
onKeyDown={e => e.key === 'Enter' && status === 'idle' && handleSearch()}
placeholder="URL einfügen (Long-Press → Einfügen)"
style={{ ...S.inp, width: '100%', boxSizing: 'border-box', marginBottom: 10 }}
/>
<div style={{ display: 'flex', gap: 8 }}>
<button onClick={handleCheck} disabled={isBusy || !inputUrl.trim()}
style={{ ...S.btn(C.accent), flex: 1, opacity: (isBusy || !inputUrl.trim()) ? 0.45 : 1, cursor: (isBusy || !inputUrl.trim()) ? 'default' : 'pointer' }}>
{isBusy ? `${statusMsg}` : '🔍 Archiv suchen'}
{status === 'idle' && (
<button onClick={handleSearch} disabled={!inputUrl.trim()}
style={{ ...S.btn(C.accent), flex: 1, opacity: !inputUrl.trim() ? 0.45 : 1, cursor: !inputUrl.trim() ? 'default' : 'pointer' }}>
🔍 Archiv öffnen
</button>
{hasResult && (
)}
{status !== 'idle' && (
<button onClick={handleReset}
style={{ background: 'rgba(255,255,255,0.06)', border: '1px solid rgba(255,255,255,0.14)', borderRadius: 6, padding: '8px 14px', color: 'rgba(255,255,255,0.6)', cursor: 'pointer', fontFamily: 'monospace', fontSize: 12, whiteSpace: 'nowrap' }}>
style={{ background: 'rgba(255,255,255,0.06)', border: '1px solid rgba(255,255,255,0.14)', borderRadius: 6, padding: '8px 14px', color: 'rgba(255,255,255,0.6)', cursor: 'pointer', fontFamily: 'monospace', fontSize: 12 }}>
Reset
</button>
)}
</div>
</div>
{isDone && (
<div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 12 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
<span>🏛</span>
<span style={{ color: C.success, fontFamily: 'monospace', fontSize: 12 }}>Wayback Machine</span>
<span style={{ background: `${C.success}20`, border: `1px solid ${C.success}44`, borderRadius: 4, padding: '1px 6px', fontSize: 10, color: C.success, fontFamily: 'monospace' }}>
{wasCreated ? 'neu archiviert ✨' : 'Snapshot gefunden'}
</span>
{/* CAPTCHA-Schritt */}
{status === 'captcha' && (
<div style={{ ...S.card, borderColor: `${C.accent}55`, marginBottom: 16 }}>
<div style={{ ...S.head, marginBottom: 8 }}>SCHRITT 1 CAPTCHA BESTÄTIGEN</div>
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginBottom: 12, lineHeight: 1.6 }}>
archive.ph wird unten geladen. Falls ein CAPTCHA erscheint: abhaken, warten bis der Artikel sichtbar ist, dann auf "Artikel bereit → PDF erstellen" klicken.
</div>
<div style={{ background: 'rgba(255,255,255,0.03)', border: '1px solid rgba(255,255,255,0.08)', borderRadius: 6, padding: '7px 10px', marginBottom: 10, wordBreak: 'break-all' }}>
<a href={waybackUrl} target="_blank" rel="noreferrer" style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>{waybackUrl}</a>
{/* archive.ph iframe */}
<div style={{ borderRadius: 8, overflow: 'hidden', border: '1px solid rgba(255,255,255,0.1)', marginBottom: 12, background: '#fff' }}>
<iframe
ref={iframeRef}
src={archiveUrl}
style={{ width: '100%', height: 400, border: 'none', display: 'block' }}
title="archive.ph"
/>
</div>
<div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}>
<a href={waybackUrl} target="_blank" rel="noreferrer" style={{ ...S.btn(C.success), textDecoration: 'none', fontSize: 12 }}>🌐 Öffnen</a>
<button onClick={handlePdf} disabled={pdfLoading}
style={{ ...S.btn(C.accent), opacity: pdfLoading ? 0.5 : 1, cursor: pdfLoading ? 'default' : 'pointer', fontSize: 12 }}>
{pdfLoading ? '⏳ PDF...' : '⬇ Als PDF'}
<div style={{ display: 'flex', gap: 8 }}>
<a href={archiveUrl} target="_blank" rel="noreferrer"
style={{ ...S.btn(C.muted), textDecoration: 'none', fontSize: 12 }}>
🌐 Im Browser öffnen
</a>
<button onClick={handleCaptchaDone}
style={{ ...S.btn(C.success), flex: 1, fontSize: 12 }}>
Artikel bereit PDF erstellen
</button>
</div>
{pdfLoading && <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 8 }}>Seite wird gerendert (1530s)...</div>}
</div>
)}
{status === 'notfound' && (
<div style={{ ...S.card, borderColor: `${C.error}44`, marginBottom: 12 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
<span>📭</span>
<span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>
Kein Snapshot gefunden {getDomain(inputUrl)} blockiert möglicherweise die Wayback Machine.
</span>
{/* PDF-Schritt */}
{status === 'ready' && (
<div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 16 }}>
<div style={{ ...S.head, marginBottom: 8 }}>SCHRITT 2 PDF ERSTELLEN</div>
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginBottom: 12, lineHeight: 1.6 }}>
Der Server lädt den Artikel jetzt über archive.ph und erstellt ein PDF. Das dauert 1530 Sekunden.
</div>
<div style={{ background: 'rgba(255,255,255,0.03)', border: '1px solid rgba(255,255,255,0.08)', borderRadius: 6, padding: '7px 10px', marginBottom: 12, wordBreak: 'break-all' }}>
<span style={{ color: C.muted, fontSize: 10, fontFamily: 'monospace' }}>{archiveUrl}</span>
</div>
)}
{hasResult && inputUrl.trim() && (
<div style={{ ...S.card, borderColor: 'rgba(255,255,255,0.1)', marginBottom: 12 }}>
<div style={{ ...S.head, marginBottom: 10 }}>WEITERE OPTIONEN</div>
{[
{
icon: '🔓',
label: '12ft.io',
desc: 'Paywall-Bypasser (kein CAPTCHA)',
url: `https://12ft.io/proxy?q=${encodeURIComponent(getTargetUrl())}`,
},
{
icon: '📦',
label: 'archive.ph',
desc: 'Nach CAPTCHA vollen Artikel sichtbar',
url: archivePhUrl,
},
{
icon: '🗑',
label: 'removepaywall.com',
desc: 'Alternativer Bypasser',
url: `https://www.removepaywall.com/search?url=${encodeURIComponent(getTargetUrl())}`,
},
].map(({ icon, label, desc, url }) => (
<div key={label} style={{ display: 'flex', alignItems: 'center', gap: 10, marginBottom: 8 }}>
<a href={url} target="_blank" rel="noreferrer"
style={{ ...S.btn(C.muted), textDecoration: 'none', fontSize: 11, whiteSpace: 'nowrap', flexShrink: 0 }}>
{icon} {label}
<div style={{ display: 'flex', gap: 8 }}>
<a href={archiveUrl} target="_blank" rel="noreferrer"
style={{ ...S.btn(C.success), textDecoration: 'none', fontSize: 12 }}>
🌐 Öffnen
</a>
<span style={{ color: C.muted, fontSize: 10, fontFamily: 'monospace' }}>{desc}</span>
<button onClick={handlePdf} disabled={pdfLoading}
style={{ ...S.btn(C.accent), flex: 1, opacity: pdfLoading ? 0.5 : 1, cursor: pdfLoading ? 'default' : 'pointer', fontSize: 12 }}>
{pdfLoading ? '⏳ Erstelle PDF...' : '⬇ Als PDF speichern'}
</button>
</div>
))}
{pdfLoading && (
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 8 }}>
Seite wird serverseitig gerendert (1530s)...
</div>
)}
</div>
)}
@@ -251,10 +197,10 @@ export default function PaywallKiller() {
<div style={{ ...S.card }}>
<div style={{ ...S.head, marginBottom: 10 }}>SO FUNKTIONIERT ES</div>
{[
['1', 'URL einfügen (Long-Press → Einfügen)'],
['2', 'Wayback Machine wird serverseitig durchsucht'],
['3', 'Falls kein Snapshot: Artikel wird automatisch archiviert'],
['4', 'Snapshot öffnen oder als PDF speichern'],
['1', 'URL des Artikels einfügen'],
['2', 'archive.ph wird geladen CAPTCHA einmalig bestätigen'],
['3', 'Wenn der Artikel sichtbar ist: "PDF erstellen" klicken'],
['4', 'PDF wird automatisch heruntergeladen'],
].map(([num, text]) => (
<div key={num} style={{ display: 'flex', gap: 10, alignItems: 'flex-start', marginBottom: 8 }}>
<span style={{ background: `${C.accent}20`, border: `1px solid ${C.accent}44`, borderRadius: '50%', width: 20, height: 20, flexShrink: 0, display: 'flex', alignItems: 'center', justifyContent: 'center', fontSize: 10, color: C.accent, fontFamily: 'monospace', marginTop: 2 }}>{num}</span>