Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: Wayback CDX API im Frontend + archive.ph immer anzeigen

Browse Source
This commit is contained in:
Dicken
2026-06-12 08:53:58 +02:00
parent 373bb3b6a0
commit c83f78595f
2 changed files with 104 additions and 163 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
backend/src/tools/paywallkiller/routes.js
View File

@@ -166,8 +166,8 @@ router.post('/check', authenticate, async (req, res) => {
router.post('/pdf', authenticate, async (req, res) => { router.post('/pdf', authenticate, async (req, res) => {
const { archiveUrl } = req.body; const { archiveUrl } = req.body;
if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' }); if (!archiveUrl || typeof archiveUrl !== 'string') return res.status(400).json({ error: 'archiveUrl fehlt' });
if (!/^https?:\/\/archive\.(is|ph|today|fo|li|vn|md|gg)\//.test(archiveUrl)) { if (!/^https?:\/\/(archive\.(is|ph|today|fo|li|vn|md|gg)|web\.archive\.org)\//.test(archiveUrl)) {
return res.status(400).json({ error: 'Ungültige archive.is URL' }); return res.status(400).json({ error: 'Ungültige Archiv-URL' });
} }
let browser; let browser;

263
frontend/src/tools/paywallkiller.jsx
View File

@@ -1,4 +1,4 @@
import { useState, useRef, useEffect } from 'react'; import { useState, useRef } from 'react';
import { S } from '../lib.js'; import { S } from '../lib.js';
const C = { const C = {
@@ -13,131 +13,56 @@ function getDomain(url) {
catch { return url; } catch { return url; }
} }
// ── Iframe-basierte Archivsuche (umgeht Server-IP-Block) ────────────────── // ── Wayback Machine CDX API – direkt im Browser (CORS: *) ─────────────────
// Öffnet archive.ph/?url=... in einem versteckten iframe und liest async function checkWayback(targetUrl) {
// nach dem Load die finale URL aus (Snapshot-Hash-URL). // Neuesten Snapshot suchen (limit=1, sortiert newest-first via fl+limit)
function useArchiveSearch() { const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
const iframeRef = useRef(null); const res = await fetch(cdxUrl, { signal: AbortSignal.timeout(10000) });
const resolveRef = useRef(null); if (!res.ok) return null;
const timeoutRef = useRef(null); const data = await res.json();
// data = [["timestamp"],["20260608201434"]] oder [] wenn nichts gefunden
useEffect(() => { if (!Array.isArray(data) || data.length < 2) return null;
return () => { const timestamp = data[1]?.[0];
if (timeoutRef.current) clearTimeout(timeoutRef.current); if (!timestamp) return null;
}; return `https://web.archive.org/web/${timestamp}/${targetUrl}`;
}, []);
function search(targetUrl) {
return new Promise((resolve, reject) => {
resolveRef.current = { resolve, reject };
// Altes iframe entfernen
if (iframeRef.current) {
document.body.removeChild(iframeRef.current);
iframeRef.current = null;
}
const iframe = document.createElement('iframe');
iframe.style.cssText = 'position:fixed;top:-9999px;left:-9999px;width:1px;height:1px;opacity:0;pointer-events:none;border:none;';
iframe.sandbox = 'allow-scripts allow-same-origin allow-forms';
iframe.onload = () => {
if (timeoutRef.current) clearTimeout(timeoutRef.current);
try {
// Nach dem Load: finale URL des iframes prüfen
const finalUrl = iframe.contentWindow?.location?.href || '';
cleanup();
if (!finalUrl || finalUrl === 'about:blank') {
return resolve({ found: false });
}
// Snapshot-URL hat Form https://archive.ph/HASH (4-10 Zeichen)
if (/archive\.(is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10}$/.test(finalUrl)) {
return resolve({ found: true, url: finalUrl });
}
// Suchergebnis-Seite geladen: Links nach Snapshots durchsuchen
try {
const doc = iframe.contentDocument || iframe.contentWindow?.document;
if (doc) {
// Alle Links durchsuchen
const links = Array.from(doc.querySelectorAll('a[href]'));
for (const a of links) {
const href = a.getAttribute('href') || '';
const abs = href.startsWith('http') ? href : `https://archive.ph${href}`;
if (/archive\.(is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10}$/.test(abs)) {
const h = abs.split('/').pop();
const blacklist = ['newest','oldest','submit','search','about','rss','api','timemap'];
if (!blacklist.includes(h) && !/^\d+$/.test(h)) {
return resolve({ found: true, url: abs });
}
}
}
}
} catch {
// Cross-Origin Zugriff blockiert – trotzdem finalUrl prüfen
}
resolve({ found: false });
} catch (e) {
cleanup();
resolve({ found: false });
}
};
iframe.onerror = () => {
cleanup();
resolve({ found: false });
};
// Timeout nach 15s
timeoutRef.current = setTimeout(() => {
cleanup();
reject(new Error('Timeout beim Laden von archive.ph'));
}, 15000);
const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`;
iframe.src = searchUrl;
document.body.appendChild(iframe);
iframeRef.current = iframe;
function cleanup() {
if (timeoutRef.current) clearTimeout(timeoutRef.current);
if (iframeRef.current) {
try { document.body.removeChild(iframeRef.current); } catch {}
iframeRef.current = null;
}
}
});
}
return { search };
} }
export default function PaywallKiller() { export default function PaywallKiller() {
const [inputUrl, setInputUrl] = useState(''); const [inputUrl, setInputUrl] = useState('');
const [status, setStatus] = useState('idle'); const [status, setStatus] = useState('idle');
const [archiveUrl, setArchiveUrl] = useState(''); const [waybackUrl, setWaybackUrl] = useState('');
const [errMsg, setErrMsg] = useState(''); const [archiveUrl, setArchiveUrl] = useState('');
const [pdfLoading, setPdfLoading] = useState(false); const [errMsg, setErrMsg] = useState('');
const [pdfLoading, setPdfLoading] = useState(false);
const [pdfTarget, setPdfTarget] = useState('');
const inputRef = useRef(null); const inputRef = useRef(null);
const { search } = useArchiveSearch();
async function handleCheck() { async function handleCheck() {
const url = inputUrl.trim(); const url = inputUrl.trim();
if (!url) return; if (!url) return;
let targetUrl = url.startsWith('http') ? url : 'https://' + url; let targetUrl = url.startsWith('http') ? url : 'https://' + url;
setStatus('checking'); setStatus('checking');
setErrMsg(''); setErrMsg('');
setWaybackUrl('');
setArchiveUrl(''); setArchiveUrl('');
try { try {
const result = await search(targetUrl); // archive.ph URL immer konstruieren (ohne Prüfung – direkt nutzbar)
if (result.found) { const archivePh = `https://archive.ph/newest/${targetUrl}`;
setArchiveUrl(result.url);
setStatus('found'); // Wayback CDX parallel prüfen
let wb = null;
try { wb = await checkWayback(targetUrl); } catch { /* ignorieren */ }
setWaybackUrl(wb || '');
setArchiveUrl(archivePh);
if (wb) {
setStatus('found_both');
} else { } else {
setStatus('notfound'); // Auch ohne Wayback-Treffer archive.ph anbieten
setStatus('found_archive');
} }
} catch (err) { } catch (err) {
setErrMsg(err.message || 'Fehler beim Suchen'); setErrMsg(err.message || 'Fehler beim Suchen');
@@ -145,7 +70,8 @@ export default function PaywallKiller() {
} }
} }
async function handlePdf() { async function handlePdf(url) {
setPdfTarget(url);
setPdfLoading(true); setPdfLoading(true);
try { try {
const token = localStorage.getItem('sk_token'); const token = localStorage.getItem('sk_token');
@@ -155,7 +81,7 @@ export default function PaywallKiller() {
'Content-Type': 'application/json', 'Content-Type': 'application/json',
...(token ? { Authorization: `Bearer ${token}` } : {}), ...(token ? { Authorization: `Bearer ${token}` } : {}),
}, },
body: JSON.stringify({ archiveUrl }), body: JSON.stringify({ archiveUrl: url }),
}); });
if (!res.ok) { if (!res.ok) {
const data = await res.json().catch(() => ({})); const data = await res.json().catch(() => ({}));
@@ -172,15 +98,16 @@ export default function PaywallKiller() {
setTimeout(() => URL.revokeObjectURL(objUrl), 5000); setTimeout(() => URL.revokeObjectURL(objUrl), 5000);
} catch (err) { } catch (err) {
setErrMsg(err.message || 'PDF konnte nicht erstellt werden'); setErrMsg(err.message || 'PDF konnte nicht erstellt werden');
setStatus('error');
} finally { } finally {
setPdfLoading(false); setPdfLoading(false);
setPdfTarget('');
} }
} }
function handleReset() { function handleReset() {
setInputUrl(''); setInputUrl('');
setStatus('idle'); setStatus('idle');
setWaybackUrl('');
setArchiveUrl(''); setArchiveUrl('');
setErrMsg(''); setErrMsg('');
setPdfLoading(false); setPdfLoading(false);
@@ -195,10 +122,20 @@ export default function PaywallKiller() {
} }
const isChecking = status === 'checking'; const isChecking = status === 'checking';
const isFound = status === 'found'; const hasResult = status.startsWith('found') || status === 'error';
const isNotFound = status === 'notfound'; const hasWayback = status === 'found_both' && waybackUrl;
const isError = status === 'error'; const hasArchive = (status === 'found_both' || status === 'found_archive') && archiveUrl;
const hasResult = isFound || isNotFound || isError;
// PDF-Button für eine bestimmte URL
function PdfBtn({ url }) {
const loading = pdfLoading && pdfTarget === url;
return (
<button onClick={() => handlePdf(url)} disabled={pdfLoading}
style={{ ...S.btn(C.accent), opacity: pdfLoading ? 0.5 : 1, cursor: pdfLoading ? 'default' : 'pointer', fontSize: 12 }}>
{loading ? '⏳ PDF...' : '⬇ PDF'}
</button>
);
}
return ( return (
<div style={{ padding: '20px 16px', maxWidth: 640, margin: '0 auto' }}> <div style={{ padding: '20px 16px', maxWidth: 640, margin: '0 auto' }}>
@@ -209,10 +146,11 @@ export default function PaywallKiller() {
<span style={{ color: C.accent, fontFamily: 'monospace', fontSize: 13, letterSpacing: 1 }}>PAYWALL-KILLER</span> <span style={{ color: C.accent, fontFamily: 'monospace', fontSize: 13, letterSpacing: 1 }}>PAYWALL-KILLER</span>
</div> </div>
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace' }}> <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace' }}>
Artikel über archive.ph abrufen · Als PDF speichern Wayback Machine · archive.ph · Als PDF speichern
</div> </div>
</div> </div>
{/* Eingabe */}
<div style={{ ...S.card, marginBottom: 16 }}> <div style={{ ...S.card, marginBottom: 16 }}>
<div style={{ ...S.head, marginBottom: 10 }}>ARTIKEL-URL</div> <div style={{ ...S.head, marginBottom: 10 }}>ARTIKEL-URL</div>
<div style={{ display: 'flex', gap: 8, marginBottom: 10 }}> <div style={{ display: 'flex', gap: 8, marginBottom: 10 }}>
@@ -232,7 +170,7 @@ export default function PaywallKiller() {
<div style={{ display: 'flex', gap: 8 }}> <div style={{ display: 'flex', gap: 8 }}>
<button onClick={handleCheck} disabled={isChecking || !inputUrl.trim()} <button onClick={handleCheck} disabled={isChecking || !inputUrl.trim()}
style={{ ...S.btn(C.accent), flex: 1, opacity: (isChecking || !inputUrl.trim()) ? 0.45 : 1, cursor: (isChecking || !inputUrl.trim()) ? 'default' : 'pointer' }}> style={{ ...S.btn(C.accent), flex: 1, opacity: (isChecking || !inputUrl.trim()) ? 0.45 : 1, cursor: (isChecking || !inputUrl.trim()) ? 'default' : 'pointer' }}>
{isChecking ? '⏳ Lade archive.ph...' : '🔍 Archiv suchen'} {isChecking ? '⏳ Suche Archiv...' : '🔍 Archiv suchen'}
</button> </button>
{hasResult && ( {hasResult && (
<button onClick={handleReset} <button onClick={handleReset}
@@ -241,59 +179,61 @@ export default function PaywallKiller() {
</button> </button>
)} )}
</div> </div>
{isChecking && (
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 10 }}>
archive.ph wird direkt im Browser abgefragt...
</div>
)}
</div> </div>
{isFound && ( {/* Wayback Machine Treffer */}
<div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 16 }}> {hasWayback && (
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 12 }}> <div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 12 }}>
<span>✅</span> <div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
<span style={{ color: C.success, fontFamily: 'monospace', fontSize: 12 }}>Archiv gefunden</span> <span>🏛</span>
<span style={{ color: C.success, fontFamily: 'monospace', fontSize: 12 }}>Wayback Machine</span>
<span style={{ background: `${C.success}20`, border: `1px solid ${C.success}44`, borderRadius: 4, padding: '1px 6px', fontSize: 10, color: C.success, fontFamily: 'monospace' }}>gefunden</span>
</div> </div>
<div style={{ background: 'rgba(255,255,255,0.03)', border: '1px solid rgba(255,255,255,0.08)', borderRadius: 6, padding: '8px 10px', marginBottom: 14, wordBreak: 'break-all' }}> <div style={{ background: 'rgba(255,255,255,0.03)', border: '1px solid rgba(255,255,255,0.08)', borderRadius: 6, padding: '7px 10px', marginBottom: 10, wordBreak: 'break-all' }}>
<a href={archiveUrl} target="_blank" rel="noreferrer" style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}> <a href={waybackUrl} target="_blank" rel="noreferrer" style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>{waybackUrl}</a>
{archiveUrl}
</a>
</div> </div>
<div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}> <div style={{ display: 'flex', gap: 8 }}>
<a href={archiveUrl} target="_blank" rel="noreferrer" style={{ ...S.btn(C.success), textDecoration: 'none' }}> <a href={waybackUrl} target="_blank" rel="noreferrer" style={{ ...S.btn(C.success), textDecoration: 'none', fontSize: 12 }}>🌐 Öffnen</a>
🌐 Im Browser öffnen <PdfBtn url={waybackUrl} />
</a>
<button onClick={handlePdf} disabled={pdfLoading}
style={{ ...S.btn(C.accent), opacity: pdfLoading ? 0.5 : 1, cursor: pdfLoading ? 'default' : 'pointer' }}>
{pdfLoading ? '⏳ Erstelle PDF...' : '⬇ Als PDF speichern'}
</button>
</div> </div>
{pdfLoading && ( </div>
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 10 }}> )}
Seite wird serverseitig gerendert, dauert 15–30 Sekunden...
{/* archive.ph Link */}
{hasArchive && (
<div style={{ ...S.card, borderColor: hasWayback ? 'rgba(255,255,255,0.1)' : `${C.success}55`, marginBottom: 12 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
<span>📦</span>
<span style={{ color: hasWayback ? 'rgba(255,255,255,0.6)' : C.success, fontFamily: 'monospace', fontSize: 12 }}>archive.ph</span>
{!hasWayback && <span style={{ background: `${C.success}20`, border: `1px solid ${C.success}44`, borderRadius: 4, padding: '1px 6px', fontSize: 10, color: C.success, fontFamily: 'monospace' }}>neuester Snapshot</span>}
</div>
<div style={{ background: 'rgba(255,255,255,0.03)', border: '1px solid rgba(255,255,255,0.08)', borderRadius: 6, padding: '7px 10px', marginBottom: 10, wordBreak: 'break-all' }}>
<a href={archiveUrl} target="_blank" rel="noreferrer" style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>{archiveUrl}</a>
</div>
<div style={{ display: 'flex', gap: 8 }}>
<a href={archiveUrl} target="_blank" rel="noreferrer" style={{ ...S.btn(C.success), textDecoration: 'none', fontSize: 12 }}>🌐 Öffnen</a>
<PdfBtn url={archiveUrl} />
</div>
{!hasWayback && (
<div style={{ color: C.muted, fontSize: 10, fontFamily: 'monospace', marginTop: 8 }}>
Kein Wayback-Snapshot gefunden. archive.ph-Link öffnet direkt den neuesten Snapshot (falls vorhanden).
</div> </div>
)} )}
</div> </div>
)} )}
{isNotFound && ( {/* PDF-Ladehinweis */}
<div style={{ ...S.card, borderColor: `${C.error}44` }}> {pdfLoading && (
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 8 }}> <div style={{ ...S.card, borderColor: `${C.accent}44`, marginBottom: 12 }}>
<span>📭</span> <div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace' }}>
<span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>Kein Archiv gefunden</span> ⏳ Seite wird serverseitig gerendert, bitte warten (15–30s)...
</div> </div>
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', lineHeight: 1.6, marginBottom: 12 }}>
Für <strong style={{ color: 'rgba(255,255,255,0.55)' }}>{getDomain(inputUrl)}</strong> wurde kein Snapshot auf archive.ph gefunden.
</div>
<a href={`https://archive.ph/${inputUrl.trim()}`} target="_blank" rel="noreferrer"
style={{ ...S.btn(C.muted), textDecoration: 'none', display: 'inline-block', fontSize: 11 }}>
🔗 Manuell auf archive.ph prüfen
</a>
</div> </div>
)} )}
{isError && ( {/* Fehler */}
<div style={{ ...S.card, borderColor: `${C.error}44` }}> {errMsg && (
<div style={{ ...S.card, borderColor: `${C.error}44`, marginBottom: 12 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8 }}> <div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
<span>⚠️</span> <span>⚠️</span>
<span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>{errMsg}</span> <span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>{errMsg}</span>
@@ -301,12 +241,13 @@ export default function PaywallKiller() {
</div> </div>
)} )}
{/* Idle-Info */}
{status === 'idle' && ( {status === 'idle' && (
<div style={{ ...S.card }}> <div style={{ ...S.card }}>
<div style={{ ...S.head, marginBottom: 10 }}>SO FUNKTIONIERT ES</div> <div style={{ ...S.head, marginBottom: 10 }}>SO FUNKTIONIERT ES</div>
{[ {[
['1', 'URL des gesperrten Artikels einfügen'], ['1', 'URL des gesperrten Artikels einfügen'],
['2', 'Archiv auf archive.ph suchen (direkt im Browser)'], ['2', 'Wayback Machine + archive.ph werden geprüft'],
['3', 'Im Browser öffnen oder als PDF speichern'], ['3', 'Im Browser öffnen oder als PDF speichern'],
].map(([num, text]) => ( ].map(([num, text]) => (
<div key={num} style={{ display: 'flex', gap: 10, alignItems: 'center', marginBottom: 8 }}> <div key={num} style={{ display: 'flex', gap: 10, alignItems: 'center', marginBottom: 8 }}>