Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: Wayback auto-save wenn kein Snapshot existiert

Browse Source
This commit is contained in:
Dicken
2026-06-12 12:06:44 +02:00
parent 51e52d76c4
commit a712d495d7
2 changed files with 130 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

69
backend/src/tools/paywallkiller/routes.js
View File

@@ -231,7 +231,8 @@ router.post('/pdf', authenticate, async (req, res) => {
// ── GET /api/tools/paywallkiller/wayback?url=... ──────────────────────────
// Proxyt die Wayback CDX API – umgeht CORS/CSP Probleme im Browser
// 1. Sucht vorhandenen Snapshot via CDX API
// 2. Falls keiner → erstellt neuen Snapshot via Save API
router.get('/wayback', authenticate, async (req, res) => {
const { url } = req.query;
if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url Parameter fehlt' });
@@ -239,24 +240,62 @@ router.get('/wayback', authenticate, async (req, res) => {
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
// ── Schritt 1: Vorhandenen Snapshot suchen ──────────────────────────────
try {
const result = await fetchPage(cdxUrl);
if (result.statusCode !== 200) {
return res.status(502).json({ error: `Wayback API Fehler: ${result.statusCode}` });
}
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
const cdxResult = await fetchPage(cdxUrl);
if (cdxResult.statusCode === 200 && cdxResult.body) {
let data;
try { data = JSON.parse(result.body); } catch { return res.status(502).json({ error: 'Ungültige Wayback-Antwort' }); }
// data = [["timestamp"],["20260608201434"]] oder [] wenn nichts gefunden
if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) {
return res.json({ found: false });
}
try { data = JSON.parse(cdxResult.body); } catch { data = null; }
if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
const timestamp = data[1][0];
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${timestamp}/${targetUrl}` });
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${timestamp}/${targetUrl}`, created: false });
}
}
} catch { /* weiter zu Save API */ }
// ── Schritt 2: Keinen Snapshot gefunden → neu erstellen ─────────────────
return res.json({ found: false, canSave: true });
});
// ── POST /api/tools/paywallkiller/wayback-save ────────────────────────────
// Erstellt einen neuen Wayback-Snapshot und wartet auf das Ergebnis
router.post('/wayback-save', authenticate, async (req, res) => {
const { url } = req.body;
if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url fehlt' });
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
// Save API aufrufen: GET https://web.archive.org/save/URL
// Antwort: 302 Redirect auf https://web.archive.org/web/TIMESTAMP/URL
try {
const saveUrl = `https://web.archive.org/save/${targetUrl}`;
const saveResult = await fetchPage(saveUrl); // fetchPage folgt Redirects
// Nach Redirect landet man auf web.archive.org/web/TIMESTAMP/URL
const final = saveResult.finalUrl || '';
const match = final.match(/web\.archive\.org\/web\/(\d{14})\//);
if (match) {
return res.json({ success: true, waybackUrl: final, timestamp: match[1] });
}
// Fallback: CDX nochmal prüfen (Save kann asynchron sein)
await new Promise(r => setTimeout(r, 3000));
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
const cdxResult = await fetchPage(cdxUrl);
if (cdxResult.statusCode === 200) {
let data;
try { data = JSON.parse(cdxResult.body); } catch { data = null; }
if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
const timestamp = data[1][0];
return res.json({ success: true, waybackUrl: `https://web.archive.org/web/${timestamp}/${targetUrl}`, timestamp });
}
}
return res.status(502).json({ error: 'Snapshot konnte nicht erstellt werden. Die Seite blockiert möglicherweise die Wayback Machine.' });
} catch (err) {
return res.status(500).json({ error: 'Fehler: ' + err.message });
return res.status(500).json({ error: 'Save fehlgeschlagen: ' + err.message });
}
});

92
frontend/src/tools/paywallkiller.jsx
View File

@@ -6,6 +6,7 @@ const C = {
success: '#4ade80',
error: '#f87171',
muted: '#6b7280',
info: '#60a5fa',
};
function getDomain(url) {
@@ -15,29 +16,59 @@ function getDomain(url) {
export default function PaywallKiller() {
const [inputUrl, setInputUrl] = useState('');
// status: idle | checking | saving | done | error
const [status, setStatus] = useState('idle');
const [waybackUrl, setWaybackUrl] = useState('');
const [wasCreated, setWasCreated] = useState(false);
const [errMsg, setErrMsg] = useState('');
const [pdfLoading, setPdfLoading] = useState(false);
const inputRef = useRef(null);
function getTargetUrl() {
const url = inputUrl.trim();
return url.startsWith('http') ? url : 'https://' + url;
}
async function handleCheck() {
const url = inputUrl.trim();
if (!url) return;
const targetUrl = url.startsWith('http') ? url : 'https://' + url;
setStatus('checking');
setErrMsg('');
setWaybackUrl('');
setWasCreated(false);
try {
const data = await api(`/tools/paywallkiller/wayback?url=${encodeURIComponent(targetUrl)}`, { method: 'GET' });
setWaybackUrl(data.found ? data.waybackUrl : '');
const data = await api(`/tools/paywallkiller/wayback?url=${encodeURIComponent(getTargetUrl())}`, { method: 'GET' });
if (data.found) {
setWaybackUrl(data.waybackUrl);
setStatus('done');
} else {
// Kein Snapshot → direkt Save starten
await handleSave();
}
} catch (err) {
setErrMsg(err.message || 'Fehler beim Suchen');
setStatus('error');
}
}
async function handleSave() {
setStatus('saving');
setErrMsg('');
try {
const data = await api('/tools/paywallkiller/wayback-save', { body: { url: getTargetUrl() } });
if (data.success) {
setWaybackUrl(data.waybackUrl);
setWasCreated(true);
setStatus('done');
} else {
throw new Error(data.error || 'Snapshot konnte nicht erstellt werden');
}
} catch (err) {
setErrMsg(err.message || 'Snapshot-Erstellung fehlgeschlagen');
setStatus('error');
}
}
async function handlePdf() {
if (!waybackUrl) return;
setPdfLoading(true);
@@ -76,16 +107,18 @@ export default function PaywallKiller() {
setInputUrl('');
setStatus('idle');
setWaybackUrl('');
setWasCreated(false);
setErrMsg('');
setPdfLoading(false);
setTimeout(() => inputRef.current?.focus(), 50);
}
const isChecking = status === 'checking';
const isSaving = status === 'saving';
const isBusy = isChecking || isSaving;
const isDone = status === 'done';
const hasWayback = isDone && !!waybackUrl;
const archivePhUrl = isDone
? `https://archive.ph/newest/${(inputUrl.startsWith('http') ? inputUrl : 'https://' + inputUrl).trim()}`
const archivePhUrl = inputUrl.trim()
? `https://archive.ph/newest/${getTargetUrl()}`
: '';
return (
@@ -109,14 +142,14 @@ export default function PaywallKiller() {
ref={inputRef}
value={inputUrl}
onChange={e => setInputUrl(e.target.value)}
onKeyDown={e => e.key === 'Enter' && !isChecking && handleCheck()}
placeholder="URL hier einfügen (Long-Press → Einfügen)"
onKeyDown={e => e.key === 'Enter' && !isBusy && handleCheck()}
placeholder="URL einfügen (Long-Press → Einfügen)"
style={{ ...S.inp, width: '100%', boxSizing: 'border-box', marginBottom: 10 }}
/>
<div style={{ display: 'flex', gap: 8 }}>
<button onClick={handleCheck} disabled={isChecking || !inputUrl.trim()}
style={{ ...S.btn(C.accent), flex: 1, opacity: (isChecking || !inputUrl.trim()) ? 0.45 : 1, cursor: (isChecking || !inputUrl.trim()) ? 'default' : 'pointer' }}>
{isChecking ? '⏳ Suche in Wayback Machine...' : '🔍 Archiv suchen'}
<button onClick={handleCheck} disabled={isBusy || !inputUrl.trim()}
style={{ ...S.btn(C.accent), flex: 1, opacity: (isBusy || !inputUrl.trim()) ? 0.45 : 1, cursor: (isBusy || !inputUrl.trim()) ? 'default' : 'pointer' }}>
{isChecking ? '⏳ Suche Archiv...' : isSaving ? '⏳ Erstelle Snapshot...' : '🔍 Archiv suchen'}
</button>
{(isDone || status === 'error') && (
<button onClick={handleReset}
@@ -125,27 +158,24 @@ export default function PaywallKiller() {
</button>
)}
</div>
{isSaving && (
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 10, lineHeight: 1.5 }}>
Kein Snapshot gefunden – die Wayback Machine archiviert den Artikel jetzt. Das dauert 15–30 Sekunden...
</div>
)}
</div>
{/* Ergebnisse */}
{/* Ergebnis: Wayback Machine */}
{isDone && (
<>
{/* Wayback Machine */}
<div style={{ ...S.card, borderColor: hasWayback ? `${C.success}55` : 'rgba(255,255,255,0.1)', marginBottom: 12 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: hasWayback ? 10 : 0 }}>
<div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 12 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
<span>🏛</span>
<span style={{ color: hasWayback ? C.success : C.muted, fontFamily: 'monospace', fontSize: 12 }}>Wayback Machine</span>
<span style={{
background: hasWayback ? `${C.success}20` : 'rgba(255,255,255,0.06)',
border: `1px solid ${hasWayback ? C.success + '44' : 'rgba(255,255,255,0.12)'}`,
borderRadius: 4, padding: '1px 6px', fontSize: 10,
color: hasWayback ? C.success : C.muted, fontFamily: 'monospace',
}}>
{hasWayback ? 'Snapshot gefunden' : 'kein Snapshot'}
<span style={{ color: C.success, fontFamily: 'monospace', fontSize: 12 }}>Wayback Machine</span>
<span style={{ background: `${C.success}20`, border: `1px solid ${C.success}44`, borderRadius: 4, padding: '1px 6px', fontSize: 10, color: C.success, fontFamily: 'monospace' }}>
{wasCreated ? 'neu archiviert ✨' : 'Snapshot gefunden'}
</span>
</div>
{hasWayback && (
<>
<div style={{ background: 'rgba(255,255,255,0.03)', border: '1px solid rgba(255,255,255,0.08)', borderRadius: 6, padding: '7px 10px', marginBottom: 10, wordBreak: 'break-all' }}>
<a href={waybackUrl} target="_blank" rel="noreferrer"
style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>{waybackUrl}</a>
@@ -163,11 +193,9 @@ export default function PaywallKiller() {
Seite wird serverseitig gerendert (15–30s)...
</div>
)}
</>
)}
</div>
{/* archive.ph */}
{/* archive.ph – immer anzeigen, nur Öffnen */}
<div style={{ ...S.card, borderColor: 'rgba(255,255,255,0.1)', marginBottom: 12 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
<span>📦</span>
@@ -197,10 +225,10 @@ export default function PaywallKiller() {
<div style={{ ...S.card }}>
<div style={{ ...S.head, marginBottom: 10 }}>SO FUNKTIONIERT ES</div>
{[
['1', 'URL des gesperrten Artikels ins Textfeld einfügen (Long-Press → Einfügen)'],
['2', 'Wayback Machine (web.archive.org) wird serverseitig durchsucht'],
['3', 'Snapshot öffnen oder als PDF speichern'],
['4', 'Alternativ: archive.ph direkt im Browser öffnen'],
['1', 'URL des gesperrten Artikels einfügen (Long-Press → Einfügen)'],
['2', 'Vorhandenen Snapshot in der Wayback Machine suchen'],
['3', 'Falls keiner existiert: Artikel wird automatisch archiviert'],
['4', 'Snapshot öffnen oder als PDF speichern'],
].map(([num, text]) => (
<div key={num} style={{ display: 'flex', gap: 10, alignItems: 'flex-start', marginBottom: 8 }}>
<span style={{ background: `${C.accent}20`, border: `1px solid ${C.accent}44`, borderRadius: '50%', width: 20, height: 20, flexShrink: 0, display: 'flex', alignItems: 'center', justifyContent: 'center', fontSize: 10, color: C.accent, fontFamily: 'monospace', marginTop: 2 }}>{num}</span>