Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: Wayback auto-save wenn kein Snapshot existiert

Browse Source
This commit is contained in:
Dicken
2026-06-12 12:06:44 +02:00
parent 51e52d76c4
commit a712d495d7
2 changed files with 130 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
backend/src/tools/paywallkiller/routes.js
View File

@@ -231,7 +231,8 @@ router.post('/pdf', authenticate, async (req, res) => {
// ── GET /api/tools/paywallkiller/wayback?url=... ──────────────────────────
// Proxyt die Wayback CDX API – umgeht CORS/CSP Probleme im Browser
// 1. Sucht vorhandenen Snapshot via CDX API
// 2. Falls keiner → erstellt neuen Snapshot via Save API
router.get('/wayback', authenticate, async (req, res) => {
const { url } = req.query;
if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url Parameter fehlt' });
@@ -239,24 +240,62 @@ router.get('/wayback', authenticate, async (req, res) => {
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
// ── Schritt 1: Vorhandenen Snapshot suchen ──────────────────────────────
try {
const result = await fetchPage(cdxUrl);
if (result.statusCode !== 200) {
return res.status(502).json({ error: `Wayback API Fehler: ${result.statusCode}` });
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
const cdxResult = await fetchPage(cdxUrl);
if (cdxResult.statusCode === 200 && cdxResult.body) {
let data;
try { data = JSON.parse(cdxResult.body); } catch { data = null; }
if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
const timestamp = data[1][0];
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${timestamp}/${targetUrl}`, created: false });
}
}
let data;
try { data = JSON.parse(result.body); } catch { return res.status(502).json({ error: 'Ungültige Wayback-Antwort' }); }
} catch { /* weiter zu Save API */ }
// data = [["timestamp"],["20260608201434"]] oder [] wenn nichts gefunden
if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) {
return res.json({ found: false });
// ── Schritt 2: Keinen Snapshot gefunden → neu erstellen ─────────────────
return res.json({ found: false, canSave: true });
});
// ── POST /api/tools/paywallkiller/wayback-save ────────────────────────────
// Erstellt einen neuen Wayback-Snapshot und wartet auf das Ergebnis
router.post('/wayback-save', authenticate, async (req, res) => {
const { url } = req.body;
if (!url || typeof url !== 'string') return res.status(400).json({ error: 'url fehlt' });
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
// Save API aufrufen: GET https://web.archive.org/save/URL
// Antwort: 302 Redirect auf https://web.archive.org/web/TIMESTAMP/URL
try {
const saveUrl = `https://web.archive.org/save/${targetUrl}`;
const saveResult = await fetchPage(saveUrl); // fetchPage folgt Redirects
// Nach Redirect landet man auf web.archive.org/web/TIMESTAMP/URL
const final = saveResult.finalUrl || '';
const match = final.match(/web\.archive\.org\/web\/(\d{14})\//);
if (match) {
return res.json({ success: true, waybackUrl: final, timestamp: match[1] });
}
const timestamp = data[1][0];
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${timestamp}/${targetUrl}` });
// Fallback: CDX nochmal prüfen (Save kann asynchron sein)
await new Promise(r => setTimeout(r, 3000));
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
const cdxResult = await fetchPage(cdxUrl);
if (cdxResult.statusCode === 200) {
let data;
try { data = JSON.parse(cdxResult.body); } catch { data = null; }
if (Array.isArray(data) && data.length >= 2 && data[1]?.[0]) {
const timestamp = data[1][0];
return res.json({ success: true, waybackUrl: `https://web.archive.org/web/${timestamp}/${targetUrl}`, timestamp });
}
}
return res.status(502).json({ error: 'Snapshot konnte nicht erstellt werden. Die Seite blockiert möglicherweise die Wayback Machine.' });
} catch (err) {
return res.status(500).json({ error: 'Fehler: ' + err.message });
return res.status(500).json({ error: 'Save fehlgeschlagen: ' + err.message });
}
});