fix: CDX via Backend-Proxy /cdx statt Browser-fetch

This commit is contained in:
2026-06-12 13:03:40 +02:00
parent 321f65a802
commit a415d3c691
2 changed files with 86 additions and 57 deletions

View File

@@ -1,12 +1,60 @@
const express = require('express');
const router = express.Router();
const { authenticate } = require('../../middleware/auth');
const https = require('https');
const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
// ── Einfacher HTTPS GET Helper ─────────────────────────────────────────────
function httpsGet(url, timeoutMs = 20000) {
return new Promise((resolve, reject) => {
const req = https.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)',
'Accept': '*/*',
},
timeout: timeoutMs,
}, (res) => {
let body = '';
res.setEncoding('utf8');
res.on('data', d => { if (body.length < 200000) body += d; });
res.on('end', () => resolve({ status: res.statusCode, body, headers: res.headers }));
res.on('error', reject);
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
});
}
// ── GET /api/tools/paywallkiller/cdx?url=... ──────────────────────────────
// Proxy für Wayback CDX API löst CORS/Origin-Probleme der PWA
router.get('/cdx', authenticate, async (req, res) => {
const { url } = req.query;
if (!url) return res.status(400).json({ error: 'url fehlt' });
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
try {
const result = await httpsGet(cdxUrl);
if (result.status !== 200) return res.status(502).json({ error: `CDX Status: ${result.status}` });
let data;
try { data = JSON.parse(result.body); } catch { return res.status(502).json({ error: 'Ungültige CDX-Antwort' }); }
if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) {
return res.json({ found: false });
}
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` });
} catch (err) {
return res.status(500).json({ error: err.message });
}
});
// ── POST /api/tools/paywallkiller/pdf ─────────────────────────────────────
// Lädt Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF
router.post('/pdf', authenticate, async (req, res) => {
const { originalUrl, archiveUrl } = req.body;
const targetUrl = originalUrl || archiveUrl;
@@ -39,7 +87,6 @@ router.post('/pdf', authenticate, async (req, res) => {
await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 });
await new Promise(r => setTimeout(r, 1500));
// Paywall-Overlays, Cookie-Banner, Blur-Filter entfernen
await page.addStyleTag({ content: `
[class*="paywall"],[class*="Paywall"],[id*="paywall"],
[class*="piano"],[class*="Piano"],[id*="piano"],