fix: CDX via Backend-Proxy /cdx statt Browser-fetch
This commit is contained in:
@@ -1,12 +1,60 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { authenticate } = require('../../middleware/auth');
|
||||
const https = require('https');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const crypto = require('crypto');
|
||||
|
||||
// ── Einfacher HTTPS GET Helper ─────────────────────────────────────────────
|
||||
function httpsGet(url, timeoutMs = 20000) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.get(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)',
|
||||
'Accept': '*/*',
|
||||
},
|
||||
timeout: timeoutMs,
|
||||
}, (res) => {
|
||||
let body = '';
|
||||
res.setEncoding('utf8');
|
||||
res.on('data', d => { if (body.length < 200000) body += d; });
|
||||
res.on('end', () => resolve({ status: res.statusCode, body, headers: res.headers }));
|
||||
res.on('error', reject);
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
|
||||
});
|
||||
}
|
||||
|
||||
// ── GET /api/tools/paywallkiller/cdx?url=... ──────────────────────────────
|
||||
// Proxy für Wayback CDX API – löst CORS/Origin-Probleme der PWA
|
||||
router.get('/cdx', authenticate, async (req, res) => {
|
||||
const { url } = req.query;
|
||||
if (!url) return res.status(400).json({ error: 'url fehlt' });
|
||||
|
||||
let targetUrl = url.trim();
|
||||
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
|
||||
|
||||
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
|
||||
|
||||
try {
|
||||
const result = await httpsGet(cdxUrl);
|
||||
if (result.status !== 200) return res.status(502).json({ error: `CDX Status: ${result.status}` });
|
||||
|
||||
let data;
|
||||
try { data = JSON.parse(result.body); } catch { return res.status(502).json({ error: 'Ungültige CDX-Antwort' }); }
|
||||
|
||||
if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) {
|
||||
return res.json({ found: false });
|
||||
}
|
||||
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ── POST /api/tools/paywallkiller/pdf ─────────────────────────────────────
|
||||
// Lädt Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF
|
||||
router.post('/pdf', authenticate, async (req, res) => {
|
||||
const { originalUrl, archiveUrl } = req.body;
|
||||
const targetUrl = originalUrl || archiveUrl;
|
||||
@@ -39,7 +87,6 @@ router.post('/pdf', authenticate, async (req, res) => {
|
||||
await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 });
|
||||
await new Promise(r => setTimeout(r, 1500));
|
||||
|
||||
// Paywall-Overlays, Cookie-Banner, Blur-Filter entfernen
|
||||
await page.addStyleTag({ content: `
|
||||
[class*="paywall"],[class*="Paywall"],[id*="paywall"],
|
||||
[class*="piano"],[class*="Piano"],[id*="piano"],
|
||||
|
||||
Reference in New Issue
Block a user