fix: CDX via Backend-Proxy /cdx statt Browser-fetch
This commit is contained in:
@@ -1,12 +1,60 @@
|
||||
const express = require('express');
|
||||
const router = express.Router();
|
||||
const { authenticate } = require('../../middleware/auth');
|
||||
const https = require('https');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const crypto = require('crypto');
|
||||
|
||||
// ── Einfacher HTTPS GET Helper ─────────────────────────────────────────────
|
||||
function httpsGet(url, timeoutMs = 20000) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const req = https.get(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)',
|
||||
'Accept': '*/*',
|
||||
},
|
||||
timeout: timeoutMs,
|
||||
}, (res) => {
|
||||
let body = '';
|
||||
res.setEncoding('utf8');
|
||||
res.on('data', d => { if (body.length < 200000) body += d; });
|
||||
res.on('end', () => resolve({ status: res.statusCode, body, headers: res.headers }));
|
||||
res.on('error', reject);
|
||||
});
|
||||
req.on('error', reject);
|
||||
req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
|
||||
});
|
||||
}
|
||||
|
||||
// ── GET /api/tools/paywallkiller/cdx?url=... ──────────────────────────────
|
||||
// Proxy für Wayback CDX API – löst CORS/Origin-Probleme der PWA
|
||||
router.get('/cdx', authenticate, async (req, res) => {
|
||||
const { url } = req.query;
|
||||
if (!url) return res.status(400).json({ error: 'url fehlt' });
|
||||
|
||||
let targetUrl = url.trim();
|
||||
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
|
||||
|
||||
const cdxUrl = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
|
||||
|
||||
try {
|
||||
const result = await httpsGet(cdxUrl);
|
||||
if (result.status !== 200) return res.status(502).json({ error: `CDX Status: ${result.status}` });
|
||||
|
||||
let data;
|
||||
try { data = JSON.parse(result.body); } catch { return res.status(502).json({ error: 'Ungültige CDX-Antwort' }); }
|
||||
|
||||
if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) {
|
||||
return res.json({ found: false });
|
||||
}
|
||||
return res.json({ found: true, waybackUrl: `https://web.archive.org/web/${data[1][0]}/${targetUrl}` });
|
||||
} catch (err) {
|
||||
return res.status(500).json({ error: err.message });
|
||||
}
|
||||
});
|
||||
|
||||
// ── POST /api/tools/paywallkiller/pdf ─────────────────────────────────────
|
||||
// Lädt Original-URL mit Puppeteer, blendet Paywall-Overlays aus, erstellt PDF
|
||||
router.post('/pdf', authenticate, async (req, res) => {
|
||||
const { originalUrl, archiveUrl } = req.body;
|
||||
const targetUrl = originalUrl || archiveUrl;
|
||||
@@ -39,7 +87,6 @@ router.post('/pdf', authenticate, async (req, res) => {
|
||||
await page.goto(targetUrl, { waitUntil: 'networkidle0', timeout: 30000 });
|
||||
await new Promise(r => setTimeout(r, 1500));
|
||||
|
||||
// Paywall-Overlays, Cookie-Banner, Blur-Filter entfernen
|
||||
await page.addStyleTag({ content: `
|
||||
[class*="paywall"],[class*="Paywall"],[id*="paywall"],
|
||||
[class*="piano"],[class*="Piano"],[id*="piano"],
|
||||
|
||||
@@ -13,52 +13,14 @@ function getDomain(url) {
|
||||
catch { return url; }
|
||||
}
|
||||
|
||||
// Wayback CDX – direkt im Browser
|
||||
async function searchWayback(targetUrl) {
|
||||
const cdx = `https://web.archive.org/cdx/search/cdx?url=${encodeURIComponent(targetUrl)}&output=json&limit=1&fl=timestamp&filter=statuscode:200&fastLatest=true`;
|
||||
let res;
|
||||
try {
|
||||
// Ohne custom headers – minimaler Request damit CORS-Preflight entfällt
|
||||
res = await fetch(cdx);
|
||||
} catch (e) {
|
||||
throw new Error('Netzwerkfehler: ' + e.message);
|
||||
}
|
||||
if (!res.ok) throw new Error(`CDX ${res.status}`);
|
||||
const data = await res.json();
|
||||
if (!Array.isArray(data) || data.length < 2 || !data[1]?.[0]) return null;
|
||||
return `https://web.archive.org/web/${data[1][0]}/${targetUrl}`;
|
||||
}
|
||||
|
||||
// Wayback Save + CDX-Polling
|
||||
async function saveAndPollWayback(targetUrl, onStatus) {
|
||||
// Save via no-cors (kein lesbarer Response, aber Archivierung wird getriggert)
|
||||
try {
|
||||
fetch(`https://web.archive.org/save/${targetUrl}`, {
|
||||
method: 'GET',
|
||||
mode: 'no-cors',
|
||||
}).catch(() => {});
|
||||
} catch {}
|
||||
|
||||
// CDX pollen: 8x alle 5s = max 40s
|
||||
for (let i = 0; i < 8; i++) {
|
||||
await new Promise(r => setTimeout(r, 5000));
|
||||
onStatus(`Warte auf Snapshot... (${(i + 1) * 5}s)`);
|
||||
try {
|
||||
const url = await searchWayback(targetUrl);
|
||||
if (url) return url;
|
||||
} catch {}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export default function PaywallKiller() {
|
||||
const [inputUrl, setInputUrl] = useState('');
|
||||
const [status, setStatus] = useState('idle');
|
||||
const [statusMsg, setStatusMsg] = useState('');
|
||||
const [waybackUrl, setWaybackUrl] = useState('');
|
||||
const [wasCreated, setWasCreated] = useState(false);
|
||||
const [errMsg, setErrMsg] = useState('');
|
||||
const [pdfLoading, setPdfLoading] = useState(false);
|
||||
const [inputUrl, setInputUrl] = useState('');
|
||||
const [status, setStatus] = useState('idle');
|
||||
const [statusMsg, setStatusMsg] = useState('');
|
||||
const [waybackUrl, setWaybackUrl] = useState('');
|
||||
const [wasCreated, setWasCreated] = useState(false);
|
||||
const [errMsg, setErrMsg] = useState('');
|
||||
const [pdfLoading, setPdfLoading] = useState(false);
|
||||
const inputRef = useRef(null);
|
||||
|
||||
function getTargetUrl() {
|
||||
@@ -66,6 +28,31 @@ export default function PaywallKiller() {
|
||||
return url.startsWith('http') ? url : 'https://' + url;
|
||||
}
|
||||
|
||||
// CDX-Suche über Backend-Proxy (umgeht CORS/Origin-Problem der PWA)
|
||||
async function searchWayback(targetUrl) {
|
||||
const data = await api(`/tools/paywallkiller/cdx?url=${encodeURIComponent(targetUrl)}`, { method: 'GET' });
|
||||
return data.found ? data.waybackUrl : null;
|
||||
}
|
||||
|
||||
// Save via Browser (no-cors, fire-and-forget), dann CDX-Polling via Backend
|
||||
async function saveAndPoll(targetUrl) {
|
||||
// Save-Request abschießen (Browser, kein CORS-Response nötig)
|
||||
try {
|
||||
fetch(`https://web.archive.org/save/${targetUrl}`, { mode: 'no-cors' }).catch(() => {});
|
||||
} catch {}
|
||||
|
||||
// CDX pollen via Backend-Proxy
|
||||
for (let i = 0; i < 8; i++) {
|
||||
await new Promise(r => setTimeout(r, 5000));
|
||||
setStatusMsg(`Warte auf Snapshot... (${(i + 1) * 5}s)`);
|
||||
try {
|
||||
const url = await searchWayback(targetUrl);
|
||||
if (url) return url;
|
||||
} catch {}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function handleCheck() {
|
||||
if (!inputUrl.trim()) return;
|
||||
const targetUrl = getTargetUrl();
|
||||
@@ -76,18 +63,16 @@ export default function PaywallKiller() {
|
||||
setWasCreated(false);
|
||||
|
||||
try {
|
||||
// Schritt 1: vorhandenen Snapshot suchen
|
||||
const existing = await searchWayback(targetUrl);
|
||||
if (existing) {
|
||||
setWaybackUrl(existing);
|
||||
setStatus('done');
|
||||
return;
|
||||
}
|
||||
|
||||
// Schritt 2: Kein Snapshot → neu erstellen + pollen
|
||||
// Kein Snapshot → archivieren
|
||||
setStatus('saving');
|
||||
setStatusMsg('Kein Snapshot gefunden – archiviere jetzt...');
|
||||
const newUrl = await saveAndPollWayback(targetUrl, msg => setStatusMsg(msg));
|
||||
setStatusMsg('Kein Snapshot – archiviere jetzt...');
|
||||
const newUrl = await saveAndPoll(targetUrl);
|
||||
if (newUrl) {
|
||||
setWaybackUrl(newUrl);
|
||||
setWasCreated(true);
|
||||
@@ -185,7 +170,6 @@ export default function PaywallKiller() {
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Wayback-Ergebnis */}
|
||||
{isDone && (
|
||||
<div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 12 }}>
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
|
||||
@@ -209,19 +193,17 @@ export default function PaywallKiller() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Kein Snapshot erstellt */}
|
||||
{status === 'notfound' && (
|
||||
<div style={{ ...S.card, borderColor: `${C.error}44`, marginBottom: 12 }}>
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
|
||||
<span>📭</span>
|
||||
<span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>
|
||||
Archivierung fehlgeschlagen – {getDomain(inputUrl)} blockiert möglicherweise die Wayback Machine.
|
||||
Kein Snapshot gefunden – {getDomain(inputUrl)} blockiert möglicherweise die Wayback Machine.
|
||||
</span>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* archive.ph – immer nach Suche anzeigen */}
|
||||
{hasResult && archivePhUrl && (
|
||||
<div style={{ ...S.card, borderColor: 'rgba(255,255,255,0.1)', marginBottom: 12 }}>
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 10 }}>
|
||||
@@ -249,7 +231,7 @@ export default function PaywallKiller() {
|
||||
<div style={{ ...S.head, marginBottom: 10 }}>SO FUNKTIONIERT ES</div>
|
||||
{[
|
||||
['1', 'URL einfügen (Long-Press → Einfügen)'],
|
||||
['2', 'Wayback Machine wird direkt im Browser durchsucht'],
|
||||
['2', 'Wayback Machine wird serverseitig durchsucht'],
|
||||
['3', 'Falls kein Snapshot: Artikel wird automatisch archiviert'],
|
||||
['4', 'Snapshot öffnen oder als PDF speichern'],
|
||||
].map(([num, text]) => (
|
||||
|
||||
Reference in New Issue
Block a user