Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: archive.is Suche via HTML-Parse, Button-Farben gefixt

Browse Source
This commit is contained in:
Dicken
2026-06-12 01:42:42 +02:00
parent 38c27ac5bb
commit 902d5fd99e
2 changed files with 146 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

100
backend/src/tools/paywallkiller/routes.js
View File

@@ -7,60 +7,71 @@ const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
// ── Hilfsfunktion: folgt Redirects, gibt finale URL zurück ───────────────
function fetchFollowRedirects(url, redirectCount = 0) {
// ── HTTP GET, gibt finalUrl + statusCode + body (optional) zurück ─────────
function fetchPage(url, opts = {}) {
return new Promise((resolve, reject) => {
if (redirectCount > 8) return reject(new Error('Zu viele Redirects'));
if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects'));
const mod = url.startsWith('https') ? https : http;
const req = mod.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml',
'Accept-Language': 'de,en;q=0.5',
},
timeout: 12000,
}, (res) => {
if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) {
let location = res.headers.location;
if (!location.startsWith('http')) {
try {
const base = new URL(url);
location = location.startsWith('/')
? `${base.protocol}//${base.host}${location}`
: `${base.protocol}//${base.host}/${location}`;
} catch { return reject(new Error('Ungültige Redirect-URL')); }
let loc = res.headers.location;
if (!loc.startsWith('http')) {
try { const b = new URL(url); loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; }
catch { return reject(new Error('Ungültige Redirect-URL')); }
}
res.resume();
return fetchFollowRedirects(location, redirectCount + 1).then(resolve).catch(reject);
return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1 }).then(resolve).catch(reject);
}
if (opts.bodyNeeded) {
let body = '';
res.setEncoding('utf8');
res.on('data', d => { if (body.length < 50000) body += d; });
res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body }));
} else {
res.resume();
resolve({ statusCode: res.statusCode, finalUrl: url });
}
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
});
}
// ── Prüfe ob archive.is einen Snapshot hat ────────────────────────────────
// archive.is/newest/URL leitet auf archive.is/HASH weiter wenn ein Snapshot existiert.
// Wenn kein Snapshot → Redirect zurück auf archive.is/ (Root) oder /newest/... bleibt stehen.
async function checkArchive(archiveUrl) {
// ── Archivsuche über archive.is ───────────────────────────────────────────
// archive.is/?url=ZIEL gibt HTML zurück mit Links zu archivierten Snapshots.
// Wir parsen den ersten Snapshot-Link daraus.
async function findArchiveSnapshot(targetUrl) {
const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`;
try {
const result = await fetchFollowRedirects(archiveUrl);
const final = result.finalUrl;
// Kein Redirect = nichts gefunden (blieb auf gleicher URL)
if (final === archiveUrl) return { found: false };
// Snapshot-URL hat die Form https://archive.is/ABCDE (alphanumeric hash, 4-8 Zeichen)
if (/https?:\/\/archive\.(is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10}$/.test(final)) {
return { found: true, resolvedUrl: final };
const result = await fetchPage(searchUrl, { bodyNeeded: true });
if (result.statusCode !== 200 || !result.body) return null;
// Snapshots stehen als Links in der Form href="/ABCDE" oder href="https://archive.is/ABCDE"
// Wir suchen den neuesten (ersten in der Liste – archive.is sortiert newest first)
const hashPattern = /href="(?:https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg))?\/([a-zA-Z0-9]{4,10})"/g;
let match;
const hashes = new Set();
while ((match = hashPattern.exec(result.body)) !== null) {
const h = match[1];
// Blacklist bekannter Nicht-Snapshot-Pfade
if (['newest','oldest','submit','search','about','faq','rss'].includes(h)) continue;
hashes.add(h);
}
// archive.is Root oder Submit-Seite = kein Treffer
if (/archive\.[a-z]+\/?$/.test(final)) return { found: false };
if (final.includes('/submit')) return { found: false };
// Sonst: irgendwohin weitergeleitet → als Treffer werten
return { found: true, resolvedUrl: final };
if (hashes.size === 0) return null;
// Erster Hash = neuester Snapshot
const newestHash = [...hashes][0];
return `https://archive.is/${newestHash}`;
} catch {
return { found: false };
return null;
}
}
@@ -72,17 +83,10 @@ router.post('/check', authenticate, async (req, res) => {
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
const newestUrl = `https://archive.is/newest/${targetUrl}`;
const oldestUrl = `https://archive.is/oldest/${targetUrl}`;
try {
const newest = await checkArchive(newestUrl);
if (newest.found) {
return res.json({ archiveUrl: newest.resolvedUrl || newestUrl, type: 'newest' });
}
const oldest = await checkArchive(oldestUrl);
if (oldest.found) {
return res.json({ archiveUrl: oldest.resolvedUrl || oldestUrl, type: 'oldest' });
const snapshotUrl = await findArchiveSnapshot(targetUrl);
if (snapshotUrl) {
return res.json({ archiveUrl: snapshotUrl, type: 'newest' });
}
return res.status(404).json({ error: 'Kein Archiv für diese URL gefunden' });
} catch (err) {
@@ -103,17 +107,12 @@ router.post('/pdf', authenticate, async (req, res) => {
try {
let puppeteer, chromium;
try {
puppeteer = require('puppeteer-core');
chromium = require('@sparticuz/chromium');
} catch {
try {
puppeteer = require('puppeteer');
chromium = null;
} catch {
return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar – puppeteer nicht installiert' });
}
try { puppeteer = require('puppeteer'); chromium = null; }
catch { return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar – puppeteer nicht installiert' }); }
}
const launchOptions = chromium
@@ -130,9 +129,7 @@ router.post('/pdf', authenticate, async (req, res) => {
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
await page.goto(archiveUrl, { waitUntil: 'networkidle2', timeout: 30000 });
await page.pdf({
path: tmpFile,
format: 'A4',
printBackground: true,
path: tmpFile, format: 'A4', printBackground: true,
margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' },
});
await browser.close();
@@ -144,7 +141,6 @@ router.post('/pdf', authenticate, async (req, res) => {
stream.pipe(res);
stream.on('end', () => fs.unlink(tmpFile, () => {}));
stream.on('error', () => fs.unlink(tmpFile, () => {}));
} catch (err) {
if (browser) try { await browser.close(); } catch {}
fs.unlink(tmpFile, () => {});

127
frontend/src/tools/paywallkiller.jsx
View File

@@ -1,16 +1,14 @@
import { useState, useRef } from 'react';
import { S, api } from '../lib.js';
// ── Farben ─────────────────────────────────────────────────────────────────
const C = {
accent: '#f59e0b',
success: '#4ade80',
error: '#f87171',
muted: 'rgba(255,255,255,0.4)',
dim: 'rgba(255,255,255,0.07)',
muted: '#6b7280',
dim: '#1f2937',
};
// ── Hilfsfunktion: Domain aus URL extrahieren ──────────────────────────────
function getDomain(url) {
try { return new URL(url).hostname.replace(/^www\./, ''); }
catch { return url; }
@@ -18,14 +16,13 @@ function getDomain(url) {
export default function PaywallKiller() {
const [inputUrl, setInputUrl] = useState('');
const [status, setStatus] = useState('idle'); // idle | checking | found | notfound | pdf | error
const [status, setStatus] = useState('idle');
const [archiveUrl, setArchiveUrl] = useState('');
const [archiveType, setArchiveType] = useState('');
const [errMsg, setErrMsg] = useState('');
const [pdfLoading, setPdfLoading] = useState(false);
const inputRef = useRef(null);
// ── URL prüfen ────────────────────────────────────────────────────────────
async function handleCheck() {
const url = inputUrl.trim();
if (!url) return;
@@ -48,7 +45,6 @@ export default function PaywallKiller() {
}
}
// ── PDF herunterladen ─────────────────────────────────────────────────────
async function handlePdf() {
setPdfLoading(true);
try {
@@ -61,12 +57,10 @@ export default function PaywallKiller() {
},
body: JSON.stringify({ archiveUrl }),
});
if (!res.ok) {
const data = await res.json().catch(() => ({}));
throw new Error(data.error || 'PDF-Fehler');
}
const blob = await res.blob();
const objUrl = URL.createObjectURL(blob);
const a = document.createElement('a');
@@ -84,7 +78,6 @@ export default function PaywallKiller() {
}
}
// ── Reset ─────────────────────────────────────────────────────────────────
function handleReset() {
setInputUrl('');
setStatus('idle');
@@ -95,7 +88,6 @@ export default function PaywallKiller() {
setTimeout(() => inputRef.current?.focus(), 50);
}
// ── Paste aus Clipboard ───────────────────────────────────────────────────
async function handlePaste() {
try {
const text = await navigator.clipboard.readText();
@@ -109,14 +101,15 @@ export default function PaywallKiller() {
const isFound = status === 'found';
const isNotFound = status === 'notfound';
const isError = status === 'error';
const hasResult = isFound || isNotFound || isError;
return (
<div style={{ padding: '20px 16px', maxWidth: 660, margin: '0 auto' }}>
<div style={{ padding: '20px 16px', maxWidth: 640, margin: '0 auto' }}>
{/* Header */}
<div style={{ marginBottom: 24 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 10, marginBottom: 4 }}>
<span style={{ fontSize: 22 }}>🔓</span>
<span style={{ fontSize: 20 }}>🔓</span>
<span style={{ color: C.accent, fontFamily: 'monospace', fontSize: 13, letterSpacing: 1 }}>
PAYWALL-KILLER
</span>
@@ -126,9 +119,11 @@ export default function PaywallKiller() {
</div>
</div>
{/* URL-Eingabe */}
{/* Eingabe */}
<div style={{ ...S.card, marginBottom: 16 }}>
<div style={{ ...S.head, marginBottom: 10 }}>ARTIKEL-URL</div>
{/* Input + Paste-Button */}
<div style={{ display: 'flex', gap: 8, marginBottom: 10 }}>
<input
ref={inputRef}
@@ -138,10 +133,25 @@ export default function PaywallKiller() {
placeholder="https://www.zeitung.de/artikel/..."
style={{ ...S.inp, flex: 1 }}
/>
<button onClick={handlePaste} style={{ ...S.btn(C.muted, true), padding: '5px 12px' }} title="Aus Zwischenablage einfügen">
<button
onClick={handlePaste}
title="Aus Zwischenablage einfügen"
style={{
background: 'rgba(255,255,255,0.06)',
border: '1px solid rgba(255,255,255,0.12)',
borderRadius: 6,
padding: '0 12px',
color: '#fff',
cursor: 'pointer',
fontSize: 14,
flexShrink: 0,
}}
>
📋
</button>
</div>
{/* Action-Buttons */}
<div style={{ display: 'flex', gap: 8 }}>
<button
onClick={handleCheck}
@@ -149,15 +159,28 @@ export default function PaywallKiller() {
style={{
...S.btn(C.accent),
flex: 1,
opacity: (isChecking || !inputUrl.trim()) ? 0.5 : 1,
opacity: (isChecking || !inputUrl.trim()) ? 0.45 : 1,
cursor: (isChecking || !inputUrl.trim()) ? 'default' : 'pointer',
}}
>
{isChecking ? '⏳ Suche Archiv...' : '🔍 Archiv suchen'}
</button>
{(isFound || isNotFound || isError) && (
<button onClick={handleReset} style={S.btn(C.muted, false)}>
✕ Zurücksetzen
{hasResult && (
<button
onClick={handleReset}
style={{
background: 'rgba(255,255,255,0.06)',
border: '1px solid rgba(255,255,255,0.14)',
borderRadius: 6,
padding: '8px 14px',
color: 'rgba(255,255,255,0.6)',
cursor: 'pointer',
fontFamily: 'monospace',
fontSize: 12,
whiteSpace: 'nowrap',
}}
>
✕ Reset
</button>
)}
</div>
@@ -165,26 +188,25 @@ export default function PaywallKiller() {
{/* Gefunden */}
{isFound && (
<div style={{ ...S.card, borderColor: `${C.success}44`, marginBottom: 16 }}>
<div style={{ ...S.card, borderColor: `${C.success}55`, marginBottom: 16 }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 12 }}>
<span style={{ fontSize: 16 }}>✅</span>
<span>✅</span>
<span style={{ color: C.success, fontFamily: 'monospace', fontSize: 12 }}>
Archiv gefunden
</span>
<span style={{
marginLeft: 8,
background: `${C.accent}22`,
border: `1px solid ${C.accent}44`,
background: `${C.accent}20`,
border: `1px solid ${C.accent}55`,
borderRadius: 4,
padding: '1px 6px',
padding: '1px 7px',
fontSize: 10,
color: C.accent,
fontFamily: 'monospace',
}}>
{archiveType === 'newest' ? 'neuestes' : 'ältestes'} Archiv
</span>
{archiveType === 'newest' ? 'neuester Snapshot' : 'ältester Snapshot'}
</span>
</div>
{/* Archive-URL anzeigen */}
<div style={{
background: 'rgba(255,255,255,0.03)',
border: '1px solid rgba(255,255,255,0.08)',
@@ -193,18 +215,18 @@ export default function PaywallKiller() {
marginBottom: 14,
wordBreak: 'break-all',
}}>
<a href={archiveUrl} target="_blank" rel="noreferrer" style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>
<a href={archiveUrl} target="_blank" rel="noreferrer"
style={{ color: C.accent, fontSize: 11, fontFamily: 'monospace' }}>
{archiveUrl}
</a>
</div>
{/* Aktions-Buttons */}
<div style={{ display: 'flex', gap: 8, flexWrap: 'wrap' }}>
<a
href={archiveUrl}
target="_blank"
rel="noreferrer"
style={{ ...S.btn(C.success), textDecoration: 'none', display: 'inline-block' }}
style={{ ...S.btn(C.success), textDecoration: 'none' }}
>
🌐 Im Browser öffnen
</a>
@@ -221,10 +243,9 @@ export default function PaywallKiller() {
</button>
</div>
{/* PDF-Hinweis */}
{pdfLoading && (
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', marginTop: 10 }}>
Seite wird geladen und als PDF gerendert. Das kann 15–30 Sekunden dauern...
Seite wird gerendert, das dauert 15–30 Sekunden...
</div>
)}
</div>
@@ -234,16 +255,15 @@ export default function PaywallKiller() {
{isNotFound && (
<div style={{ ...S.card, borderColor: `${C.error}44` }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8, marginBottom: 8 }}>
<span style={{ fontSize: 16 }}>📭</span>
<span>📭</span>
<span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>
Kein Archiv gefunden
</span>
</div>
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', lineHeight: 1.6 }}>
Für <strong style={{ color: 'rgba(255,255,255,0.6)' }}>{getDomain(inputUrl)}</strong> existiert
kein Eintrag auf archive.is. Der Artikel wurde möglicherweise noch nie archiviert.
<div style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', lineHeight: 1.6, marginBottom: 12 }}>
Für <strong style={{ color: 'rgba(255,255,255,0.55)' }}>{getDomain(inputUrl)}</strong> wurde
kein Snapshot auf archive.is gefunden.
</div>
<div style={{ marginTop: 12 }}>
<a
href={`https://archive.is/${inputUrl.trim()}`}
target="_blank"
@@ -253,14 +273,13 @@ export default function PaywallKiller() {
🔗 Manuell auf archive.is prüfen
</a>
</div>
</div>
)}
{/* Fehler */}
{isError && (
<div style={{ ...S.card, borderColor: `${C.error}44` }}>
<div style={{ display: 'flex', alignItems: 'center', gap: 8 }}>
<span style={{ fontSize: 16 }}>⚠️</span>
<span>⚠️</span>
<span style={{ color: C.error, fontFamily: 'monospace', fontSize: 12 }}>
{errMsg}
</span>
@@ -268,34 +287,28 @@ export default function PaywallKiller() {
</div>
)}
{/* Info-Sektion */}
{/* Idle-Info */}
{status === 'idle' && (
<div style={{ ...S.card, marginTop: 8 }}>
<div style={{ ...S.head, marginBottom: 10 }}>WIE ES FUNKTIONIERT</div>
<div style={{ display: 'flex', flexDirection: 'column', gap: 8 }}>
<div style={{ ...S.card }}>
<div style={{ ...S.head, marginBottom: 10 }}>SO FUNKTIONIERT ES</div>
{[
['1', 'URL des gesperrten Artikels einfügen'],
['2', 'Archiv auf archive.is suchen (newest → oldest)'],
['3', 'Archivierte Version im Browser öffnen oder als PDF speichern'],
['2', 'Archiv auf archive.is suchen'],
['3', 'Im Browser öffnen oder als PDF speichern'],
].map(([num, text]) => (
<div key={num} style={{ display: 'flex', gap: 10, alignItems: 'flex-start' }}>
<div key={num} style={{ display: 'flex', gap: 10, alignItems: 'center', marginBottom: 8 }}>
<span style={{
background: `${C.accent}22`,
background: `${C.accent}20`,
border: `1px solid ${C.accent}44`,
borderRadius: '50%',
width: 20, height: 20,
width: 20, height: 20, flexShrink: 0,
display: 'flex', alignItems: 'center', justifyContent: 'center',
fontSize: 10, color: C.accent, fontFamily: 'monospace', flexShrink: 0,
}}>
{num}
</span>
<span style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace', lineHeight: 1.6 }}>
{text}
</span>
fontSize: 10, color: C.accent, fontFamily: 'monospace',
}}>{num}</span>
<span style={{ color: C.muted, fontSize: 11, fontFamily: 'monospace' }}>{text}</span>
</div>
))}
</div>
</div>
)}
</div>
);