Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: archive.is Suche via HTML-Parse, Button-Farben gefixt

Browse Source
This commit is contained in:
Dicken
2026-06-12 01:42:42 +02:00
parent 38c27ac5bb
commit 902d5fd99e
2 changed files with 146 additions and 137 deletions
Download Patch File Download Diff File Expand all files Collapse all files

106
backend/src/tools/paywallkiller/routes.js
View File

@@ -7,60 +7,71 @@ const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
// ── Hilfsfunktion: folgt Redirects, gibt finale URL zurück ───────────────
function fetchFollowRedirects(url, redirectCount = 0) {
// ── HTTP GET, gibt finalUrl + statusCode + body (optional) zurück ─────────
function fetchPage(url, opts = {}) {
return new Promise((resolve, reject) => {
if (redirectCount > 8) return reject(new Error('Zu viele Redirects'));
if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects'));
const mod = url.startsWith('https') ? https : http;
const req = mod.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml',
'Accept-Language': 'de,en;q=0.5',
},
timeout: 12000,
}, (res) => {
if ([301, 302, 303, 307, 308].includes(res.statusCode) && res.headers.location) {
let location = res.headers.location;
if (!location.startsWith('http')) {
try {
const base = new URL(url);
location = location.startsWith('/')
? `${base.protocol}//${base.host}${location}`
: `${base.protocol}//${base.host}/${location}`;
} catch { return reject(new Error('Ungültige Redirect-URL')); }
if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) {
let loc = res.headers.location;
if (!loc.startsWith('http')) {
try { const b = new URL(url); loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; }
catch { return reject(new Error('Ungültige Redirect-URL')); }
}
res.resume();
return fetchFollowRedirects(location, redirectCount + 1).then(resolve).catch(reject);
return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1 }).then(resolve).catch(reject);
}
if (opts.bodyNeeded) {
let body = '';
res.setEncoding('utf8');
res.on('data', d => { if (body.length < 50000) body += d; });
res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body }));
} else {
res.resume();
resolve({ statusCode: res.statusCode, finalUrl: url });
}
res.resume();
resolve({ statusCode: res.statusCode, finalUrl: url });
});
req.on('error', reject);
req.on('timeout', () => { req.destroy(); reject(new Error('Timeout')); });
});
}
// ── Prüfe ob archive.is einen Snapshot hat ────────────────────────────────
// archive.is/newest/URL leitet auf archive.is/HASH weiter wenn ein Snapshot existiert.
// Wenn kein Snapshot → Redirect zurück auf archive.is/ (Root) oder /newest/... bleibt stehen.
async function checkArchive(archiveUrl) {
// ── Archivsuche über archive.is ───────────────────────────────────────────
// archive.is/?url=ZIEL gibt HTML zurück mit Links zu archivierten Snapshots.
// Wir parsen den ersten Snapshot-Link daraus.
async function findArchiveSnapshot(targetUrl) {
const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`;
try {
const result = await fetchFollowRedirects(archiveUrl);
const final = result.finalUrl;
// Kein Redirect = nichts gefunden (blieb auf gleicher URL)
if (final === archiveUrl) return { found: false };
// Snapshot-URL hat die Form https://archive.is/ABCDE (alphanumeric hash, 4-8 Zeichen)
if (/https?:\/\/archive\.(is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10}$/.test(final)) {
return { found: true, resolvedUrl: final };
const result = await fetchPage(searchUrl, { bodyNeeded: true });
if (result.statusCode !== 200 || !result.body) return null;
// Snapshots stehen als Links in der Form href="/ABCDE" oder href="https://archive.is/ABCDE"
// Wir suchen den neuesten (ersten in der Liste – archive.is sortiert newest first)
const hashPattern = /href="(?:https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg))?\/([a-zA-Z0-9]{4,10})"/g;
let match;
const hashes = new Set();
while ((match = hashPattern.exec(result.body)) !== null) {
const h = match[1];
// Blacklist bekannter Nicht-Snapshot-Pfade
if (['newest','oldest','submit','search','about','faq','rss'].includes(h)) continue;
hashes.add(h);
}
// archive.is Root oder Submit-Seite = kein Treffer
if (/archive\.[a-z]+\/?$/.test(final)) return { found: false };
if (final.includes('/submit')) return { found: false };
// Sonst: irgendwohin weitergeleitet → als Treffer werten
return { found: true, resolvedUrl: final };
if (hashes.size === 0) return null;
// Erster Hash = neuester Snapshot
const newestHash = [...hashes][0];
return `https://archive.is/${newestHash}`;
} catch {
return { found: false };
return null;
}
}
@@ -72,17 +83,10 @@ router.post('/check', authenticate, async (req, res) => {
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
const newestUrl = `https://archive.is/newest/${targetUrl}`;
const oldestUrl = `https://archive.is/oldest/${targetUrl}`;
try {
const newest = await checkArchive(newestUrl);
if (newest.found) {
return res.json({ archiveUrl: newest.resolvedUrl || newestUrl, type: 'newest' });
}
const oldest = await checkArchive(oldestUrl);
if (oldest.found) {
return res.json({ archiveUrl: oldest.resolvedUrl || oldestUrl, type: 'oldest' });
const snapshotUrl = await findArchiveSnapshot(targetUrl);
if (snapshotUrl) {
return res.json({ archiveUrl: snapshotUrl, type: 'newest' });
}
return res.status(404).json({ error: 'Kein Archiv für diese URL gefunden' });
} catch (err) {
@@ -103,17 +107,12 @@ router.post('/pdf', authenticate, async (req, res) => {
try {
let puppeteer, chromium;
try {
puppeteer = require('puppeteer-core');
chromium = require('@sparticuz/chromium');
} catch {
try {
puppeteer = require('puppeteer');
chromium = null;
} catch {
return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar – puppeteer nicht installiert' });
}
try { puppeteer = require('puppeteer'); chromium = null; }
catch { return res.status(500).json({ error: 'PDF-Rendering nicht verfügbar – puppeteer nicht installiert' }); }
}
const launchOptions = chromium
@@ -130,9 +129,7 @@ router.post('/pdf', authenticate, async (req, res) => {
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36');
await page.goto(archiveUrl, { waitUntil: 'networkidle2', timeout: 30000 });
await page.pdf({
path: tmpFile,
format: 'A4',
printBackground: true,
path: tmpFile, format: 'A4', printBackground: true,
margin: { top: '10mm', bottom: '10mm', left: '10mm', right: '10mm' },
});
await browser.close();
@@ -144,7 +141,6 @@ router.post('/pdf', authenticate, async (req, res) => {
stream.pipe(res);
stream.on('end', () => fs.unlink(tmpFile, () => {}));
stream.on('error', () => fs.unlink(tmpFile, () => {}));
} catch (err) {
if (browser) try { await browser.close(); } catch {}
fs.unlink(tmpFile, () => {});