Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

fix: archive.ph Session-Cookie + Browser-Headers gegen 429

Browse Source
This commit is contained in:
Dicken
2026-06-12 02:43:33 +02:00
parent b26800aa4e
commit 77c48bb9d0
1 changed files with 87 additions and 40 deletions
Download Patch File Download Diff File Expand all files Collapse all files

121
backend/src/tools/paywallkiller/routes.js
View File

@@ -7,37 +7,55 @@ const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
// ── HTTP GET mit Redirect-Follow, gibt Body zurück ────────────────────────
// ── HTTP GET mit Redirect-Follow + Cookie-Jar ─────────────────────────────
function fetchPage(url, opts = {}) {
return new Promise((resolve, reject) => {
if ((opts.redirects || 0) > 8) return reject(new Error('Zu viele Redirects'));
const mod = url.startsWith('https') ? https : http;
const req = mod.get(url, {
headers: {
// Cookies aus jar zusammenbauen
const cookieHeader = opts.cookieJar && Object.keys(opts.cookieJar).length
? Object.entries(opts.cookieJar).map(([k,v]) => `${k}=${v}`).join('; ')
: undefined;
const headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
'Accept-Language': 'de,en;q=0.5',
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language': 'de-DE,de;q=0.9,en;q=0.8',
'Accept-Encoding': 'identity',
'Connection': 'keep-alive',
},
timeout: 15000,
}, (res) => {
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': opts.referer ? 'same-origin' : 'none',
'Upgrade-Insecure-Requests': '1',
...(opts.referer ? { 'Referer': opts.referer } : {}),
...(cookieHeader ? { 'Cookie': cookieHeader } : {}),
};
const req = mod.get(url, { headers, timeout: 15000 }, (res) => {
// Cookies sammeln
const jar = opts.cookieJar || {};
const setCookies = res.headers['set-cookie'] || [];
for (const c of setCookies) {
const m = c.match(/^([^=]+)=([^;]*)/);
if (m) jar[m[1].trim()] = m[2].trim();
}
if ([301,302,303,307,308].includes(res.statusCode) && res.headers.location) {
let loc = res.headers.location;
if (!loc.startsWith('http')) {
try {
const b = new URL(url);
loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`;
} catch { return reject(new Error('Ungültige Redirect-URL')); }
try { const b = new URL(url); loc = loc.startsWith('/') ? `${b.protocol}//${b.host}${loc}` : `${b.protocol}//${b.host}/${loc}`; }
catch { return reject(new Error('Ungültige Redirect-URL')); }
}
res.resume();
return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1 }).then(resolve).catch(reject);
return fetchPage(loc, { ...opts, redirects: (opts.redirects||0)+1, cookieJar: jar, referer: url })
.then(resolve).catch(reject);
}
let body = '';
res.setEncoding('utf8');
res.on('data', d => { if (body.length < 100000) body += d; });
res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body, headers: res.headers }));
res.on('data', d => { if (body.length < 150000) body += d; });
res.on('end', () => resolve({ statusCode: res.statusCode, finalUrl: url, body, cookieJar: jar }));
res.on('error', reject);
});
req.on('error', reject);
@@ -45,44 +63,82 @@ function fetchPage(url, opts = {}) {
});
}
// ── Session initialisieren (Startseite besuchen für Cookies) ──────────────
async function initSession() {
const result = await fetchPage('https://archive.ph/', { cookieJar: {} });
return result.cookieJar;
}
// ── Snapshot-URLs aus archive.is HTML extrahieren ─────────────────────────
function extractSnapshotUrls(html) {
const urls = [];
// Format 1: href="/HASH" - relativer Pfad (4-8 alphanumerische Zeichen)
const rel = /href="\/([a-zA-Z0-9]{4,8})(?:"|\?|\/)/g;
// Format: href="/HASH" oder href="https://archive.XX/HASH"
// Hash = 4-10 alphanumerische Zeichen, keine bekannten Pfade
const blacklist = new Set(['newest','oldest','submit','search','about','rss','api','timemap','http','www.']);
const rel = /href="\/([a-zA-Z0-9]{4,10})(?:"|\/|\?)/g;
let m;
while ((m = rel.exec(html)) !== null) {
const h = m[1];
if (!['http','www.','subm','sear','abou','time','rss_','feed','api_'].some(x => h.startsWith(x))) {
if (!blacklist.has(h) && !/^\d+$/.test(h)) {
urls.push(`https://archive.ph/${h}`);
}
}
// Format 2: href="https://archive.XX/HASH"
const abs = /href="(https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,8})(?:"|\?)/g;
const abs = /href="(https?:\/\/archive\.(?:is|ph|today|fo|li|vn|md|gg)\/[a-zA-Z0-9]{4,10})(?:"|\/|\?)/g;
while ((m = abs.exec(html)) !== null) {
urls.push(m[1]);
const h = m[1].split('/').pop();
if (!blacklist.has(h)) urls.push(m[1]);
}
return [...new Set(urls)];
}
// ── DEBUG: Roher Response von archive.is ──────────────────────────────────
// ── Archivsuche mit Session-Cookie ────────────────────────────────────────
async function findArchiveSnapshot(targetUrl) {
// Schritt 1: Session-Cookie holen
let cookieJar;
try { cookieJar = await initSession(); }
catch { cookieJar = {}; }
// Schritt 2: Suche mit Cookie und Referer
const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`;
const result = await fetchPage(searchUrl, {
cookieJar,
referer: 'https://archive.ph/',
});
if (result.statusCode === 429) {
throw new Error('archive.is Rate-Limit erreicht – bitte kurz warten und nochmal versuchen');
}
if (result.statusCode !== 200 || !result.body) return null;
const snapshots = extractSnapshotUrls(result.body);
return snapshots.length > 0 ? snapshots[0] : null;
}
// ── DEBUG endpoint ─────────────────────────────────────────────────────────
router.post('/debug', authenticate, async (req, res) => {
const { url } = req.body;
if (!url) return res.status(400).json({ error: 'URL fehlt' });
let targetUrl = url.trim();
if (!targetUrl.startsWith('http')) targetUrl = 'https://' + targetUrl;
const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`;
try {
const result = await fetchPage(searchUrl);
let cookieJar;
try { cookieJar = await initSession(); } catch(e) { cookieJar = {}; }
const searchUrl = `https://archive.ph/?url=${encodeURIComponent(targetUrl)}`;
const result = await fetchPage(searchUrl, { cookieJar, referer: 'https://archive.ph/' });
const extracted = extractSnapshotUrls(result.body);
res.json({
statusCode: result.statusCode,
finalUrl: result.finalUrl,
cookiesObtained: Object.keys(cookieJar),
bodyLength: result.body.length,
bodyPreview: result.body.substring(0, 2000),
// Alle href-Werte zur Analyse
allHrefs: (result.body.match(/href="[^"]{1,60}"/g) || []).slice(0, 40),
bodyPreview: result.body.substring(0, 3000),
allHrefs: (result.body.match(/href="[^"]{1,80}"/g) || []).slice(0, 50),
extractedSnapshots: extracted,
});
} catch (err) {
@@ -90,15 +146,6 @@ router.post('/debug', authenticate, async (req, res) => {
}
});
// ── Archivsuche ────────────────────────────────────────────────────────────
async function findArchiveSnapshot(targetUrl) {
const searchUrl = `https://archive.is/?url=${encodeURIComponent(targetUrl)}`;
const result = await fetchPage(searchUrl);
if (result.statusCode !== 200 || !result.body) return null;
const snapshots = extractSnapshotUrls(result.body);
return snapshots.length > 0 ? snapshots[0] : null;
}
// ── POST /check ────────────────────────────────────────────────────────────
router.post('/check', authenticate, async (req, res) => {
const { url } = req.body;
@@ -111,7 +158,7 @@ router.post('/check', authenticate, async (req, res) => {
if (snapshotUrl) return res.json({ archiveUrl: snapshotUrl, type: 'newest' });
return res.status(404).json({ error: 'Kein Archiv für diese URL gefunden' });
} catch (err) {
return res.status(500).json({ error: 'Fehler beim Prüfen: ' + err.message });
return res.status(500).json({ error: err.message });
}
});