revert: Tudum-HTML-Parsing entfernt - echtes Markup ist React/Next.js, kein Plain-Markdown wie angenommen
This commit is contained in:
@@ -496,81 +496,12 @@ router.get('/xrel-cached', authenticate, (req, res) => {
|
|||||||
// Netflix läuft daher wie alle anderen Anbieter über die leichtgewichtige
|
// Netflix läuft daher wie alle anderen Anbieter über die leichtgewichtige
|
||||||
// TMDb-Näherung weiter unten.
|
// TMDb-Näherung weiter unten.
|
||||||
|
|
||||||
// ── Netflix: echte Top-10-Daten direkt von der Tudum-Seite (HTML, klein) ──────
|
// HINWEIS: Ein Versuch, die Netflix-Tudum-HTML-Seite zu parsen, wurde wieder
|
||||||
// Anders als die riesige XLSX-Datei (5 Jahre, alle Länder) ist diese Seite pro
|
// entfernt – das echte Markup ist modernes React/Next.js mit CSS-in-JS
|
||||||
// Abruf nur eine einzelne, kleine, server-gerenderte Seite – die enthält die
|
// (emotion-Klassen), keine saubere serverseitig gerenderte Tabelle wie
|
||||||
// "Ranking"-Tabelle bereits fertig im HTML, kein client-seitiges Nachladen,
|
// angenommen. Das Regex-Parsing fand 0 Treffer und wäre zudem bei jedem
|
||||||
// kein großer Download. Wird 24h gecacht (Netflix aktualisiert nur wöchentlich).
|
// Netflix-Frontend-Update erneut kaputt gegangen. Netflix läuft daher wie
|
||||||
async function getNetflixTudumTop10(mediaKind /* 'movie' | 'tv' */) {
|
// alle anderen Anbieter über die TMDb-Näherung weiter unten.
|
||||||
const cacheKey = `netflix-tudum:${mediaKind}`;
|
|
||||||
const cached = cacheGet(cacheKey);
|
|
||||||
if (cached !== undefined) return cached;
|
|
||||||
|
|
||||||
const path = mediaKind === 'tv' ? '/germany/tv' : '/germany';
|
|
||||||
const resp = await fetch(`https://www.netflix.com/tudum/top10${path}`, {
|
|
||||||
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)' },
|
|
||||||
signal: AbortSignal.timeout(8000),
|
|
||||||
});
|
|
||||||
if (!resp.ok) throw new Error(`Tudum nicht erreichbar (HTTP ${resp.status})`);
|
|
||||||
const html = await resp.text();
|
|
||||||
|
|
||||||
// Die Ranking-Tabelle steht als Markdown-ähnliche Tabelle im Seiten-Markup:
|
|
||||||
// "01![...]Titelname: Season X | 2" – wir extrahieren Rang + Titel daraus.
|
|
||||||
// Robusterer Weg: die Zeilen direkt nach "## Top 10 ... overview" parsen.
|
|
||||||
const rows = [];
|
|
||||||
const tableSection = html.split(/Top 10 (?:Shows|Movies) in Germany overview/i)[1] || '';
|
|
||||||
const lineMatches = tableSection.matchAll(/\|\s*(\d{2})!\[[^\]]*\]\([^)]*\)([^|]+?)\s*\|\s*(\d+)\s*\|/g);
|
|
||||||
for (const m of lineMatches) {
|
|
||||||
const rank = parseInt(m[1], 10);
|
|
||||||
const title = m[2].trim();
|
|
||||||
if (rank >= 1 && rank <= 10 && title) rows.push({ rank, title, weeksInTop10: parseInt(m[3], 10) || null });
|
|
||||||
}
|
|
||||||
|
|
||||||
// Fallback: falls das Tabellen-Format sich ändert, aus der Bild-Alt-Text-Liste lesen
|
|
||||||
if (rows.length < 5) {
|
|
||||||
const altMatches = html.matchAll(/!\[([^\]]+?)(?::\s*(?:Limited Series|Season \d+))?\]\([^)]*\)[\s\S]{0,80}?#(\d{1,2}) in (?:Shows|Movies)/g);
|
|
||||||
let i = 0;
|
|
||||||
for (const m of altMatches) {
|
|
||||||
i++;
|
|
||||||
const rank = parseInt(m[2], 10);
|
|
||||||
if (rank >= 1 && rank <= 10 && !rows.find(r => r.rank === rank)) {
|
|
||||||
rows.push({ rank, title: m[1].trim(), weeksInTop10: null });
|
|
||||||
}
|
|
||||||
if (i > 30) break; // Sicherheitslimit
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
rows.sort((a, b) => a.rank - b.rank);
|
|
||||||
const top10 = rows.slice(0, 10);
|
|
||||||
cacheSet(cacheKey, top10, 24 * 60 * 60 * 1000); // 24h Cache
|
|
||||||
return top10;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Netflix-Titel (nur Name, z.B. "Nemesis: Season 1") mit TMDb abgleichen
|
|
||||||
async function matchTudumTitleToTmdb(rawTitle, mediaKind, genreMap) {
|
|
||||||
// "Nemesis: Season 1" / "The Witness: Limited Series" → reiner Titel ohne Zusatz
|
|
||||||
const cleanTitle = rawTitle.replace(/:\s*(Season\s*\d+|Limited Series|Part\s*\d+).*$/i, '').trim();
|
|
||||||
try {
|
|
||||||
const searchPath = mediaKind === 'tv' ? '/search/tv' : '/search/movie';
|
|
||||||
const data = await tmdb(searchPath, { query: cleanTitle });
|
|
||||||
const hit = (data.results || [])[0];
|
|
||||||
if (!hit) return null;
|
|
||||||
if (mediaKind === 'tv') {
|
|
||||||
return {
|
|
||||||
id: hit.id, media_type: 'tv',
|
|
||||||
title: hit.name || hit.original_name,
|
|
||||||
original_title: hit.original_name,
|
|
||||||
release_date: hit.first_air_date || '',
|
|
||||||
poster_path: hit.poster_path || '',
|
|
||||||
genres: (hit.genre_ids||[]).map(gid => genreMap[gid]).filter(Boolean),
|
|
||||||
vote_average: hit.vote_average,
|
|
||||||
original_language: hit.original_language,
|
|
||||||
origin_country: hit.origin_country || [],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return { ...enrichMovie(hit, genreMap, null, null), media_type: 'movie' };
|
|
||||||
} catch { return null; }
|
|
||||||
}
|
|
||||||
|
|
||||||
const STREAMING_PROVIDERS = [
|
const STREAMING_PROVIDERS = [
|
||||||
{ id: 'netflix', label: 'Netflix', tmdbId: '8' },
|
{ id: 'netflix', label: 'Netflix', tmdbId: '8' },
|
||||||
@@ -614,45 +545,6 @@ router.get('/streaming/:providerId', authenticate, async (req, res) => {
|
|||||||
const cachedResult = cacheGet(streamCacheKey);
|
const cachedResult = cacheGet(streamCacheKey);
|
||||||
if (cachedResult !== undefined) return res.json(cachedResult);
|
if (cachedResult !== undefined) return res.json(cachedResult);
|
||||||
|
|
||||||
// ── NETFLIX: echte Tudum-Charts, leichtgewichtiges HTML, kein großer Download ──
|
|
||||||
if (provider.id === 'netflix') {
|
|
||||||
try {
|
|
||||||
const genreMap = await getGenreMap();
|
|
||||||
const [tudumMovies, tudumShows] = await Promise.all([
|
|
||||||
getNetflixTudumTop10('movie'),
|
|
||||||
getNetflixTudumTop10('tv'),
|
|
||||||
]);
|
|
||||||
|
|
||||||
if (tudumMovies.length >= 5 && tudumShows.length >= 5) {
|
|
||||||
const movies = (await Promise.all(
|
|
||||||
tudumMovies.map(async t => {
|
|
||||||
const m = await matchTudumTitleToTmdb(t.title, 'movie', genreMap);
|
|
||||||
return m ? { ...m, rank: t.rank, weeksInTop10: t.weeksInTop10, tudumTitle: t.title } : null;
|
|
||||||
})
|
|
||||||
)).filter(Boolean);
|
|
||||||
const tvShows = (await Promise.all(
|
|
||||||
tudumShows.map(async t => {
|
|
||||||
const m = await matchTudumTitleToTmdb(t.title, 'tv', genreMap);
|
|
||||||
return m ? { ...m, rank: t.rank, weeksInTop10: t.weeksInTop10, tudumTitle: t.title } : null;
|
|
||||||
})
|
|
||||||
)).filter(Boolean);
|
|
||||||
|
|
||||||
const result = {
|
|
||||||
provider: { id: provider.id, label: provider.label },
|
|
||||||
movies, tvShows,
|
|
||||||
isOfficialData: true,
|
|
||||||
filterInfo: null,
|
|
||||||
};
|
|
||||||
cacheSet(streamCacheKey, result, 6 * 60 * 60 * 1000);
|
|
||||||
return res.json(result);
|
|
||||||
}
|
|
||||||
// Zu wenig geparst → unten zur TMDb-Näherung durchfallen lassen
|
|
||||||
} catch (tudumErr) {
|
|
||||||
// Tudum nicht erreichbar/Format geändert → unten zur TMDb-Näherung durchfallen lassen
|
|
||||||
}
|
|
||||||
}
|
|
||||||
// ── ENDE NETFLIX-Sonderfall (bei Erfolg wurde oben bereits geantwortet) ──────
|
|
||||||
|
|
||||||
// Nur Titel der letzten 2 Jahre (aktuelles Jahr - 1)
|
// Nur Titel der letzten 2 Jahre (aktuelles Jahr - 1)
|
||||||
const minYear = new Date().getFullYear() - 1;
|
const minYear = new Date().getFullYear() - 1;
|
||||||
const minDate = `${minYear}-01-01`;
|
const minDate = `${minYear}-01-01`;
|
||||||
@@ -945,37 +837,6 @@ router.get('/streaming/debug/:providerId', authenticate, requireAdmin, async (re
|
|||||||
} catch(e) { res.status(500).json({ error: e.message }); }
|
} catch(e) { res.status(500).json({ error: e.message }); }
|
||||||
});
|
});
|
||||||
|
|
||||||
// Debug: rohen Tudum-Abruf + Parsing-Ergebnis prüfen
|
|
||||||
router.get('/streaming/debug-tudum/:mediaKind', authenticate, requireAdmin, async (req, res) => {
|
|
||||||
const mediaKind = req.params.mediaKind === 'tv' ? 'tv' : 'movie';
|
|
||||||
try {
|
|
||||||
const path = mediaKind === 'tv' ? '/germany/tv' : '/germany';
|
|
||||||
const resp = await fetch(`https://www.netflix.com/tudum/top10${path}`, {
|
|
||||||
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)' },
|
|
||||||
signal: AbortSignal.timeout(8000),
|
|
||||||
});
|
|
||||||
const html = await resp.text();
|
|
||||||
|
|
||||||
const tableSection = html.split(/Top 10 (?:Shows|Movies) in Germany overview/i)[1] || '';
|
|
||||||
const lineMatches = [...tableSection.matchAll(/\|\s*(\d{2})!\[[^\]]*\]\([^)]*\)([^|]+?)\s*\|\s*(\d+)\s*\|/g)];
|
|
||||||
|
|
||||||
const altMatches = [...html.matchAll(/!\[([^\]]+?)(?::\s*(?:Limited Series|Season \d+))?\]\([^)]*\)[\s\S]{0,80}?#(\d{1,2}) in (?:Shows|Movies)/g)];
|
|
||||||
|
|
||||||
res.json({
|
|
||||||
httpStatus: resp.status,
|
|
||||||
htmlLength: html.length,
|
|
||||||
hasOverviewSection: tableSection.length > 0,
|
|
||||||
tableSectionPreview: tableSection.slice(0, 500),
|
|
||||||
lineMatchesCount: lineMatches.length,
|
|
||||||
lineMatchesSample: lineMatches.slice(0, 3).map(m => ({ rank: m[1], title: m[2], weeks: m[3] })),
|
|
||||||
altMatchesCount: altMatches.length,
|
|
||||||
altMatchesSample: altMatches.slice(0, 5).map(m => ({ title: m[1], rank: m[2] })),
|
|
||||||
});
|
|
||||||
} catch (e) {
|
|
||||||
res.status(500).json({ error: e.message, stack: e.stack });
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
router.post('/streaming/clear-cache', authenticate, requireAdmin, (req, res) => {
|
router.post('/streaming/clear-cache', authenticate, requireAdmin, (req, res) => {
|
||||||
const r = db.prepare("DELETE FROM xrel_cache WHERE cache_key LIKE 'streaming-top10:%'").run();
|
const r = db.prepare("DELETE FROM xrel_cache WHERE cache_key LIKE 'streaming-top10:%'").run();
|
||||||
res.json({ ok: true, cleared: r.changes });
|
res.json({ ok: true, cleared: r.changes });
|
||||||
|
|||||||
@@ -476,16 +476,10 @@ function StreamingGrid({ onOpenModal, xrelIds, onXrelLoaded, favIds: favIdsProp,
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div>
|
<div>
|
||||||
{data?.isOfficialData ? (
|
<div style={{color:'rgba(255,255,255,0.35)',fontFamily:'monospace',fontSize:11,marginBottom:12,lineHeight:1.6}}>
|
||||||
<div style={{color:'#4ecdc4',fontFamily:'monospace',fontSize:11,marginBottom:12,lineHeight:1.6}}>
|
Top 10 Filme & Serien pro Anbieter (DE) · nur Titel ab {new Date().getFullYear()-1} · sortiert nach Popularität (Näherung, keine offiziellen Anbieter-Daten)<br/>
|
||||||
📊 Original Netflix Top 10 (DE) – echte, offizielle Charts direkt von Netflix (netflix.com/tudum)
|
Deutsch & Englisch unbegrenzt · alle anderen Sprachen zusammen max. 2 Plätze in den Top 10
|
||||||
</div>
|
</div>
|
||||||
) : (
|
|
||||||
<div style={{color:'rgba(255,255,255,0.35)',fontFamily:'monospace',fontSize:11,marginBottom:12,lineHeight:1.6}}>
|
|
||||||
Top 10 Filme & Serien pro Anbieter (DE) · nur Titel ab {new Date().getFullYear()-1} · sortiert nach Popularität (Näherung, keine offiziellen Anbieter-Daten)<br/>
|
|
||||||
Deutsch & Englisch unbegrenzt · alle anderen Sprachen zusammen max. 2 Plätze in den Top 10
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
{/* Anbieter-Auswahl */}
|
{/* Anbieter-Auswahl */}
|
||||||
<div style={{display:'flex',gap:6,flexWrap:'wrap',marginBottom:16}}>
|
<div style={{display:'flex',gap:6,flexWrap:'wrap',marginBottom:16}}>
|
||||||
{providers.map(p => (
|
{providers.map(p => (
|
||||||
|
|||||||
Reference in New Issue
Block a user