revert: Netflix XLSX entfernt - überlastet Container (CPU 90%, RAM voll). Netflix läuft wieder über TMDb-Näherung
This commit is contained in:
@@ -7,7 +7,6 @@
|
|||||||
"better-sqlite3": "^9.4.3",
|
"better-sqlite3": "^9.4.3",
|
||||||
"express": "^4.18.2",
|
"express": "^4.18.2",
|
||||||
"jsonwebtoken": "^9.0.2",
|
"jsonwebtoken": "^9.0.2",
|
||||||
"multer": "^1.4.5-lts.1",
|
"multer": "^1.4.5-lts.1"
|
||||||
"xlsx": "^0.18.5"
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -2,13 +2,11 @@ const express = require('express');
|
|||||||
const router = express.Router();
|
const router = express.Router();
|
||||||
const db = require('../../db');
|
const db = require('../../db');
|
||||||
const { authenticate, requireAdmin } = require('../../middleware/auth');
|
const { authenticate, requireAdmin } = require('../../middleware/auth');
|
||||||
const XLSX = require('xlsx');
|
|
||||||
|
|
||||||
const TMDB_BASE = 'https://api.themoviedb.org/3';
|
const TMDB_BASE = 'https://api.themoviedb.org/3';
|
||||||
const XREL_BASE = 'https://api.xrel.to/v2';
|
const XREL_BASE = 'https://api.xrel.to/v2';
|
||||||
const THIS_YEAR = new Date().getFullYear();
|
const THIS_YEAR = new Date().getFullYear();
|
||||||
const MIN_YEAR = THIS_YEAR - 1;
|
const MIN_YEAR = THIS_YEAR - 1;
|
||||||
const NETFLIX_TOP10_URL = 'https://www.netflix.com/tudum/top10/data/all-weeks-countries.xlsx';
|
|
||||||
|
|
||||||
// Sprachen die wir NICHT wollen
|
// Sprachen die wir NICHT wollen
|
||||||
const BLOCKED_LANGS = new Set(['tr', 'hi', 'te', 'ta', 'ml', 'kn', 'bn', 'mr', 'pa']);
|
const BLOCKED_LANGS = new Set(['tr', 'hi', 'te', 'ta', 'ml', 'kn', 'bn', 'mr', 'pa']);
|
||||||
@@ -491,69 +489,12 @@ router.get('/xrel-cached', authenticate, (req, res) => {
|
|||||||
|
|
||||||
// ── Streaming Top 10 pro Anbieter ──────────────────────────────────────────────
|
// ── Streaming Top 10 pro Anbieter ──────────────────────────────────────────────
|
||||||
// TMDb Watch-Provider-IDs für Deutschland (watch_region=DE)
|
// TMDb Watch-Provider-IDs für Deutschland (watch_region=DE)
|
||||||
// ── Netflix offizielle Top-10-Daten (keine Näherung – echte Zahlen von Netflix selbst) ──
|
// HINWEIS: Eine frühere Version hat hier versucht die offizielle Netflix-XLSX-Datei
|
||||||
// Quelle: https://www.netflix.com/tudum/top10/data/all-weeks-countries.xlsx
|
// (komplette Historie seit 2021, alle ~90 Länder) direkt zu laden und zu parsen.
|
||||||
// Wird wöchentlich (dienstags) von Netflix aktualisiert → 20h Cache reicht völlig
|
// Das hat den Container überlastet (CPU ~90%, RAM voll, Container kurzzeitig nicht
|
||||||
async function getNetflixTop10(category /* 'Films' | 'TV' */) {
|
// erreichbar) – die Datei ist für ein synchrones Parsing im selben Prozess zu groß.
|
||||||
const cacheKey = `netflix-top10:${category}`;
|
// Netflix läuft daher wie alle anderen Anbieter über die leichtgewichtige
|
||||||
const cached = cacheGet(cacheKey);
|
// TMDb-Näherung weiter unten.
|
||||||
if (cached !== undefined) return cached;
|
|
||||||
|
|
||||||
const resp = await fetch(NETFLIX_TOP10_URL, {
|
|
||||||
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; DickenDock/1.0)' },
|
|
||||||
});
|
|
||||||
if (!resp.ok) throw new Error(`Netflix-Daten nicht erreichbar (HTTP ${resp.status})`);
|
|
||||||
const buf = await resp.arrayBuffer();
|
|
||||||
const wb = XLSX.read(buf, { type: 'array' });
|
|
||||||
const sheet = wb.Sheets[wb.SheetNames[0]];
|
|
||||||
const rows = XLSX.utils.sheet_to_json(sheet);
|
|
||||||
|
|
||||||
// Spalten (Stand der Netflix-Datei): country_name, country_iso2, category, week, weekly_rank,
|
|
||||||
// show_title, season_title, cumulative_weeks_in_top10, ...
|
|
||||||
const deRows = rows.filter(r => r.country_iso2 === 'DE' && r.category === category);
|
|
||||||
if (!deRows.length) { cacheSet(cacheKey, [], 60*60*1000); return []; }
|
|
||||||
|
|
||||||
// Neueste Woche finden
|
|
||||||
const latestWeek = deRows.reduce((max, r) => (r.week > max ? r.week : max), deRows[0].week);
|
|
||||||
const top10 = deRows
|
|
||||||
.filter(r => r.week === latestWeek)
|
|
||||||
.sort((a, b) => a.weekly_rank - b.weekly_rank)
|
|
||||||
.slice(0, 10)
|
|
||||||
.map(r => ({
|
|
||||||
rank: r.weekly_rank,
|
|
||||||
title: r.show_title,
|
|
||||||
seasonTitle: r.season_title || '',
|
|
||||||
weeksInTop10: r.cumulative_weeks_in_top10 || null,
|
|
||||||
week: r.week,
|
|
||||||
}));
|
|
||||||
|
|
||||||
cacheSet(cacheKey, top10, 20 * 60 * 60 * 1000); // 20h Cache
|
|
||||||
return top10;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Netflix-Titel mit TMDb abgleichen (für Poster, Genres, Modal-Infos)
|
|
||||||
async function matchNetflixTitleToTmdb(title, mediaType, genreMap) {
|
|
||||||
try {
|
|
||||||
const searchPath = mediaType === 'tv' ? '/search/tv' : '/search/movie';
|
|
||||||
const data = await tmdb(searchPath, { query: title });
|
|
||||||
const hit = (data.results || [])[0];
|
|
||||||
if (!hit) return null;
|
|
||||||
if (mediaType === 'tv') {
|
|
||||||
return {
|
|
||||||
id: hit.id, media_type: 'tv',
|
|
||||||
title: hit.name || hit.original_name,
|
|
||||||
original_title: hit.original_name,
|
|
||||||
release_date: hit.first_air_date || '',
|
|
||||||
poster_path: hit.poster_path || '',
|
|
||||||
genres: (hit.genre_ids||[]).map(gid => genreMap[gid]).filter(Boolean),
|
|
||||||
vote_average: hit.vote_average,
|
|
||||||
original_language: hit.original_language,
|
|
||||||
origin_country: hit.origin_country || [],
|
|
||||||
};
|
|
||||||
}
|
|
||||||
return { ...enrichMovie(hit, genreMap, null, null), media_type: 'movie' };
|
|
||||||
} catch { return null; }
|
|
||||||
}
|
|
||||||
|
|
||||||
const STREAMING_PROVIDERS = [
|
const STREAMING_PROVIDERS = [
|
||||||
{ id: 'netflix', label: 'Netflix', tmdbId: '8' },
|
{ id: 'netflix', label: 'Netflix', tmdbId: '8' },
|
||||||
@@ -593,37 +534,6 @@ router.get('/streaming/:providerId', authenticate, async (req, res) => {
|
|||||||
const provider = STREAMING_PROVIDERS.find(p => p.id === req.params.providerId);
|
const provider = STREAMING_PROVIDERS.find(p => p.id === req.params.providerId);
|
||||||
if (!provider) return res.status(404).json({ error: 'Unbekannter Anbieter' });
|
if (!provider) return res.status(404).json({ error: 'Unbekannter Anbieter' });
|
||||||
|
|
||||||
// ── NETFLIX: echte offizielle Daten, KEIN Filter ─────────────────────────
|
|
||||||
if (provider.id === 'netflix') {
|
|
||||||
const genreMap = await getGenreMap();
|
|
||||||
const [nfFilms, nfShows] = await Promise.all([
|
|
||||||
getNetflixTop10('Films'),
|
|
||||||
getNetflixTop10('TV'),
|
|
||||||
]);
|
|
||||||
|
|
||||||
const movies = (await Promise.all(
|
|
||||||
nfFilms.map(async nf => {
|
|
||||||
const matched = await matchNetflixTitleToTmdb(nf.title, 'movie', genreMap);
|
|
||||||
return matched ? { ...matched, rank: nf.rank, weeksInTop10: nf.weeksInTop10, netflixTitle: nf.title } : null;
|
|
||||||
})
|
|
||||||
)).filter(Boolean);
|
|
||||||
|
|
||||||
const tvShows = (await Promise.all(
|
|
||||||
nfShows.map(async nf => {
|
|
||||||
const matched = await matchNetflixTitleToTmdb(nf.title, 'tv', genreMap);
|
|
||||||
return matched ? { ...matched, rank: nf.rank, weeksInTop10: nf.weeksInTop10, netflixTitle: nf.title } : null;
|
|
||||||
})
|
|
||||||
)).filter(Boolean);
|
|
||||||
|
|
||||||
return res.json({
|
|
||||||
provider: { id: provider.id, label: provider.label },
|
|
||||||
movies, tvShows,
|
|
||||||
isOfficialData: true,
|
|
||||||
filterInfo: null, // keine Filter – Original Netflix Top 10
|
|
||||||
});
|
|
||||||
}
|
|
||||||
// ── ENDE NETFLIX-Sonderfall ───────────────────────────────────────────────
|
|
||||||
|
|
||||||
// Nur Titel der letzten 2 Jahre (aktuelles Jahr - 1)
|
// Nur Titel der letzten 2 Jahre (aktuelles Jahr - 1)
|
||||||
const minYear = new Date().getFullYear() - 1;
|
const minYear = new Date().getFullYear() - 1;
|
||||||
const minDate = `${minYear}-01-01`;
|
const minDate = `${minYear}-01-01`;
|
||||||
@@ -721,7 +631,6 @@ router.get('/streaming/:providerId', authenticate, async (req, res) => {
|
|||||||
res.json({
|
res.json({
|
||||||
provider: { id: provider.id, label: provider.label },
|
provider: { id: provider.id, label: provider.label },
|
||||||
movies, tvShows,
|
movies, tvShows,
|
||||||
isOfficialData: false,
|
|
||||||
filterInfo: {
|
filterInfo: {
|
||||||
minYear,
|
minYear,
|
||||||
minVotes: MIN_VOTES,
|
minVotes: MIN_VOTES,
|
||||||
|
|||||||
@@ -476,16 +476,10 @@ function StreamingGrid({ onOpenModal, xrelIds, onXrelLoaded, favIds: favIdsProp,
|
|||||||
|
|
||||||
return (
|
return (
|
||||||
<div>
|
<div>
|
||||||
{data?.isOfficialData ? (
|
<div style={{color:'rgba(255,255,255,0.35)',fontFamily:'monospace',fontSize:11,marginBottom:12,lineHeight:1.6}}>
|
||||||
<div style={{color:'#4ecdc4',fontFamily:'monospace',fontSize:11,marginBottom:12,lineHeight:1.6}}>
|
Top 10 Filme & Serien pro Anbieter (DE) · nur Titel ab {new Date().getFullYear()-1} · sortiert nach Popularität (Näherung, keine offiziellen Anbieter-Daten)<br/>
|
||||||
📊 Original Netflix Top 10 (DE) – echte, offizielle Daten direkt von Netflix, keine Filterung
|
Nur wirklich bekannte Titel · Deutsch & Englisch unbegrenzt, andere Sprachen max. 2 Plätze – echte Hits wie „Squid Game" rutschen trotzdem rein
|
||||||
</div>
|
</div>
|
||||||
) : (
|
|
||||||
<div style={{color:'rgba(255,255,255,0.35)',fontFamily:'monospace',fontSize:11,marginBottom:12,lineHeight:1.6}}>
|
|
||||||
Top 10 Filme & Serien pro Anbieter (DE) · nur Titel ab {new Date().getFullYear()-1} · sortiert nach Popularität (Näherung, keine offiziellen Anbieter-Daten)<br/>
|
|
||||||
Nur wirklich bekannte Titel · Deutsch & Englisch unbegrenzt, andere Sprachen max. 2 Plätze – echte Hits wie „Squid Game" rutschen trotzdem rein
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
{/* Anbieter-Auswahl */}
|
{/* Anbieter-Auswahl */}
|
||||||
<div style={{display:'flex',gap:6,flexWrap:'wrap',marginBottom:16}}>
|
<div style={{display:'flex',gap:6,flexWrap:'wrap',marginBottom:16}}>
|
||||||
{providers.map(p => (
|
{providers.map(p => (
|
||||||
|
|||||||
Reference in New Issue
Block a user