diff --git a/backend/src/tools/koepi/routes.js b/backend/src/tools/koepi/routes.js new file mode 100644 index 0000000..b9788f6 --- /dev/null +++ b/backend/src/tools/koepi/routes.js @@ -0,0 +1,275 @@ +const express = require('express'); +const https = require('https'); +const { authenticate } = require('../../middleware/auth'); +const router = express.Router(); + +// ── Cache (in-memory, 1h) ───────────────────────────────────────────────────── +const cache = {}; +function cacheGet(key) { + const e = cache[key]; + if (!e) return null; + if (Date.now() > e.expires) { delete cache[key]; return null; } + return e.data; +} +function cacheSet(key, data, ttlMs = 60 * 60 * 1000) { + cache[key] = { data, expires: Date.now() + ttlMs }; +} + +// ── HTTP-Helper mit Redirect-Follow ────────────────────────────────────────── +function fetchUrl(url, maxRedirects = 5) { + return new Promise((resolve, reject) => { + const doGet = (u, remaining) => { + const req = https.get(u, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html,application/xhtml+xml', + 'Accept-Language': 'de-DE,de;q=0.9', + 'Accept-Encoding': 'identity', + } + }, (res) => { + if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && remaining > 0) { + const next = res.headers.location.startsWith('http') + ? res.headers.location + : `https://www.kaufda.de${res.headers.location}`; + res.resume(); + doGet(next, remaining - 1); + return; + } + let d = ''; + res.on('data', c => d += c); + res.on('end', () => resolve({ status: res.statusCode, body: d })); + }); + req.setTimeout(12000, () => { req.destroy(); reject(new Error('Timeout')); }); + req.on('error', reject); + }; + doGet(url, maxRedirects); + }); +} + +// ── NEXT_DATA aus HTML extrahieren ──────────────────────────────────────────── +function extractNextData(html) { + const marker = 'application/json">'; + const nidx = html.indexOf('NEXT_DATA'); + if (nidx === -1) return null; + const start = html.indexOf(marker, nidx) + marker.length; + const end = html.indexOf('', start); + if (start <= marker.length || end === -1) return null; + try { return JSON.parse(html.slice(start, end)); } catch { return null; } +} + +// ── Händler-Filter (Prospekte) ──────────────────────────────────────────────── +const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl','real']; + +function isTargetPublisher(name) { + if (!name) return false; + const n = name.toLowerCase(); + return TARGET_PUBLISHERS.some(p => n.includes(p)); +} + +// ── Angebote scrapen (Produktseite) ────────────────────────────────────────── +async function scrapeOffers() { + const cacheKey = 'koepi:offers'; + const cached = cacheGet(cacheKey); + if (cached) return cached; + + const { body } = await fetchUrl( + 'https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg' + ); + + const offers = []; + + // Methode 1: Produktkacheln aus HTML parsen + // Suche nach offer-card Patterns im HTML + const offerPatterns = [ + // Schema.org JSON-LD + /