diff --git a/Dockerfile b/Dockerfile index 3b0beb4..6bb801d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -14,7 +14,10 @@ RUN npm run build # ── Stage 2: Express liefert API + React in einem Container ────────────────── FROM node:20-alpine -RUN apk add --no-cache tzdata python3 make g++ +# Puppeteer/Chromium für KöPi Web-Scraping +RUN apk add --no-cache tzdata python3 make g++ chromium nss freetype harfbuzz ca-certificates ttf-freefont +ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true +ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser # Zeitzone fest auf Europe/Berlin setzen, damit Server und Browser (DE) immer # dieselbe lokale Zeit berechnen – inkl. automatischer Sommer-/Winterzeit-Umstellung ENV TZ=Europe/Berlin diff --git a/backend/package.json b/backend/package.json index 212e9ba..274b73c 100644 --- a/backend/package.json +++ b/backend/package.json @@ -8,6 +8,7 @@ "express": "^4.18.2", "jsonwebtoken": "^9.0.2", "multer": "^1.4.5-lts.1", - "ws": "^8.17.1" + "ws": "^8.17.1", + "puppeteer-core": "^22.0.0" } } \ No newline at end of file diff --git a/backend/src/tools/koepi/routes.js b/backend/src/tools/koepi/routes.js index 6f29d9f..7c0dc6a 100644 --- a/backend/src/tools/koepi/routes.js +++ b/backend/src/tools/koepi/routes.js @@ -1,7 +1,7 @@ -const express = require('express'); -const https = require('https'); +const express = require('express'); +const https = require('https'); const { authenticate } = require('../../middleware/auth'); -const router = express.Router(); +const router = express.Router(); // ── Cache (in-memory, 1h) ───────────────────────────────────────────────────── const cache = {}; @@ -15,7 +15,7 @@ function cacheSet(key, data, ttlMs = 60 * 60 * 1000) { cache[key] = { data, expires: Date.now() + ttlMs }; } -// ── HTTP-Helper mit Redirect-Follow ────────────────────────────────────────── +// ── HTTP-Helper ─────────────────────────────────────────────────────────────── function fetchUrl(url, maxRedirects = 5) { return new Promise((resolve, reject) => { const doGet = (u, remaining) => { @@ -24,7 +24,6 @@ function fetchUrl(url, maxRedirects = 5) { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept': 'text/html,application/xhtml+xml', 'Accept-Language': 'de-DE,de;q=0.9', - 'Accept-Encoding': 'identity', } }, (res) => { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && remaining > 0) { @@ -39,149 +38,128 @@ function fetchUrl(url, maxRedirects = 5) { res.on('data', c => d += c); res.on('end', () => resolve({ status: res.statusCode, body: d })); }); - req.setTimeout(12000, () => { req.destroy(); reject(new Error('Timeout')); }); + req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); }); req.on('error', reject); }; doGet(url, maxRedirects); }); } -// ── NEXT_DATA aus HTML extrahieren ──────────────────────────────────────────── +// ── NEXT_DATA extrahieren ───────────────────────────────────────────────────── function extractNextData(html) { - const marker = 'application/json">'; - const nidx = html.indexOf('NEXT_DATA'); + const marker = 'application/json">'; + const nidx = html.indexOf('NEXT_DATA'); if (nidx === -1) return null; - const start = html.indexOf(marker, nidx) + marker.length; - const end = html.indexOf('', start); + const start = html.indexOf(marker, nidx) + marker.length; + const end = html.indexOf('', start); if (start <= marker.length || end === -1) return null; try { return JSON.parse(html.slice(start, end)); } catch { return null; } } -// ── Händler-Filter (Prospekte) ──────────────────────────────────────────────── -const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl','real']; - +// ── Händler-Filter ──────────────────────────────────────────────────────────── +const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl']; function isTargetPublisher(name) { if (!name) return false; const n = name.toLowerCase(); return TARGET_PUBLISHERS.some(p => n.includes(p)); } -// ── Angebote scrapen (Produktseite) ────────────────────────────────────────── -async function scrapeOffers() { +// ── Angebote via Puppeteer ──────────────────────────────────────────────────── +async function scrapeOffersWithPuppeteer() { const cacheKey = 'koepi:offers'; const cached = cacheGet(cacheKey); if (cached) return cached; - const { body } = await fetchUrl( - 'https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg' - ); - - const offers = []; - - // Methode 1: Produktkacheln aus HTML parsen - // Suche nach offer-card Patterns im HTML - const offerPatterns = [ - // Schema.org JSON-LD - /