feat: KöPi - Puppeteer/Chromium für echte Produktangebote

backend/package.json

@@ -8,6 +8,7 @@
     "express": "^4.18.2",
     "jsonwebtoken": "^9.0.2",
     "multer": "^1.4.5-lts.1",
-    "ws": "^8.17.1"
+    "ws": "^8.17.1",
+    "puppeteer-core": "^22.0.0"
   }
 }

backend/src/tools/koepi/routes.js

@@ -1,7 +1,7 @@
-const express  = require('express');
-const https    = require('https');
+const express = require('express');
+const https   = require('https');
 const { authenticate } = require('../../middleware/auth');
-const router   = express.Router();
+const router  = express.Router();
 
 // ── Cache (in-memory, 1h) ─────────────────────────────────────────────────────
 const cache = {};
@@ -15,7 +15,7 @@ function cacheSet(key, data, ttlMs = 60 * 60 * 1000) {
   cache[key] = { data, expires: Date.now() + ttlMs };
 }
 
-// ── HTTP-Helper mit Redirect-Follow ──────────────────────────────────────────
+// ── HTTP-Helper ───────────────────────────────────────────────────────────────
 function fetchUrl(url, maxRedirects = 5) {
   return new Promise((resolve, reject) => {
     const doGet = (u, remaining) => {
@@ -24,7 +24,6 @@ function fetchUrl(url, maxRedirects = 5) {
           'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
           'Accept': 'text/html,application/xhtml+xml',
           'Accept-Language': 'de-DE,de;q=0.9',
-          'Accept-Encoding': 'identity',
         }
       }, (res) => {
         if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && remaining > 0) {
@@ -39,149 +38,128 @@ function fetchUrl(url, maxRedirects = 5) {
         res.on('data', c => d += c);
         res.on('end', () => resolve({ status: res.statusCode, body: d }));
       });
-      req.setTimeout(12000, () => { req.destroy(); reject(new Error('Timeout')); });
+      req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); });
       req.on('error', reject);
     };
     doGet(url, maxRedirects);
   });
 }
 
-// ── NEXT_DATA aus HTML extrahieren ────────────────────────────────────────────
+// ── NEXT_DATA extrahieren ─────────────────────────────────────────────────────
 function extractNextData(html) {
-  const marker  = 'application/json">';
-  const nidx    = html.indexOf('NEXT_DATA');
+  const marker = 'application/json">';
+  const nidx   = html.indexOf('NEXT_DATA');
   if (nidx === -1) return null;
-  const start   = html.indexOf(marker, nidx) + marker.length;
-  const end     = html.indexOf('</script>', start);
+  const start  = html.indexOf(marker, nidx) + marker.length;
+  const end    = html.indexOf('</script>', start);
   if (start <= marker.length || end === -1) return null;
   try { return JSON.parse(html.slice(start, end)); } catch { return null; }
 }
 
-// ── Händler-Filter (Prospekte) ────────────────────────────────────────────────
-const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl','real'];
-
+// ── Händler-Filter ────────────────────────────────────────────────────────────
+const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl'];
 function isTargetPublisher(name) {
   if (!name) return false;
   const n = name.toLowerCase();
   return TARGET_PUBLISHERS.some(p => n.includes(p));
 }
 
-// ── Angebote scrapen (Produktseite) ──────────────────────────────────────────
-async function scrapeOffers() {
+// ── Angebote via Puppeteer ────────────────────────────────────────────────────
+async function scrapeOffersWithPuppeteer() {
   const cacheKey = 'koepi:offers';
   const cached = cacheGet(cacheKey);
   if (cached) return cached;
 
-  const { body } = await fetchUrl(
-    'https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg'
-  );
-
-  const offers = [];
-
-  // Methode 1: Produktkacheln aus HTML parsen
-  // Suche nach offer-card Patterns im HTML
-  const offerPatterns = [
-    // Schema.org JSON-LD
-    /<script type="application\/ld\+json">([\s\S]*?)<\/script>/g,
-    // Meta-Tags mit Produktinfos
-    /data-offer-id="([^"]+)"[^>]*data-price="([^"]+)"[^>]*data-publisher="([^"]+)"/g,
-  ];
-
-  // Schema.org JSON-LD extrahieren
-  let match;
-  const ldReg = /<script type="application\/ld\+json">([\s\S]*?)<\/script>/g;
-  while ((match = ldReg.exec(body)) !== null) {
-    try {
-      const ld = JSON.parse(match[1]);
-      const items = Array.isArray(ld) ? ld : [ld];
-      for (const item of items) {
-        if (item['@type'] === 'ItemList' && item.itemListElement) {
-          for (const el of item.itemListElement) {
-            const thing = el.item || el;
-            if (thing.name?.toLowerCase().includes('könig') || thing.name?.toLowerCase().includes('pilsener') || thing.name?.toLowerCase().includes('pilsen')) {
-              offers.push({
-                name:      thing.name,
-                price:     thing.offers?.price || thing.offers?.lowPrice || null,
-                currency:  'EUR',
-                store:     thing.offers?.seller?.name || null,
-                url:       thing.url || 'https://www.kaufda.de/Angebote/Koenig-Pilsener',
-                image:     thing.image || null,
-                validFrom: thing.offers?.validFrom || null,
-                validTo:   thing.offers?.validThrough || null,
-              });
-            }
-          }
-        }
-        if (item['@type'] === 'Product' && item.name) {
-          if (item.name.toLowerCase().includes('könig') || item.name.toLowerCase().includes('pilsen')) {
-            const offer = Array.isArray(item.offers) ? item.offers[0] : item.offers;
-            offers.push({
-              name:      item.name,
-              price:     offer?.price || offer?.lowPrice || null,
-              currency:  'EUR',
-              store:     offer?.seller?.name || null,
-              url:       item.url || 'https://www.kaufda.de/Angebote/Koenig-Pilsener',
-              image:     Array.isArray(item.image) ? item.image[0] : item.image || null,
-              validFrom: offer?.validFrom || null,
-              validTo:   offer?.validThrough || null,
-            });
-          }
-        }
-      }
-    } catch {}
+  let puppeteer;
+  try { puppeteer = require('puppeteer-core'); } catch(e) {
+    return { offers: [], error: 'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() };
   }
 
-  // Methode 2: NEXT_DATA
-  const nextData = extractNextData(body);
-  if (nextData) {
-    const pp = nextData.props?.pageProps;
-    // Suche nach Angeboten in allen möglichen Keys
-    const str = JSON.stringify(pp || {});
-    // Extrahiere strukturierte Angebotsdaten wenn vorhanden
-    const offerData = pp?.offers || pp?.pageInformation?.template?.content?.offers || [];
-    if (Array.isArray(offerData)) {
-      for (const o of offerData) {
-        if (o.name || o.title) {
-          offers.push({
-            name:      o.name || o.title,
-            price:     o.price || o.minPrice || null,
-            currency:  'EUR',
-            store:     o.publisher?.name || o.retailer || null,
-            image:     o.image?.url || o.imageUrl || null,
-            validFrom: o.validFrom || null,
-            validTo:   o.validUntil || null,
-            url:       `https://www.kaufda.de/Angebote/Koenig-Pilsener`,
-          });
-        }
-      }
-    }
-  }
-
-  // Deduplizieren
-  const seen = new Set();
-  const unique = offers.filter(o => {
-    const key = `${o.name}|${o.store}|${o.price}`;
-    if (seen.has(key)) return false;
-    seen.add(key);
-    return true;
+  const browser = await puppeteer.launch({
+    executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium-browser',
+    args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'],
+    headless: true,
   });
 
-  const result = { offers: unique, scrapedAt: new Date().toISOString(), source: 'kaufda.de' };
-  cacheSet(cacheKey, result);
-  return result;
+  try {
+    const page = await browser.newPage();
+    await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
+    await page.setExtraHTTPHeaders({ 'Accept-Language': 'de-DE,de;q=0.9' });
+
+    await page.goto('https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg', {
+      waitUntil: 'networkidle2',
+      timeout: 30000,
+    });
+
+    // Warte auf Produktkacheln
+    await page.waitForSelector('[data-testid="offer-card"], .offer-card, article, [class*="offer"]', {
+      timeout: 10000,
+    }).catch(() => {});
+
+    // Alle Angebotsdaten extrahieren
+    const offers = await page.evaluate(() => {
+      const results = [];
+
+      // Methode 1: Schema.org
+      document.querySelectorAll('script[type="application/ld+json"]').forEach(s => {
+        try {
+          const data = JSON.parse(s.textContent);
+          const items = Array.isArray(data) ? data : [data];
+          items.forEach(item => {
+            if (item['@type'] === 'Product') {
+              const offer = Array.isArray(item.offers) ? item.offers[0] : item.offers;
+              results.push({
+                name:      item.name,
+                price:     offer?.price || null,
+                store:     offer?.seller?.name || null,
+                image:     Array.isArray(item.image) ? item.image[0] : item.image || null,
+                validFrom: offer?.validFrom || null,
+                validTo:   offer?.validThrough || null,
+                url:       item.url || window.location.href,
+              });
+            }
+          });
+        } catch {}
+      });
+
+      // Methode 2: DOM-Kacheln
+      const cards = document.querySelectorAll('[class*="offer-card"], [class*="OfferCard"], [data-testid*="offer"], article[class*="offer"]');
+      cards.forEach(card => {
+        const name   = card.querySelector('[class*="title"], [class*="name"], h2, h3')?.textContent?.trim();
+        const price  = card.querySelector('[class*="price"], [class*="Price"]')?.textContent?.trim();
+        const store  = card.querySelector('[class*="retailer"], [class*="publisher"], [class*="store"]')?.textContent?.trim();
+        const img    = card.querySelector('img')?.src;
+        if (name) results.push({ name, price, store, image: img, url: window.location.href });
+      });
+
+      return results;
+    });
+
+    const result = {
+      offers: offers.filter((o, i, arr) =>
+        arr.findIndex(x => x.name === o.name && x.store === o.store) === i
+      ),
+      scrapedAt: new Date().toISOString(),
+      source: 'kaufda.de (puppeteer)',
+    };
+
+    cacheSet(cacheKey, result);
+    return result;
+  } finally {
+    await browser.close();
+  }
 }
 
-// ── Prospekte scrapen (shelf-Seite) ──────────────────────────────────────────
+// ── Prospekte scrapen ─────────────────────────────────────────────────────────
 async function scrapeProspekte() {
   const cacheKey = 'koepi:prospekte';
   const cached = cacheGet(cacheKey);
   if (cached) return cached;
 
-  // Mehrere PLZ/Koordinaten für Umkreis
   const locations = [
-    { lat: 51.4332, lng: 6.7625, city: 'Duisburg', zip: '47051' },
-    { lat: 51.5167, lng: 6.9833, city: 'Essen',    zip: '45127' },
-    { lat: 51.2217, lng: 6.7762, city: 'Düsseldorf', zip: '40213' },
+    { lat: 51.4332, lng: 6.7625, city: 'Duisburg',    zip: '47051' },
+    { lat: 51.5167, lng: 6.9833, city: 'Essen',       zip: '45127' },
+    { lat: 51.2217, lng: 6.7762, city: 'Düsseldorf',  zip: '40213' },
   ];
 
   const allBrochures = new Map();
@@ -191,56 +169,48 @@ async function scrapeProspekte() {
       const { body } = await fetchUrl(
         `https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=${loc.lat}&lng=${loc.lng}&zip=${loc.zip}&city=${encodeURIComponent(loc.city)}`
       );
-
       const nextData = extractNextData(body);
       if (!nextData) continue;
 
-      const pp = nextData.props?.pageProps;
-      const contents = pp?.pageInformation?.shelfContents?.contents
-        || pp?.pageInformation?.template?.shelfContents?.contents
-        || pp?.pageInformation?.template?.content?.shelfContents?.contents
-        || [];
+      const pp       = nextData.props?.pageProps;
+      const contents = pp?.pageInformation?.shelfContents?.contents || [];
 
       for (const item of contents) {
         const c = item.content;
         if (!c || item.contentType !== 'brochure') continue;
+        const id = c.contentId;
+        if (!id || allBrochures.has(id)) continue;
 
         const publisher = c.publisher?.name || '';
         const store     = c.closestStore;
-        const id        = c.contentId;
 
-        if (!allBrochures.has(id)) {
-          allBrochures.set(id, {
-            id,
-            publisher,
-            title:      c.title || '',
-            store:      store?.name || publisher,
-            city:       store?.city || loc.city,
-            street:     store ? `${store.street || ''} ${store.streetNumber || ''}`.trim() : '',
-            zip:        store?.zip || loc.zip,
-            image:      c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null,
-            validFrom:  c.validFrom || null,
-            validTo:    c.validUntil || null,
-            pageCount:  c.pageCount || null,
-            url:        `https://www.kaufda.de/webapp/brochure/${id}`,
-            isTarget:   isTargetPublisher(publisher),
-            badges:     c.contentBadges?.map(b => b.name) || [],
-          });
-        }
+        allBrochures.set(id, {
+          id,
+          publisher,
+          title:     c.title || '',
+          store:     store?.name || publisher,
+          city:      store?.city || loc.city,
+          street:    store ? `${store.street || ''} ${store.streetNumber || ''}`.trim() : '',
+          zip:       store?.zip || loc.zip,
+          image:     c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null,
+          validFrom: c.validFrom || null,
+          validTo:   c.validUntil || null,
+          pageCount: c.pageCount || null,
+          url:       `https://www.kaufda.de/webapp/brochure/${id}`,
+          isTarget:  isTargetPublisher(publisher),
+          badges:    c.contentBadges?.map(b => b.name) || [],
+        });
       }
     } catch(e) {
-      console.error('KöPi scrape error:', loc.city, e.message);
+      console.error('KöPi prospekte error:', loc.city, e.message);
     }
   }
 
-  const brochures  = [...allBrochures.values()];
-  const targeted   = brochures.filter(b => b.isTarget);
-  const others     = brochures.filter(b => !b.isTarget);
-
+  const brochures = [...allBrochures.values()];
   const result = {
-    targeted,
-    others,
-    total: brochures.length,
+    targeted: brochures.filter(b => b.isTarget),
+    others:   brochures.filter(b => !b.isTarget),
+    total:    brochures.length,
     scrapedAt: new Date().toISOString(),
   };
 
@@ -251,9 +221,10 @@ async function scrapeProspekte() {
 // ── Routes ────────────────────────────────────────────────────────────────────
 router.get('/offers', authenticate, async (req, res) => {
   try {
-    const data = await scrapeOffers();
+    const data = await scrapeOffersWithPuppeteer();
     res.json(data);
   } catch(e) {
+    console.error('KöPi offers error:', e.message);
     res.status(500).json({ error: e.message });
   }
 });
@@ -263,6 +234,7 @@ router.get('/prospekte', authenticate, async (req, res) => {
     const data = await scrapeProspekte();
     res.json(data);
   } catch(e) {
+    console.error('KöPi prospekte error:', e.message);
     res.status(500).json({ error: e.message });
   }
 });