feat: KoePi nutzt jetzt primaer die marktguru-JSON-API statt Puppeteer, mit automatischem Fallback

backend/src/tools/koepi/routes.js

@@ -81,17 +81,76 @@ function isLocalBranch(retailer, realAddress) {
   return expectedParts.some(p => real.includes(p) || (p && p.includes(real)));
 }
 
-// ── Angebote via Puppeteer von marktguru ─────────────────────────────────────
+// ── Angebote von marktguru ────────────────────────────────────────────────────
-async function scrapeMarktguru(forceRefresh = false) {
+// Primär über die öffentlich bekannte marktguru-JSON-API (schnell, kein Browser
-  const cacheKey = 'koepi:mg';
+// nötig, kein Cloudflare-Risiko). Schlägt das fehl (z.B. weil die Keys mal
-  if (!forceRefresh) {
+// ungültig werden), fällt der Code automatisch auf das alte Puppeteer/HTML-
-    const cached = cacheGet(cacheKey);
+// Scraping zurück, damit KöPi nicht komplett ausfällt.
-    if (cached) return cached;
+const MARKTGURU_API_URL    = 'https://api.marktguru.de/api/v1/offers/search';
+const MARKTGURU_CLIENT_KEY = 'WU/RH+PMGDi+gkZer3WbMelt6zcYHSTytNB7VpTia90=';
+const MARKTGURU_API_KEY    = '8Kk+pmbf7TgJ9nVj2cXeA7P5zBGv8iuutVVMRfOfvNE=';
+const MG_ZIP = '47259';
+const MG_QUERY = 'könig pilsener';
+
+function formatIsoDate(iso) {
+  if (!iso) return '';
+  const d = new Date(iso);
+  if (Number.isNaN(d.getTime())) return '';
+  const pad = n => String(n).padStart(2, '0');
+  return `${pad(d.getDate())}.${pad(d.getMonth() + 1)}.`;
+}
+
+async function fetchOffersViaApi() {
+  const url = `${MARKTGURU_API_URL}?as=web&limit=24&offset=0&q=${encodeURIComponent(MG_QUERY)}&zipCode=${MG_ZIP}`;
+  const res = await fetch(url, {
+    headers: {
+      'x-clientkey': MARKTGURU_CLIENT_KEY,
+      'x-apikey': MARKTGURU_API_KEY,
+      'Accept': 'application/json',
+      'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+    },
+  });
+  if (!res.ok) {
+    const text = await res.text().catch(() => '');
+    throw new Error(`API HTTP ${res.status}: ${text.slice(0, 200)}`);
+  }
+  const json = await res.json();
+
+  // Diagnose: rohe Struktur des ersten Treffers loggen, damit sich Feldnamen
+  // (z.B. für eine echte Store-Adresse) im Zweifel nachträglich anpassen lassen
+  console.log('🍺 KöPi API: totalResults=', json.totalResults, '| results.length=', json.results?.length);
+  if (json.results?.[0]) {
+    console.log('🍺 KöPi API Rohdaten (erster Treffer):', JSON.stringify(json.results[0]).slice(0, 1500));
   }
 
+  return (json.results || []).map(r => {
+    const advertiser = r.advertisers?.[0] || {};
+    const store = r.closestStore || r.store || {};
+    const priceNum = typeof r.price === 'number' ? r.price : parseFloat(r.price);
+    const oldPriceNum = typeof r.oldPrice === 'number' ? r.oldPrice : parseFloat(r.oldPrice);
+    const realAddress = [store.street, store.streetNumber].filter(Boolean).join(' ').trim()
+      || store.address || advertiser.address || '';
+    return {
+      name: r.description || r.title || advertiser.name || '',
+      brand: r.brand?.name || '',
+      price: Number.isFinite(priceNum) ? `€ ${priceNum.toFixed(2).replace('.', ',')}` : (r.price || ''),
+      oldPrice: Number.isFinite(oldPriceNum) ? `€ ${oldPriceNum.toFixed(2).replace('.', ',')}` : (r.oldPrice || ''),
+      retailer: advertiser.name || '',
+      dateRange: r.validFrom && r.validUntil ? `${formatIsoDate(r.validFrom)} - ${formatIsoDate(r.validUntil)}` : formatIsoDate(r.validUntil),
+      validity: '',
+      description: r.description || '',
+      badge: '',
+      address: '',
+      realAddress,
+      image: r.image?.url || r.images?.[0]?.url || null,
+    };
+  });
+}
+
+async function fetchOffersViaPuppeteer() {
   let puppeteer;
   try { puppeteer = require('puppeteer-core'); }
-  catch(e) { return { offers:[], error:'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() }; }
+  catch(e) { throw new Error('puppeteer-core nicht installiert'); }
 
   const browser = await puppeteer.launch({
     executablePath: '/usr/bin/chromium-browser',
@@ -105,7 +164,7 @@ async function scrapeMarktguru(forceRefresh = false) {
     await page.setViewport({ width:1280, height:800 });
     await page.setCookie({ name:'mg_user-settings', value:MG_SETTINGS, domain:'.marktguru.de', path:'/' });
 
-await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', {
+    await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', {
       waitUntil: 'networkidle2', timeout: 30000,
     });
     // Seite nach unten scrollen damit lazy-loaded Bilder geladen werden
@@ -136,18 +195,13 @@ await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=4
         const retailer = card.querySelector('dd.retailer-name a')?.textContent?.trim() || '';
         const dateRange= card.querySelector('dd.valid')?.textContent?.trim() || '';
         const daysLeft = card.querySelector('dd.time-left span')?.textContent?.trim() || '';
-        const daysText = card.querySelector('dd.time-left')?.textContent?.trim() || '';
+        const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` :
-        const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` : 
                          card.querySelector('dt.time-left')?.textContent?.includes('Brandneu') ? 'Neu' : '';
         const descEl   = card.querySelector('.info div, .info');
         const description = descEl?.textContent?.trim() || '';
         const img      = card.querySelector('img.offer-list-item-img');
         const badge    = card.querySelector('.badge, .discount-badge')?.textContent?.trim() || '';
         const imgSrc   = img?.src?.replace('/medium.webp', '/large.webp') || img?.src || null;
-        // Adresse steht bei marktguru direkt nach dem Händlernamen, aber nicht
-        // zwingend als direktes DOM-Geschwisterelement (z.B. wenn dazwischen
-        // noch ein <dt>-Label liegt) — daher über die Position in der Liste
-        // ALLER <dd>-Elemente der Karte suchen statt über nextElementSibling
         const allDds = Array.from(card.querySelectorAll('dd'));
         const retailerIdx = allDds.findIndex(d => d.classList.contains('retailer-name'));
         let realAddress = '';
@@ -159,84 +213,103 @@ await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=4
         return { name, brand, price, oldPrice, retailer, dateRange, validity, description, badge, address: '', realAddress, image: imgSrc };
       });
     });
-
+    return offers;
-    // Lokale Bilder auswählen basierend auf Beschreibung
-    for (const offer of offers) {
-      const desc = (offer.description || '').toLowerCase();
-      console.log('DESC:', JSON.stringify(desc), '| RETAILER:', offer.retailer);
-      if (desc.includes('steini')) {
-        offer.imageLocal = '/koepi/kasten_steini.png';
-      } else if (desc.includes('11 x 0,5') || desc.includes('11x0,5')) {
-        offer.imageLocal = '/koepi/kasten_lang.png';
-      } else if (desc.includes('6 x') || desc.includes('6x') || (desc.includes('flasche') && !desc.includes('kasten'))) {
-        offer.imageLocal = '/koepi/traeger.png';
-      } else if (desc.includes('24 x 0,33') || desc.includes('24x0,33') || desc.includes('24 x 0,33')) {
-        offer.imageLocal = '/koepi/kasten_033.png';
-      } else if (desc.includes('0,5')) {
-        offer.imageLocal = '/koepi/kasten_050.png';
-      } else if (desc.includes('0,33')) {
-        offer.imageLocal = '/koepi/kasten_033.png';
-      } else {
-        offer.imageLocal = '/koepi/kasten_050.png';
-      }
-      offer.image = null; // CDN nicht mehr nötig
-      console.log('IMAGE:', offer.imageLocal, '| RETAILER:', offer.retailer);
-    }
-
-    // Filtern auf Ziel-Händler + nur König Pilsener
-    const filtered = offers.filter(o => {
-      const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase();
-      const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener');
-      return hasKP && isTargetRetailer(o.retailer);
-    });
-
-    // Nur Angebote der tatsächlich bekannten, lokalen Filialen behalten. Die
-    // Adress-basierte Prüfung (isLocalBranch) greift nur, wenn marktguru für
-    // ein Angebot eine echte Adresse liefert — auf der aktuell genutzten
-    // Such-URL ist das nicht der Fall, daher bleibt sie hier wirkungslos
-    // (siehe Kommentar an isLocalBranch). Verlässlich funktioniert aktuell nur
-    // der explizite Namens-Ausschluss (z.B. "REWE Dortmund").
-    const excludedNow = filtered.filter(o => isExcludedRetailer(o.retailer));
-    if (excludedNow.length) {
-      console.log('🍺 KöPi: Angebote per Namens-Filter ausgeschlossen:', excludedNow.map(o => o.retailer).join(', '));
-    }
-
-    const localOnly = filtered
-      .filter(o => !isExcludedRetailer(o.retailer))
-      .filter(o => isLocalBranch(o.retailer, o.realAddress));
-
-    // Händler normalisieren: E center = EDEKA
-    for (const o of localOnly) {
-      if (o.retailer?.toLowerCase().includes('e center')) o.retailerDisplay = o.retailer;
-      else o.retailerDisplay = o.retailer;
-    }
-
-    // Deduplizieren: gleicher normalisierter Händler + gleicher Preis → ein Eintrag
-    // Kürzeste Beschreibung nehmen (sauberste)
-    const merged = new Map();
-    for (const o of localOnly) {
-      const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || '';
-      const key = `${normalizedRetailer}|${o.price}`;
-      if (merged.has(key)) {
-        const existing = merged.get(key);
-        // Kürzere Beschreibung bevorzugen
-        if (o.description && (!existing.description || o.description.length < existing.description.length)) {
-          existing.description = o.description;
-        }
-      } else {
-        merged.set(key, { ...o });
-      }
-    }
-    const unique = [...merged.values()].map(o => ({ ...o, address: getAddress(o.retailer) }));
-
-    const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source: 'marktguru.de' };
-    cacheSet(cacheKey, result);
-    return result;
   } finally {
     await browser.close();
   }
 }
 
+// Gemeinsame Weiterverarbeitung, egal ob die Rohdaten über die API oder per
+// Puppeteer/HTML-Scraping ermittelt wurden
+function processOffers(offers) {
+  // Lokale Bilder auswählen basierend auf Beschreibung
+  for (const offer of offers) {
+    const desc = (offer.description || '').toLowerCase();
+    if (desc.includes('steini')) {
+      offer.imageLocal = '/koepi/kasten_steini.png';
+    } else if (desc.includes('11 x 0,5') || desc.includes('11x0,5')) {
+      offer.imageLocal = '/koepi/kasten_lang.png';
+    } else if (desc.includes('6 x') || desc.includes('6x') || (desc.includes('flasche') && !desc.includes('kasten'))) {
+      offer.imageLocal = '/koepi/traeger.png';
+    } else if (desc.includes('24 x 0,33') || desc.includes('24x0,33')) {
+      offer.imageLocal = '/koepi/kasten_033.png';
+    } else if (desc.includes('0,5')) {
+      offer.imageLocal = '/koepi/kasten_050.png';
+    } else if (desc.includes('0,33')) {
+      offer.imageLocal = '/koepi/kasten_033.png';
+    } else {
+      offer.imageLocal = '/koepi/kasten_050.png';
+    }
+    offer.image = null; // CDN nicht mehr nötig
+  }
+
+  // Filtern auf Ziel-Händler + nur König Pilsener
+  const filtered = offers.filter(o => {
+    const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase();
+    const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener');
+    return hasKP && isTargetRetailer(o.retailer);
+  });
+
+  const excludedNow = filtered.filter(o => isExcludedRetailer(o.retailer));
+  if (excludedNow.length) {
+    console.log('🍺 KöPi: Angebote per Namens-Filter ausgeschlossen:', excludedNow.map(o => o.retailer).join(', '));
+  }
+
+  const localOnly = filtered
+    .filter(o => !isExcludedRetailer(o.retailer))
+    .filter(o => isLocalBranch(o.retailer, o.realAddress));
+
+  for (const o of localOnly) {
+    o.retailerDisplay = o.retailer;
+  }
+
+  // Deduplizieren: gleicher normalisierter Händler + gleicher Preis → ein Eintrag
+  const merged = new Map();
+  for (const o of localOnly) {
+    const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || '';
+    const key = `${normalizedRetailer}|${o.price}`;
+    if (merged.has(key)) {
+      const existing = merged.get(key);
+      if (o.description && (!existing.description || o.description.length < existing.description.length)) {
+        existing.description = o.description;
+      }
+    } else {
+      merged.set(key, { ...o });
+    }
+  }
+  return [...merged.values()].map(o => ({ ...o, address: getAddress(o.retailer) }));
+}
+
+async function scrapeMarktguru(forceRefresh = false) {
+  const cacheKey = 'koepi:mg';
+  if (!forceRefresh) {
+    const cached = cacheGet(cacheKey);
+    if (cached) return cached;
+  }
+
+  let offers, source;
+  try {
+    offers = await fetchOffersViaApi();
+    source = 'marktguru.de (API)';
+    console.log(`🍺 KöPi: ${offers.length} Rohtreffer über die marktguru-API`);
+  } catch (apiErr) {
+    console.error('🍺 KöPi: API-Weg fehlgeschlagen, falle zurück auf Puppeteer:', apiErr.message);
+    try {
+      offers = await fetchOffersViaPuppeteer();
+      source = 'marktguru.de (Puppeteer-Fallback)';
+      console.log(`🍺 KöPi: ${offers.length} Rohtreffer per Puppeteer (Fallback)`);
+    } catch (ppErr) {
+      console.error('🍺 KöPi: auch Puppeteer-Fallback fehlgeschlagen:', ppErr.message);
+      return { offers: [], error: `API: ${apiErr.message} | Puppeteer: ${ppErr.message}`, scrapedAt: new Date().toISOString() };
+    }
+  }
+
+  const unique = processOffers(offers);
+  const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source };
+  cacheSet(cacheKey, result);
+  return result;
+}
+
 // ── Prospekte von kaufda ──────────────────────────────────────────────────────
 function fetchUrl(url) {
   return new Promise((resolve, reject) => {