diff --git a/backend/src/tools/koepi/routes.js b/backend/src/tools/koepi/routes.js
index 8965428..61d61a0 100644
--- a/backend/src/tools/koepi/routes.js
+++ b/backend/src/tools/koepi/routes.js
@@ -81,17 +81,76 @@ function isLocalBranch(retailer, realAddress) {
return expectedParts.some(p => real.includes(p) || (p && p.includes(real)));
}
-// ── Angebote via Puppeteer von marktguru ─────────────────────────────────────
-async function scrapeMarktguru(forceRefresh = false) {
- const cacheKey = 'koepi:mg';
- if (!forceRefresh) {
- const cached = cacheGet(cacheKey);
- if (cached) return cached;
+// ── Angebote von marktguru ────────────────────────────────────────────────────
+// Primär über die öffentlich bekannte marktguru-JSON-API (schnell, kein Browser
+// nötig, kein Cloudflare-Risiko). Schlägt das fehl (z.B. weil die Keys mal
+// ungültig werden), fällt der Code automatisch auf das alte Puppeteer/HTML-
+// Scraping zurück, damit KöPi nicht komplett ausfällt.
+const MARKTGURU_API_URL = 'https://api.marktguru.de/api/v1/offers/search';
+const MARKTGURU_CLIENT_KEY = 'WU/RH+PMGDi+gkZer3WbMelt6zcYHSTytNB7VpTia90=';
+const MARKTGURU_API_KEY = '8Kk+pmbf7TgJ9nVj2cXeA7P5zBGv8iuutVVMRfOfvNE=';
+const MG_ZIP = '47259';
+const MG_QUERY = 'könig pilsener';
+
+function formatIsoDate(iso) {
+ if (!iso) return '';
+ const d = new Date(iso);
+ if (Number.isNaN(d.getTime())) return '';
+ const pad = n => String(n).padStart(2, '0');
+ return `${pad(d.getDate())}.${pad(d.getMonth() + 1)}.`;
+}
+
+async function fetchOffersViaApi() {
+ const url = `${MARKTGURU_API_URL}?as=web&limit=24&offset=0&q=${encodeURIComponent(MG_QUERY)}&zipCode=${MG_ZIP}`;
+ const res = await fetch(url, {
+ headers: {
+ 'x-clientkey': MARKTGURU_CLIENT_KEY,
+ 'x-apikey': MARKTGURU_API_KEY,
+ 'Accept': 'application/json',
+ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
+ },
+ });
+ if (!res.ok) {
+ const text = await res.text().catch(() => '');
+ throw new Error(`API HTTP ${res.status}: ${text.slice(0, 200)}`);
+ }
+ const json = await res.json();
+
+ // Diagnose: rohe Struktur des ersten Treffers loggen, damit sich Feldnamen
+ // (z.B. für eine echte Store-Adresse) im Zweifel nachträglich anpassen lassen
+ console.log('🍺 KöPi API: totalResults=', json.totalResults, '| results.length=', json.results?.length);
+ if (json.results?.[0]) {
+ console.log('🍺 KöPi API Rohdaten (erster Treffer):', JSON.stringify(json.results[0]).slice(0, 1500));
}
+ return (json.results || []).map(r => {
+ const advertiser = r.advertisers?.[0] || {};
+ const store = r.closestStore || r.store || {};
+ const priceNum = typeof r.price === 'number' ? r.price : parseFloat(r.price);
+ const oldPriceNum = typeof r.oldPrice === 'number' ? r.oldPrice : parseFloat(r.oldPrice);
+ const realAddress = [store.street, store.streetNumber].filter(Boolean).join(' ').trim()
+ || store.address || advertiser.address || '';
+ return {
+ name: r.description || r.title || advertiser.name || '',
+ brand: r.brand?.name || '',
+ price: Number.isFinite(priceNum) ? `€ ${priceNum.toFixed(2).replace('.', ',')}` : (r.price || ''),
+ oldPrice: Number.isFinite(oldPriceNum) ? `€ ${oldPriceNum.toFixed(2).replace('.', ',')}` : (r.oldPrice || ''),
+ retailer: advertiser.name || '',
+ dateRange: r.validFrom && r.validUntil ? `${formatIsoDate(r.validFrom)} - ${formatIsoDate(r.validUntil)}` : formatIsoDate(r.validUntil),
+ validity: '',
+ description: r.description || '',
+ badge: '',
+ address: '',
+ realAddress,
+ image: r.image?.url || r.images?.[0]?.url || null,
+ };
+ });
+}
+
+async function fetchOffersViaPuppeteer() {
let puppeteer;
try { puppeteer = require('puppeteer-core'); }
- catch(e) { return { offers:[], error:'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() }; }
+ catch(e) { throw new Error('puppeteer-core nicht installiert'); }
const browser = await puppeteer.launch({
executablePath: '/usr/bin/chromium-browser',
@@ -105,7 +164,7 @@ async function scrapeMarktguru(forceRefresh = false) {
await page.setViewport({ width:1280, height:800 });
await page.setCookie({ name:'mg_user-settings', value:MG_SETTINGS, domain:'.marktguru.de', path:'/' });
-await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', {
+ await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', {
waitUntil: 'networkidle2', timeout: 30000,
});
// Seite nach unten scrollen damit lazy-loaded Bilder geladen werden
@@ -136,18 +195,13 @@ await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=4
const retailer = card.querySelector('dd.retailer-name a')?.textContent?.trim() || '';
const dateRange= card.querySelector('dd.valid')?.textContent?.trim() || '';
const daysLeft = card.querySelector('dd.time-left span')?.textContent?.trim() || '';
- const daysText = card.querySelector('dd.time-left')?.textContent?.trim() || '';
- const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` :
+ const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` :
card.querySelector('dt.time-left')?.textContent?.includes('Brandneu') ? 'Neu' : '';
const descEl = card.querySelector('.info div, .info');
const description = descEl?.textContent?.trim() || '';
const img = card.querySelector('img.offer-list-item-img');
const badge = card.querySelector('.badge, .discount-badge')?.textContent?.trim() || '';
const imgSrc = img?.src?.replace('/medium.webp', '/large.webp') || img?.src || null;
- // Adresse steht bei marktguru direkt nach dem Händlernamen, aber nicht
- // zwingend als direktes DOM-Geschwisterelement (z.B. wenn dazwischen
- // noch ein
-Label liegt) — daher über die Position in der Liste
- // ALLER -Elemente der Karte suchen statt über nextElementSibling
const allDds = Array.from(card.querySelectorAll('dd'));
const retailerIdx = allDds.findIndex(d => d.classList.contains('retailer-name'));
let realAddress = '';
@@ -159,84 +213,103 @@ await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=4
return { name, brand, price, oldPrice, retailer, dateRange, validity, description, badge, address: '', realAddress, image: imgSrc };
});
});
-
- // Lokale Bilder auswählen basierend auf Beschreibung
- for (const offer of offers) {
- const desc = (offer.description || '').toLowerCase();
- console.log('DESC:', JSON.stringify(desc), '| RETAILER:', offer.retailer);
- if (desc.includes('steini')) {
- offer.imageLocal = '/koepi/kasten_steini.png';
- } else if (desc.includes('11 x 0,5') || desc.includes('11x0,5')) {
- offer.imageLocal = '/koepi/kasten_lang.png';
- } else if (desc.includes('6 x') || desc.includes('6x') || (desc.includes('flasche') && !desc.includes('kasten'))) {
- offer.imageLocal = '/koepi/traeger.png';
- } else if (desc.includes('24 x 0,33') || desc.includes('24x0,33') || desc.includes('24 x 0,33')) {
- offer.imageLocal = '/koepi/kasten_033.png';
- } else if (desc.includes('0,5')) {
- offer.imageLocal = '/koepi/kasten_050.png';
- } else if (desc.includes('0,33')) {
- offer.imageLocal = '/koepi/kasten_033.png';
- } else {
- offer.imageLocal = '/koepi/kasten_050.png';
- }
- offer.image = null; // CDN nicht mehr nötig
- console.log('IMAGE:', offer.imageLocal, '| RETAILER:', offer.retailer);
- }
-
- // Filtern auf Ziel-Händler + nur König Pilsener
- const filtered = offers.filter(o => {
- const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase();
- const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener');
- return hasKP && isTargetRetailer(o.retailer);
- });
-
- // Nur Angebote der tatsächlich bekannten, lokalen Filialen behalten. Die
- // Adress-basierte Prüfung (isLocalBranch) greift nur, wenn marktguru für
- // ein Angebot eine echte Adresse liefert — auf der aktuell genutzten
- // Such-URL ist das nicht der Fall, daher bleibt sie hier wirkungslos
- // (siehe Kommentar an isLocalBranch). Verlässlich funktioniert aktuell nur
- // der explizite Namens-Ausschluss (z.B. "REWE Dortmund").
- const excludedNow = filtered.filter(o => isExcludedRetailer(o.retailer));
- if (excludedNow.length) {
- console.log('🍺 KöPi: Angebote per Namens-Filter ausgeschlossen:', excludedNow.map(o => o.retailer).join(', '));
- }
-
- const localOnly = filtered
- .filter(o => !isExcludedRetailer(o.retailer))
- .filter(o => isLocalBranch(o.retailer, o.realAddress));
-
- // Händler normalisieren: E center = EDEKA
- for (const o of localOnly) {
- if (o.retailer?.toLowerCase().includes('e center')) o.retailerDisplay = o.retailer;
- else o.retailerDisplay = o.retailer;
- }
-
- // Deduplizieren: gleicher normalisierter Händler + gleicher Preis → ein Eintrag
- // Kürzeste Beschreibung nehmen (sauberste)
- const merged = new Map();
- for (const o of localOnly) {
- const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || '';
- const key = `${normalizedRetailer}|${o.price}`;
- if (merged.has(key)) {
- const existing = merged.get(key);
- // Kürzere Beschreibung bevorzugen
- if (o.description && (!existing.description || o.description.length < existing.description.length)) {
- existing.description = o.description;
- }
- } else {
- merged.set(key, { ...o });
- }
- }
- const unique = [...merged.values()].map(o => ({ ...o, address: getAddress(o.retailer) }));
-
- const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source: 'marktguru.de' };
- cacheSet(cacheKey, result);
- return result;
+ return offers;
} finally {
await browser.close();
}
}
+// Gemeinsame Weiterverarbeitung, egal ob die Rohdaten über die API oder per
+// Puppeteer/HTML-Scraping ermittelt wurden
+function processOffers(offers) {
+ // Lokale Bilder auswählen basierend auf Beschreibung
+ for (const offer of offers) {
+ const desc = (offer.description || '').toLowerCase();
+ if (desc.includes('steini')) {
+ offer.imageLocal = '/koepi/kasten_steini.png';
+ } else if (desc.includes('11 x 0,5') || desc.includes('11x0,5')) {
+ offer.imageLocal = '/koepi/kasten_lang.png';
+ } else if (desc.includes('6 x') || desc.includes('6x') || (desc.includes('flasche') && !desc.includes('kasten'))) {
+ offer.imageLocal = '/koepi/traeger.png';
+ } else if (desc.includes('24 x 0,33') || desc.includes('24x0,33')) {
+ offer.imageLocal = '/koepi/kasten_033.png';
+ } else if (desc.includes('0,5')) {
+ offer.imageLocal = '/koepi/kasten_050.png';
+ } else if (desc.includes('0,33')) {
+ offer.imageLocal = '/koepi/kasten_033.png';
+ } else {
+ offer.imageLocal = '/koepi/kasten_050.png';
+ }
+ offer.image = null; // CDN nicht mehr nötig
+ }
+
+ // Filtern auf Ziel-Händler + nur König Pilsener
+ const filtered = offers.filter(o => {
+ const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase();
+ const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener');
+ return hasKP && isTargetRetailer(o.retailer);
+ });
+
+ const excludedNow = filtered.filter(o => isExcludedRetailer(o.retailer));
+ if (excludedNow.length) {
+ console.log('🍺 KöPi: Angebote per Namens-Filter ausgeschlossen:', excludedNow.map(o => o.retailer).join(', '));
+ }
+
+ const localOnly = filtered
+ .filter(o => !isExcludedRetailer(o.retailer))
+ .filter(o => isLocalBranch(o.retailer, o.realAddress));
+
+ for (const o of localOnly) {
+ o.retailerDisplay = o.retailer;
+ }
+
+ // Deduplizieren: gleicher normalisierter Händler + gleicher Preis → ein Eintrag
+ const merged = new Map();
+ for (const o of localOnly) {
+ const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || '';
+ const key = `${normalizedRetailer}|${o.price}`;
+ if (merged.has(key)) {
+ const existing = merged.get(key);
+ if (o.description && (!existing.description || o.description.length < existing.description.length)) {
+ existing.description = o.description;
+ }
+ } else {
+ merged.set(key, { ...o });
+ }
+ }
+ return [...merged.values()].map(o => ({ ...o, address: getAddress(o.retailer) }));
+}
+
+async function scrapeMarktguru(forceRefresh = false) {
+ const cacheKey = 'koepi:mg';
+ if (!forceRefresh) {
+ const cached = cacheGet(cacheKey);
+ if (cached) return cached;
+ }
+
+ let offers, source;
+ try {
+ offers = await fetchOffersViaApi();
+ source = 'marktguru.de (API)';
+ console.log(`🍺 KöPi: ${offers.length} Rohtreffer über die marktguru-API`);
+ } catch (apiErr) {
+ console.error('🍺 KöPi: API-Weg fehlgeschlagen, falle zurück auf Puppeteer:', apiErr.message);
+ try {
+ offers = await fetchOffersViaPuppeteer();
+ source = 'marktguru.de (Puppeteer-Fallback)';
+ console.log(`🍺 KöPi: ${offers.length} Rohtreffer per Puppeteer (Fallback)`);
+ } catch (ppErr) {
+ console.error('🍺 KöPi: auch Puppeteer-Fallback fehlgeschlagen:', ppErr.message);
+ return { offers: [], error: `API: ${apiErr.message} | Puppeteer: ${ppErr.message}`, scrapedAt: new Date().toISOString() };
+ }
+ }
+
+ const unique = processOffers(offers);
+ const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source };
+ cacheSet(cacheKey, result);
+ return result;
+}
+
// ── Prospekte von kaufda ──────────────────────────────────────────────────────
function fetchUrl(url) {
return new Promise((resolve, reject) => {