diff --git a/backend/src/tools/koepi/routes.js b/backend/src/tools/koepi/routes.js index 8965428..61d61a0 100644 --- a/backend/src/tools/koepi/routes.js +++ b/backend/src/tools/koepi/routes.js @@ -81,17 +81,76 @@ function isLocalBranch(retailer, realAddress) { return expectedParts.some(p => real.includes(p) || (p && p.includes(real))); } -// ── Angebote via Puppeteer von marktguru ───────────────────────────────────── -async function scrapeMarktguru(forceRefresh = false) { - const cacheKey = 'koepi:mg'; - if (!forceRefresh) { - const cached = cacheGet(cacheKey); - if (cached) return cached; +// ── Angebote von marktguru ──────────────────────────────────────────────────── +// Primär über die öffentlich bekannte marktguru-JSON-API (schnell, kein Browser +// nötig, kein Cloudflare-Risiko). Schlägt das fehl (z.B. weil die Keys mal +// ungültig werden), fällt der Code automatisch auf das alte Puppeteer/HTML- +// Scraping zurück, damit KöPi nicht komplett ausfällt. +const MARKTGURU_API_URL = 'https://api.marktguru.de/api/v1/offers/search'; +const MARKTGURU_CLIENT_KEY = 'WU/RH+PMGDi+gkZer3WbMelt6zcYHSTytNB7VpTia90='; +const MARKTGURU_API_KEY = '8Kk+pmbf7TgJ9nVj2cXeA7P5zBGv8iuutVVMRfOfvNE='; +const MG_ZIP = '47259'; +const MG_QUERY = 'könig pilsener'; + +function formatIsoDate(iso) { + if (!iso) return ''; + const d = new Date(iso); + if (Number.isNaN(d.getTime())) return ''; + const pad = n => String(n).padStart(2, '0'); + return `${pad(d.getDate())}.${pad(d.getMonth() + 1)}.`; +} + +async function fetchOffersViaApi() { + const url = `${MARKTGURU_API_URL}?as=web&limit=24&offset=0&q=${encodeURIComponent(MG_QUERY)}&zipCode=${MG_ZIP}`; + const res = await fetch(url, { + headers: { + 'x-clientkey': MARKTGURU_CLIENT_KEY, + 'x-apikey': MARKTGURU_API_KEY, + 'Accept': 'application/json', + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + }, + }); + if (!res.ok) { + const text = await res.text().catch(() => ''); + throw new Error(`API HTTP ${res.status}: ${text.slice(0, 200)}`); + } + const json = await res.json(); + + // Diagnose: rohe Struktur des ersten Treffers loggen, damit sich Feldnamen + // (z.B. für eine echte Store-Adresse) im Zweifel nachträglich anpassen lassen + console.log('🍺 KöPi API: totalResults=', json.totalResults, '| results.length=', json.results?.length); + if (json.results?.[0]) { + console.log('🍺 KöPi API Rohdaten (erster Treffer):', JSON.stringify(json.results[0]).slice(0, 1500)); } + return (json.results || []).map(r => { + const advertiser = r.advertisers?.[0] || {}; + const store = r.closestStore || r.store || {}; + const priceNum = typeof r.price === 'number' ? r.price : parseFloat(r.price); + const oldPriceNum = typeof r.oldPrice === 'number' ? r.oldPrice : parseFloat(r.oldPrice); + const realAddress = [store.street, store.streetNumber].filter(Boolean).join(' ').trim() + || store.address || advertiser.address || ''; + return { + name: r.description || r.title || advertiser.name || '', + brand: r.brand?.name || '', + price: Number.isFinite(priceNum) ? `€ ${priceNum.toFixed(2).replace('.', ',')}` : (r.price || ''), + oldPrice: Number.isFinite(oldPriceNum) ? `€ ${oldPriceNum.toFixed(2).replace('.', ',')}` : (r.oldPrice || ''), + retailer: advertiser.name || '', + dateRange: r.validFrom && r.validUntil ? `${formatIsoDate(r.validFrom)} - ${formatIsoDate(r.validUntil)}` : formatIsoDate(r.validUntil), + validity: '', + description: r.description || '', + badge: '', + address: '', + realAddress, + image: r.image?.url || r.images?.[0]?.url || null, + }; + }); +} + +async function fetchOffersViaPuppeteer() { let puppeteer; try { puppeteer = require('puppeteer-core'); } - catch(e) { return { offers:[], error:'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() }; } + catch(e) { throw new Error('puppeteer-core nicht installiert'); } const browser = await puppeteer.launch({ executablePath: '/usr/bin/chromium-browser', @@ -105,7 +164,7 @@ async function scrapeMarktguru(forceRefresh = false) { await page.setViewport({ width:1280, height:800 }); await page.setCookie({ name:'mg_user-settings', value:MG_SETTINGS, domain:'.marktguru.de', path:'/' }); -await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', { + await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', { waitUntil: 'networkidle2', timeout: 30000, }); // Seite nach unten scrollen damit lazy-loaded Bilder geladen werden @@ -136,18 +195,13 @@ await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=4 const retailer = card.querySelector('dd.retailer-name a')?.textContent?.trim() || ''; const dateRange= card.querySelector('dd.valid')?.textContent?.trim() || ''; const daysLeft = card.querySelector('dd.time-left span')?.textContent?.trim() || ''; - const daysText = card.querySelector('dd.time-left')?.textContent?.trim() || ''; - const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` : + const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` : card.querySelector('dt.time-left')?.textContent?.includes('Brandneu') ? 'Neu' : ''; const descEl = card.querySelector('.info div, .info'); const description = descEl?.textContent?.trim() || ''; const img = card.querySelector('img.offer-list-item-img'); const badge = card.querySelector('.badge, .discount-badge')?.textContent?.trim() || ''; const imgSrc = img?.src?.replace('/medium.webp', '/large.webp') || img?.src || null; - // Adresse steht bei marktguru direkt nach dem Händlernamen, aber nicht - // zwingend als direktes DOM-Geschwisterelement (z.B. wenn dazwischen - // noch ein
-Label liegt) — daher über die Position in der Liste - // ALLER
-Elemente der Karte suchen statt über nextElementSibling const allDds = Array.from(card.querySelectorAll('dd')); const retailerIdx = allDds.findIndex(d => d.classList.contains('retailer-name')); let realAddress = ''; @@ -159,84 +213,103 @@ await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=4 return { name, brand, price, oldPrice, retailer, dateRange, validity, description, badge, address: '', realAddress, image: imgSrc }; }); }); - - // Lokale Bilder auswählen basierend auf Beschreibung - for (const offer of offers) { - const desc = (offer.description || '').toLowerCase(); - console.log('DESC:', JSON.stringify(desc), '| RETAILER:', offer.retailer); - if (desc.includes('steini')) { - offer.imageLocal = '/koepi/kasten_steini.png'; - } else if (desc.includes('11 x 0,5') || desc.includes('11x0,5')) { - offer.imageLocal = '/koepi/kasten_lang.png'; - } else if (desc.includes('6 x') || desc.includes('6x') || (desc.includes('flasche') && !desc.includes('kasten'))) { - offer.imageLocal = '/koepi/traeger.png'; - } else if (desc.includes('24 x 0,33') || desc.includes('24x0,33') || desc.includes('24 x 0,33')) { - offer.imageLocal = '/koepi/kasten_033.png'; - } else if (desc.includes('0,5')) { - offer.imageLocal = '/koepi/kasten_050.png'; - } else if (desc.includes('0,33')) { - offer.imageLocal = '/koepi/kasten_033.png'; - } else { - offer.imageLocal = '/koepi/kasten_050.png'; - } - offer.image = null; // CDN nicht mehr nötig - console.log('IMAGE:', offer.imageLocal, '| RETAILER:', offer.retailer); - } - - // Filtern auf Ziel-Händler + nur König Pilsener - const filtered = offers.filter(o => { - const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase(); - const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener'); - return hasKP && isTargetRetailer(o.retailer); - }); - - // Nur Angebote der tatsächlich bekannten, lokalen Filialen behalten. Die - // Adress-basierte Prüfung (isLocalBranch) greift nur, wenn marktguru für - // ein Angebot eine echte Adresse liefert — auf der aktuell genutzten - // Such-URL ist das nicht der Fall, daher bleibt sie hier wirkungslos - // (siehe Kommentar an isLocalBranch). Verlässlich funktioniert aktuell nur - // der explizite Namens-Ausschluss (z.B. "REWE Dortmund"). - const excludedNow = filtered.filter(o => isExcludedRetailer(o.retailer)); - if (excludedNow.length) { - console.log('🍺 KöPi: Angebote per Namens-Filter ausgeschlossen:', excludedNow.map(o => o.retailer).join(', ')); - } - - const localOnly = filtered - .filter(o => !isExcludedRetailer(o.retailer)) - .filter(o => isLocalBranch(o.retailer, o.realAddress)); - - // Händler normalisieren: E center = EDEKA - for (const o of localOnly) { - if (o.retailer?.toLowerCase().includes('e center')) o.retailerDisplay = o.retailer; - else o.retailerDisplay = o.retailer; - } - - // Deduplizieren: gleicher normalisierter Händler + gleicher Preis → ein Eintrag - // Kürzeste Beschreibung nehmen (sauberste) - const merged = new Map(); - for (const o of localOnly) { - const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || ''; - const key = `${normalizedRetailer}|${o.price}`; - if (merged.has(key)) { - const existing = merged.get(key); - // Kürzere Beschreibung bevorzugen - if (o.description && (!existing.description || o.description.length < existing.description.length)) { - existing.description = o.description; - } - } else { - merged.set(key, { ...o }); - } - } - const unique = [...merged.values()].map(o => ({ ...o, address: getAddress(o.retailer) })); - - const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source: 'marktguru.de' }; - cacheSet(cacheKey, result); - return result; + return offers; } finally { await browser.close(); } } +// Gemeinsame Weiterverarbeitung, egal ob die Rohdaten über die API oder per +// Puppeteer/HTML-Scraping ermittelt wurden +function processOffers(offers) { + // Lokale Bilder auswählen basierend auf Beschreibung + for (const offer of offers) { + const desc = (offer.description || '').toLowerCase(); + if (desc.includes('steini')) { + offer.imageLocal = '/koepi/kasten_steini.png'; + } else if (desc.includes('11 x 0,5') || desc.includes('11x0,5')) { + offer.imageLocal = '/koepi/kasten_lang.png'; + } else if (desc.includes('6 x') || desc.includes('6x') || (desc.includes('flasche') && !desc.includes('kasten'))) { + offer.imageLocal = '/koepi/traeger.png'; + } else if (desc.includes('24 x 0,33') || desc.includes('24x0,33')) { + offer.imageLocal = '/koepi/kasten_033.png'; + } else if (desc.includes('0,5')) { + offer.imageLocal = '/koepi/kasten_050.png'; + } else if (desc.includes('0,33')) { + offer.imageLocal = '/koepi/kasten_033.png'; + } else { + offer.imageLocal = '/koepi/kasten_050.png'; + } + offer.image = null; // CDN nicht mehr nötig + } + + // Filtern auf Ziel-Händler + nur König Pilsener + const filtered = offers.filter(o => { + const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase(); + const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener'); + return hasKP && isTargetRetailer(o.retailer); + }); + + const excludedNow = filtered.filter(o => isExcludedRetailer(o.retailer)); + if (excludedNow.length) { + console.log('🍺 KöPi: Angebote per Namens-Filter ausgeschlossen:', excludedNow.map(o => o.retailer).join(', ')); + } + + const localOnly = filtered + .filter(o => !isExcludedRetailer(o.retailer)) + .filter(o => isLocalBranch(o.retailer, o.realAddress)); + + for (const o of localOnly) { + o.retailerDisplay = o.retailer; + } + + // Deduplizieren: gleicher normalisierter Händler + gleicher Preis → ein Eintrag + const merged = new Map(); + for (const o of localOnly) { + const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || ''; + const key = `${normalizedRetailer}|${o.price}`; + if (merged.has(key)) { + const existing = merged.get(key); + if (o.description && (!existing.description || o.description.length < existing.description.length)) { + existing.description = o.description; + } + } else { + merged.set(key, { ...o }); + } + } + return [...merged.values()].map(o => ({ ...o, address: getAddress(o.retailer) })); +} + +async function scrapeMarktguru(forceRefresh = false) { + const cacheKey = 'koepi:mg'; + if (!forceRefresh) { + const cached = cacheGet(cacheKey); + if (cached) return cached; + } + + let offers, source; + try { + offers = await fetchOffersViaApi(); + source = 'marktguru.de (API)'; + console.log(`🍺 KöPi: ${offers.length} Rohtreffer über die marktguru-API`); + } catch (apiErr) { + console.error('🍺 KöPi: API-Weg fehlgeschlagen, falle zurück auf Puppeteer:', apiErr.message); + try { + offers = await fetchOffersViaPuppeteer(); + source = 'marktguru.de (Puppeteer-Fallback)'; + console.log(`🍺 KöPi: ${offers.length} Rohtreffer per Puppeteer (Fallback)`); + } catch (ppErr) { + console.error('🍺 KöPi: auch Puppeteer-Fallback fehlgeschlagen:', ppErr.message); + return { offers: [], error: `API: ${apiErr.message} | Puppeteer: ${ppErr.message}`, scrapedAt: new Date().toISOString() }; + } + } + + const unique = processOffers(offers); + const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source }; + cacheSet(cacheKey, result); + return result; +} + // ── Prospekte von kaufda ────────────────────────────────────────────────────── function fetchUrl(url) { return new Promise((resolve, reject) => {