From 6ffbb9069961034b0884942e0fcb3d09a8d85d72 Mon Sep 17 00:00:00 2001 From: Dicken Date: Fri, 17 Jul 2026 21:57:55 +0200 Subject: [PATCH] feat: KoePi Prospekte-Tab auf marktguru+Whitelist umgestellt, KW/Wochentage, seiten-genaue Direktlinks --- backend/src/tools/koepi/routes.js | 161 ++++++++++++++++-------------- frontend/src/tools/koepi.jsx | 21 ++-- 2 files changed, 98 insertions(+), 84 deletions(-) diff --git a/backend/src/tools/koepi/routes.js b/backend/src/tools/koepi/routes.js index 44de872..b6c79c6 100644 --- a/backend/src/tools/koepi/routes.js +++ b/backend/src/tools/koepi/routes.js @@ -101,34 +101,44 @@ async function fetchLeafletFlightToLeafletMap() { return map; } -// Holt für ein einzelnes Leaflet die closestStore-Adresse — das ist die -// eigentliche, bisher fehlende Information, um Filialen derselben Kette zu -// unterscheiden (z.B. "Netto Im Bonnefeld" vs. "Netto Hochstraße 10"). -// Holt für ein einzelnes Leaflet den kompletten closestStore-Datensatz von -// marktguru (id, address, name, distanceInMeters) — die Store-ID ist die -// stabile, exakte Grundlage für die Filial-Whitelist. -async function fetchClosestStore(leafletId, cache) { +// Holt für ein einzelnes Leaflet den kompletten Datensatz von marktguru: +// closestStore (id/address/distanceInMeters — Grundlage der Filial-Whitelist), +// name (enthält oft "(KW29 Do-Sa)" o.ä.) und children (Liste aller Angebote +// mit ihrem pageIndex, für den seiten-genauen Direktlink). +async function fetchLeafletDetail(leafletId, cache) { if (cache.has(leafletId)) return cache.get(leafletId); - const empty = { id: null, address: '', name: '', distanceMeters: null }; + const empty = { id: leafletId, name: '', closestStore: { id: null, address: '', distanceMeters: null }, children: [] }; try { const url = `https://api.marktguru.de/api/v1/leaflets/${leafletId}?as=mobiledetailed&latitude=${MG_LAT}&longitude=${MG_LON}&zipCode=${MG_ZIP}`; const res = await fetch(url, { headers: mgHeaders }); if (!res.ok) { cache.set(leafletId, empty); return empty; } const json = await res.json(); const store = json.closestStore; - const result = store - ? { id: store.id ?? null, address: store.address || '', name: store.name || '', distanceMeters: store.distanceInMeters ?? null } - : empty; - console.log(`🍺 KöPi: Leaflet ${leafletId} → closestStore: ${store ? `id=${store.id} "${store.address}" (${store.name}, ${Math.round(store.distanceInMeters||0)}m entfernt)` : 'keine'}`); + const result = { + id: json.id ?? leafletId, + name: json.name || '', + closestStore: store + ? { id: store.id ?? null, address: store.address || '', distanceMeters: store.distanceInMeters ?? null } + : { id: null, address: '', distanceMeters: null }, + children: json.children || [], + }; + console.log(`🍺 KöPi: Leaflet ${leafletId} ("${result.name}") → closestStore: ${store ? `id=${store.id} "${store.address}" (${Math.round(store.distanceInMeters||0)}m entfernt)` : 'keine'}`); cache.set(leafletId, result); return result; } catch (e) { - console.log(`🍺 KöPi: closestStore-Abruf für Leaflet ${leafletId} fehlgeschlagen: ${e.message}`); + console.log(`🍺 KöPi: Leaflet-Detail-Abruf für ${leafletId} fehlgeschlagen: ${e.message}`); cache.set(leafletId, empty); return empty; } } +// Extrahiert "(KW29 Do-Sa)"-artige Klammerzusätze aus dem Leaflet-Namen, falls +// vorhanden (nicht jede Filiale/jeder Prospekt hat sowas) +function extractWeekInfo(name) { + const m = (name || '').match(/\([^)]*KW\s*\d+[^)]*\)/i); + return m ? m[0] : ''; +} + // Löst die feste Liste gewünschter Filialen (LOCAL_STORE_SEED_LEAFLETS) einmalig // zu echten Store-IDs auf und speichert sie in der DB als Whitelist. Wird beim // ersten Scrape automatisch ausgeführt (falls die Tabelle leer ist) und kann @@ -138,7 +148,8 @@ async function resolveLocalStores() { const cache = new Map(); let resolved = 0; for (const seed of LOCAL_STORE_SEED_LEAFLETS) { - const store = await fetchClosestStore(seed.leafletId, cache); + const detail = await fetchLeafletDetail(seed.leafletId, cache); + const store = detail.closestStore; if (store.id != null) { db.prepare(` INSERT OR REPLACE INTO koepi_local_stores (store_id, retailer, address, resolved_at) @@ -173,12 +184,12 @@ async function fetchOffersViaApi() { console.log('🍺 KöPi: leafletflights-Abruf fehlgeschlagen, Filial-Filter bleibt diesmal wirkungslos:', e.message); } const leafletCache = new Map(); - const storeInfoByOfferId = new Map(); + const detailByOfferId = new Map(); for (const r of rawOffers) { const leafletId = flightToLeaflet.get(r.leafletFlightId); if (leafletId) { - const info = await fetchClosestStore(leafletId, leafletCache); - storeInfoByOfferId.set(r.id, info); + const detail = await fetchLeafletDetail(leafletId, leafletCache); + detailByOfferId.set(r.id, detail); } } @@ -186,15 +197,29 @@ async function fetchOffersViaApi() { const advertiser = r.advertisers?.[0] || {}; const priceNum = typeof r.price === 'number' ? r.price : parseFloat(r.price); const oldPriceNum = typeof r.oldPrice === 'number' ? r.oldPrice : parseFloat(r.oldPrice); - const storeInfo = storeInfoByOfferId.get(r.id) || { id: null, address: '', distanceMeters: null }; + const detail = detailByOfferId.get(r.id); + const storeInfo = detail?.closestStore || { id: null, address: '', distanceMeters: null }; const validity0 = r.validityDates?.[0] || {}; + const weekInfo = extractWeekInfo(detail?.name); + const baseDateRange = validity0.from && validity0.to + ? `${formatIsoDate(validity0.from)} - ${formatIsoDate(validity0.to)}` + : formatIsoDate(validity0.to); + // Seiten-genauer Direktlink zum Angebot innerhalb des Prospekts — wird bei + // jedem Scrape frisch aus den aktuellen API-Daten ermittelt (children[].id + // == "offers/{id}"), funktioniert also automatisch weiter bei neuen + // Prospekten, ohne dass irgendwas hartkodiert werden muss. + const child = detail?.children?.find(c => c.id === `offers/${r.id}`); + const leafletUrl = detail + ? `https://www.marktguru.de/leaflets/${detail.id}${child ? `/page/${child.pageIndex}` : ''}` + : null; return { name: r.description || r.title || advertiser.name || '', brand: r.brand?.name || '', price: Number.isFinite(priceNum) ? `€ ${priceNum.toFixed(2).replace('.', ',')}` : (r.price || ''), oldPrice: Number.isFinite(oldPriceNum) ? `€ ${oldPriceNum.toFixed(2).replace('.', ',')}` : (r.oldPrice || ''), retailer: advertiser.name || '', - dateRange: validity0.from && validity0.to ? `${formatIsoDate(validity0.from)} - ${formatIsoDate(validity0.to)}` : formatIsoDate(validity0.to), + dateRange: weekInfo ? `${baseDateRange} ${weekInfo}` : baseDateRange, + weekInfo, validity: '', description: r.description || '', badge: '', @@ -205,6 +230,7 @@ async function fetchOffersViaApi() { storeDistanceMeters: storeInfo.distanceMeters, leafletFlightId: r.leafletFlightId || null, offerId: r.id || null, + leafletUrl, image: r.image?.url || r.images?.[0]?.url || null, }; }); @@ -386,80 +412,65 @@ async function scrapeMarktguru(forceRefresh = false) { return result; } -// ── Prospekte von kaufda ────────────────────────────────────────────────────── -function fetchUrl(url) { - return new Promise((resolve, reject) => { - const req = https.get(url, { - headers: { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Accept': 'text/html', 'Accept-Language': 'de-DE,de;q=0.9', - } - }, res => { - let d = ''; - res.on('data', c => d += c); - res.on('end', () => resolve({ status: res.statusCode, body: d })); - }); - req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); }); - req.on('error', reject); - }); -} - -function extractNextData(html) { - const marker = 'application/json">'; - const ni = html.indexOf('NEXT_DATA'); - if (ni === -1) return null; - const st = html.indexOf(marker, ni) + marker.length; - const en = html.indexOf('', st); - try { return JSON.parse(html.slice(st, en)); } catch { return null; } -} - +// ── Prospekte von marktguru (dieselbe Filial-Whitelist wie bei den Angeboten) ── async function scrapeProspekte() { const cacheKey = 'koepi:prospekte'; const cached = cacheGet(cacheKey); if (cached) return cached; - const { body } = await fetchUrl( - 'https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=51.3750&lng=6.7680&zip=47259&city=Duisburg' - ); - const nd = extractNextData(body); - const contents = nd?.props?.pageProps?.pageInformation?.shelfContents?.contents || []; + const knownCount = db.prepare('SELECT COUNT(*) AS n FROM koepi_local_stores').get().n; + if (knownCount === 0) { + console.log('🍺 KöPi Prospekte: Filial-Whitelist ist leer, löse sie jetzt einmalig auf...'); + try { await resolveLocalStores(); } + catch (e) { console.error('🍺 KöPi Prospekte: Auflösen der Filial-Whitelist fehlgeschlagen:', e.message); } + } - const seen = new Map(); - for (const item of contents) { - const c = item.content; - if (!c || item.contentType !== 'brochure') continue; - const id = c.contentId; - if (!id || seen.has(id)) continue; - const pub = c.publisher?.name || ''; - const store = c.closestStore; - seen.set(id, { - id, publisher: pub, - title: c.title || '', - store: store?.name || pub, - city: store?.city || 'Duisburg', - street: store ? `${store.street||''} ${store.streetNumber||''}`.trim() : '', - zip: store?.zip || '47259', - image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null, - validFrom: c.validFrom || null, - validTo: c.validUntil || null, - pageCount: c.pageCount || null, - url: `https://www.kaufda.de/Geschaefte/${encodeURIComponent(pub.replace(/\s+/g,'-'))}`, - isTarget: isTargetRetailer(pub), - badges: c.contentBadges?.map(b => b.name) || [], + const flightsUrl = `https://api.marktguru.de/api/v1/leafletflights?as=mobile&limit=100&zipCode=${MG_ZIP}`; + const flightsRes = await fetch(flightsUrl, { headers: mgHeaders }); + if (!flightsRes.ok) throw new Error(`leafletflights HTTP ${flightsRes.status}`); + const flightsJson = await flightsRes.json(); + console.log(`🍺 KöPi Prospekte: ${flightsJson.results?.length} von ${flightsJson.totalResults} Prospekt-Kampagnen geladen`); + + // Nur Kampagnen der gewünschten Ketten überhaupt im Detail auflösen (spart API-Calls) + const candidates = (flightsJson.results || []).filter(f => isTargetRetailer(f.advertiser?.name)); + + const leafletCache = new Map(); + const all = []; + for (const f of candidates) { + if (!f.mainLeafletId) continue; + const detail = await fetchLeafletDetail(f.mainLeafletId, leafletCache); + const store = detail.closestStore; + const weekInfo = extractWeekInfo(detail.name); + all.push({ + id: f.id, + publisher: f.advertiser?.name || '', + title: detail.name || f.advertiser?.name || '', + weekInfo, + street: store.address || '', + zip: MG_ZIP, + city: 'Duisburg', + storeId: store.id, + image: null, + validFrom: f.validFrom || null, + validTo: f.validTo || null, + pageCount: f.pageCount || null, + url: `https://www.marktguru.de/leaflets/${f.mainLeafletId}`, + isTarget: isKnownLocalStore(store.id) && store.id != null, + badges: [], }); } - const all = [...seen.values()]; const result = { targeted: all.filter(b => b.isTarget), others: all.filter(b => !b.isTarget), scrapedAt: new Date().toISOString(), }; + console.log(`🍺 KöPi Prospekte: ${result.targeted.length} eigene Filiale(n), ${result.others.length} andere Filialen derselben Ketten`); cacheSet(cacheKey, result); return result; } -// ── Bild-Proxy (für kaufda Bilder) ─────────────────────────────────────────── +// ── Bild-Proxy (für evtl. externe Bilder) ─────────────────────────────────── router.get('/img', authenticate, (req, res) => { const url = req.query.url; if (!url || !url.startsWith('https://')) return res.status(400).end(); diff --git a/frontend/src/tools/koepi.jsx b/frontend/src/tools/koepi.jsx index d8a0901..6465b4b 100644 --- a/frontend/src/tools/koepi.jsx +++ b/frontend/src/tools/koepi.jsx @@ -79,25 +79,29 @@ function OfferCard({ offer }) { {/* Gültigkeit */} -
+
{offer.dateRange && ( 📅 {offer.dateRange} )} + {offer.leafletUrl && ( + + 🔗 Im Prospekt ansehen + + )}
); } -// ── Prospekt-Karte (kaufda) ─────────────────────────────────────────────────── +// ── Prospekt-Karte (marktguru) ──────────────────────────────────────────────── function ProspektCard({ b }) { const color = publisherColor(b.publisher); - const isNew = b.badges?.includes('new'); - const expiring = b.badges?.includes('expiring_soon'); return (
{b.image && ( {b.publisher}
- {expiring && ENDET BALD} -
{b.title}
- 📍 {b.street || b.store} + 📍 {b.street}
- {(b.validFrom || b.validTo) && ( + {(b.validFrom || b.validTo || b.weekInfo) && (
{b.validFrom && `Ab ${fmtDate(b.validFrom)}`}{b.validFrom&&b.validTo&&' · '}{b.validTo&&`Bis ${fmtDate(b.validTo)}`} + {b.weekInfo && ` ${b.weekInfo}`}
)}