feat: KoePi Prospekte-Tab auf marktguru+Whitelist umgestellt, KW/Wochentage, seiten-genaue Direktlinks

2026-07-17 21:57:55 +02:00
parent 8e8a5ffea5
commit 6ffbb90699
2 changed files with 98 additions and 84 deletions
--- a/backend/src/tools/koepi/routes.js
+++ b/backend/src/tools/koepi/routes.js
@@ -101,34 +101,44 @@ async function fetchLeafletFlightToLeafletMap() {
  return map;
 }

-// Holt für ein einzelnes Leaflet die closestStore-Adresse — das ist die
-// eigentliche, bisher fehlende Information, um Filialen derselben Kette zu
-// unterscheiden (z.B. "Netto Im Bonnefeld" vs. "Netto Hochstraße 10").
-// Holt für ein einzelnes Leaflet den kompletten closestStore-Datensatz von
-// marktguru (id, address, name, distanceInMeters) — die Store-ID ist die
-// stabile, exakte Grundlage für die Filial-Whitelist.
-async function fetchClosestStore(leafletId, cache) {
+// Holt für ein einzelnes Leaflet den kompletten Datensatz von marktguru:
+// closestStore (id/address/distanceInMeters — Grundlage der Filial-Whitelist),
+// name (enthält oft "(KW29 Do-Sa)" o.ä.) und children (Liste aller Angebote
+// mit ihrem pageIndex, für den seiten-genauen Direktlink).
+async function fetchLeafletDetail(leafletId, cache) {
  if (cache.has(leafletId)) return cache.get(leafletId);
-  const empty = { id: null, address: '', name: '', distanceMeters: null };
+  const empty = { id: leafletId, name: '', closestStore: { id: null, address: '', distanceMeters: null }, children: [] };
  try {
    const url = `https://api.marktguru.de/api/v1/leaflets/${leafletId}?as=mobiledetailed&latitude=${MG_LAT}&longitude=${MG_LON}&zipCode=${MG_ZIP}`;
    const res = await fetch(url, { headers: mgHeaders });
    if (!res.ok) { cache.set(leafletId, empty); return empty; }
    const json = await res.json();
    const store = json.closestStore;
-    const result = store
-      ? { id: store.id ?? null, address: store.address || '', name: store.name || '', distanceMeters: store.distanceInMeters ?? null }
-      : empty;
-    console.log(`🍺 KöPi: Leaflet ${leafletId} → closestStore: ${store ? `id=${store.id} "${store.address}" (${store.name}, ${Math.round(store.distanceInMeters||0)}m entfernt)` : 'keine'}`);
+    const result = {
+      id: json.id ?? leafletId,
+      name: json.name || '',
+      closestStore: store
+        ? { id: store.id ?? null, address: store.address || '', distanceMeters: store.distanceInMeters ?? null }
+        : { id: null, address: '', distanceMeters: null },
+      children: json.children || [],
+    };
+    console.log(`🍺 KöPi: Leaflet ${leafletId} ("${result.name}") → closestStore: ${store ? `id=${store.id} "${store.address}" (${Math.round(store.distanceInMeters||0)}m entfernt)` : 'keine'}`);
    cache.set(leafletId, result);
    return result;
  } catch (e) {
-    console.log(`🍺 KöPi: closestStore-Abruf für Leaflet ${leafletId} fehlgeschlagen: ${e.message}`);
+    console.log(`🍺 KöPi: Leaflet-Detail-Abruf für ${leafletId} fehlgeschlagen: ${e.message}`);
    cache.set(leafletId, empty);
    return empty;
  }
 }

+// Extrahiert "(KW29 Do-Sa)"-artige Klammerzusätze aus dem Leaflet-Namen, falls
+// vorhanden (nicht jede Filiale/jeder Prospekt hat sowas)
+function extractWeekInfo(name) {
+  const m = (name || '').match(/\([^)]*KW\s*\d+[^)]*\)/i);
+  return m ? m[0] : '';
+}
+
 // Löst die feste Liste gewünschter Filialen (LOCAL_STORE_SEED_LEAFLETS) einmalig
 // zu echten Store-IDs auf und speichert sie in der DB als Whitelist. Wird beim
 // ersten Scrape automatisch ausgeführt (falls die Tabelle leer ist) und kann
@@ -138,7 +148,8 @@ async function resolveLocalStores() {
  const cache = new Map();
  let resolved = 0;
  for (const seed of LOCAL_STORE_SEED_LEAFLETS) {
-    const store = await fetchClosestStore(seed.leafletId, cache);
+    const detail = await fetchLeafletDetail(seed.leafletId, cache);
+    const store = detail.closestStore;
    if (store.id != null) {
      db.prepare(`
        INSERT OR REPLACE INTO koepi_local_stores (store_id, retailer, address, resolved_at)
@@ -173,12 +184,12 @@ async function fetchOffersViaApi() {
    console.log('🍺 KöPi: leafletflights-Abruf fehlgeschlagen, Filial-Filter bleibt diesmal wirkungslos:', e.message);
  }
  const leafletCache = new Map();
-  const storeInfoByOfferId = new Map();
+  const detailByOfferId = new Map();
  for (const r of rawOffers) {
    const leafletId = flightToLeaflet.get(r.leafletFlightId);
    if (leafletId) {
-      const info = await fetchClosestStore(leafletId, leafletCache);
-      storeInfoByOfferId.set(r.id, info);
+      const detail = await fetchLeafletDetail(leafletId, leafletCache);
+      detailByOfferId.set(r.id, detail);
    }
  }

@@ -186,15 +197,29 @@ async function fetchOffersViaApi() {
    const advertiser = r.advertisers?.[0] || {};
    const priceNum = typeof r.price === 'number' ? r.price : parseFloat(r.price);
    const oldPriceNum = typeof r.oldPrice === 'number' ? r.oldPrice : parseFloat(r.oldPrice);
-    const storeInfo = storeInfoByOfferId.get(r.id) || { id: null, address: '', distanceMeters: null };
+    const detail = detailByOfferId.get(r.id);
+    const storeInfo = detail?.closestStore || { id: null, address: '', distanceMeters: null };
    const validity0 = r.validityDates?.[0] || {};
+    const weekInfo = extractWeekInfo(detail?.name);
+    const baseDateRange = validity0.from && validity0.to
+      ? `${formatIsoDate(validity0.from)} - ${formatIsoDate(validity0.to)}`
+      : formatIsoDate(validity0.to);
+    // Seiten-genauer Direktlink zum Angebot innerhalb des Prospekts — wird bei
+    // jedem Scrape frisch aus den aktuellen API-Daten ermittelt (children[].id
+    // == "offers/{id}"), funktioniert also automatisch weiter bei neuen
+    // Prospekten, ohne dass irgendwas hartkodiert werden muss.
+    const child = detail?.children?.find(c => c.id === `offers/${r.id}`);
+    const leafletUrl = detail
+      ? `https://www.marktguru.de/leaflets/${detail.id}${child ? `/page/${child.pageIndex}` : ''}`
+      : null;
    return {
      name: r.description || r.title || advertiser.name || '',
      brand: r.brand?.name || '',
      price: Number.isFinite(priceNum) ? `€ ${priceNum.toFixed(2).replace('.', ',')}` : (r.price || ''),
      oldPrice: Number.isFinite(oldPriceNum) ? `€ ${oldPriceNum.toFixed(2).replace('.', ',')}` : (r.oldPrice || ''),
      retailer: advertiser.name || '',
-      dateRange: validity0.from && validity0.to ? `${formatIsoDate(validity0.from)} - ${formatIsoDate(validity0.to)}` : formatIsoDate(validity0.to),
+      dateRange: weekInfo ? `${baseDateRange} ${weekInfo}` : baseDateRange,
+      weekInfo,
      validity: '',
      description: r.description || '',
      badge: '',
@@ -205,6 +230,7 @@ async function fetchOffersViaApi() {
      storeDistanceMeters: storeInfo.distanceMeters,
      leafletFlightId: r.leafletFlightId || null,
      offerId: r.id || null,
+      leafletUrl,
      image: r.image?.url || r.images?.[0]?.url || null,
    };
  });
@@ -386,80 +412,65 @@ async function scrapeMarktguru(forceRefresh = false) {
  return result;
 }

-// ── Prospekte von kaufda ──────────────────────────────────────────────────────
-function fetchUrl(url) {
-  return new Promise((resolve, reject) => {
-    const req = https.get(url, {
-      headers: {
-        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-        'Accept': 'text/html', 'Accept-Language': 'de-DE,de;q=0.9',
-      }
-    }, res => {
-      let d = '';
-      res.on('data', c => d += c);
-      res.on('end', () => resolve({ status: res.statusCode, body: d }));
-    });
-    req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); });
-    req.on('error', reject);
-  });
-}
-
-function extractNextData(html) {
-  const marker = 'application/json">';
-  const ni = html.indexOf('NEXT_DATA');
-  if (ni === -1) return null;
-  const st = html.indexOf(marker, ni) + marker.length;
-  const en = html.indexOf('</script>', st);
-  try { return JSON.parse(html.slice(st, en)); } catch { return null; }
-}
-
+// ── Prospekte von marktguru (dieselbe Filial-Whitelist wie bei den Angeboten) ──
 async function scrapeProspekte() {
  const cacheKey = 'koepi:prospekte';
  const cached = cacheGet(cacheKey);
  if (cached) return cached;

-  const { body } = await fetchUrl(
-    'https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=51.3750&lng=6.7680&zip=47259&city=Duisburg'
-  );
-  const nd = extractNextData(body);
-  const contents = nd?.props?.pageProps?.pageInformation?.shelfContents?.contents || [];
+  const knownCount = db.prepare('SELECT COUNT(*) AS n FROM koepi_local_stores').get().n;
+  if (knownCount === 0) {
+    console.log('🍺 KöPi Prospekte: Filial-Whitelist ist leer, löse sie jetzt einmalig auf...');
+    try { await resolveLocalStores(); }
+    catch (e) { console.error('🍺 KöPi Prospekte: Auflösen der Filial-Whitelist fehlgeschlagen:', e.message); }
+  }

-  const seen = new Map();
-  for (const item of contents) {
-    const c = item.content;
-    if (!c || item.contentType !== 'brochure') continue;
-    const id = c.contentId;
-    if (!id || seen.has(id)) continue;
-    const pub   = c.publisher?.name || '';
-    const store = c.closestStore;
-    seen.set(id, {
-      id, publisher: pub,
-      title:    c.title || '',
-      store:    store?.name || pub,
-      city:     store?.city || 'Duisburg',
-      street:   store ? `${store.street||''} ${store.streetNumber||''}`.trim() : '',
-      zip:      store?.zip || '47259',
-      image:    c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null,
-      validFrom: c.validFrom || null,
-      validTo:   c.validUntil || null,
-      pageCount: c.pageCount || null,
-      url:      `https://www.kaufda.de/Geschaefte/${encodeURIComponent(pub.replace(/\s+/g,'-'))}`,
-      isTarget: isTargetRetailer(pub),
-      badges:   c.contentBadges?.map(b => b.name) || [],
+  const flightsUrl = `https://api.marktguru.de/api/v1/leafletflights?as=mobile&limit=100&zipCode=${MG_ZIP}`;
+  const flightsRes = await fetch(flightsUrl, { headers: mgHeaders });
+  if (!flightsRes.ok) throw new Error(`leafletflights HTTP ${flightsRes.status}`);
+  const flightsJson = await flightsRes.json();
+  console.log(`🍺 KöPi Prospekte: ${flightsJson.results?.length} von ${flightsJson.totalResults} Prospekt-Kampagnen geladen`);
+
+  // Nur Kampagnen der gewünschten Ketten überhaupt im Detail auflösen (spart API-Calls)
+  const candidates = (flightsJson.results || []).filter(f => isTargetRetailer(f.advertiser?.name));
+
+  const leafletCache = new Map();
+  const all = [];
+  for (const f of candidates) {
+    if (!f.mainLeafletId) continue;
+    const detail = await fetchLeafletDetail(f.mainLeafletId, leafletCache);
+    const store = detail.closestStore;
+    const weekInfo = extractWeekInfo(detail.name);
+    all.push({
+      id: f.id,
+      publisher: f.advertiser?.name || '',
+      title: detail.name || f.advertiser?.name || '',
+      weekInfo,
+      street: store.address || '',
+      zip: MG_ZIP,
+      city: 'Duisburg',
+      storeId: store.id,
+      image: null,
+      validFrom: f.validFrom || null,
+      validTo: f.validTo || null,
+      pageCount: f.pageCount || null,
+      url: `https://www.marktguru.de/leaflets/${f.mainLeafletId}`,
+      isTarget: isKnownLocalStore(store.id) && store.id != null,
+      badges: [],
    });
  }

-  const all = [...seen.values()];
  const result = {
    targeted: all.filter(b => b.isTarget),
    others:   all.filter(b => !b.isTarget),
    scrapedAt: new Date().toISOString(),
  };
+  console.log(`🍺 KöPi Prospekte: ${result.targeted.length} eigene Filiale(n), ${result.others.length} andere Filialen derselben Ketten`);
  cacheSet(cacheKey, result);
  return result;
 }

-// ── Bild-Proxy (für kaufda Bilder) ───────────────────────────────────────────
+// ── Bild-Proxy (für evtl. externe Bilder) ───────────────────────────────────
 router.get('/img', authenticate, (req, res) => {
  const url = req.query.url;
  if (!url || !url.startsWith('https://')) return res.status(400).end();
--- a/frontend/src/tools/koepi.jsx
+++ b/frontend/src/tools/koepi.jsx
@@ -79,25 +79,29 @@ function OfferCard({ offer }) {
      </div>

      {/* Gültigkeit */}
-      <div style={{ borderTop:'1px solid rgba(255,255,255,0.06)', paddingTop:6, display:'flex', gap:12, flexWrap:'wrap' }}>
+      <div style={{ borderTop:'1px solid rgba(255,255,255,0.06)', paddingTop:6, display:'flex', gap:12, flexWrap:'wrap', alignItems:'center' }}>
        {offer.dateRange && (
          <span style={{ color:'rgba(255,255,255,0.35)', fontFamily:'monospace', fontSize:10 }}>📅 {offer.dateRange}</span>
        )}
+        {offer.leafletUrl && (
+          <a href={offer.leafletUrl} target="_blank" rel="noopener noreferrer"
+            style={{ color:'#4ecdc4', fontFamily:'monospace', fontSize:10, textDecoration:'none', marginLeft:'auto' }}>
+            🔗 Im Prospekt ansehen
+          </a>
+        )}
      </div>
    </div>
  );
 }

-// ── Prospekt-Karte (kaufda) ───────────────────────────────────────────────────
+// ── Prospekt-Karte (marktguru) ────────────────────────────────────────────────
 function ProspektCard({ b }) {
  const color = publisherColor(b.publisher);
-  const isNew = b.badges?.includes('new');
-  const expiring = b.badges?.includes('expiring_soon');
  return (
    <a href={b.url} target="_blank" rel="noopener noreferrer" style={{ textDecoration:'none' }}>
      <div style={{
        ...S.card, padding:0, overflow:'hidden', cursor:'pointer',
-        border: expiring ? '1px solid rgba(239,68,68,0.4)' : '1px solid rgba(255,255,255,0.08)',
+        border: '1px solid rgba(255,255,255,0.08)',
      }}>
        {b.image && (
          <img src={`/api/tools/koepi/img?url=${encodeURIComponent(b.image)}`}
@@ -111,15 +115,14 @@ function ProspektCard({ b }) {
              borderRadius:4, padding:'2px 8px', color, fontFamily:'monospace', fontSize:10, fontWeight:700 }}>
              {b.publisher}
            </div>
-            {expiring && <span style={{ background:'#ef4444', color:'#fff', borderRadius:4, padding:'2px 6px', fontSize:9, fontFamily:'monospace', fontWeight:700 }}>ENDET BALD</span>}
          </div>
-          <div style={{ color:'#fff', fontFamily:'monospace', fontSize:11, marginBottom:3 }}>{b.title}</div>
          <div style={{ color:'rgba(255,255,255,0.4)', fontFamily:'monospace', fontSize:10, marginBottom:2 }}>
-            📍 {b.street || b.store}
+            📍 {b.street}
          </div>
-          {(b.validFrom || b.validTo) && (
+          {(b.validFrom || b.validTo || b.weekInfo) && (
            <div style={{ color:'rgba(255,255,255,0.3)', fontFamily:'monospace', fontSize:9, marginTop:4 }}>
              {b.validFrom && `Ab ${fmtDate(b.validFrom)}`}{b.validFrom&&b.validTo&&' · '}{b.validTo&&`Bis ${fmtDate(b.validTo)}`}
+              {b.weekInfo && ` ${b.weekInfo}`}
            </div>
          )}
        </div>