Dicken/dickendock
Template
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: KoePi Prospekte-Tab auf marktguru+Whitelist umgestellt, KW/Wochentage, seiten-genaue Direktlinks

Browse Source
This commit is contained in:
Dicken
2026-07-17 21:57:55 +02:00
parent 8e8a5ffea5
commit 6ffbb90699
2 changed files with 98 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

161
backend/src/tools/koepi/routes.js
View File

@@ -101,34 +101,44 @@ async function fetchLeafletFlightToLeafletMap() {
return map; return map;
} }
// Holt für ein einzelnes Leaflet die closestStore-Adresse — das ist die // Holt für ein einzelnes Leaflet den kompletten Datensatz von marktguru:
// eigentliche, bisher fehlende Information, um Filialen derselben Kette zu // closestStore (id/address/distanceInMeters — Grundlage der Filial-Whitelist),
// unterscheiden (z.B. "Netto Im Bonnefeld" vs. "Netto Hochstraße 10"). // name (enthält oft "(KW29 Do-Sa)" o.ä.) und children (Liste aller Angebote
// Holt für ein einzelnes Leaflet den kompletten closestStore-Datensatz von // mit ihrem pageIndex, für den seiten-genauen Direktlink).
// marktguru (id, address, name, distanceInMeters) — die Store-ID ist die async function fetchLeafletDetail(leafletId, cache) {
// stabile, exakte Grundlage für die Filial-Whitelist.
async function fetchClosestStore(leafletId, cache) {
if (cache.has(leafletId)) return cache.get(leafletId); if (cache.has(leafletId)) return cache.get(leafletId);
const empty = { id: null, address: '', name: '', distanceMeters: null }; const empty = { id: leafletId, name: '', closestStore: { id: null, address: '', distanceMeters: null }, children: [] };
try { try {
const url = `https://api.marktguru.de/api/v1/leaflets/${leafletId}?as=mobiledetailed&latitude=${MG_LAT}&longitude=${MG_LON}&zipCode=${MG_ZIP}`; const url = `https://api.marktguru.de/api/v1/leaflets/${leafletId}?as=mobiledetailed&latitude=${MG_LAT}&longitude=${MG_LON}&zipCode=${MG_ZIP}`;
const res = await fetch(url, { headers: mgHeaders }); const res = await fetch(url, { headers: mgHeaders });
if (!res.ok) { cache.set(leafletId, empty); return empty; } if (!res.ok) { cache.set(leafletId, empty); return empty; }
const json = await res.json(); const json = await res.json();
const store = json.closestStore; const store = json.closestStore;
const result = store const result = {
? { id: store.id ?? null, address: store.address || '', name: store.name || '', distanceMeters: store.distanceInMeters ?? null } id: json.id ?? leafletId,
: empty; name: json.name || '',
console.log(`🍺 KöPi: Leaflet ${leafletId} → closestStore: ${store ? `id=${store.id} "${store.address}" (${store.name}, ${Math.round(store.distanceInMeters||0)}m entfernt)` : 'keine'}`); closestStore: store
? { id: store.id ?? null, address: store.address || '', distanceMeters: store.distanceInMeters ?? null }
: { id: null, address: '', distanceMeters: null },
children: json.children || [],
};
console.log(`🍺 KöPi: Leaflet ${leafletId} ("${result.name}") → closestStore: ${store ? `id=${store.id} "${store.address}" (${Math.round(store.distanceInMeters||0)}m entfernt)` : 'keine'}`);
cache.set(leafletId, result); cache.set(leafletId, result);
return result; return result;
} catch (e) { } catch (e) {
console.log(`🍺 KöPi: closestStore-Abruf für Leaflet ${leafletId} fehlgeschlagen: ${e.message}`); console.log(`🍺 KöPi: Leaflet-Detail-Abruf für ${leafletId} fehlgeschlagen: ${e.message}`);
cache.set(leafletId, empty); cache.set(leafletId, empty);
return empty; return empty;
} }
} }
// Extrahiert "(KW29 Do-Sa)"-artige Klammerzusätze aus dem Leaflet-Namen, falls
// vorhanden (nicht jede Filiale/jeder Prospekt hat sowas)
function extractWeekInfo(name) {
const m = (name || '').match(/\([^)]*KW\s*\d+[^)]*\)/i);
return m ? m[0] : '';
}
// Löst die feste Liste gewünschter Filialen (LOCAL_STORE_SEED_LEAFLETS) einmalig // Löst die feste Liste gewünschter Filialen (LOCAL_STORE_SEED_LEAFLETS) einmalig
// zu echten Store-IDs auf und speichert sie in der DB als Whitelist. Wird beim // zu echten Store-IDs auf und speichert sie in der DB als Whitelist. Wird beim
// ersten Scrape automatisch ausgeführt (falls die Tabelle leer ist) und kann // ersten Scrape automatisch ausgeführt (falls die Tabelle leer ist) und kann
@@ -138,7 +148,8 @@ async function resolveLocalStores() {
const cache = new Map(); const cache = new Map();
let resolved = 0; let resolved = 0;
for (const seed of LOCAL_STORE_SEED_LEAFLETS) { for (const seed of LOCAL_STORE_SEED_LEAFLETS) {
const store = await fetchClosestStore(seed.leafletId, cache); const detail = await fetchLeafletDetail(seed.leafletId, cache);
const store = detail.closestStore;
if (store.id != null) { if (store.id != null) {
db.prepare(` db.prepare(`
INSERT OR REPLACE INTO koepi_local_stores (store_id, retailer, address, resolved_at) INSERT OR REPLACE INTO koepi_local_stores (store_id, retailer, address, resolved_at)
@@ -173,12 +184,12 @@ async function fetchOffersViaApi() {
console.log('🍺 KöPi: leafletflights-Abruf fehlgeschlagen, Filial-Filter bleibt diesmal wirkungslos:', e.message); console.log('🍺 KöPi: leafletflights-Abruf fehlgeschlagen, Filial-Filter bleibt diesmal wirkungslos:', e.message);
} }
const leafletCache = new Map(); const leafletCache = new Map();
const storeInfoByOfferId = new Map(); const detailByOfferId = new Map();
for (const r of rawOffers) { for (const r of rawOffers) {
const leafletId = flightToLeaflet.get(r.leafletFlightId); const leafletId = flightToLeaflet.get(r.leafletFlightId);
if (leafletId) { if (leafletId) {
const info = await fetchClosestStore(leafletId, leafletCache); const detail = await fetchLeafletDetail(leafletId, leafletCache);
storeInfoByOfferId.set(r.id, info); detailByOfferId.set(r.id, detail);
} }
} }
@@ -186,15 +197,29 @@ async function fetchOffersViaApi() {
const advertiser = r.advertisers?.[0] || {}; const advertiser = r.advertisers?.[0] || {};
const priceNum = typeof r.price === 'number' ? r.price : parseFloat(r.price); const priceNum = typeof r.price === 'number' ? r.price : parseFloat(r.price);
const oldPriceNum = typeof r.oldPrice === 'number' ? r.oldPrice : parseFloat(r.oldPrice); const oldPriceNum = typeof r.oldPrice === 'number' ? r.oldPrice : parseFloat(r.oldPrice);
const storeInfo = storeInfoByOfferId.get(r.id) || { id: null, address: '', distanceMeters: null }; const detail = detailByOfferId.get(r.id);
const storeInfo = detail?.closestStore || { id: null, address: '', distanceMeters: null };
const validity0 = r.validityDates?.[0] || {}; const validity0 = r.validityDates?.[0] || {};
const weekInfo = extractWeekInfo(detail?.name);
const baseDateRange = validity0.from && validity0.to
? `${formatIsoDate(validity0.from)} - ${formatIsoDate(validity0.to)}`
: formatIsoDate(validity0.to);
// Seiten-genauer Direktlink zum Angebot innerhalb des Prospekts — wird bei
// jedem Scrape frisch aus den aktuellen API-Daten ermittelt (children[].id
// == "offers/{id}"), funktioniert also automatisch weiter bei neuen
// Prospekten, ohne dass irgendwas hartkodiert werden muss.
const child = detail?.children?.find(c => c.id === `offers/${r.id}`);
const leafletUrl = detail
? `https://www.marktguru.de/leaflets/${detail.id}${child ? `/page/${child.pageIndex}` : ''}`
: null;
return { return {
name: r.description || r.title || advertiser.name || '', name: r.description || r.title || advertiser.name || '',
brand: r.brand?.name || '', brand: r.brand?.name || '',
price: Number.isFinite(priceNum) ? `€ ${priceNum.toFixed(2).replace('.', ',')}` : (r.price || ''), price: Number.isFinite(priceNum) ? `€ ${priceNum.toFixed(2).replace('.', ',')}` : (r.price || ''),
oldPrice: Number.isFinite(oldPriceNum) ? `€ ${oldPriceNum.toFixed(2).replace('.', ',')}` : (r.oldPrice || ''), oldPrice: Number.isFinite(oldPriceNum) ? `€ ${oldPriceNum.toFixed(2).replace('.', ',')}` : (r.oldPrice || ''),
retailer: advertiser.name || '', retailer: advertiser.name || '',
dateRange: validity0.from && validity0.to ? `${formatIsoDate(validity0.from)} - ${formatIsoDate(validity0.to)}` : formatIsoDate(validity0.to), dateRange: weekInfo ? `${baseDateRange} ${weekInfo}` : baseDateRange,
weekInfo,
validity: '', validity: '',
description: r.description || '', description: r.description || '',
badge: '', badge: '',
@@ -205,6 +230,7 @@ async function fetchOffersViaApi() {
storeDistanceMeters: storeInfo.distanceMeters, storeDistanceMeters: storeInfo.distanceMeters,
leafletFlightId: r.leafletFlightId || null, leafletFlightId: r.leafletFlightId || null,
offerId: r.id || null, offerId: r.id || null,
leafletUrl,
image: r.image?.url || r.images?.[0]?.url || null, image: r.image?.url || r.images?.[0]?.url || null,
}; };
}); });
@@ -386,80 +412,65 @@ async function scrapeMarktguru(forceRefresh = false) {
return result; return result;
} }
// ── Prospekte von kaufda ────────────────────────────────────────────────────── // ── Prospekte von marktguru (dieselbe Filial-Whitelist wie bei den Angeboten) ──
function fetchUrl(url) {
return new Promise((resolve, reject) => {
const req = https.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html', 'Accept-Language': 'de-DE,de;q=0.9',
}
}, res => {
let d = '';
res.on('data', c => d += c);
res.on('end', () => resolve({ status: res.statusCode, body: d }));
});
req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); });
req.on('error', reject);
});
}
function extractNextData(html) {
const marker = 'application/json">';
const ni = html.indexOf('NEXT_DATA');
if (ni === -1) return null;
const st = html.indexOf(marker, ni) + marker.length;
const en = html.indexOf('</script>', st);
try { return JSON.parse(html.slice(st, en)); } catch { return null; }
}
async function scrapeProspekte() { async function scrapeProspekte() {
const cacheKey = 'koepi:prospekte'; const cacheKey = 'koepi:prospekte';
const cached = cacheGet(cacheKey); const cached = cacheGet(cacheKey);
if (cached) return cached; if (cached) return cached;
const { body } = await fetchUrl( const knownCount = db.prepare('SELECT COUNT(*) AS n FROM koepi_local_stores').get().n;
'https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=51.3750&lng=6.7680&zip=47259&city=Duisburg' if (knownCount === 0) {
); console.log('🍺 KöPi Prospekte: Filial-Whitelist ist leer, löse sie jetzt einmalig auf...');
const nd = extractNextData(body); try { await resolveLocalStores(); }
const contents = nd?.props?.pageProps?.pageInformation?.shelfContents?.contents || []; catch (e) { console.error('🍺 KöPi Prospekte: Auflösen der Filial-Whitelist fehlgeschlagen:', e.message); }
}
const seen = new Map(); const flightsUrl = `https://api.marktguru.de/api/v1/leafletflights?as=mobile&limit=100&zipCode=${MG_ZIP}`;
for (const item of contents) { const flightsRes = await fetch(flightsUrl, { headers: mgHeaders });
const c = item.content; if (!flightsRes.ok) throw new Error(`leafletflights HTTP ${flightsRes.status}`);
if (!c || item.contentType !== 'brochure') continue; const flightsJson = await flightsRes.json();
const id = c.contentId; console.log(`🍺 KöPi Prospekte: ${flightsJson.results?.length} von ${flightsJson.totalResults} Prospekt-Kampagnen geladen`);
if (!id || seen.has(id)) continue;
const pub = c.publisher?.name || ''; // Nur Kampagnen der gewünschten Ketten überhaupt im Detail auflösen (spart API-Calls)
const store = c.closestStore; const candidates = (flightsJson.results || []).filter(f => isTargetRetailer(f.advertiser?.name));
seen.set(id, {
id, publisher: pub, const leafletCache = new Map();
title: c.title || '', const all = [];
store: store?.name || pub, for (const f of candidates) {
city: store?.city || 'Duisburg', if (!f.mainLeafletId) continue;
street: store ? `${store.street||''} ${store.streetNumber||''}`.trim() : '', const detail = await fetchLeafletDetail(f.mainLeafletId, leafletCache);
zip: store?.zip || '47259', const store = detail.closestStore;
image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null, const weekInfo = extractWeekInfo(detail.name);
validFrom: c.validFrom || null, all.push({
validTo: c.validUntil || null, id: f.id,
pageCount: c.pageCount || null, publisher: f.advertiser?.name || '',
url: `https://www.kaufda.de/Geschaefte/${encodeURIComponent(pub.replace(/\s+/g,'-'))}`, title: detail.name || f.advertiser?.name || '',
isTarget: isTargetRetailer(pub), weekInfo,
badges: c.contentBadges?.map(b => b.name) || [], street: store.address || '',
zip: MG_ZIP,
city: 'Duisburg',
storeId: store.id,
image: null,
validFrom: f.validFrom || null,
validTo: f.validTo || null,
pageCount: f.pageCount || null,
url: `https://www.marktguru.de/leaflets/${f.mainLeafletId}`,
isTarget: isKnownLocalStore(store.id) && store.id != null,
badges: [],
}); });
} }
const all = [...seen.values()];
const result = { const result = {
targeted: all.filter(b => b.isTarget), targeted: all.filter(b => b.isTarget),
others: all.filter(b => !b.isTarget), others: all.filter(b => !b.isTarget),
scrapedAt: new Date().toISOString(), scrapedAt: new Date().toISOString(),
}; };
console.log(`🍺 KöPi Prospekte: ${result.targeted.length} eigene Filiale(n), ${result.others.length} andere Filialen derselben Ketten`);
cacheSet(cacheKey, result); cacheSet(cacheKey, result);
return result; return result;
} }
// ── Bild-Proxy (für kaufda Bilder) ─────────────────────────────────────────── // ── Bild-Proxy (für evtl. externe Bilder) ───────────────────────────────────
router.get('/img', authenticate, (req, res) => { router.get('/img', authenticate, (req, res) => {
const url = req.query.url; const url = req.query.url;
if (!url || !url.startsWith('https://')) return res.status(400).end(); if (!url || !url.startsWith('https://')) return res.status(400).end();

21
frontend/src/tools/koepi.jsx
View File

@@ -79,25 +79,29 @@ function OfferCard({ offer }) {
</div> </div>
{/* Gültigkeit */} {/* Gültigkeit */}
<div style={{ borderTop:'1px solid rgba(255,255,255,0.06)', paddingTop:6, display:'flex', gap:12, flexWrap:'wrap' }}> <div style={{ borderTop:'1px solid rgba(255,255,255,0.06)', paddingTop:6, display:'flex', gap:12, flexWrap:'wrap', alignItems:'center' }}>
{offer.dateRange && ( {offer.dateRange && (
<span style={{ color:'rgba(255,255,255,0.35)', fontFamily:'monospace', fontSize:10 }}>📅 {offer.dateRange}</span> <span style={{ color:'rgba(255,255,255,0.35)', fontFamily:'monospace', fontSize:10 }}>📅 {offer.dateRange}</span>
)} )}
{offer.leafletUrl && (
<a href={offer.leafletUrl} target="_blank" rel="noopener noreferrer"
style={{ color:'#4ecdc4', fontFamily:'monospace', fontSize:10, textDecoration:'none', marginLeft:'auto' }}>
🔗 Im Prospekt ansehen
</a>
)}
</div> </div>
</div> </div>
); );
} }
// ── Prospekt-Karte (kaufda) ─────────────────────────────────────────────────── // ── Prospekt-Karte (marktguru) ────────────────────────────────────────────────
function ProspektCard({ b }) { function ProspektCard({ b }) {
const color = publisherColor(b.publisher); const color = publisherColor(b.publisher);
const isNew = b.badges?.includes('new');
const expiring = b.badges?.includes('expiring_soon');
return ( return (
<a href={b.url} target="_blank" rel="noopener noreferrer" style={{ textDecoration:'none' }}> <a href={b.url} target="_blank" rel="noopener noreferrer" style={{ textDecoration:'none' }}>
<div style={{ <div style={{
...S.card, padding:0, overflow:'hidden', cursor:'pointer', ...S.card, padding:0, overflow:'hidden', cursor:'pointer',
border: expiring ? '1px solid rgba(239,68,68,0.4)' : '1px solid rgba(255,255,255,0.08)', border: '1px solid rgba(255,255,255,0.08)',
}}> }}>
{b.image && ( {b.image && (
<img src={`/api/tools/koepi/img?url=${encodeURIComponent(b.image)}`} <img src={`/api/tools/koepi/img?url=${encodeURIComponent(b.image)}`}
@@ -111,15 +115,14 @@ function ProspektCard({ b }) {
borderRadius:4, padding:'2px 8px', color, fontFamily:'monospace', fontSize:10, fontWeight:700 }}> borderRadius:4, padding:'2px 8px', color, fontFamily:'monospace', fontSize:10, fontWeight:700 }}>
{b.publisher} {b.publisher}
</div> </div>
{expiring && <span style={{ background:'#ef4444', color:'#fff', borderRadius:4, padding:'2px 6px', fontSize:9, fontFamily:'monospace', fontWeight:700 }}>ENDET BALD</span>}
</div> </div>
<div style={{ color:'#fff', fontFamily:'monospace', fontSize:11, marginBottom:3 }}>{b.title}</div>
<div style={{ color:'rgba(255,255,255,0.4)', fontFamily:'monospace', fontSize:10, marginBottom:2 }}> <div style={{ color:'rgba(255,255,255,0.4)', fontFamily:'monospace', fontSize:10, marginBottom:2 }}>
📍 {b.street || b.store} 📍 {b.street}
</div> </div>
{(b.validFrom || b.validTo) && ( {(b.validFrom || b.validTo || b.weekInfo) && (
<div style={{ color:'rgba(255,255,255,0.3)', fontFamily:'monospace', fontSize:9, marginTop:4 }}> <div style={{ color:'rgba(255,255,255,0.3)', fontFamily:'monospace', fontSize:9, marginTop:4 }}>
{b.validFrom && `Ab ${fmtDate(b.validFrom)}`}{b.validFrom&&b.validTo&&' · '}{b.validTo&&`Bis ${fmtDate(b.validTo)}`} {b.validFrom && `Ab ${fmtDate(b.validFrom)}`}{b.validFrom&&b.validTo&&' · '}{b.validTo&&`Bis ${fmtDate(b.validTo)}`}
{b.weekInfo && ` ${b.weekInfo}`}
</div> </div>
)} )}
</div> </div>