feat: KoePi komplett auf zuverlaessige Pro-Kette-API umgestellt (publishers/retailer/.../leaflets), ersetzt alle Umkreis-Rate-Mechanismen
This commit is contained in:
@@ -56,24 +56,23 @@ function addressesMatch(a, b) {
|
||||
return false;
|
||||
}
|
||||
|
||||
// Die gewünschten Filialen. Zwei Wege, wie eine Filiale hier eingetragen wird:
|
||||
// - leafletId bekannt (Prospekt-Link bestätigt): wird direkt aufgelöst.
|
||||
// - nur addressMatch bekannt: wird automatisch unter allen AKTUELL laufenden
|
||||
// Prospekten dieser Kette in der Umgebung gesucht — die closestStore-Adresse
|
||||
// wird gegen die angegebene Zieladresse abgeglichen, kein Prospekt-Link nötig.
|
||||
const LOCAL_STORE_SEED_LEAFLETS = [
|
||||
{ retailer: 'Netto Marken-Discount', leafletId: 5653691 }, // Im Bonnefeld 19
|
||||
{ retailer: 'Netto Getränke-Discount', leafletId: 5627267 }, // Im Bonnefeld 21
|
||||
{ retailer: 'trinkgut', leafletId: 5634593 }, // Keniastraße 39
|
||||
{ retailer: 'REWE', leafletId: 5616164 }, // Fischerstr. 110-112
|
||||
{ retailer: 'Kaufland', leafletId: 5628827 }, // Auf der Höhe 20
|
||||
{ retailer: 'Penny', leafletId: 5638286 }, // Sittardsberger Allee 10
|
||||
{ retailer: 'E center', leafletId: 5637005 }, // Düsseldorfer Landstr. 361
|
||||
];
|
||||
const LOCAL_STORE_SEED_BY_ADDRESS = [
|
||||
{ retailer: 'Lidl', matchRetailerKey: 'lidl', addressMatch: 'Mündelheimer Str. 184' },
|
||||
{ retailer: 'ALDI Süd', matchRetailerKey: 'aldi', addressMatch: 'Mündelheimer Straße 173' },
|
||||
{ retailer: 'HORNBACH', matchRetailerKey: 'hornbach', addressMatch: 'Düsseldorfer Straße 400' },
|
||||
// Die gewünschten Filialen — jede über ihren offiziellen marktguru
|
||||
// "uniqueName" (z.B. "lidl", "netto-marken-discount") direkt abgefragt via
|
||||
// /publishers/retailer/{uniqueName}/leaflets — liefert zuverlässig ALLE
|
||||
// aktuellen und kürzlich abgelaufenen Prospekte GENAU dieser Kette, ganz ohne
|
||||
// Umkreis-Raten. addressMatch bestimmt, welche der (ggf. mehreren
|
||||
// bundesweiten) Filialen dieser Kette tatsächlich eure ist.
|
||||
const LOCAL_STORE_TARGETS = [
|
||||
{ retailer: 'Netto Marken-Discount', uniqueName: 'netto-marken-discount', addressMatch: 'Im Bonnefeld 19' },
|
||||
{ retailer: 'Netto Getränke-Discount', uniqueName: 'netto-getraenke-discount', addressMatch: 'Im Bonnefeld 21' },
|
||||
{ retailer: 'trinkgut', uniqueName: 'trinkgut-discount', addressMatch: 'Keniastraße 39' },
|
||||
{ retailer: 'REWE', uniqueName: 'rewe', addressMatch: 'Fischerstr. 110-112' },
|
||||
{ retailer: 'Kaufland', uniqueName: 'kaufland', addressMatch: 'Auf der Höhe 20' },
|
||||
{ retailer: 'Penny', uniqueName: 'penny', addressMatch: 'Sittardsberger Allee 10' },
|
||||
{ retailer: 'E center', uniqueName: 'edeka-center', addressMatch: 'Düsseldorfer Landstr. 361' },
|
||||
{ retailer: 'Lidl', uniqueName: 'lidl', addressMatch: 'Mündelheimer Str. 184' },
|
||||
{ retailer: 'ALDI Süd', uniqueName: 'aldi-sued', addressMatch: 'Mündelheimer Straße 173' },
|
||||
{ retailer: 'HORNBACH', uniqueName: 'hornbach', addressMatch: 'Düsseldorfer Straße 400' },
|
||||
];
|
||||
|
||||
function isKnownLocalStore(storeId) {
|
||||
@@ -121,10 +120,25 @@ const mgHeaders = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
};
|
||||
|
||||
// Holt alle (aktuellen + kürzlich abgelaufenen) Prospekte GENAU einer Kette
|
||||
// direkt über deren offiziellen marktguru-Namen — zuverlässiger als die
|
||||
// Umkreis-Suche, die manche laufenden Kampagnen nicht mit anzeigt.
|
||||
async function fetchLeafletsForRetailer(uniqueName) {
|
||||
try {
|
||||
const url = `https://api.marktguru.de/api/v1/publishers/retailer/${uniqueName}/leaflets?as=mobile&limit=8&offset=0&zipCode=${MG_ZIP}`;
|
||||
const res = await fetch(url, { headers: mgHeaders });
|
||||
if (!res.ok) return [];
|
||||
const json = await res.json();
|
||||
return json.results || [];
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
// Holt für eine PLZ alle laufenden Prospekt-Kampagnen (leafletFlights) und baut
|
||||
// eine Zuordnung leafletFlightId -> mainLeafletId, die wir brauchen, um darüber
|
||||
// später das leaflet-Detail (inkl. closestStore) abzurufen.
|
||||
async function fetchLeafletFlightToLeafletMap() {
|
||||
async function fetchLeafletFlightToLeafletMap(extraSearchKeys = []) {
|
||||
const url = `https://api.marktguru.de/api/v1/leafletflights?as=mobile&limit=100&zipCode=${MG_ZIP}`;
|
||||
const res = await fetch(url, { headers: mgHeaders });
|
||||
if (!res.ok) throw new Error(`leafletflights HTTP ${res.status}`);
|
||||
@@ -133,7 +147,23 @@ async function fetchLeafletFlightToLeafletMap() {
|
||||
for (const f of (json.results || [])) {
|
||||
if (f.id && f.mainLeafletId) map.set(f.id, f.mainLeafletId);
|
||||
}
|
||||
console.log(`🍺 KöPi: ${map.size} leafletFlight→Leaflet Zuordnungen geladen (von ${json.totalResults} gesamt)`);
|
||||
console.log(`🍺 KöPi: ${map.size} leafletFlight→Leaflet Zuordnungen über die breite Suche geladen (von ${json.totalResults} gesamt)`);
|
||||
|
||||
// Die breite Umkreis-Suche zeigt manchmal nicht jede laufende Kampagne einer
|
||||
// Kette (siehe Lidl-Fall). Für die Ketten, die gerade tatsächlich unter den
|
||||
// Angeboten auftauchen, zusätzlich direkt über ihre eigene Prospekt-Liste
|
||||
// nachfragen (zuverlässiger als Umkreis-Raten).
|
||||
if (extraSearchKeys.length) {
|
||||
const relevantTargets = LOCAL_STORE_TARGETS.filter(t =>
|
||||
extraSearchKeys.some(key => t.retailer.toLowerCase().includes(key))
|
||||
);
|
||||
const extraResults = await Promise.all(relevantTargets.map(t => fetchLeafletsForRetailer(t.uniqueName)));
|
||||
let added = 0;
|
||||
for (const f of extraResults.flat()) {
|
||||
if (f.id && f.mainLeafletId && !map.has(f.id)) { map.set(f.id, f.mainLeafletId); added++; }
|
||||
}
|
||||
if (added) console.log(`🍺 KöPi: +${added} zusätzliche Zuordnungen durch direkte Ketten-Abfrage (${map.size} insgesamt)`);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
@@ -210,7 +240,7 @@ function weekInfoWithFallback(name, validFromIso) {
|
||||
return week ? `(KW${week})` : '';
|
||||
}
|
||||
|
||||
// Löst die feste Liste gewünschter Filialen (LOCAL_STORE_SEED_LEAFLETS) einmalig
|
||||
// Löst die feste Liste gewünschter Filialen (LOCAL_STORE_TARGETS) einmalig
|
||||
// zu echten Store-IDs auf und speichert sie in der DB als Whitelist. Wird beim
|
||||
// ersten Scrape automatisch ausgeführt (falls die Tabelle leer ist) und kann
|
||||
// über den Admin-Button jederzeit erneut angestoßen werden, falls sich mal ein
|
||||
@@ -219,58 +249,28 @@ async function resolveLocalStores() {
|
||||
const cache = new Map();
|
||||
let resolved = 0;
|
||||
|
||||
// 1) Direkt bekannte Leaflet-IDs (per Prospekt-Link bestätigt)
|
||||
for (const seed of LOCAL_STORE_SEED_LEAFLETS) {
|
||||
const detail = await fetchLeafletDetail(seed.leafletId, cache);
|
||||
const store = detail.closestStore;
|
||||
if (store.id != null) {
|
||||
for (const target of LOCAL_STORE_TARGETS) {
|
||||
const leaflets = await fetchLeafletsForRetailer(target.uniqueName);
|
||||
console.log(`🍺 KöPi: "${target.retailer}" (${target.uniqueName}) — ${leaflets.length} Prospekt(e) gefunden`);
|
||||
let found = null;
|
||||
for (const l of leaflets) {
|
||||
if (!l.mainLeafletId) continue;
|
||||
const detail = await fetchLeafletDetail(l.mainLeafletId, cache);
|
||||
const store = detail.closestStore;
|
||||
if (store.id != null && addressesMatch(store.address, target.addressMatch)) {
|
||||
found = store;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (found) {
|
||||
db.prepare(`
|
||||
INSERT OR REPLACE INTO koepi_local_stores (store_id, retailer, address, resolved_at)
|
||||
VALUES (?, ?, ?, datetime('now','localtime'))
|
||||
`).run(store.id, seed.retailer, store.address);
|
||||
console.log(`🍺 KöPi: Filiale aufgelöst — ${seed.retailer} → Store-ID ${store.id} ("${store.address}")`);
|
||||
`).run(found.id, target.retailer, found.address);
|
||||
console.log(`🍺 KöPi: Filiale gefunden — ${target.retailer} → Store-ID ${found.id} ("${found.address}")`);
|
||||
resolved++;
|
||||
} else {
|
||||
console.log(`🍺 KöPi: Filiale NICHT auflösbar — ${seed.retailer} (Leaflet ${seed.leafletId}) — evtl. Prospekt abgelaufen, Link erneuern`);
|
||||
}
|
||||
}
|
||||
|
||||
// 2) Nur Adresse bekannt — unter allen aktuell laufenden Prospekten dieser
|
||||
// Kette in der Umgebung suchen, closestStore-Adresse gegen Zieladresse prüfen
|
||||
if (LOCAL_STORE_SEED_BY_ADDRESS.length) {
|
||||
try {
|
||||
const flightsUrl = `https://api.marktguru.de/api/v1/leafletflights?as=mobile&limit=100&zipCode=${MG_ZIP}`;
|
||||
const flightsRes = await fetch(flightsUrl, { headers: mgHeaders });
|
||||
const flightsJson = flightsRes.ok ? await flightsRes.json() : { results: [] };
|
||||
|
||||
for (const target of LOCAL_STORE_SEED_BY_ADDRESS) {
|
||||
const candidates = (flightsJson.results || []).filter(f =>
|
||||
(f.advertiser?.name || '').toLowerCase().includes(target.matchRetailerKey)
|
||||
);
|
||||
console.log(`🍺 KöPi: Suche "${target.retailer}" per Adresse — ${candidates.length} laufende Prospekt(e) dieser Kette gefunden`);
|
||||
let found = null;
|
||||
for (const c of candidates) {
|
||||
if (!c.mainLeafletId) continue;
|
||||
const detail = await fetchLeafletDetail(c.mainLeafletId, cache);
|
||||
const store = detail.closestStore;
|
||||
if (store.id != null && addressesMatch(store.address, target.addressMatch)) {
|
||||
found = store;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (found) {
|
||||
db.prepare(`
|
||||
INSERT OR REPLACE INTO koepi_local_stores (store_id, retailer, address, resolved_at)
|
||||
VALUES (?, ?, ?, datetime('now','localtime'))
|
||||
`).run(found.id, target.retailer, found.address);
|
||||
console.log(`🍺 KöPi: Filiale per Adresse gefunden — ${target.retailer} → Store-ID ${found.id} ("${found.address}")`);
|
||||
resolved++;
|
||||
} else {
|
||||
console.log(`🍺 KöPi: Filiale per Adresse NICHT gefunden — ${target.retailer} (gesucht: "${target.addressMatch}") — evtl. gerade kein laufender Prospekt dieser Kette in der Gegend`);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.log('🍺 KöPi: Adress-basierte Filial-Suche fehlgeschlagen:', e.message);
|
||||
console.log(`🍺 KöPi: Filiale NICHT gefunden — ${target.retailer} (gesucht: "${target.addressMatch}" unter ${leaflets.length} Prospekten)`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -290,9 +290,13 @@ async function fetchOffersViaApi() {
|
||||
const rawOffers = json.results || [];
|
||||
|
||||
// Echte Filial-ID + Adresse pro Angebot auflösen (leafletFlightId -> mainLeafletId -> closestStore)
|
||||
const offerRetailerKeys = [...new Set(rawOffers.map(r => {
|
||||
const name = (r.advertisers?.[0]?.name || '').toLowerCase();
|
||||
return TARGET_RETAILERS.find(k => name.includes(k));
|
||||
}).filter(Boolean))];
|
||||
let flightToLeaflet = new Map();
|
||||
try {
|
||||
flightToLeaflet = await fetchLeafletFlightToLeafletMap();
|
||||
flightToLeaflet = await fetchLeafletFlightToLeafletMap(offerRetailerKeys);
|
||||
} catch (e) {
|
||||
console.log('🍺 KöPi: leafletflights-Abruf fehlgeschlagen, Filial-Filter bleibt diesmal wirkungslos:', e.message);
|
||||
}
|
||||
@@ -554,46 +558,41 @@ async function scrapeProspekte() {
|
||||
catch (e) { console.error('🍺 KöPi Prospekte: Auflösen der Filial-Whitelist fehlgeschlagen:', e.message); }
|
||||
}
|
||||
|
||||
const flightsUrl = `https://api.marktguru.de/api/v1/leafletflights?as=mobile&limit=100&zipCode=${MG_ZIP}`;
|
||||
const flightsRes = await fetch(flightsUrl, { headers: mgHeaders });
|
||||
if (!flightsRes.ok) throw new Error(`leafletflights HTTP ${flightsRes.status}`);
|
||||
const flightsJson = await flightsRes.json();
|
||||
console.log(`🍺 KöPi Prospekte: ${flightsJson.results?.length} von ${flightsJson.totalResults} Prospekt-Kampagnen geladen`);
|
||||
|
||||
// Nur Kampagnen der gewünschten Ketten überhaupt im Detail auflösen (spart API-Calls)
|
||||
const candidates = (flightsJson.results || []).filter(f => isTargetRetailer(f.advertiser?.name));
|
||||
|
||||
// Für jede der 10 Zielfilialen direkt und zuverlässig deren eigene Prospekte
|
||||
// abfragen — kein Umkreis-Raten mehr nötig
|
||||
const leafletCache = new Map();
|
||||
const all = [];
|
||||
for (const f of candidates) {
|
||||
if (!f.mainLeafletId) continue;
|
||||
const detail = await fetchLeafletDetail(f.mainLeafletId, leafletCache);
|
||||
const store = detail.closestStore;
|
||||
const weekInfo = weekInfoWithFallback(detail.name, f.validFrom);
|
||||
// Store-spezifische Bezeichnung aus der Whitelist bevorzugen (z.B. "Netto
|
||||
// Getränke-Discount") statt dem generischen advertiser.name von marktguru
|
||||
// (liefert für diese Filiale fälschlich nur "Netto Marken-Discount")
|
||||
const knownStore = store.id != null
|
||||
? db.prepare('SELECT retailer FROM koepi_local_stores WHERE store_id=?').get(store.id)
|
||||
: null;
|
||||
all.push({
|
||||
id: f.id,
|
||||
publisher: knownStore?.retailer || f.advertiser?.name || '',
|
||||
title: f.advertiser?.name || cleanTitle(detail.name) || '',
|
||||
rawName: detail.name || '', // nur intern, für den Ausschluss-Filter unten
|
||||
weekInfo,
|
||||
street: store.address || '',
|
||||
zip: MG_ZIP,
|
||||
city: 'Duisburg',
|
||||
storeId: store.id,
|
||||
image: null,
|
||||
validFrom: f.validFrom || null,
|
||||
validTo: f.validTo || null,
|
||||
pageCount: f.pageCount || null,
|
||||
url: `https://www.marktguru.de/leaflets/${f.mainLeafletId}`,
|
||||
isTarget: isKnownLocalStore(store.id) && store.id != null,
|
||||
badges: [],
|
||||
});
|
||||
for (const target of LOCAL_STORE_TARGETS) {
|
||||
const leaflets = await fetchLeafletsForRetailer(target.uniqueName);
|
||||
for (const f of leaflets) {
|
||||
if (!f.mainLeafletId) continue;
|
||||
const detail = await fetchLeafletDetail(f.mainLeafletId, leafletCache);
|
||||
const store = detail.closestStore;
|
||||
const weekInfo = weekInfoWithFallback(detail.name, f.validFrom);
|
||||
// Store-spezifische Bezeichnung aus der Whitelist bevorzugen (z.B. "Netto
|
||||
// Getränke-Discount") statt dem generischen Kettennamen
|
||||
const knownStore = store.id != null
|
||||
? db.prepare('SELECT retailer FROM koepi_local_stores WHERE store_id=?').get(store.id)
|
||||
: null;
|
||||
all.push({
|
||||
id: f.id,
|
||||
publisher: knownStore?.retailer || target.retailer,
|
||||
title: knownStore?.retailer || target.retailer,
|
||||
rawName: detail.name || '', // nur intern, für den Ausschluss-Filter unten
|
||||
weekInfo,
|
||||
street: store.address || '',
|
||||
zip: MG_ZIP,
|
||||
city: 'Duisburg',
|
||||
storeId: store.id,
|
||||
image: null,
|
||||
validFrom: f.validFrom || null,
|
||||
validTo: f.validTo || null,
|
||||
pageCount: f.pageCount || null,
|
||||
url: `https://www.marktguru.de/leaflets/${f.mainLeafletId}`,
|
||||
isTarget: isKnownLocalStore(store.id) && store.id != null,
|
||||
badges: [],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Bier-Icon: mit den aktuellen König-Pilsener-Angeboten abgleichen, welche
|
||||
@@ -806,7 +805,7 @@ router.post('/clear-cache', authenticate, (req, res) => {
|
||||
router.get('/local-stores', authenticate, (req, res) => {
|
||||
if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' });
|
||||
const stores = db.prepare('SELECT * FROM koepi_local_stores ORDER BY retailer').all();
|
||||
res.json({ stores, seedCount: LOCAL_STORE_SEED_LEAFLETS.length + LOCAL_STORE_SEED_BY_ADDRESS.length });
|
||||
res.json({ stores, seedCount: LOCAL_STORE_TARGETS.length });
|
||||
});
|
||||
|
||||
// POST /local-stores/resolve – Filial-Whitelist neu auflösen (Admin, z.B. wenn
|
||||
@@ -816,7 +815,7 @@ router.post('/local-stores/resolve', authenticate, async (req, res) => {
|
||||
try {
|
||||
const resolved = await resolveLocalStores();
|
||||
const stores = db.prepare('SELECT * FROM koepi_local_stores ORDER BY retailer').all();
|
||||
res.json({ ok: true, resolved, total: LOCAL_STORE_SEED_LEAFLETS.length + LOCAL_STORE_SEED_BY_ADDRESS.length, stores });
|
||||
res.json({ ok: true, resolved, total: LOCAL_STORE_TARGETS.length, stores });
|
||||
} catch(e) { res.status(500).json({ error: e.message }); }
|
||||
});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user