733 lines
34 KiB
JavaScript
733 lines
34 KiB
JavaScript
const express = require('express');
|
||
const https = require('https');
|
||
const db = require('../../db');
|
||
const mqttClient = require('../../mqtt');
|
||
const { logPush } = require('../../pushLog');
|
||
const { authenticate } = require('../../middleware/auth');
|
||
const router = express.Router();
|
||
|
||
// ── Cache ─────────────────────────────────────────────────────────────────────
|
||
const cache = {};
|
||
function cacheGet(k) { const e=cache[k]; if(!e||Date.now()>e.ex) { delete cache[k]; return null; } return e.d; }
|
||
function cacheSet(k,d,ms=60*60*1000) { cache[k]={d,ex:Date.now()+ms}; }
|
||
|
||
// ── marktguru Cookie (Standort Duisburg 47259) ────────────────────────────────
|
||
const MG_SETTINGS = encodeURIComponent(JSON.stringify({
|
||
location: { name:'Duisburg', uniqueName:'duisburg', longitude:6.7625, latitude:51.4332, zipCodes:['47259'] },
|
||
locationSource: 'manual', hidePageflipMenu: false, cookieAccept: true, userGroup: 35,
|
||
}));
|
||
|
||
// ── Gewünschte Händler (grobe Vorfilterung, bevor die genaue Filiale geprüft wird) ─
|
||
const TARGET_RETAILERS = ['edeka','e center','rewe','netto','kaufland','penny','trinkgut','lidl','aldi','hornbach'];
|
||
function isTargetRetailer(name) {
|
||
if (!name) return false;
|
||
const n = name.toLowerCase();
|
||
return TARGET_RETAILERS.some(r => n.includes(r));
|
||
}
|
||
|
||
// Straßennamen robust vergleichbar machen (Groß-/Kleinschreibung, "straße"/"str.",
|
||
// Kommas/Leerzeichen/Punkte egal)
|
||
function normalizeAddress(s) {
|
||
return (s || '')
|
||
.toLowerCase()
|
||
.replace(/straße|strasse/g, 'str')
|
||
.replace(/[^a-zäöüß0-9]/g, '');
|
||
}
|
||
// Adressen mit mehreren kommagetrennten Hausnummern (z.B. "Straße 173,173a")
|
||
// in einzelne vollständige Varianten zerlegen ("Straße 173" / "Straße 173a"),
|
||
// damit ein Treffer gegen nur eine der beiden Hausnummern trotzdem zählt
|
||
function expandAddressVariants(addr) {
|
||
const m = (addr || '').match(/^(.*?)(\d.*)$/);
|
||
if (!m) return [addr];
|
||
const streetPart = m[1].trim();
|
||
const numbers = m[2].split(',').map(s => s.trim()).filter(Boolean);
|
||
if (numbers.length <= 1) return [addr];
|
||
return numbers.map(n => `${streetPart} ${n}`);
|
||
}
|
||
function addressesMatch(a, b) {
|
||
for (const va of expandAddressVariants(a)) {
|
||
for (const vb of expandAddressVariants(b)) {
|
||
const na = normalizeAddress(va), nb = normalizeAddress(vb);
|
||
if (na && nb && (na.includes(nb) || nb.includes(na))) return true;
|
||
}
|
||
}
|
||
return false;
|
||
}
|
||
|
||
// Die gewünschten Filialen. Zwei Wege, wie eine Filiale hier eingetragen wird:
|
||
// - leafletId bekannt (Prospekt-Link bestätigt): wird direkt aufgelöst.
|
||
// - nur addressMatch bekannt: wird automatisch unter allen AKTUELL laufenden
|
||
// Prospekten dieser Kette in der Umgebung gesucht — die closestStore-Adresse
|
||
// wird gegen die angegebene Zieladresse abgeglichen, kein Prospekt-Link nötig.
|
||
const LOCAL_STORE_SEED_LEAFLETS = [
|
||
{ retailer: 'Netto Marken-Discount', leafletId: 5653691 }, // Im Bonnefeld 19
|
||
{ retailer: 'Netto Getränke-Discount', leafletId: 5627267 }, // Im Bonnefeld 21
|
||
{ retailer: 'trinkgut', leafletId: 5634593 }, // Keniastraße 39
|
||
{ retailer: 'REWE', leafletId: 5616164 }, // Fischerstr. 110-112
|
||
{ retailer: 'Kaufland', leafletId: 5628827 }, // Auf der Höhe 20
|
||
{ retailer: 'Penny', leafletId: 5638286 }, // Sittardsberger Allee 10
|
||
{ retailer: 'E center', leafletId: 5637005 }, // Düsseldorfer Landstr. 361
|
||
];
|
||
const LOCAL_STORE_SEED_BY_ADDRESS = [
|
||
{ retailer: 'Lidl', matchRetailerKey: 'lidl', addressMatch: 'Mündelheimer Str. 184' },
|
||
{ retailer: 'ALDI Süd', matchRetailerKey: 'aldi', addressMatch: 'Mündelheimer Straße 173' },
|
||
{ retailer: 'HORNBACH', matchRetailerKey: 'hornbach', addressMatch: 'Düsseldorfer Straße 400' },
|
||
];
|
||
|
||
function isKnownLocalStore(storeId) {
|
||
if (storeId == null) return true; // Store konnte nicht ermittelt werden -> lieber behalten als faelschlich verstecken
|
||
return !!db.prepare('SELECT 1 FROM koepi_local_stores WHERE store_id=?').get(storeId);
|
||
}
|
||
|
||
// Explizit unerwünschte Marken/Franchise-Label, unabhängig von der Filiale —
|
||
// zusätzliches Sicherheitsnetz, greift aber durch die ID-Whitelist ohnehin kaum
|
||
// noch (REWE Dortmund taucht dort schlicht nicht auf)
|
||
const EXCLUDED_RETAILER_NAMES = ['rewe dortmund'];
|
||
function isExcludedRetailer(retailer) {
|
||
const n = (retailer || '').toLowerCase();
|
||
return EXCLUDED_RETAILER_NAMES.some(x => n.includes(x));
|
||
}
|
||
|
||
// ── Angebote von marktguru ────────────────────────────────────────────────────
|
||
// Primär über die öffentlich bekannte marktguru-JSON-API (schnell, kein Browser
|
||
// nötig, kein Cloudflare-Risiko). Schlägt das fehl (z.B. weil die Keys mal
|
||
// ungültig werden), fällt der Code automatisch auf das alte Puppeteer/HTML-
|
||
// Scraping zurück, damit KöPi nicht komplett ausfällt.
|
||
const MARKTGURU_API_URL = 'https://api.marktguru.de/api/v1/offers/search';
|
||
const MARKTGURU_CLIENT_KEY = 'WU/RH+PMGDi+gkZer3WbMelt6zcYHSTytNB7VpTia90=';
|
||
const MARKTGURU_API_KEY = '8Kk+pmbf7TgJ9nVj2cXeA7P5zBGv8iuutVVMRfOfvNE=';
|
||
const MG_ZIP = '47259';
|
||
const MG_QUERY = 'könig pilsener';
|
||
|
||
function formatIsoDate(iso) {
|
||
if (!iso) return '';
|
||
const d = new Date(iso);
|
||
if (Number.isNaN(d.getTime())) return '';
|
||
const pad = n => String(n).padStart(2, '0');
|
||
return `${pad(d.getDate())}.${pad(d.getMonth() + 1)}.`;
|
||
}
|
||
|
||
// Duisburg 47259 Zentrum — für den closestStore-Abgleich im leaflets-Detail-Call.
|
||
// Innerhalb der zipCode-Filterung spielt der genaue Punkt kaum eine Rolle.
|
||
const MG_LAT = '51.3543213';
|
||
const MG_LON = '6.7151937';
|
||
|
||
const mgHeaders = {
|
||
'x-clientkey': MARKTGURU_CLIENT_KEY,
|
||
'x-apikey': MARKTGURU_API_KEY,
|
||
'Accept': 'application/json',
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
};
|
||
|
||
// Holt für eine PLZ alle laufenden Prospekt-Kampagnen (leafletFlights) und baut
|
||
// eine Zuordnung leafletFlightId -> mainLeafletId, die wir brauchen, um darüber
|
||
// später das leaflet-Detail (inkl. closestStore) abzurufen.
|
||
async function fetchLeafletFlightToLeafletMap() {
|
||
const url = `https://api.marktguru.de/api/v1/leafletflights?as=mobile&limit=100&zipCode=${MG_ZIP}`;
|
||
const res = await fetch(url, { headers: mgHeaders });
|
||
if (!res.ok) throw new Error(`leafletflights HTTP ${res.status}`);
|
||
const json = await res.json();
|
||
const map = new Map();
|
||
for (const f of (json.results || [])) {
|
||
if (f.id && f.mainLeafletId) map.set(f.id, f.mainLeafletId);
|
||
}
|
||
console.log(`🍺 KöPi: ${map.size} leafletFlight→Leaflet Zuordnungen geladen (von ${json.totalResults} gesamt)`);
|
||
return map;
|
||
}
|
||
|
||
// Holt für ein einzelnes Leaflet den kompletten Datensatz von marktguru:
|
||
// closestStore (id/address/distanceInMeters — Grundlage der Filial-Whitelist),
|
||
// name (enthält oft "(KW29 Do-Sa)" o.ä.) und children (Liste aller Angebote
|
||
// mit ihrem pageIndex, für den seiten-genauen Direktlink).
|
||
async function fetchLeafletDetail(leafletId, cache) {
|
||
if (cache.has(leafletId)) return cache.get(leafletId);
|
||
const empty = { id: leafletId, name: '', closestStore: { id: null, address: '', distanceMeters: null }, children: [] };
|
||
try {
|
||
const url = `https://api.marktguru.de/api/v1/leaflets/${leafletId}?as=mobiledetailed&latitude=${MG_LAT}&longitude=${MG_LON}&zipCode=${MG_ZIP}`;
|
||
const res = await fetch(url, { headers: mgHeaders });
|
||
if (!res.ok) { cache.set(leafletId, empty); return empty; }
|
||
const json = await res.json();
|
||
const store = json.closestStore;
|
||
const result = {
|
||
id: json.id ?? leafletId,
|
||
name: json.name || '',
|
||
closestStore: store
|
||
? { id: store.id ?? null, address: store.address || '', distanceMeters: store.distanceInMeters ?? null }
|
||
: { id: null, address: '', distanceMeters: null },
|
||
children: json.children || [],
|
||
};
|
||
console.log(`🍺 KöPi: Leaflet ${leafletId} ("${result.name}") → closestStore: ${store ? `id=${store.id} "${store.address}" (${Math.round(store.distanceInMeters||0)}m entfernt)` : 'keine'}`);
|
||
cache.set(leafletId, result);
|
||
return result;
|
||
} catch (e) {
|
||
console.log(`🍺 KöPi: Leaflet-Detail-Abruf für ${leafletId} fehlgeschlagen: ${e.message}`);
|
||
cache.set(leafletId, empty);
|
||
return empty;
|
||
}
|
||
}
|
||
|
||
// Extrahiert "(KW29 Do-Sa)"-artige Klammerzusätze aus dem Leaflet-Namen, falls
|
||
// vorhanden (nicht jede Filiale/jeder Prospekt hat sowas)
|
||
function extractWeekInfo(name) {
|
||
const m = (name || '').match(/\([^)]*KW\s*\d+[^)]*\)/i);
|
||
return m ? m[0] : '';
|
||
}
|
||
|
||
// Löst die feste Liste gewünschter Filialen (LOCAL_STORE_SEED_LEAFLETS) einmalig
|
||
// zu echten Store-IDs auf und speichert sie in der DB als Whitelist. Wird beim
|
||
// ersten Scrape automatisch ausgeführt (falls die Tabelle leer ist) und kann
|
||
// über den Admin-Button jederzeit erneut angestoßen werden, falls sich mal ein
|
||
// Prospekt-Link ändert oder eine Filiale ergänzt werden soll.
|
||
async function resolveLocalStores() {
|
||
const cache = new Map();
|
||
let resolved = 0;
|
||
|
||
// 1) Direkt bekannte Leaflet-IDs (per Prospekt-Link bestätigt)
|
||
for (const seed of LOCAL_STORE_SEED_LEAFLETS) {
|
||
const detail = await fetchLeafletDetail(seed.leafletId, cache);
|
||
const store = detail.closestStore;
|
||
if (store.id != null) {
|
||
db.prepare(`
|
||
INSERT OR REPLACE INTO koepi_local_stores (store_id, retailer, address, resolved_at)
|
||
VALUES (?, ?, ?, datetime('now','localtime'))
|
||
`).run(store.id, seed.retailer, store.address);
|
||
console.log(`🍺 KöPi: Filiale aufgelöst — ${seed.retailer} → Store-ID ${store.id} ("${store.address}")`);
|
||
resolved++;
|
||
} else {
|
||
console.log(`🍺 KöPi: Filiale NICHT auflösbar — ${seed.retailer} (Leaflet ${seed.leafletId}) — evtl. Prospekt abgelaufen, Link erneuern`);
|
||
}
|
||
}
|
||
|
||
// 2) Nur Adresse bekannt — unter allen aktuell laufenden Prospekten dieser
|
||
// Kette in der Umgebung suchen, closestStore-Adresse gegen Zieladresse prüfen
|
||
if (LOCAL_STORE_SEED_BY_ADDRESS.length) {
|
||
try {
|
||
const flightsUrl = `https://api.marktguru.de/api/v1/leafletflights?as=mobile&limit=100&zipCode=${MG_ZIP}`;
|
||
const flightsRes = await fetch(flightsUrl, { headers: mgHeaders });
|
||
const flightsJson = flightsRes.ok ? await flightsRes.json() : { results: [] };
|
||
|
||
for (const target of LOCAL_STORE_SEED_BY_ADDRESS) {
|
||
const candidates = (flightsJson.results || []).filter(f =>
|
||
(f.advertiser?.name || '').toLowerCase().includes(target.matchRetailerKey)
|
||
);
|
||
console.log(`🍺 KöPi: Suche "${target.retailer}" per Adresse — ${candidates.length} laufende Prospekt(e) dieser Kette gefunden`);
|
||
let found = null;
|
||
for (const c of candidates) {
|
||
if (!c.mainLeafletId) continue;
|
||
const detail = await fetchLeafletDetail(c.mainLeafletId, cache);
|
||
const store = detail.closestStore;
|
||
if (store.id != null && addressesMatch(store.address, target.addressMatch)) {
|
||
found = store;
|
||
break;
|
||
}
|
||
}
|
||
if (found) {
|
||
db.prepare(`
|
||
INSERT OR REPLACE INTO koepi_local_stores (store_id, retailer, address, resolved_at)
|
||
VALUES (?, ?, ?, datetime('now','localtime'))
|
||
`).run(found.id, target.retailer, found.address);
|
||
console.log(`🍺 KöPi: Filiale per Adresse gefunden — ${target.retailer} → Store-ID ${found.id} ("${found.address}")`);
|
||
resolved++;
|
||
} else {
|
||
console.log(`🍺 KöPi: Filiale per Adresse NICHT gefunden — ${target.retailer} (gesucht: "${target.addressMatch}") — evtl. gerade kein laufender Prospekt dieser Kette in der Gegend`);
|
||
}
|
||
}
|
||
} catch (e) {
|
||
console.log('🍺 KöPi: Adress-basierte Filial-Suche fehlgeschlagen:', e.message);
|
||
}
|
||
}
|
||
|
||
return resolved;
|
||
}
|
||
|
||
async function fetchOffersViaApi() {
|
||
const url = `${MARKTGURU_API_URL}?as=web&limit=24&offset=0&q=${encodeURIComponent(MG_QUERY)}&zipCode=${MG_ZIP}`;
|
||
const res = await fetch(url, { headers: mgHeaders });
|
||
if (!res.ok) {
|
||
const text = await res.text().catch(() => '');
|
||
throw new Error(`API HTTP ${res.status}: ${text.slice(0, 200)}`);
|
||
}
|
||
const json = await res.json();
|
||
console.log('🍺 KöPi API: totalResults=', json.totalResults, '| results.length=', json.results?.length);
|
||
|
||
const rawOffers = json.results || [];
|
||
|
||
// Echte Filial-ID + Adresse pro Angebot auflösen (leafletFlightId -> mainLeafletId -> closestStore)
|
||
let flightToLeaflet = new Map();
|
||
try {
|
||
flightToLeaflet = await fetchLeafletFlightToLeafletMap();
|
||
} catch (e) {
|
||
console.log('🍺 KöPi: leafletflights-Abruf fehlgeschlagen, Filial-Filter bleibt diesmal wirkungslos:', e.message);
|
||
}
|
||
const leafletCache = new Map();
|
||
const detailByOfferId = new Map();
|
||
for (const r of rawOffers) {
|
||
const leafletId = flightToLeaflet.get(r.leafletFlightId);
|
||
if (leafletId) {
|
||
const detail = await fetchLeafletDetail(leafletId, leafletCache);
|
||
detailByOfferId.set(r.id, detail);
|
||
}
|
||
}
|
||
|
||
return rawOffers.map(r => {
|
||
const advertiser = r.advertisers?.[0] || {};
|
||
const priceNum = typeof r.price === 'number' ? r.price : parseFloat(r.price);
|
||
const oldPriceNum = typeof r.oldPrice === 'number' ? r.oldPrice : parseFloat(r.oldPrice);
|
||
const detail = detailByOfferId.get(r.id);
|
||
const storeInfo = detail?.closestStore || { id: null, address: '', distanceMeters: null };
|
||
const validity0 = r.validityDates?.[0] || {};
|
||
const weekInfo = extractWeekInfo(detail?.name);
|
||
const baseDateRange = validity0.from && validity0.to
|
||
? `${formatIsoDate(validity0.from)} - ${formatIsoDate(validity0.to)}`
|
||
: formatIsoDate(validity0.to);
|
||
// Seiten-genauer Direktlink zum Angebot innerhalb des Prospekts — wird bei
|
||
// jedem Scrape frisch aus den aktuellen API-Daten ermittelt (children[].id
|
||
// == "offers/{id}"), funktioniert also automatisch weiter bei neuen
|
||
// Prospekten, ohne dass irgendwas hartkodiert werden muss.
|
||
const child = detail?.children?.find(c => c.id === `offers/${r.id}`);
|
||
const leafletUrl = detail
|
||
? `https://www.marktguru.de/leaflets/${detail.id}${child ? `/page/${child.pageIndex}` : ''}`
|
||
: null;
|
||
return {
|
||
name: r.description || r.title || advertiser.name || '',
|
||
brand: r.brand?.name || '',
|
||
price: Number.isFinite(priceNum) ? `€ ${priceNum.toFixed(2).replace('.', ',')}` : (r.price || ''),
|
||
oldPrice: Number.isFinite(oldPriceNum) ? `€ ${oldPriceNum.toFixed(2).replace('.', ',')}` : (r.oldPrice || ''),
|
||
retailer: advertiser.name || '',
|
||
dateRange: weekInfo ? `${baseDateRange} ${weekInfo}` : baseDateRange,
|
||
weekInfo,
|
||
validity: '',
|
||
description: r.description || '',
|
||
badge: '',
|
||
// Adresse kommt jetzt direkt von marktguru (closestStore.address), nicht mehr fest hinterlegt
|
||
address: storeInfo.address || '',
|
||
realAddress: storeInfo.address || '',
|
||
storeId: storeInfo.id,
|
||
storeDistanceMeters: storeInfo.distanceMeters,
|
||
leafletFlightId: r.leafletFlightId || null,
|
||
offerId: r.id || null,
|
||
leafletUrl,
|
||
image: r.image?.url || r.images?.[0]?.url || null,
|
||
};
|
||
});
|
||
}
|
||
|
||
async function fetchOffersViaPuppeteer() {
|
||
let puppeteer;
|
||
try { puppeteer = require('puppeteer-core'); }
|
||
catch(e) { throw new Error('puppeteer-core nicht installiert'); }
|
||
|
||
const browser = await puppeteer.launch({
|
||
executablePath: '/usr/bin/chromium-browser',
|
||
args: ['--no-sandbox','--disable-setuid-sandbox','--disable-dev-shm-usage','--disable-gpu','--single-process'],
|
||
headless: true,
|
||
});
|
||
|
||
try {
|
||
const page = await browser.newPage();
|
||
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
|
||
await page.setViewport({ width:1280, height:800 });
|
||
await page.setCookie({ name:'mg_user-settings', value:MG_SETTINGS, domain:'.marktguru.de', path:'/' });
|
||
|
||
await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', {
|
||
waitUntil: 'networkidle2', timeout: 30000,
|
||
});
|
||
// Seite nach unten scrollen damit lazy-loaded Bilder geladen werden
|
||
await page.evaluate(async () => {
|
||
await new Promise(resolve => {
|
||
let totalHeight = 0;
|
||
const distance = 300;
|
||
const timer = setInterval(() => {
|
||
window.scrollBy(0, distance);
|
||
totalHeight += distance;
|
||
if (totalHeight >= document.body.scrollHeight) {
|
||
clearInterval(timer);
|
||
resolve();
|
||
}
|
||
}, 200);
|
||
});
|
||
});
|
||
await new Promise(r => setTimeout(r, 3000));
|
||
|
||
const offers = await page.evaluate(() => {
|
||
// Exakte Selektoren basierend auf marktguru HTML-Struktur
|
||
const cards = document.querySelectorAll('li.offer-list-item');
|
||
return Array.from(cards).map(card => {
|
||
const name = card.querySelector('h3')?.textContent?.trim() || '';
|
||
const brand = card.querySelector('dd.brand a')?.textContent?.trim() || '';
|
||
const price = card.querySelector('span.price')?.textContent?.trim() || '';
|
||
const oldPrice = card.querySelector('.price-bubble .old-price, .crossed')?.textContent?.trim() || '';
|
||
const retailer = card.querySelector('dd.retailer-name a')?.textContent?.trim() || '';
|
||
const dateRange= card.querySelector('dd.valid')?.textContent?.trim() || '';
|
||
const daysLeft = card.querySelector('dd.time-left span')?.textContent?.trim() || '';
|
||
const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` :
|
||
card.querySelector('dt.time-left')?.textContent?.includes('Brandneu') ? 'Neu' : '';
|
||
const descEl = card.querySelector('.info div, .info');
|
||
const description = descEl?.textContent?.trim() || '';
|
||
const img = card.querySelector('img.offer-list-item-img');
|
||
const badge = card.querySelector('.badge, .discount-badge')?.textContent?.trim() || '';
|
||
const imgSrc = img?.src?.replace('/medium.webp', '/large.webp') || img?.src || null;
|
||
const allDds = Array.from(card.querySelectorAll('dd'));
|
||
const retailerIdx = allDds.findIndex(d => d.classList.contains('retailer-name'));
|
||
let realAddress = '';
|
||
if (retailerIdx !== -1 && allDds[retailerIdx + 1]) {
|
||
const next = allDds[retailerIdx + 1];
|
||
const isOtherField = next.classList.contains('valid') || next.classList.contains('time-left');
|
||
if (!isOtherField) realAddress = next.textContent?.trim() || '';
|
||
}
|
||
return { name, brand, price, oldPrice, retailer, dateRange, validity, description, badge, address: '', realAddress, image: imgSrc };
|
||
});
|
||
});
|
||
return offers;
|
||
} finally {
|
||
await browser.close();
|
||
}
|
||
}
|
||
|
||
// Gemeinsame Weiterverarbeitung, egal ob die Rohdaten über die API oder per
|
||
// Puppeteer/HTML-Scraping ermittelt wurden
|
||
function processOffers(offers) {
|
||
// Lokale Bilder auswählen basierend auf Beschreibung
|
||
for (const offer of offers) {
|
||
const desc = (offer.description || '').toLowerCase();
|
||
if (desc.includes('steini')) {
|
||
offer.imageLocal = '/koepi/kasten_steini.png';
|
||
} else if (desc.includes('11 x 0,5') || desc.includes('11x0,5')) {
|
||
offer.imageLocal = '/koepi/kasten_lang.png';
|
||
} else if (desc.includes('6 x') || desc.includes('6x') || (desc.includes('flasche') && !desc.includes('kasten'))) {
|
||
offer.imageLocal = '/koepi/traeger.png';
|
||
} else if (desc.includes('24 x 0,33') || desc.includes('24x0,33')) {
|
||
offer.imageLocal = '/koepi/kasten_033.png';
|
||
} else if (desc.includes('0,5')) {
|
||
offer.imageLocal = '/koepi/kasten_050.png';
|
||
} else if (desc.includes('0,33')) {
|
||
offer.imageLocal = '/koepi/kasten_033.png';
|
||
} else {
|
||
offer.imageLocal = '/koepi/kasten_050.png';
|
||
}
|
||
offer.image = null; // CDN nicht mehr nötig
|
||
}
|
||
|
||
// Filtern auf Ziel-Händler + nur König Pilsener
|
||
const filtered = offers.filter(o => {
|
||
const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase();
|
||
const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener');
|
||
return hasKP && isTargetRetailer(o.retailer);
|
||
});
|
||
|
||
const excludedNow = filtered.filter(o => isExcludedRetailer(o.retailer));
|
||
if (excludedNow.length) {
|
||
console.log('🍺 KöPi: Angebote per Namens-Filter ausgeschlossen:', excludedNow.map(o => o.retailer).join(', '));
|
||
}
|
||
|
||
console.log('🍺 KöPi Filial-Whitelist-Diagnose:', filtered.map(o =>
|
||
`${o.retailer} | Store-ID=${o.storeId ?? 'unbekannt'} | ${o.realAddress||'keine Adresse'} | bekannt=${isKnownLocalStore(o.storeId)}`
|
||
).join('\n '));
|
||
|
||
const localOnly = filtered
|
||
.filter(o => !isExcludedRetailer(o.retailer))
|
||
.filter(o => isKnownLocalStore(o.storeId));
|
||
|
||
for (const o of localOnly) {
|
||
o.retailerDisplay = o.retailer;
|
||
}
|
||
|
||
// Deduplizieren: gleicher normalisierter Händler + gleicher Preis + gleiche
|
||
// Filiale → ein Eintrag
|
||
const merged = new Map();
|
||
for (const o of localOnly) {
|
||
const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || '';
|
||
const key = `${normalizedRetailer}|${o.price}|${o.storeId ?? ''}`;
|
||
if (merged.has(key)) {
|
||
const existing = merged.get(key);
|
||
if (o.description && (!existing.description || o.description.length < existing.description.length)) {
|
||
existing.description = o.description;
|
||
}
|
||
} else {
|
||
merged.set(key, { ...o });
|
||
}
|
||
}
|
||
return [...merged.values()];
|
||
}
|
||
|
||
async function scrapeMarktguru(forceRefresh = false) {
|
||
const cacheKey = 'koepi:mg';
|
||
if (!forceRefresh) {
|
||
const cached = cacheGet(cacheKey);
|
||
if (cached) return cached;
|
||
}
|
||
|
||
// Filial-Whitelist beim allerersten Lauf automatisch auflösen (Tabelle leer)
|
||
const knownCount = db.prepare('SELECT COUNT(*) AS n FROM koepi_local_stores').get().n;
|
||
if (knownCount === 0) {
|
||
console.log('🍺 KöPi: Filial-Whitelist ist leer, löse sie jetzt einmalig auf...');
|
||
try { await resolveLocalStores(); }
|
||
catch (e) { console.error('🍺 KöPi: Auflösen der Filial-Whitelist fehlgeschlagen:', e.message); }
|
||
}
|
||
|
||
let offers, source;
|
||
try {
|
||
offers = await fetchOffersViaApi();
|
||
source = 'marktguru.de (API)';
|
||
console.log(`🍺 KöPi: ${offers.length} Rohtreffer über die marktguru-API`);
|
||
} catch (apiErr) {
|
||
console.error('🍺 KöPi: API-Weg fehlgeschlagen, falle zurück auf Puppeteer:', apiErr.message);
|
||
try {
|
||
offers = await fetchOffersViaPuppeteer();
|
||
source = 'marktguru.de (Puppeteer-Fallback)';
|
||
console.log(`🍺 KöPi: ${offers.length} Rohtreffer per Puppeteer (Fallback)`);
|
||
} catch (ppErr) {
|
||
console.error('🍺 KöPi: auch Puppeteer-Fallback fehlgeschlagen:', ppErr.message);
|
||
return { offers: [], error: `API: ${apiErr.message} | Puppeteer: ${ppErr.message}`, scrapedAt: new Date().toISOString() };
|
||
}
|
||
}
|
||
|
||
const unique = processOffers(offers);
|
||
const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source };
|
||
cacheSet(cacheKey, result);
|
||
return result;
|
||
}
|
||
|
||
// ── Prospekte von marktguru (dieselbe Filial-Whitelist wie bei den Angeboten) ──
|
||
async function scrapeProspekte() {
|
||
const cacheKey = 'koepi:prospekte';
|
||
const cached = cacheGet(cacheKey);
|
||
if (cached) return cached;
|
||
|
||
const knownCount = db.prepare('SELECT COUNT(*) AS n FROM koepi_local_stores').get().n;
|
||
if (knownCount === 0) {
|
||
console.log('🍺 KöPi Prospekte: Filial-Whitelist ist leer, löse sie jetzt einmalig auf...');
|
||
try { await resolveLocalStores(); }
|
||
catch (e) { console.error('🍺 KöPi Prospekte: Auflösen der Filial-Whitelist fehlgeschlagen:', e.message); }
|
||
}
|
||
|
||
const flightsUrl = `https://api.marktguru.de/api/v1/leafletflights?as=mobile&limit=100&zipCode=${MG_ZIP}`;
|
||
const flightsRes = await fetch(flightsUrl, { headers: mgHeaders });
|
||
if (!flightsRes.ok) throw new Error(`leafletflights HTTP ${flightsRes.status}`);
|
||
const flightsJson = await flightsRes.json();
|
||
console.log(`🍺 KöPi Prospekte: ${flightsJson.results?.length} von ${flightsJson.totalResults} Prospekt-Kampagnen geladen`);
|
||
|
||
// Nur Kampagnen der gewünschten Ketten überhaupt im Detail auflösen (spart API-Calls)
|
||
const candidates = (flightsJson.results || []).filter(f => isTargetRetailer(f.advertiser?.name));
|
||
|
||
const leafletCache = new Map();
|
||
const all = [];
|
||
for (const f of candidates) {
|
||
if (!f.mainLeafletId) continue;
|
||
const detail = await fetchLeafletDetail(f.mainLeafletId, leafletCache);
|
||
const store = detail.closestStore;
|
||
const weekInfo = extractWeekInfo(detail.name);
|
||
all.push({
|
||
id: f.id,
|
||
publisher: f.advertiser?.name || '',
|
||
title: detail.name || f.advertiser?.name || '',
|
||
weekInfo,
|
||
street: store.address || '',
|
||
zip: MG_ZIP,
|
||
city: 'Duisburg',
|
||
storeId: store.id,
|
||
image: null,
|
||
validFrom: f.validFrom || null,
|
||
validTo: f.validTo || null,
|
||
pageCount: f.pageCount || null,
|
||
url: `https://www.marktguru.de/leaflets/${f.mainLeafletId}`,
|
||
isTarget: isKnownLocalStore(store.id) && store.id != null,
|
||
badges: [],
|
||
});
|
||
}
|
||
|
||
// Bier-Icon: mit den aktuellen König-Pilsener-Angeboten abgleichen, welche
|
||
// Filialen gerade wirklich König Pilsener im Angebot haben
|
||
try {
|
||
const offersResult = await scrapeMarktguru(false);
|
||
const storeIdsWithKoepi = new Set((offersResult.offers || []).map(o => o.storeId).filter(id => id != null));
|
||
for (const b of all) b.hasKoenigPilsener = b.storeId != null && storeIdsWithKoepi.has(b.storeId);
|
||
} catch (e) {
|
||
console.log('🍺 KöPi Prospekte: Abgleich mit aktuellen Angeboten fehlgeschlagen (ignoriert):', e.message);
|
||
for (const b of all) b.hasKoenigPilsener = false;
|
||
}
|
||
|
||
const result = {
|
||
targeted: all.filter(b => b.isTarget),
|
||
others: all.filter(b => !b.isTarget),
|
||
scrapedAt: new Date().toISOString(),
|
||
};
|
||
console.log(`🍺 KöPi Prospekte: ${result.targeted.length} eigene Filiale(n), ${result.others.length} andere Filialen derselben Ketten, ${result.targeted.filter(b=>b.hasKoenigPilsener).length} davon mit König Pilsener im Angebot`);
|
||
cacheSet(cacheKey, result);
|
||
return result;
|
||
}
|
||
|
||
// ── Bild-Proxy (für evtl. externe Bilder) ───────────────────────────────────
|
||
router.get('/img', authenticate, (req, res) => {
|
||
const url = req.query.url;
|
||
if (!url || !url.startsWith('https://')) return res.status(400).end();
|
||
try {
|
||
const parsed = new URL(url);
|
||
const allowed = ['content-media.bonial.biz','publisher-media.bonial.biz'];
|
||
if (!allowed.some(h => parsed.hostname === h)) return res.status(403).end();
|
||
const r = https.get(url, { headers: {'User-Agent':'Mozilla/5.0'} }, imgRes => {
|
||
res.setHeader('Content-Type', imgRes.headers['content-type'] || 'image/jpeg');
|
||
res.setHeader('Cache-Control', 'public, max-age=3600');
|
||
imgRes.pipe(res);
|
||
});
|
||
r.on('error', () => res.status(502).end());
|
||
} catch { res.status(400).end(); }
|
||
});
|
||
|
||
// ── Tages-Cron: neue/geänderte Angebote per Pushover melden ─────────────────
|
||
const KOEPI_SETTINGS_KEY = 'koepi_last_offers';
|
||
|
||
// Vergleichbare, stabile Signatur eines Angebots (Reihenfolge-unabhängig).
|
||
// Bewusst NUR dateRange (echtes Datum), NICHT validity ("Noch X Tage") — der
|
||
// Countdown-Text ändert sich täglich von allein und würde sonst jeden Tag
|
||
// eine Pushover-Benachrichtigung auslösen, auch wenn sich am Angebot nichts
|
||
// geändert hat.
|
||
function offerSignature(offers) {
|
||
return offers
|
||
.map(o => `${(o.retailer || '').toLowerCase().trim()}|${(o.price || '').trim()}|${(o.dateRange || '').trim()}`)
|
||
.sort()
|
||
.join('\n');
|
||
}
|
||
|
||
function formatOfferMessage(offers) {
|
||
if (!offers.length) return 'Aktuell keine König Pilsener Angebote gefunden.';
|
||
return offers
|
||
.map(o => `${o.retailer || '?'}: ${o.price || '?'} (${o.dateRange || 'Gültigkeit unbekannt'})`)
|
||
.join('\n');
|
||
}
|
||
|
||
// onlyUserId gesetzt: nur an diesen einen Nutzer schicken (z.B. Admin-Testklick),
|
||
// sonst an alle Nutzer mit hinterlegtem Pushover (echter Tages-Cron)
|
||
async function sendKoepiPushover(offers, onlyUserId = null) {
|
||
const recipients = onlyUserId
|
||
? db.prepare(`
|
||
SELECT user_id, user_key, app_token FROM pushover_settings
|
||
WHERE user_id = ? AND user_key IS NOT NULL AND app_token IS NOT NULL
|
||
`).all(onlyUserId)
|
||
: db.prepare(`
|
||
SELECT user_id, user_key, app_token FROM pushover_settings
|
||
WHERE user_key IS NOT NULL AND app_token IS NOT NULL
|
||
`).all();
|
||
const message = formatOfferMessage(offers);
|
||
const title = '🍺 KöPi Angebote';
|
||
for (const r of recipients) {
|
||
try {
|
||
await fetch('https://api.pushover.net/1/messages.json', {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||
body: new URLSearchParams({
|
||
token: r.app_token,
|
||
user: r.user_key,
|
||
title,
|
||
message,
|
||
priority: 0,
|
||
}),
|
||
});
|
||
logPush({ userId: r.user_id, title, message, priority: 0, source: 'koepi' });
|
||
} catch (e) {
|
||
console.error('KöPi Pushover Fehler:', e.message);
|
||
logPush({ userId: r.user_id, title, message, priority: 0, source: 'koepi', success: false });
|
||
}
|
||
}
|
||
}
|
||
|
||
// Reiner Anzeige-Refresh für den HA-Button "KöPi Check jetzt" — scraped frisch
|
||
// und aktualisiert nur die MQTT-Entitäten. Sendet NIE Pushover und rührt den
|
||
// in admin_settings gespeicherten Vergleichsstand nicht an (der bleibt exklusiv
|
||
// dem 06:00-Cron vorbehalten, damit dessen Diff-Erkennung korrekt bleibt).
|
||
async function refreshOffersOnly() {
|
||
try {
|
||
const { offers } = await scrapeMarktguru(true);
|
||
const currentSig = offerSignature(offers);
|
||
const lastSig = db.prepare('SELECT value FROM admin_settings WHERE key=?').get(KOEPI_SETTINGS_KEY)?.value || '';
|
||
const changed = currentSig !== lastSig; // nur zur Anzeige im "geändert"-Sensor, löst nichts aus
|
||
mqttClient.publishKoepiState({ offers, changed });
|
||
console.log(`🍺 KöPi-Refresh (HA-Button): ${offers.length} Angebot(e) angezeigt, keine Pushover.`);
|
||
return { changed, sent: false, offerCount: offers.length };
|
||
} catch (e) {
|
||
console.error('KöPi Refresh Fehler:', e.message);
|
||
return { error: e.message };
|
||
}
|
||
}
|
||
|
||
// Wird täglich um 06:00 vom Scheduler in index.js aufgerufen (ohne onlyUserId
|
||
// → geht an alle). forceNotify=true (manueller Admin-Test): Pushover wird auch
|
||
// ohne Änderung verschickt; onlyUserId gesetzt: nur an diesen einen Nutzer.
|
||
async function runDailyCheck(forceNotify = false, onlyUserId = null) {
|
||
try {
|
||
const { offers } = await scrapeMarktguru(true);
|
||
const currentSig = offerSignature(offers);
|
||
const lastSig = db.prepare('SELECT value FROM admin_settings WHERE key=?').get(KOEPI_SETTINGS_KEY)?.value || '';
|
||
const changed = currentSig !== lastSig;
|
||
|
||
if (!changed && !forceNotify) {
|
||
console.log('🍺 KöPi-Check: keine Änderungen an den Angeboten.');
|
||
mqttClient.publishKoepiState({ offers, changed: false });
|
||
return { changed: false, sent: false, offerCount: offers.length };
|
||
}
|
||
|
||
await sendKoepiPushover(offers, onlyUserId);
|
||
db.prepare('INSERT OR REPLACE INTO admin_settings (key, value) VALUES (?, ?)')
|
||
.run(KOEPI_SETTINGS_KEY, currentSig);
|
||
console.log(`🍺 KöPi-Check: ${offers.length} Angebot(e)${changed ? ' geändert' : ' (Test, unverändert)'} – Pushover versendet${onlyUserId ? ` (nur an User ${onlyUserId})` : ''}.`);
|
||
mqttClient.publishKoepiState({ offers, changed });
|
||
return { changed, sent: true, offerCount: offers.length };
|
||
} catch (e) {
|
||
console.error('KöPi Tages-Check Fehler:', e.message);
|
||
return { error: e.message };
|
||
}
|
||
}
|
||
|
||
// ── Routes ────────────────────────────────────────────────────────────────────
|
||
router.get('/offers', authenticate, async (req, res) => {
|
||
try { res.json(await scrapeMarktguru()); }
|
||
catch(e) { console.error('KöPi offers:', e.message); res.status(500).json({ error: e.message }); }
|
||
});
|
||
|
||
router.get('/prospekte', authenticate, async (req, res) => {
|
||
try { res.json(await scrapeProspekte()); }
|
||
catch(e) {
|
||
console.error('🍺 KöPi Prospekte-Fehler:', e.message);
|
||
res.status(500).json({ error: e.message });
|
||
}
|
||
});
|
||
|
||
router.post('/clear-cache', authenticate, (req, res) => {
|
||
if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' });
|
||
['koepi:mg','koepi:prospekte'].forEach(k => delete cache[k]);
|
||
res.json({ ok: true });
|
||
});
|
||
|
||
// Manueller Test des Tages-Checks (Admin) — sendet Pushover auch ohne echte
|
||
// Änderung, aber NUR an den Admin selbst (nicht an alle Nutzer)
|
||
// GET /local-stores – aktuelle Filial-Whitelist anzeigen (Admin)
|
||
router.get('/local-stores', authenticate, (req, res) => {
|
||
if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' });
|
||
const stores = db.prepare('SELECT * FROM koepi_local_stores ORDER BY retailer').all();
|
||
res.json({ stores, seedCount: LOCAL_STORE_SEED_LEAFLETS.length + LOCAL_STORE_SEED_BY_ADDRESS.length });
|
||
});
|
||
|
||
// POST /local-stores/resolve – Filial-Whitelist neu auflösen (Admin, z.B. wenn
|
||
// sich ein Prospekt-Link geändert hat oder eine Filiale ergänzt wurde)
|
||
router.post('/local-stores/resolve', authenticate, async (req, res) => {
|
||
if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' });
|
||
try {
|
||
const resolved = await resolveLocalStores();
|
||
const stores = db.prepare('SELECT * FROM koepi_local_stores ORDER BY retailer').all();
|
||
res.json({ ok: true, resolved, total: LOCAL_STORE_SEED_LEAFLETS.length + LOCAL_STORE_SEED_BY_ADDRESS.length, stores });
|
||
} catch(e) { res.status(500).json({ error: e.message }); }
|
||
});
|
||
|
||
router.post('/run-daily-check', authenticate, async (req, res) => {
|
||
if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' });
|
||
try {
|
||
const result = await runDailyCheck(true, req.user.id);
|
||
if (result.error) return res.status(500).json({ error: result.error });
|
||
res.json(result);
|
||
} catch(e) { res.status(500).json({ error: e.message }); }
|
||
});
|
||
|
||
module.exports = router;
|
||
module.exports.runDailyCheck = runDailyCheck;
|
||
module.exports.refreshOffersOnly = refreshOffersOnly;
|