432 lines
19 KiB
JavaScript
432 lines
19 KiB
JavaScript
const express = require('express');
|
||
const https = require('https');
|
||
const db = require('../../db');
|
||
const mqttClient = require('../../mqtt');
|
||
const { authenticate } = require('../../middleware/auth');
|
||
const router = express.Router();
|
||
|
||
// ── Cache ─────────────────────────────────────────────────────────────────────
|
||
const cache = {};
|
||
function cacheGet(k) { const e=cache[k]; if(!e||Date.now()>e.ex) { delete cache[k]; return null; } return e.d; }
|
||
function cacheSet(k,d,ms=60*60*1000) { cache[k]={d,ex:Date.now()+ms}; }
|
||
|
||
// ── marktguru Cookie (Standort Duisburg 47259) ────────────────────────────────
|
||
const MG_SETTINGS = encodeURIComponent(JSON.stringify({
|
||
location: { name:'Duisburg', uniqueName:'duisburg', longitude:6.7625, latitude:51.4332, zipCodes:['47259'] },
|
||
locationSource: 'manual', hidePageflipMenu: false, cookieAccept: true, userGroup: 35,
|
||
}));
|
||
|
||
// ── Gewünschte Händler ────────────────────────────────────────────────────────
|
||
const TARGET_RETAILERS = ['edeka','e center','rewe','netto','kaufland','penny','trinkgut'];
|
||
const RETAILER_ADDRESSES = {
|
||
'edeka': 'Düsseldorfer Landstr. 361',
|
||
'e center': 'Düsseldorfer Landstr. 361',
|
||
'rewe': 'Mündelheimer Str. 132 / Keniastr. 39',
|
||
'netto': 'Im Bonnefeld 19-21',
|
||
'kaufland': 'Auf der Höhe 20',
|
||
'penny': 'Sittardsberger Allee 10',
|
||
'trinkgut': 'Keniastraße 39',
|
||
};
|
||
function isTargetRetailer(name) {
|
||
if (!name) return false;
|
||
const n = name.toLowerCase();
|
||
return TARGET_RETAILERS.some(r => n.includes(r));
|
||
}
|
||
function getAddress(retailer) {
|
||
if (!retailer) return '';
|
||
const n = retailer.toLowerCase();
|
||
for (const [key, addr] of Object.entries(RETAILER_ADDRESSES)) {
|
||
if (n.includes(key)) return addr;
|
||
}
|
||
return '';
|
||
}
|
||
|
||
// Straßennamen robust vergleichbar machen (Groß-/Kleinschreibung, "straße"/"str.",
|
||
// Leerzeichen/Punkte egal — Hausnummer bleibt Teil des Vergleichs)
|
||
function normalizeAddress(s) {
|
||
return (s || '')
|
||
.toLowerCase()
|
||
.replace(/straße|strasse/g, 'str')
|
||
.replace(/[^a-zäöüß0-9]/g, '');
|
||
}
|
||
|
||
// Explizit unerwünschte Marken/Franchise-Label, unabhängig von der Adresse —
|
||
// "REWE Dortmund" ist auf marktguru ein eigener Franchise-Betreiber-Name, der
|
||
// auch dann raus soll, wenn seine Adresse zufällig mit einer lokalen Filiale
|
||
// übereinstimmt
|
||
const EXCLUDED_RETAILER_NAMES = ['rewe dortmund'];
|
||
function isExcludedRetailer(retailer) {
|
||
const n = (retailer || '').toLowerCase();
|
||
return EXCLUDED_RETAILER_NAMES.some(x => n.includes(x));
|
||
}
|
||
|
||
// Prüft, ob die tatsächlich gescrapte Adresse zu einer der bekannten LOKALEN
|
||
// Filialen für diesen Händler passt. Ohne erkannten Händler oder ohne gescrapte
|
||
// Adresse wird NICHT gefiltert (lieber ein Angebot zuviel zeigen als eins zu
|
||
// Unrecht verstecken, falls marktguru das Layout mal ändert).
|
||
function isLocalBranch(retailer, realAddress) {
|
||
const n = (retailer || '').toLowerCase();
|
||
const key = Object.keys(RETAILER_ADDRESSES).find(k => n.includes(k));
|
||
if (!key) return true;
|
||
if (!realAddress) return true;
|
||
const expectedParts = RETAILER_ADDRESSES[key].split('/').map(p => normalizeAddress(p.trim()));
|
||
const real = normalizeAddress(realAddress);
|
||
return expectedParts.some(p => real.includes(p) || (p && p.includes(real)));
|
||
}
|
||
|
||
// ── Angebote via Puppeteer von marktguru ─────────────────────────────────────
|
||
async function scrapeMarktguru(forceRefresh = false) {
|
||
const cacheKey = 'koepi:mg';
|
||
if (!forceRefresh) {
|
||
const cached = cacheGet(cacheKey);
|
||
if (cached) return cached;
|
||
}
|
||
|
||
let puppeteer;
|
||
try { puppeteer = require('puppeteer-core'); }
|
||
catch(e) { return { offers:[], error:'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() }; }
|
||
|
||
const browser = await puppeteer.launch({
|
||
executablePath: '/usr/bin/chromium-browser',
|
||
args: ['--no-sandbox','--disable-setuid-sandbox','--disable-dev-shm-usage','--disable-gpu','--single-process'],
|
||
headless: true,
|
||
});
|
||
|
||
try {
|
||
const page = await browser.newPage();
|
||
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
|
||
await page.setViewport({ width:1280, height:800 });
|
||
await page.setCookie({ name:'mg_user-settings', value:MG_SETTINGS, domain:'.marktguru.de', path:'/' });
|
||
|
||
await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', {
|
||
waitUntil: 'networkidle2', timeout: 30000,
|
||
});
|
||
// Seite nach unten scrollen damit lazy-loaded Bilder geladen werden
|
||
await page.evaluate(async () => {
|
||
await new Promise(resolve => {
|
||
let totalHeight = 0;
|
||
const distance = 300;
|
||
const timer = setInterval(() => {
|
||
window.scrollBy(0, distance);
|
||
totalHeight += distance;
|
||
if (totalHeight >= document.body.scrollHeight) {
|
||
clearInterval(timer);
|
||
resolve();
|
||
}
|
||
}, 200);
|
||
});
|
||
});
|
||
await new Promise(r => setTimeout(r, 3000));
|
||
|
||
const offers = await page.evaluate(() => {
|
||
// Exakte Selektoren basierend auf marktguru HTML-Struktur
|
||
const cards = document.querySelectorAll('li.offer-list-item');
|
||
return Array.from(cards).map(card => {
|
||
const name = card.querySelector('h3')?.textContent?.trim() || '';
|
||
const brand = card.querySelector('dd.brand a')?.textContent?.trim() || '';
|
||
const price = card.querySelector('span.price')?.textContent?.trim() || '';
|
||
const oldPrice = card.querySelector('.price-bubble .old-price, .crossed')?.textContent?.trim() || '';
|
||
const retailer = card.querySelector('dd.retailer-name a')?.textContent?.trim() || '';
|
||
const dateRange= card.querySelector('dd.valid')?.textContent?.trim() || '';
|
||
const daysLeft = card.querySelector('dd.time-left span')?.textContent?.trim() || '';
|
||
const daysText = card.querySelector('dd.time-left')?.textContent?.trim() || '';
|
||
const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` :
|
||
card.querySelector('dt.time-left')?.textContent?.includes('Brandneu') ? 'Neu' : '';
|
||
const descEl = card.querySelector('.info div, .info');
|
||
const description = descEl?.textContent?.trim() || '';
|
||
const img = card.querySelector('img.offer-list-item-img');
|
||
const badge = card.querySelector('.badge, .discount-badge')?.textContent?.trim() || '';
|
||
const imgSrc = img?.src?.replace('/medium.webp', '/large.webp') || img?.src || null;
|
||
const retailerDd = card.querySelector('dd.retailer-name');
|
||
const addressDd = retailerDd?.nextElementSibling?.tagName === 'DD' ? retailerDd.nextElementSibling : null;
|
||
const realAddress = addressDd?.textContent?.trim() || '';
|
||
return { name, brand, price, oldPrice, retailer, dateRange, validity, description, badge, address: '', realAddress, image: imgSrc };
|
||
});
|
||
});
|
||
|
||
// Lokale Bilder auswählen basierend auf Beschreibung
|
||
for (const offer of offers) {
|
||
const desc = (offer.description || '').toLowerCase();
|
||
console.log('DESC:', JSON.stringify(desc), '| RETAILER:', offer.retailer);
|
||
if (desc.includes('steini')) {
|
||
offer.imageLocal = '/koepi/kasten_steini.png';
|
||
} else if (desc.includes('11 x 0,5') || desc.includes('11x0,5')) {
|
||
offer.imageLocal = '/koepi/kasten_lang.png';
|
||
} else if (desc.includes('6 x') || desc.includes('6x') || (desc.includes('flasche') && !desc.includes('kasten'))) {
|
||
offer.imageLocal = '/koepi/traeger.png';
|
||
} else if (desc.includes('24 x 0,33') || desc.includes('24x0,33') || desc.includes('24 x 0,33')) {
|
||
offer.imageLocal = '/koepi/kasten_033.png';
|
||
} else if (desc.includes('0,5')) {
|
||
offer.imageLocal = '/koepi/kasten_050.png';
|
||
} else if (desc.includes('0,33')) {
|
||
offer.imageLocal = '/koepi/kasten_033.png';
|
||
} else {
|
||
offer.imageLocal = '/koepi/kasten_050.png';
|
||
}
|
||
offer.image = null; // CDN nicht mehr nötig
|
||
console.log('IMAGE:', offer.imageLocal, '| RETAILER:', offer.retailer);
|
||
}
|
||
|
||
// Filtern auf Ziel-Händler + nur König Pilsener
|
||
const filtered = offers.filter(o => {
|
||
const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase();
|
||
const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener');
|
||
return hasKP && isTargetRetailer(o.retailer);
|
||
});
|
||
|
||
// Nur Angebote der tatsächlich bekannten, lokalen Filialen behalten (marktguru
|
||
// zeigt teils auch weiter entfernte Filialen derselben Kette, z.B. "REWE Dortmund"
|
||
// oder einen anderen Netto/Kaufland-Standort — anhand der echten gescrapten
|
||
// Adresse rausfiltern statt sie fälschlich mit der lokalen Adresse zu überschreiben)
|
||
console.log('🍺 KöPi Adress-Diagnose:', filtered.map(o =>
|
||
`${o.retailer} | realAddress="${o.realAddress}" | excluded=${isExcludedRetailer(o.retailer)} | local=${isLocalBranch(o.retailer, o.realAddress)}`
|
||
).join('\n '));
|
||
|
||
const localOnly = filtered
|
||
.filter(o => !isExcludedRetailer(o.retailer))
|
||
.filter(o => isLocalBranch(o.retailer, o.realAddress));
|
||
|
||
// Händler normalisieren: E center = EDEKA
|
||
for (const o of localOnly) {
|
||
if (o.retailer?.toLowerCase().includes('e center')) o.retailerDisplay = o.retailer;
|
||
else o.retailerDisplay = o.retailer;
|
||
}
|
||
|
||
// Deduplizieren: gleicher normalisierter Händler + gleicher Preis → ein Eintrag
|
||
// Kürzeste Beschreibung nehmen (sauberste)
|
||
const merged = new Map();
|
||
for (const o of localOnly) {
|
||
const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || '';
|
||
const key = `${normalizedRetailer}|${o.price}`;
|
||
if (merged.has(key)) {
|
||
const existing = merged.get(key);
|
||
// Kürzere Beschreibung bevorzugen
|
||
if (o.description && (!existing.description || o.description.length < existing.description.length)) {
|
||
existing.description = o.description;
|
||
}
|
||
} else {
|
||
merged.set(key, { ...o });
|
||
}
|
||
}
|
||
const unique = [...merged.values()].map(o => ({ ...o, address: getAddress(o.retailer) }));
|
||
|
||
const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source: 'marktguru.de' };
|
||
cacheSet(cacheKey, result);
|
||
return result;
|
||
} finally {
|
||
await browser.close();
|
||
}
|
||
}
|
||
|
||
// ── Prospekte von kaufda ──────────────────────────────────────────────────────
|
||
function fetchUrl(url) {
|
||
return new Promise((resolve, reject) => {
|
||
const req = https.get(url, {
|
||
headers: {
|
||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||
'Accept': 'text/html', 'Accept-Language': 'de-DE,de;q=0.9',
|
||
}
|
||
}, res => {
|
||
let d = '';
|
||
res.on('data', c => d += c);
|
||
res.on('end', () => resolve({ status: res.statusCode, body: d }));
|
||
});
|
||
req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); });
|
||
req.on('error', reject);
|
||
});
|
||
}
|
||
|
||
function extractNextData(html) {
|
||
const marker = 'application/json">';
|
||
const ni = html.indexOf('NEXT_DATA');
|
||
if (ni === -1) return null;
|
||
const st = html.indexOf(marker, ni) + marker.length;
|
||
const en = html.indexOf('</script>', st);
|
||
try { return JSON.parse(html.slice(st, en)); } catch { return null; }
|
||
}
|
||
|
||
async function scrapeProspekte() {
|
||
const cacheKey = 'koepi:prospekte';
|
||
const cached = cacheGet(cacheKey);
|
||
if (cached) return cached;
|
||
|
||
const { body } = await fetchUrl(
|
||
'https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=51.3750&lng=6.7680&zip=47259&city=Duisburg'
|
||
);
|
||
const nd = extractNextData(body);
|
||
const contents = nd?.props?.pageProps?.pageInformation?.shelfContents?.contents || [];
|
||
|
||
const seen = new Map();
|
||
for (const item of contents) {
|
||
const c = item.content;
|
||
if (!c || item.contentType !== 'brochure') continue;
|
||
const id = c.contentId;
|
||
if (!id || seen.has(id)) continue;
|
||
const pub = c.publisher?.name || '';
|
||
const store = c.closestStore;
|
||
seen.set(id, {
|
||
id, publisher: pub,
|
||
title: c.title || '',
|
||
store: store?.name || pub,
|
||
city: store?.city || 'Duisburg',
|
||
street: store ? `${store.street||''} ${store.streetNumber||''}`.trim() : '',
|
||
zip: store?.zip || '47259',
|
||
image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null,
|
||
validFrom: c.validFrom || null,
|
||
validTo: c.validUntil || null,
|
||
pageCount: c.pageCount || null,
|
||
url: `https://www.kaufda.de/Geschaefte/${encodeURIComponent(pub.replace(/\s+/g,'-'))}`,
|
||
isTarget: isTargetRetailer(pub),
|
||
badges: c.contentBadges?.map(b => b.name) || [],
|
||
});
|
||
}
|
||
|
||
const all = [...seen.values()];
|
||
const result = {
|
||
targeted: all.filter(b => b.isTarget),
|
||
others: all.filter(b => !b.isTarget),
|
||
scrapedAt: new Date().toISOString(),
|
||
};
|
||
cacheSet(cacheKey, result);
|
||
return result;
|
||
}
|
||
|
||
// ── Bild-Proxy (für kaufda Bilder) ───────────────────────────────────────────
|
||
router.get('/img', authenticate, (req, res) => {
|
||
const url = req.query.url;
|
||
if (!url || !url.startsWith('https://')) return res.status(400).end();
|
||
try {
|
||
const parsed = new URL(url);
|
||
const allowed = ['content-media.bonial.biz','publisher-media.bonial.biz'];
|
||
if (!allowed.some(h => parsed.hostname === h)) return res.status(403).end();
|
||
const r = https.get(url, { headers: {'User-Agent':'Mozilla/5.0'} }, imgRes => {
|
||
res.setHeader('Content-Type', imgRes.headers['content-type'] || 'image/jpeg');
|
||
res.setHeader('Cache-Control', 'public, max-age=3600');
|
||
imgRes.pipe(res);
|
||
});
|
||
r.on('error', () => res.status(502).end());
|
||
} catch { res.status(400).end(); }
|
||
});
|
||
|
||
// ── Tages-Cron: neue/geänderte Angebote per Pushover melden ─────────────────
|
||
const KOEPI_SETTINGS_KEY = 'koepi_last_offers';
|
||
|
||
// Vergleichbare, stabile Signatur eines Angebots (Reihenfolge-unabhängig).
|
||
// Bewusst NUR dateRange (echtes Datum), NICHT validity ("Noch X Tage") — der
|
||
// Countdown-Text ändert sich täglich von allein und würde sonst jeden Tag
|
||
// eine Pushover-Benachrichtigung auslösen, auch wenn sich am Angebot nichts
|
||
// geändert hat.
|
||
function offerSignature(offers) {
|
||
return offers
|
||
.map(o => `${(o.retailer || '').toLowerCase().trim()}|${(o.price || '').trim()}|${(o.dateRange || '').trim()}`)
|
||
.sort()
|
||
.join('\n');
|
||
}
|
||
|
||
function formatOfferMessage(offers) {
|
||
if (!offers.length) return 'Aktuell keine König Pilsener Angebote gefunden.';
|
||
return offers
|
||
.map(o => `${o.retailer || '?'}: ${o.price || '?'} (${o.dateRange || 'Gültigkeit unbekannt'})`)
|
||
.join('\n');
|
||
}
|
||
|
||
async function sendKoepiPushover(offers) {
|
||
const recipients = db.prepare(`
|
||
SELECT user_key, app_token FROM pushover_settings
|
||
WHERE user_key IS NOT NULL AND app_token IS NOT NULL
|
||
`).all();
|
||
const message = formatOfferMessage(offers);
|
||
for (const r of recipients) {
|
||
try {
|
||
await fetch('https://api.pushover.net/1/messages.json', {
|
||
method: 'POST',
|
||
headers: { 'Content-Type': 'application/x-www-form-urlencoded' },
|
||
body: new URLSearchParams({
|
||
token: r.app_token,
|
||
user: r.user_key,
|
||
title: '🍺 KöPi Angebote',
|
||
message,
|
||
priority: 0,
|
||
}),
|
||
});
|
||
} catch (e) { console.error('KöPi Pushover Fehler:', e.message); }
|
||
}
|
||
}
|
||
|
||
// Reiner Anzeige-Refresh für den HA-Button "KöPi Check jetzt" — scraped frisch
|
||
// und aktualisiert nur die MQTT-Entitäten. Sendet NIE Pushover und rührt den
|
||
// in admin_settings gespeicherten Vergleichsstand nicht an (der bleibt exklusiv
|
||
// dem 06:00-Cron vorbehalten, damit dessen Diff-Erkennung korrekt bleibt).
|
||
async function refreshOffersOnly() {
|
||
try {
|
||
const { offers } = await scrapeMarktguru(true);
|
||
const currentSig = offerSignature(offers);
|
||
const lastSig = db.prepare('SELECT value FROM admin_settings WHERE key=?').get(KOEPI_SETTINGS_KEY)?.value || '';
|
||
const changed = currentSig !== lastSig; // nur zur Anzeige im "geändert"-Sensor, löst nichts aus
|
||
mqttClient.publishKoepiState({ offers, changed });
|
||
console.log(`🍺 KöPi-Refresh (HA-Button): ${offers.length} Angebot(e) angezeigt, keine Pushover.`);
|
||
return { changed, sent: false, offerCount: offers.length };
|
||
} catch (e) {
|
||
console.error('KöPi Refresh Fehler:', e.message);
|
||
return { error: e.message };
|
||
}
|
||
}
|
||
|
||
// Wird täglich um 06:00 vom Scheduler in index.js aufgerufen.
|
||
// forceNotify=true (manueller Admin-Test): Pushover wird auch ohne Änderung verschickt.
|
||
async function runDailyCheck(forceNotify = false) {
|
||
try {
|
||
const { offers } = await scrapeMarktguru(true);
|
||
const currentSig = offerSignature(offers);
|
||
const lastSig = db.prepare('SELECT value FROM admin_settings WHERE key=?').get(KOEPI_SETTINGS_KEY)?.value || '';
|
||
const changed = currentSig !== lastSig;
|
||
|
||
if (!changed && !forceNotify) {
|
||
console.log('🍺 KöPi-Check: keine Änderungen an den Angeboten.');
|
||
mqttClient.publishKoepiState({ offers, changed: false });
|
||
return { changed: false, sent: false, offerCount: offers.length };
|
||
}
|
||
|
||
await sendKoepiPushover(offers);
|
||
db.prepare('INSERT OR REPLACE INTO admin_settings (key, value) VALUES (?, ?)')
|
||
.run(KOEPI_SETTINGS_KEY, currentSig);
|
||
console.log(`🍺 KöPi-Check: ${offers.length} Angebot(e)${changed ? ' geändert' : ' (Test, unverändert)'} – Pushover versendet.`);
|
||
mqttClient.publishKoepiState({ offers, changed });
|
||
return { changed, sent: true, offerCount: offers.length };
|
||
} catch (e) {
|
||
console.error('KöPi Tages-Check Fehler:', e.message);
|
||
return { error: e.message };
|
||
}
|
||
}
|
||
|
||
// ── Routes ────────────────────────────────────────────────────────────────────
|
||
router.get('/offers', authenticate, async (req, res) => {
|
||
try { res.json(await scrapeMarktguru()); }
|
||
catch(e) { console.error('KöPi offers:', e.message); res.status(500).json({ error: e.message }); }
|
||
});
|
||
|
||
router.get('/prospekte', authenticate, async (req, res) => {
|
||
try { res.json(await scrapeProspekte()); }
|
||
catch(e) { res.status(500).json({ error: e.message }); }
|
||
});
|
||
|
||
router.post('/clear-cache', authenticate, (req, res) => {
|
||
if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' });
|
||
['koepi:mg','koepi:prospekte'].forEach(k => delete cache[k]);
|
||
res.json({ ok: true });
|
||
});
|
||
|
||
// Manueller Test des Tages-Checks (Admin) — sendet Pushover auch ohne echte Änderung
|
||
router.post('/run-daily-check', authenticate, async (req, res) => {
|
||
if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' });
|
||
try {
|
||
const result = await runDailyCheck(true);
|
||
if (result.error) return res.status(500).json({ error: result.error });
|
||
res.json(result);
|
||
} catch(e) { res.status(500).json({ error: e.message }); }
|
||
});
|
||
|
||
module.exports = router;
|
||
module.exports.runDailyCheck = runDailyCheck;
|
||
module.exports.refreshOffersOnly = refreshOffersOnly;
|