diff --git a/backend/src/tools/koepi/routes.js b/backend/src/tools/koepi/routes.js index 434e6cd..c04b754 100644 --- a/backend/src/tools/koepi/routes.js +++ b/backend/src/tools/koepi/routes.js @@ -16,11 +16,28 @@ const MG_SETTINGS = encodeURIComponent(JSON.stringify({ // ── Gewünschte Händler ──────────────────────────────────────────────────────── const TARGET_RETAILERS = ['edeka','e center','rewe','netto','kaufland','penny','trinkgut']; +const RETAILER_ADDRESSES = { + 'edeka': 'Düsseldorfer Landstr. 361', + 'e center': 'Düsseldorfer Landstr. 361', + 'rewe': 'Mündelheimer Str. 132 / Keniastr. 39', + 'netto': 'Im Bonnefeld 19-21', + 'kaufland': 'Auf der Höhe 20', + 'penny': 'Sittardsberger Allee 10', + 'trinkgut': 'Keniastraße 39', +}; function isTargetRetailer(name) { if (!name) return false; const n = name.toLowerCase(); return TARGET_RETAILERS.some(r => n.includes(r)); } +function getAddress(retailer) { + if (!retailer) return ''; + const n = retailer.toLowerCase(); + for (const [key, addr] of Object.entries(RETAILER_ADDRESSES)) { + if (n.includes(key)) return addr; + } + return ''; +} // ── Angebote via Puppeteer von marktguru ───────────────────────────────────── async function scrapeMarktguru() { @@ -67,13 +84,28 @@ async function scrapeMarktguru() { const description = descEl?.textContent?.trim() || ''; const img = card.querySelector('img.offer-list-item-img'); const badge = card.querySelector('.badge, .discount-badge')?.textContent?.trim() || ''; - const address = card.querySelector('.retailer-address, .address, dd.address, [class*="address"]')?.textContent?.trim() || ''; - // Bild: medium → large für bessere Qualität - const imgSrc = img?.src?.replace('/medium.webp', '/large.webp') || img?.src || null; - return { name, brand, price, oldPrice, retailer, dateRange, validity, description, badge, address, image: imgSrc }; + const imgSrc = img?.src || null; + return { name, brand, price, oldPrice, retailer, dateRange, validity, description, badge, address: '', image: imgSrc }; }); }); + // Bilder als Base64 laden (cdn.marktguru.de blockt externe Requests) + for (const offer of offers) { + if (!offer.image) continue; + try { + const imgData = await page.evaluate(async (url) => { + const resp = await fetch(url); + const blob = await resp.blob(); + return new Promise(resolve => { + const reader = new FileReader(); + reader.onloadend = () => resolve(reader.result); + reader.readAsDataURL(blob); + }); + }, offer.image); + offer.image = imgData; // base64 data URL + } catch { offer.image = null; } + } + // Filtern auf Ziel-Händler + nur König Pilsener const filtered = offers.filter(o => { const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase(); @@ -81,13 +113,23 @@ async function scrapeMarktguru() { return hasKP && isTargetRetailer(o.retailer); }); - // Deduplizieren - const seen = new Set(); - const unique = filtered.filter(o => { - const k = `${o.retailer}|${o.price}|${o.description?.slice(0,30)}`; - if (seen.has(k)) return false; - seen.add(k); return true; - }); + // Deduplizieren: gleicher Händler + gleicher Preis → zusammenfassen + const merged = new Map(); + for (const o of filtered) { + const key = `${o.retailer}|${o.price}`; + if (merged.has(key)) { + // Beschreibungen zusammenführen falls unterschiedlich + const existing = merged.get(key); + if (o.description && o.description !== existing.description) { + existing.description = existing.description + ? `${existing.description} / ${o.description}` + : o.description; + } + } else { + merged.set(key, { ...o }); + } + } + const unique = [...merged.values()].map(o => ({ ...o, address: getAddress(o.retailer) })); const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source: 'marktguru.de' }; cacheSet(cacheKey, result); @@ -170,15 +212,15 @@ async function scrapeProspekte() { return result; } -// ── Bild-Proxy ──────────────────────────────────────────────────────────────── +// ── Bild-Proxy (für kaufda Bilder) ─────────────────────────────────────────── router.get('/img', authenticate, (req, res) => { const url = req.query.url; if (!url || !url.startsWith('https://')) return res.status(400).end(); try { const parsed = new URL(url); - const allowed = ['content-media.bonial.biz','publisher-media.bonial.biz','cdn.marktguru.de','www.marktguru.de']; + const allowed = ['content-media.bonial.biz','publisher-media.bonial.biz']; if (!allowed.some(h => parsed.hostname === h)) return res.status(403).end(); - const r = https.get(url, { headers: {'User-Agent':'Mozilla/5.0','Referer':'https://www.marktguru.de/','Origin':'https://www.marktguru.de'} }, imgRes => { + const r = https.get(url, { headers: {'User-Agent':'Mozilla/5.0'} }, imgRes => { res.setHeader('Content-Type', imgRes.headers['content-type'] || 'image/jpeg'); res.setHeader('Cache-Control', 'public, max-age=3600'); imgRes.pipe(res); diff --git a/frontend/src/tools/koepi.jsx b/frontend/src/tools/koepi.jsx index dc59c7a..9651e69 100644 --- a/frontend/src/tools/koepi.jsx +++ b/frontend/src/tools/koepi.jsx @@ -38,7 +38,7 @@ function OfferCard({ offer }) { {offer.image && (
König Pilsener { e.target.parentElement.style.display='none'; }}