diff --git a/backend/src/tools/koepi/routes.js b/backend/src/tools/koepi/routes.js index 73c2834..5d1733d 100644 --- a/backend/src/tools/koepi/routes.js +++ b/backend/src/tools/koepi/routes.js @@ -3,101 +3,37 @@ const https = require('https'); const { authenticate } = require('../../middleware/auth'); const router = express.Router(); -// ── Cache (in-memory, 1h) ───────────────────────────────────────────────────── +// ── Cache ───────────────────────────────────────────────────────────────────── const cache = {}; -function cacheGet(key) { - const e = cache[key]; - if (!e) return null; - if (Date.now() > e.expires) { delete cache[key]; return null; } - return e.data; -} -function cacheSet(key, data, ttlMs = 60 * 60 * 1000) { - cache[key] = { data, expires: Date.now() + ttlMs }; -} +function cacheGet(k) { const e=cache[k]; if(!e||Date.now()>e.ex) { delete cache[k]; return null; } return e.d; } +function cacheSet(k,d,ms=60*60*1000) { cache[k]={d,ex:Date.now()+ms}; } -// ── HTTP-Helper ─────────────────────────────────────────────────────────────── -function fetchUrl(url, maxRedirects = 5) { - return new Promise((resolve, reject) => { - const doGet = (u, remaining) => { - const req = https.get(u, { - headers: { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', - 'Accept': 'text/html,application/xhtml+xml', - 'Accept-Language': 'de-DE,de;q=0.9', - } - }, (res) => { - if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && remaining > 0) { - const next = res.headers.location.startsWith('http') - ? res.headers.location - : `https://www.kaufda.de${res.headers.location}`; - res.resume(); - doGet(next, remaining - 1); - return; - } - let d = ''; - res.on('data', c => d += c); - res.on('end', () => resolve({ status: res.statusCode, body: d })); - }); - req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); }); - req.on('error', reject); - }; - doGet(url, maxRedirects); - }); -} +// ── marktguru Cookie (Standort Duisburg 47259) ──────────────────────────────── +const MG_SETTINGS = encodeURIComponent(JSON.stringify({ + location: { name:'Duisburg', uniqueName:'duisburg', longitude:6.7625, latitude:51.4332, zipCodes:['47259'] }, + locationSource: 'manual', hidePageflipMenu: false, cookieAccept: true, userGroup: 35, +})); -// ── NEXT_DATA extrahieren ───────────────────────────────────────────────────── -function extractNextData(html) { - const marker = 'application/json">'; - const nidx = html.indexOf('NEXT_DATA'); - if (nidx === -1) return null; - const start = html.indexOf(marker, nidx) + marker.length; - const end = html.indexOf('', start); - if (start <= marker.length || end === -1) return null; - try { return JSON.parse(html.slice(start, end)); } catch { return null; } -} - -// ── Gewünschte Filialen ────────────────────────────────────────────────────── -const TARGET_STORES = [ - { name: 'edeka', street: 'düsseldorfer landstr', label: 'EDEKA E center Angerbogen' }, - { name: 'edeka', street: 'angerbogen', label: 'EDEKA E center Angerbogen' }, - { name: 'e center', street: 'düsseldorfer', label: 'EDEKA E center Angerbogen' }, - { name: 'rewe', street: 'mündelheimer', label: 'REWE Mündelheimer Str.' }, - { name: 'rewe', street: 'keniastr', label: 'REWE XXL Schwinning' }, - { name: 'netto', street: 'bonnefeld', label: 'Netto Im Bonnefeld' }, - { name: 'kaufland', street: 'auf der höhe', label: 'Kaufland Kaßlerfeld' }, - { name: 'penny', street: 'sittardsberger', label: 'Penny Sittardsberger Allee' }, - { name: 'trinkgut', street: 'keniastr', label: 'Trinkgut Keniastraße' }, -]; - -function matchesTargetStore(publisherName, storeName, street) { - const p = (publisherName || '').toLowerCase(); - const s = (storeName || '').toLowerCase(); - const r = (street || '').toLowerCase(); - for (const t of TARGET_STORES) { - if ((p.includes(t.name) || s.includes(t.name)) && r.includes(t.street)) return t.label; - } - return null; -} - -function isTargetPublisher(name) { +// ── Gewünschte Händler ──────────────────────────────────────────────────────── +const TARGET_RETAILERS = ['edeka','e center','rewe','netto','kaufland','penny','trinkgut']; +function isTargetRetailer(name) { if (!name) return false; const n = name.toLowerCase(); - return ['rewe','edeka','e center','netto','kaufland','penny','trinkgut'].some(p => n.includes(p)); + return TARGET_RETAILERS.some(r => n.includes(r)); } -// ── Angebote via Puppeteer (BrochureBox_Basic + OfferGrid) ────────────────── -async function scrapeOffersWithPuppeteer() { - const cacheKey = 'koepi:offers'; +// ── Angebote via Puppeteer von marktguru ───────────────────────────────────── +async function scrapeMarktguru() { + const cacheKey = 'koepi:mg'; const cached = cacheGet(cacheKey); if (cached) return cached; let puppeteer; - try { puppeteer = require('puppeteer-core'); } catch(e) { - return { offers: [], error: 'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() }; - } + try { puppeteer = require('puppeteer-core'); } + catch(e) { return { offers:[], error:'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() }; } const browser = await puppeteer.launch({ - executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium-browser', + executablePath: '/usr/bin/chromium-browser', args: ['--no-sandbox','--disable-setuid-sandbox','--disable-dev-shm-usage','--disable-gpu','--single-process'], headless: true, }); @@ -105,80 +41,85 @@ async function scrapeOffersWithPuppeteer() { try { const page = await browser.newPage(); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'); - await page.goto( - 'https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg', - { waitUntil: 'domcontentloaded', timeout: 25000 } - ); - await new Promise(r => setTimeout(r, 3000)); + await page.setViewport({ width:1280, height:800 }); + await page.setCookie({ name:'mg_user-settings', value:MG_SETTINGS, domain:'.marktguru.de', path:'/' }); - const data = await page.evaluate(() => { - // ── BrochureBox: Prospekte wo König Pilsener drin ist ────────────────── - const brochureOffers = Array.from(document.querySelectorAll('[data-testid="BrochureBox_Basic"]')).map(b => { - const text = b.textContent.trim(); - const imgs = Array.from(b.querySelectorAll('img')); - const productImg = imgs[0]?.src || null; - const publisherImg = imgs[1]?.src || null; + await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', { + waitUntil: 'networkidle2', timeout: 30000, + }); + await new Promise(r => setTimeout(r, 4000)); - // Text parsen - const storeMatch = text.match(/bei\s+([^""„]+?)\s*[""„]/); - const titleMatch = text.match(/[""„]([^""„]+?)[""„]/); - const pageMatch = text.match(/Seite\s+(\d+)/); - const validMatch = text.match(/(Noch \d+ Tag[e]? g.ltig|G.ltig bis [\d\.]+)/); - const dateMatch = text.match(/(\d{1,2}\.\d{1,2}\.(?:\d{4})?)\s*-\s*(\w+\.?\s+\d{1,2}\.\d{1,2}\.\d{4})/); - const isNew = text.startsWith('neu'); + const offers = await page.evaluate(() => { + const results = []; + // Angebots-Karten selektieren + const cards = document.querySelectorAll('.offer-item, [class*="offer-item"], .search-offer, [class*="searchOffer"]'); - // URL aus Link holen - const link = b.querySelector('a'); - const url = link?.href || null; + // Fallback: alle Container die Preis + Händler enthalten + const containers = cards.length > 0 ? cards : + document.querySelectorAll('[class*="offer"]:not([class*="more"]):not([class*="list"]):not([class*="section"])'); - return { - type: 'brochure', - store: storeMatch?.[1]?.trim() || null, - title: titleMatch?.[1]?.trim() || null, - page: pageMatch?.[1] ? parseInt(pageMatch[1]) : null, - dateRange: dateMatch ? `${dateMatch[1]} - ${dateMatch[2]}` : null, - validity: validMatch?.[1] || null, - isNew, - url, - productImage: productImg, - publisherLogo: publisherImg, - }; + containers.forEach(card => { + const text = card.innerText || card.textContent || ''; + if (!text.includes('König') && !text.includes('Pilsen')) return; + + const img = card.querySelector('img'); + const lines = text.split('\n').map(l => l.trim()).filter(Boolean); + + // Felder aus Text extrahieren + let name='', brand='', price='', retailer='', validity='', dateRange='', description='', oldPrice=''; + + lines.forEach((line, i) => { + if (line.match(/^€\s*\d/) && !price) price = line; + if (line.match(/^\d+[,\.]\d+\s*€/) && !price) price = line; + if (line.includes('Händler:')) retailer = lines[i+1] || line.replace('Händler:','').trim(); + if (line.includes('Marke:')) brand = lines[i+1] || line.replace('Marke:','').trim(); + if (line.includes('Gültig:') || line.includes('gültig:')) dateRange = lines[i+1] || line.replace(/G.ltig:/i,'').trim(); + if (line.includes('Noch') && line.includes('Tag')) validity = line; + if (line.includes('/ l') || line.includes('/l') || line.includes('Kasten') || line.includes('Flasche')) description = line; + if (line.includes('Brandneu')) validity = 'Neu'; + }); + + // Name ist meist die erste Zeile + name = lines[0] || ''; + + // Direkte DOM-Selektion als Alternative + const priceEl = card.querySelector('[class*="price"],[class*="Price"]'); + const retailEl = card.querySelector('[class*="retailer"],[class*="Retailer"],[class*="store"],[class*="Store"]'); + const nameEl = card.querySelector('[class*="title"],[class*="name"],h3,h2'); + const brandEl = card.querySelector('[class*="brand"],[class*="Brand"]'); + const descEl = card.querySelector('[class*="desc"],[class*="info"],[class*="detail"]'); + + if (priceEl) price = priceEl.textContent.trim(); + if (retailEl) retailer = retailEl.textContent.trim(); + if (nameEl) name = nameEl.textContent.trim(); + if (brandEl) brand = brandEl.textContent.trim(); + if (descEl) description = descEl.textContent.trim(); + + results.push({ + name, brand, price, oldPrice, retailer, validity, dateRange, description, + image: img?.src || null, + }); }); - // ── OfferGrid: einzelne Produktkacheln mit Preisen ────────────────────── - const grid = document.querySelector('[data-testid="OfferGrid"]'); - const gridOffers = grid ? Array.from(grid.querySelectorAll('[role="listitem"]')).map(item => { - const img = item.querySelector('img'); - const ps = Array.from(item.querySelectorAll('p')).map(p => p.textContent.trim()); - const brand = ps[0] || null; - const name = ps[1] || null; - const store = ps[2] || null; - const price = item.querySelector('[class*="text-primary"]')?.textContent?.trim() || null; - const baseUnit = ps[5] || ps[4] || null; // z.B. "(1 L = 1,20)" - const imgTitle = img?.title || ''; - const imgAlt = img?.alt || ''; - // Flaschengröße aus imgTitle/imgAlt extrahieren - const sizeMatch = (imgTitle + ' ' + imgAlt).match(/(\d+[\.,]\d*\s*[lLxX×]\s*\d*\s*(?:Fl\.?|Flasche|Flaschen|Dose|Dosen|Pack|Kiste)?|\d+\s*(?:×|x)\s*\d+[\.,]\d*\s*[lL]|\d+\s*[lL]|\d+\s*ml)/i); - const size = sizeMatch?.[0]?.trim() || null; - return { type:'offer', brand, name, store, price, baseUnit, size, image: img?.src || null, imgTitle }; - }).filter(o => { - if (!o.name) return false; - // Nur König Pilsener — kein König Ludwig, König der Löwen etc. - const fullText = (o.imgTitle + ' ' + (o.brand||'') + ' ' + (o.name||'')).toLowerCase(); - return fullText.includes('pilsener') || fullText.includes('pilsen') || - (fullText.includes('könig') && !fullText.includes('ludwig') && !fullText.includes('löwen') && !fullText.includes('weiss')); - }) : []; - - return { brochureOffers, gridOffers }; + return results; }); - const result = { - brochureOffers: data.brochureOffers, - gridOffers: data.gridOffers, - scrapedAt: new Date().toISOString(), - source: 'kaufda.de', - }; + // Filtern auf Ziel-Händler + nur König Pilsener + const filtered = offers.filter(o => { + const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase(); + const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener'); + return hasKP && isTargetRetailer(o.retailer); + }); + // Deduplizieren + const seen = new Set(); + const unique = filtered.filter(o => { + const k = `${o.retailer}|${o.price}|${o.description?.slice(0,30)}`; + if (seen.has(k)) return false; + seen.add(k); return true; + }); + + const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source: 'marktguru.de' }; cacheSet(cacheKey, result); return result; } finally { @@ -186,121 +127,111 @@ async function scrapeOffersWithPuppeteer() { } } -// ── Prospekte scrapen ───────────────────────────────────────────────────────── +// ── Prospekte von kaufda ────────────────────────────────────────────────────── +function fetchUrl(url) { + return new Promise((resolve, reject) => { + const req = https.get(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', + 'Accept': 'text/html', 'Accept-Language': 'de-DE,de;q=0.9', + } + }, res => { + let d = ''; + res.on('data', c => d += c); + res.on('end', () => resolve({ status: res.statusCode, body: d })); + }); + req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); }); + req.on('error', reject); + }); +} + +function extractNextData(html) { + const marker = 'application/json">'; + const ni = html.indexOf('NEXT_DATA'); + if (ni === -1) return null; + const st = html.indexOf(marker, ni) + marker.length; + const en = html.indexOf('', st); + try { return JSON.parse(html.slice(st, en)); } catch { return null; } +} + async function scrapeProspekte() { const cacheKey = 'koepi:prospekte'; const cached = cacheGet(cacheKey); if (cached) return cached; - // Koordinaten rund um deine Filialen (alle im Süden Duisburgs) - const locations = [ - { lat: 51.3750, lng: 6.7680, city: 'Duisburg', zip: '47259' }, // Im Bonnefeld / Netto - { lat: 51.4020, lng: 6.7616, city: 'Duisburg', zip: '47055' }, // REWE Mündelheimer - { lat: 51.4406, lng: 6.7555, city: 'Duisburg', zip: '47059' }, // Kaufland / REWE Kenia / Trinkgut - { lat: 51.3900, lng: 6.7900, city: 'Duisburg', zip: '47259' }, // Penny Sittardsberger - ]; + const { body } = await fetchUrl( + 'https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=51.3750&lng=6.7680&zip=47259&city=Duisburg' + ); + const nd = extractNextData(body); + const contents = nd?.props?.pageProps?.pageInformation?.shelfContents?.contents || []; - const allBrochures = new Map(); - - for (const loc of locations) { - try { - const { body } = await fetchUrl( - `https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=${loc.lat}&lng=${loc.lng}&zip=${loc.zip}&city=${encodeURIComponent(loc.city)}` - ); - const nextData = extractNextData(body); - if (!nextData) continue; - - const pp = nextData.props?.pageProps; - const contents = pp?.pageInformation?.shelfContents?.contents || []; - - for (const item of contents) { - const c = item.content; - if (!c || item.contentType !== 'brochure') continue; - const id = c.contentId; - if (!id || allBrochures.has(id)) continue; - - const publisher = c.publisher?.name || ''; - const store = c.closestStore; - - const storeStreet = store ? `${store.street || ''} ${store.streetNumber || ''}`.trim() : ''; - const matchLabel = matchesTargetStore(publisher, store?.name, storeStreet); - - allBrochures.set(id, { - id, - publisher, - title: c.title || '', - store: store?.name || publisher, - storeLabel: matchLabel || store?.name || publisher, - city: store?.city || loc.city, - street: storeStreet, - zip: store?.zip || loc.zip, - image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null, - validFrom: c.validFrom || null, - validTo: c.validUntil || null, - pageCount: c.pageCount || null, - url: `https://www.kaufda.de/webapp/brochure/${id}`, - isTarget: !!matchLabel, - badges: c.contentBadges?.map(b => b.name) || [], - }); - } - } catch(e) { - console.error('KöPi prospekte error:', loc.city, e.message); - } + const seen = new Map(); + for (const item of contents) { + const c = item.content; + if (!c || item.contentType !== 'brochure') continue; + const id = c.contentId; + if (!id || seen.has(id)) continue; + const pub = c.publisher?.name || ''; + const store = c.closestStore; + seen.set(id, { + id, publisher: pub, + title: c.title || '', + store: store?.name || pub, + city: store?.city || 'Duisburg', + street: store ? `${store.street||''} ${store.streetNumber||''}`.trim() : '', + zip: store?.zip || '47259', + image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null, + validFrom: c.validFrom || null, + validTo: c.validUntil || null, + pageCount: c.pageCount || null, + url: `https://www.kaufda.de/webapp/brochure/${id}`, + isTarget: isTargetRetailer(pub), + badges: c.contentBadges?.map(b => b.name) || [], + }); } - const brochures = [...allBrochures.values()]; + const all = [...seen.values()]; const result = { - targeted: brochures.filter(b => b.isTarget), - others: brochures.filter(b => !b.isTarget), - total: brochures.length, + targeted: all.filter(b => b.isTarget), + others: all.filter(b => !b.isTarget), scrapedAt: new Date().toISOString(), }; - cacheSet(cacheKey, result); return result; } +// ── Bild-Proxy ──────────────────────────────────────────────────────────────── +router.get('/img', authenticate, (req, res) => { + const url = req.query.url; + if (!url || !url.startsWith('https://')) return res.status(400).end(); + try { + const parsed = new URL(url); + const allowed = ['content-media.bonial.biz','publisher-media.bonial.biz','cdn.marktguru.de','www.marktguru.de']; + if (!allowed.some(h => parsed.hostname === h)) return res.status(403).end(); + const r = https.get(url, { headers: {'User-Agent':'Mozilla/5.0'} }, imgRes => { + res.setHeader('Content-Type', imgRes.headers['content-type'] || 'image/jpeg'); + res.setHeader('Cache-Control', 'public, max-age=3600'); + imgRes.pipe(res); + }); + r.on('error', () => res.status(502).end()); + } catch { res.status(400).end(); } +}); + // ── Routes ──────────────────────────────────────────────────────────────────── router.get('/offers', authenticate, async (req, res) => { - try { - const data = await scrapeOffersWithPuppeteer(); - res.json(data); - } catch(e) { - console.error('KöPi offers error:', e.message); - res.status(500).json({ error: e.message }); - } + try { res.json(await scrapeMarktguru()); } + catch(e) { console.error('KöPi offers:', e.message); res.status(500).json({ error: e.message }); } }); router.get('/prospekte', authenticate, async (req, res) => { - try { - const data = await scrapeProspekte(); - res.json(data); - } catch(e) { - console.error('KöPi prospekte error:', e.message); - res.status(500).json({ error: e.message }); - } + try { res.json(await scrapeProspekte()); } + catch(e) { res.status(500).json({ error: e.message }); } }); router.post('/clear-cache', authenticate, (req, res) => { if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' }); - delete cache['koepi:offers']; - delete cache['koepi:prospekte']; + ['koepi:mg','koepi:prospekte'].forEach(k => delete cache[k]); res.json({ ok: true }); }); -// ── Bild-Proxy (umgeht CSP) ────────────────────────────────────────────────── -router.get('/img', authenticate, (req, res) => { - const url = req.query.url; - if (!url || !url.startsWith('https://')) return res.status(400).end(); - const parsed = new URL(url); - const allowed = ['content-media.bonial.biz','publisher-media.bonial.biz','web-assets.kaufda.de']; - if (!allowed.some(h => parsed.hostname === h)) return res.status(403).end(); - const reqImg = https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0' } }, (imgRes) => { - res.setHeader('Content-Type', imgRes.headers['content-type'] || 'image/jpeg'); - res.setHeader('Cache-Control', 'public, max-age=3600'); - imgRes.pipe(res); - }); - reqImg.on('error', () => res.status(502).end()); -}); - module.exports = router; diff --git a/frontend/src/tools/koepi.jsx b/frontend/src/tools/koepi.jsx index 0ee39a0..c28844a 100644 --- a/frontend/src/tools/koepi.jsx +++ b/frontend/src/tools/koepi.jsx @@ -7,160 +7,103 @@ function getMyRole() { try { return JSON.parse(atob(localStorage.getItem('sk_token').split('.')[1])).role; } catch { return null; } } +function publisherColor(name) { + if (!name) return '#555'; + const n = name.toLowerCase(); + if (n.includes('rewe')) return '#e2001a'; + if (n.includes('edeka') || n.includes('e center')) return '#e8a800'; + if (n.includes('netto')) return '#0057a8'; + if (n.includes('trinkgut')) return '#e87722'; + if (n.includes('penny')) return '#e2001a'; + if (n.includes('kaufland')) return '#e2001a'; + return '#888'; +} + function fmtDate(str) { if (!str) return null; try { return new Date(str).toLocaleDateString('de-DE', { day:'2-digit', month:'2-digit', year:'numeric' }); } catch { return str; } } -function publisherColor(name) { - if (!name) return '#555'; - const n = name.toLowerCase(); - if (n.includes('rewe')) return '#e2001a'; - if (n.includes('edeka')) return '#e8a800'; - if (n.includes('e center')) return '#e8a800'; - if (n.includes('netto')) return '#0057a8'; - if (n.includes('trinkgut')) return '#e87722'; - if (n.includes('penny')) return '#e2001a'; - if (n.includes('aldi')) return '#00538f'; - if (n.includes('lidl')) return '#f5c900'; - if (n.includes('hit')) return '#e2001a'; - if (n.includes('alldrink')) return '#2ecc71'; - if (n.includes('getränke')) return '#3498db'; - return '#666'; -} - -// ── Prospekt-Angebot Karte ──────────────────────────────────────────────────── -function BrochureOfferCard({ offer }) { - const color = publisherColor(offer.store); - const Wrapper = offer.url - ? ({children}) => {children} - : ({children}) =>