const express = require('express'); const https = require('https'); const { authenticate } = require('../../middleware/auth'); const router = express.Router(); // ── Cache ───────────────────────────────────────────────────────────────────── const cache = {}; function cacheGet(k) { const e=cache[k]; if(!e||Date.now()>e.ex) { delete cache[k]; return null; } return e.d; } function cacheSet(k,d,ms=60*60*1000) { cache[k]={d,ex:Date.now()+ms}; } // ── marktguru Cookie (Standort Duisburg 47259) ──────────────────────────────── const MG_SETTINGS = encodeURIComponent(JSON.stringify({ location: { name:'Duisburg', uniqueName:'duisburg', longitude:6.7625, latitude:51.4332, zipCodes:['47259'] }, locationSource: 'manual', hidePageflipMenu: false, cookieAccept: true, userGroup: 35, })); // ── Gewünschte Händler ──────────────────────────────────────────────────────── const TARGET_RETAILERS = ['edeka','e center','rewe','netto','kaufland','penny','trinkgut']; function isTargetRetailer(name) { if (!name) return false; const n = name.toLowerCase(); return TARGET_RETAILERS.some(r => n.includes(r)); } // ── Angebote via Puppeteer von marktguru ───────────────────────────────────── async function scrapeMarktguru() { const cacheKey = 'koepi:mg'; const cached = cacheGet(cacheKey); if (cached) return cached; let puppeteer; try { puppeteer = require('puppeteer-core'); } catch(e) { return { offers:[], error:'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() }; } const browser = await puppeteer.launch({ executablePath: '/usr/bin/chromium-browser', args: ['--no-sandbox','--disable-setuid-sandbox','--disable-dev-shm-usage','--disable-gpu','--single-process'], headless: true, }); try { const page = await browser.newPage(); await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'); await page.setViewport({ width:1280, height:800 }); await page.setCookie({ name:'mg_user-settings', value:MG_SETTINGS, domain:'.marktguru.de', path:'/' }); await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', { waitUntil: 'networkidle2', timeout: 30000, }); await new Promise(r => setTimeout(r, 4000)); const offers = await page.evaluate(() => { // Exakte Selektoren basierend auf marktguru HTML-Struktur const cards = document.querySelectorAll('li.offer-list-item'); return Array.from(cards).map(card => { const name = card.querySelector('h3')?.textContent?.trim() || ''; const brand = card.querySelector('dd.brand a')?.textContent?.trim() || ''; const price = card.querySelector('span.price')?.textContent?.trim() || ''; const oldPrice = card.querySelector('.price-bubble .old-price, .crossed')?.textContent?.trim() || ''; const retailer = card.querySelector('dd.retailer-name a')?.textContent?.trim() || ''; const dateRange= card.querySelector('dd.valid')?.textContent?.trim() || ''; const daysLeft = card.querySelector('dd.time-left span')?.textContent?.trim() || ''; const daysText = card.querySelector('dd.time-left')?.textContent?.trim() || ''; const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` : card.querySelector('dt.time-left')?.textContent?.includes('Brandneu') ? 'Neu' : ''; const descEl = card.querySelector('.info div, .info'); const description = descEl?.textContent?.trim() || ''; const img = card.querySelector('img.offer-list-item-img'); const badge = card.querySelector('.badge, .discount-badge')?.textContent?.trim() || ''; const address = card.querySelector('.retailer-address, .address, dd.address, [class*="address"]')?.textContent?.trim() || ''; // Bild: medium → large für bessere Qualität const imgSrc = img?.src?.replace('/medium.webp', '/large.webp') || img?.src || null; return { name, brand, price, oldPrice, retailer, dateRange, validity, description, badge, address, image: imgSrc }; }); }); // Filtern auf Ziel-Händler + nur König Pilsener const filtered = offers.filter(o => { const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase(); const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener'); return hasKP && isTargetRetailer(o.retailer); }); // Deduplizieren const seen = new Set(); const unique = filtered.filter(o => { const k = `${o.retailer}|${o.price}|${o.description?.slice(0,30)}`; if (seen.has(k)) return false; seen.add(k); return true; }); const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source: 'marktguru.de' }; cacheSet(cacheKey, result); return result; } finally { await browser.close(); } } // ── Prospekte von kaufda ────────────────────────────────────────────────────── function fetchUrl(url) { return new Promise((resolve, reject) => { const req = https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept': 'text/html', 'Accept-Language': 'de-DE,de;q=0.9', } }, res => { let d = ''; res.on('data', c => d += c); res.on('end', () => resolve({ status: res.statusCode, body: d })); }); req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); }); req.on('error', reject); }); } function extractNextData(html) { const marker = 'application/json">'; const ni = html.indexOf('NEXT_DATA'); if (ni === -1) return null; const st = html.indexOf(marker, ni) + marker.length; const en = html.indexOf('', st); try { return JSON.parse(html.slice(st, en)); } catch { return null; } } async function scrapeProspekte() { const cacheKey = 'koepi:prospekte'; const cached = cacheGet(cacheKey); if (cached) return cached; const { body } = await fetchUrl( 'https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=51.3750&lng=6.7680&zip=47259&city=Duisburg' ); const nd = extractNextData(body); const contents = nd?.props?.pageProps?.pageInformation?.shelfContents?.contents || []; const seen = new Map(); for (const item of contents) { const c = item.content; if (!c || item.contentType !== 'brochure') continue; const id = c.contentId; if (!id || seen.has(id)) continue; const pub = c.publisher?.name || ''; const store = c.closestStore; seen.set(id, { id, publisher: pub, title: c.title || '', store: store?.name || pub, city: store?.city || 'Duisburg', street: store ? `${store.street||''} ${store.streetNumber||''}`.trim() : '', zip: store?.zip || '47259', image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null, validFrom: c.validFrom || null, validTo: c.validUntil || null, pageCount: c.pageCount || null, url: `https://www.kaufda.de/webapp/brochure/${id}`, isTarget: isTargetRetailer(pub), badges: c.contentBadges?.map(b => b.name) || [], }); } const all = [...seen.values()]; const result = { targeted: all.filter(b => b.isTarget), others: all.filter(b => !b.isTarget), scrapedAt: new Date().toISOString(), }; cacheSet(cacheKey, result); return result; } // ── Bild-Proxy ──────────────────────────────────────────────────────────────── router.get('/img', authenticate, (req, res) => { const url = req.query.url; if (!url || !url.startsWith('https://')) return res.status(400).end(); try { const parsed = new URL(url); const allowed = ['content-media.bonial.biz','publisher-media.bonial.biz','cdn.marktguru.de','www.marktguru.de']; if (!allowed.some(h => parsed.hostname === h)) return res.status(403).end(); const r = https.get(url, { headers: {'User-Agent':'Mozilla/5.0','Referer':'https://www.marktguru.de/','Origin':'https://www.marktguru.de'} }, imgRes => { res.setHeader('Content-Type', imgRes.headers['content-type'] || 'image/jpeg'); res.setHeader('Cache-Control', 'public, max-age=3600'); imgRes.pipe(res); }); r.on('error', () => res.status(502).end()); } catch { res.status(400).end(); } }); // ── Routes ──────────────────────────────────────────────────────────────────── router.get('/offers', authenticate, async (req, res) => { try { res.json(await scrapeMarktguru()); } catch(e) { console.error('KöPi offers:', e.message); res.status(500).json({ error: e.message }); } }); router.get('/prospekte', authenticate, async (req, res) => { try { res.json(await scrapeProspekte()); } catch(e) { res.status(500).json({ error: e.message }); } }); router.post('/clear-cache', authenticate, (req, res) => { if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' }); ['koepi:mg','koepi:prospekte'].forEach(k => delete cache[k]); res.json({ ok: true }); }); module.exports = router;