Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: KöPi - marktguru.de als Quelle, echte Angebote mit Flaschengröße

Browse Source
This commit is contained in:
Dicken
2026-07-09 00:41:58 +02:00
parent 73ce4264ae
commit d2b8706b26
2 changed files with 288 additions and 431 deletions
Download Patch File Download Diff File Expand all files Collapse all files

405
backend/src/tools/koepi/routes.js
View File

@@ -3,101 +3,37 @@ const https = require('https');
const { authenticate } = require('../../middleware/auth');
const router = express.Router();
// ── Cache (in-memory, 1h) ─────────────────────────────────────────────────────
// ── Cache ─────────────────────────────────────────────────────────────────────
const cache = {};
function cacheGet(key) {
const e = cache[key];
if (!e) return null;
if (Date.now() > e.expires) { delete cache[key]; return null; }
return e.data;
}
function cacheSet(key, data, ttlMs = 60 * 60 * 1000) {
cache[key] = { data, expires: Date.now() + ttlMs };
}
function cacheGet(k) { const e=cache[k]; if(!e||Date.now()>e.ex) { delete cache[k]; return null; } return e.d; }
function cacheSet(k,d,ms=60*60*1000) { cache[k]={d,ex:Date.now()+ms}; }
// ── HTTP-Helper ───────────────────────────────────────────────────────────────
function fetchUrl(url, maxRedirects = 5) {
return new Promise((resolve, reject) => {
const doGet = (u, remaining) => {
const req = https.get(u, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml',
'Accept-Language': 'de-DE,de;q=0.9',
}
}, (res) => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && remaining > 0) {
const next = res.headers.location.startsWith('http')
? res.headers.location
: `https://www.kaufda.de${res.headers.location}`;
res.resume();
doGet(next, remaining - 1);
return;
}
let d = '';
res.on('data', c => d += c);
res.on('end', () => resolve({ status: res.statusCode, body: d }));
});
req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); });
req.on('error', reject);
};
doGet(url, maxRedirects);
});
}
// ── marktguru Cookie (Standort Duisburg 47259) ────────────────────────────────
const MG_SETTINGS = encodeURIComponent(JSON.stringify({
location: { name:'Duisburg', uniqueName:'duisburg', longitude:6.7625, latitude:51.4332, zipCodes:['47259'] },
locationSource: 'manual', hidePageflipMenu: false, cookieAccept: true, userGroup: 35,
}));
// ── NEXT_DATA extrahieren ─────────────────────────────────────────────────────
function extractNextData(html) {
const marker = 'application/json">';
const nidx = html.indexOf('NEXT_DATA');
if (nidx === -1) return null;
const start = html.indexOf(marker, nidx) + marker.length;
const end = html.indexOf('</script>', start);
if (start <= marker.length || end === -1) return null;
try { return JSON.parse(html.slice(start, end)); } catch { return null; }
}
// ── Gewünschte Filialen ──────────────────────────────────────────────────────
const TARGET_STORES = [
{ name: 'edeka', street: 'düsseldorfer landstr', label: 'EDEKA E center Angerbogen' },
{ name: 'edeka', street: 'angerbogen', label: 'EDEKA E center Angerbogen' },
{ name: 'e center', street: 'düsseldorfer', label: 'EDEKA E center Angerbogen' },
{ name: 'rewe', street: 'mündelheimer', label: 'REWE Mündelheimer Str.' },
{ name: 'rewe', street: 'keniastr', label: 'REWE XXL Schwinning' },
{ name: 'netto', street: 'bonnefeld', label: 'Netto Im Bonnefeld' },
{ name: 'kaufland', street: 'auf der höhe', label: 'Kaufland Kaßlerfeld' },
{ name: 'penny', street: 'sittardsberger', label: 'Penny Sittardsberger Allee' },
{ name: 'trinkgut', street: 'keniastr', label: 'Trinkgut Keniastraße' },
];
function matchesTargetStore(publisherName, storeName, street) {
const p = (publisherName || '').toLowerCase();
const s = (storeName || '').toLowerCase();
const r = (street || '').toLowerCase();
for (const t of TARGET_STORES) {
if ((p.includes(t.name) || s.includes(t.name)) && r.includes(t.street)) return t.label;
}
return null;
}
function isTargetPublisher(name) {
// ── Gewünschte Händler ────────────────────────────────────────────────────────
const TARGET_RETAILERS = ['edeka','e center','rewe','netto','kaufland','penny','trinkgut'];
function isTargetRetailer(name) {
if (!name) return false;
const n = name.toLowerCase();
return ['rewe','edeka','e center','netto','kaufland','penny','trinkgut'].some(p => n.includes(p));
return TARGET_RETAILERS.some(r => n.includes(r));
}
// ── Angebote via Puppeteer (BrochureBox_Basic + OfferGrid) ──────────────────
async function scrapeOffersWithPuppeteer() {
const cacheKey = 'koepi:offers';
// ── Angebote via Puppeteer von marktguru ─────────────────────────────────────
async function scrapeMarktguru() {
const cacheKey = 'koepi:mg';
const cached = cacheGet(cacheKey);
if (cached) return cached;
let puppeteer;
try { puppeteer = require('puppeteer-core'); } catch(e) {
return { offers: [], error: 'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() };
}
try { puppeteer = require('puppeteer-core'); }
catch(e) { return { offers:[], error:'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() }; }
const browser = await puppeteer.launch({
executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium-browser',
executablePath: '/usr/bin/chromium-browser',
args: ['--no-sandbox','--disable-setuid-sandbox','--disable-dev-shm-usage','--disable-gpu','--single-process'],
headless: true,
});
@@ -105,80 +41,85 @@ async function scrapeOffersWithPuppeteer() {
try {
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
await page.goto(
'https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg',
{ waitUntil: 'domcontentloaded', timeout: 25000 }
);
await new Promise(r => setTimeout(r, 3000));
await page.setViewport({ width:1280, height:800 });
await page.setCookie({ name:'mg_user-settings', value:MG_SETTINGS, domain:'.marktguru.de', path:'/' });
const data = await page.evaluate(() => {
// ── BrochureBox: Prospekte wo König Pilsener drin ist ──────────────────
const brochureOffers = Array.from(document.querySelectorAll('[data-testid="BrochureBox_Basic"]')).map(b => {
const text = b.textContent.trim();
const imgs = Array.from(b.querySelectorAll('img'));
const productImg = imgs[0]?.src || null;
const publisherImg = imgs[1]?.src || null;
await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', {
waitUntil: 'networkidle2', timeout: 30000,
});
await new Promise(r => setTimeout(r, 4000));
// Text parsen
const storeMatch = text.match(/bei\s+([^""„]+?)\s*[""„]/);
const titleMatch = text.match(/[""„]([^""„]+?)[""„]/);
const pageMatch = text.match(/Seite\s+(\d+)/);
const validMatch = text.match(/(Noch \d+ Tag[e]? g.ltig|G.ltig bis [\d\.]+)/);
const dateMatch = text.match(/(\d{1,2}\.\d{1,2}\.(?:\d{4})?)\s*-\s*(\w+\.?\s+\d{1,2}\.\d{1,2}\.\d{4})/);
const isNew = text.startsWith('neu');
const offers = await page.evaluate(() => {
const results = [];
// Angebots-Karten selektieren
const cards = document.querySelectorAll('.offer-item, [class*="offer-item"], .search-offer, [class*="searchOffer"]');
// URL aus Link holen
const link = b.querySelector('a');
const url = link?.href || null;
// Fallback: alle Container die Preis + Händler enthalten
const containers = cards.length > 0 ? cards :
document.querySelectorAll('[class*="offer"]:not([class*="more"]):not([class*="list"]):not([class*="section"])');
return {
type: 'brochure',
store: storeMatch?.[1]?.trim() || null,
title: titleMatch?.[1]?.trim() || null,
page: pageMatch?.[1] ? parseInt(pageMatch[1]) : null,
dateRange: dateMatch ? `${dateMatch[1]} - ${dateMatch[2]}` : null,
validity: validMatch?.[1] || null,
isNew,
url,
productImage: productImg,
publisherLogo: publisherImg,
};
containers.forEach(card => {
const text = card.innerText || card.textContent || '';
if (!text.includes('König') && !text.includes('Pilsen')) return;
const img = card.querySelector('img');
const lines = text.split('\n').map(l => l.trim()).filter(Boolean);
// Felder aus Text extrahieren
let name='', brand='', price='', retailer='', validity='', dateRange='', description='', oldPrice='';
lines.forEach((line, i) => {
if (line.match(/^€\s*\d/) && !price) price = line;
if (line.match(/^\d+[,\.]\d+\s*€/) && !price) price = line;
if (line.includes('Händler:')) retailer = lines[i+1] || line.replace('Händler:','').trim();
if (line.includes('Marke:')) brand = lines[i+1] || line.replace('Marke:','').trim();
if (line.includes('Gültig:') || line.includes('gültig:')) dateRange = lines[i+1] || line.replace(/G.ltig:/i,'').trim();
if (line.includes('Noch') && line.includes('Tag')) validity = line;
if (line.includes('/ l') || line.includes('/l') || line.includes('Kasten') || line.includes('Flasche')) description = line;
if (line.includes('Brandneu')) validity = 'Neu';
});
// Name ist meist die erste Zeile
name = lines[0] || '';
// Direkte DOM-Selektion als Alternative
const priceEl = card.querySelector('[class*="price"],[class*="Price"]');
const retailEl = card.querySelector('[class*="retailer"],[class*="Retailer"],[class*="store"],[class*="Store"]');
const nameEl = card.querySelector('[class*="title"],[class*="name"],h3,h2');
const brandEl = card.querySelector('[class*="brand"],[class*="Brand"]');
const descEl = card.querySelector('[class*="desc"],[class*="info"],[class*="detail"]');
if (priceEl) price = priceEl.textContent.trim();
if (retailEl) retailer = retailEl.textContent.trim();
if (nameEl) name = nameEl.textContent.trim();
if (brandEl) brand = brandEl.textContent.trim();
if (descEl) description = descEl.textContent.trim();
results.push({
name, brand, price, oldPrice, retailer, validity, dateRange, description,
image: img?.src || null,
});
});
// ── OfferGrid: einzelne Produktkacheln mit Preisen ──────────────────────
const grid = document.querySelector('[data-testid="OfferGrid"]');
const gridOffers = grid ? Array.from(grid.querySelectorAll('[role="listitem"]')).map(item => {
const img = item.querySelector('img');
const ps = Array.from(item.querySelectorAll('p')).map(p => p.textContent.trim());
const brand = ps[0] || null;
const name = ps[1] || null;
const store = ps[2] || null;
const price = item.querySelector('[class*="text-primary"]')?.textContent?.trim() || null;
const baseUnit = ps[5] || ps[4] || null; // z.B. "(1 L = 1,20)"
const imgTitle = img?.title || '';
const imgAlt = img?.alt || '';
// Flaschengröße aus imgTitle/imgAlt extrahieren
const sizeMatch = (imgTitle + ' ' + imgAlt).match(/(\d+[\.,]\d*\s*[lLxX×]\s*\d*\s*(?:Fl\.?|Flasche|Flaschen|Dose|Dosen|Pack|Kiste)?|\d+\s*(?:×|x)\s*\d+[\.,]\d*\s*[lL]|\d+\s*[lL]|\d+\s*ml)/i);
const size = sizeMatch?.[0]?.trim() || null;
return { type:'offer', brand, name, store, price, baseUnit, size, image: img?.src || null, imgTitle };
}).filter(o => {
if (!o.name) return false;
// Nur König Pilsener — kein König Ludwig, König der Löwen etc.
const fullText = (o.imgTitle + ' ' + (o.brand||'') + ' ' + (o.name||'')).toLowerCase();
return fullText.includes('pilsener') || fullText.includes('pilsen') ||
(fullText.includes('könig') && !fullText.includes('ludwig') && !fullText.includes('löwen') && !fullText.includes('weiss'));
}) : [];
return { brochureOffers, gridOffers };
return results;
});
const result = {
brochureOffers: data.brochureOffers,
gridOffers: data.gridOffers,
scrapedAt: new Date().toISOString(),
source: 'kaufda.de',
};
// Filtern auf Ziel-Händler + nur König Pilsener
const filtered = offers.filter(o => {
const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase();
const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener');
return hasKP && isTargetRetailer(o.retailer);
});
// Deduplizieren
const seen = new Set();
const unique = filtered.filter(o => {
const k = `${o.retailer}|${o.price}|${o.description?.slice(0,30)}`;
if (seen.has(k)) return false;
seen.add(k); return true;
});
const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source: 'marktguru.de' };
cacheSet(cacheKey, result);
return result;
} finally {
@@ -186,121 +127,111 @@ async function scrapeOffersWithPuppeteer() {
}
}
// ── Prospekte scrapen ─────────────────────────────────────────────────────────
// ── Prospekte von kaufda ──────────────────────────────────────────────────────
function fetchUrl(url) {
return new Promise((resolve, reject) => {
const req = https.get(url, {
headers: {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html', 'Accept-Language': 'de-DE,de;q=0.9',
}
}, res => {
let d = '';
res.on('data', c => d += c);
res.on('end', () => resolve({ status: res.statusCode, body: d }));
});
req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); });
req.on('error', reject);
});
}
function extractNextData(html) {
const marker = 'application/json">';
const ni = html.indexOf('NEXT_DATA');
if (ni === -1) return null;
const st = html.indexOf(marker, ni) + marker.length;
const en = html.indexOf('</script>', st);
try { return JSON.parse(html.slice(st, en)); } catch { return null; }
}
async function scrapeProspekte() {
const cacheKey = 'koepi:prospekte';
const cached = cacheGet(cacheKey);
if (cached) return cached;
// Koordinaten rund um deine Filialen (alle im Süden Duisburgs)
const locations = [
{ lat: 51.3750, lng: 6.7680, city: 'Duisburg', zip: '47259' }, // Im Bonnefeld / Netto
{ lat: 51.4020, lng: 6.7616, city: 'Duisburg', zip: '47055' }, // REWE Mündelheimer
{ lat: 51.4406, lng: 6.7555, city: 'Duisburg', zip: '47059' }, // Kaufland / REWE Kenia / Trinkgut
{ lat: 51.3900, lng: 6.7900, city: 'Duisburg', zip: '47259' }, // Penny Sittardsberger
];
const { body } = await fetchUrl(
'https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=51.3750&lng=6.7680&zip=47259&city=Duisburg'
);
const nd = extractNextData(body);
const contents = nd?.props?.pageProps?.pageInformation?.shelfContents?.contents || [];
const allBrochures = new Map();
for (const loc of locations) {
try {
const { body } = await fetchUrl(
`https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=${loc.lat}&lng=${loc.lng}&zip=${loc.zip}&city=${encodeURIComponent(loc.city)}`
);
const nextData = extractNextData(body);
if (!nextData) continue;
const pp = nextData.props?.pageProps;
const contents = pp?.pageInformation?.shelfContents?.contents || [];
for (const item of contents) {
const c = item.content;
if (!c || item.contentType !== 'brochure') continue;
const id = c.contentId;
if (!id || allBrochures.has(id)) continue;
const publisher = c.publisher?.name || '';
const store = c.closestStore;
const storeStreet = store ? `${store.street || ''} ${store.streetNumber || ''}`.trim() : '';
const matchLabel = matchesTargetStore(publisher, store?.name, storeStreet);
allBrochures.set(id, {
id,
publisher,
title: c.title || '',
store: store?.name || publisher,
storeLabel: matchLabel || store?.name || publisher,
city: store?.city || loc.city,
street: storeStreet,
zip: store?.zip || loc.zip,
image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null,
validFrom: c.validFrom || null,
validTo: c.validUntil || null,
pageCount: c.pageCount || null,
url: `https://www.kaufda.de/webapp/brochure/${id}`,
isTarget: !!matchLabel,
badges: c.contentBadges?.map(b => b.name) || [],
});
}
} catch(e) {
console.error('KöPi prospekte error:', loc.city, e.message);
}
const seen = new Map();
for (const item of contents) {
const c = item.content;
if (!c || item.contentType !== 'brochure') continue;
const id = c.contentId;
if (!id || seen.has(id)) continue;
const pub = c.publisher?.name || '';
const store = c.closestStore;
seen.set(id, {
id, publisher: pub,
title: c.title || '',
store: store?.name || pub,
city: store?.city || 'Duisburg',
street: store ? `${store.street||''} ${store.streetNumber||''}`.trim() : '',
zip: store?.zip || '47259',
image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null,
validFrom: c.validFrom || null,
validTo: c.validUntil || null,
pageCount: c.pageCount || null,
url: `https://www.kaufda.de/webapp/brochure/${id}`,
isTarget: isTargetRetailer(pub),
badges: c.contentBadges?.map(b => b.name) || [],
});
}
const brochures = [...allBrochures.values()];
const all = [...seen.values()];
const result = {
targeted: brochures.filter(b => b.isTarget),
others: brochures.filter(b => !b.isTarget),
total: brochures.length,
targeted: all.filter(b => b.isTarget),
others: all.filter(b => !b.isTarget),
scrapedAt: new Date().toISOString(),
};
cacheSet(cacheKey, result);
return result;
}
// ── Bild-Proxy ────────────────────────────────────────────────────────────────
router.get('/img', authenticate, (req, res) => {
const url = req.query.url;
if (!url || !url.startsWith('https://')) return res.status(400).end();
try {
const parsed = new URL(url);
const allowed = ['content-media.bonial.biz','publisher-media.bonial.biz','cdn.marktguru.de','www.marktguru.de'];
if (!allowed.some(h => parsed.hostname === h)) return res.status(403).end();
const r = https.get(url, { headers: {'User-Agent':'Mozilla/5.0'} }, imgRes => {
res.setHeader('Content-Type', imgRes.headers['content-type'] || 'image/jpeg');
res.setHeader('Cache-Control', 'public, max-age=3600');
imgRes.pipe(res);
});
r.on('error', () => res.status(502).end());
} catch { res.status(400).end(); }
});
// ── Routes ────────────────────────────────────────────────────────────────────
router.get('/offers', authenticate, async (req, res) => {
try {
const data = await scrapeOffersWithPuppeteer();
res.json(data);
} catch(e) {
console.error('KöPi offers error:', e.message);
res.status(500).json({ error: e.message });
}
try { res.json(await scrapeMarktguru()); }
catch(e) { console.error('KöPi offers:', e.message); res.status(500).json({ error: e.message }); }
});
router.get('/prospekte', authenticate, async (req, res) => {
try {
const data = await scrapeProspekte();
res.json(data);
} catch(e) {
console.error('KöPi prospekte error:', e.message);
res.status(500).json({ error: e.message });
}
try { res.json(await scrapeProspekte()); }
catch(e) { res.status(500).json({ error: e.message }); }
});
router.post('/clear-cache', authenticate, (req, res) => {
if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' });
delete cache['koepi:offers'];
delete cache['koepi:prospekte'];
['koepi:mg','koepi:prospekte'].forEach(k => delete cache[k]);
res.json({ ok: true });
});
// ── Bild-Proxy (umgeht CSP) ──────────────────────────────────────────────────
router.get('/img', authenticate, (req, res) => {
const url = req.query.url;
if (!url || !url.startsWith('https://')) return res.status(400).end();
const parsed = new URL(url);
const allowed = ['content-media.bonial.biz','publisher-media.bonial.biz','web-assets.kaufda.de'];
if (!allowed.some(h => parsed.hostname === h)) return res.status(403).end();
const reqImg = https.get(url, { headers: { 'User-Agent': 'Mozilla/5.0' } }, (imgRes) => {
res.setHeader('Content-Type', imgRes.headers['content-type'] || 'image/jpeg');
res.setHeader('Cache-Control', 'public, max-age=3600');
imgRes.pipe(res);
});
reqImg.on('error', () => res.status(502).end());
});
module.exports = router;