257 lines
12 KiB
JavaScript
257 lines
12 KiB
JavaScript
const express = require('express');
|
|
const https = require('https');
|
|
const { authenticate } = require('../../middleware/auth');
|
|
const router = express.Router();
|
|
|
|
// ── Cache ─────────────────────────────────────────────────────────────────────
|
|
const cache = {};
|
|
function cacheGet(k) { const e=cache[k]; if(!e||Date.now()>e.ex) { delete cache[k]; return null; } return e.d; }
|
|
function cacheSet(k,d,ms=60*60*1000) { cache[k]={d,ex:Date.now()+ms}; }
|
|
|
|
// ── marktguru Cookie (Standort Duisburg 47259) ────────────────────────────────
|
|
const MG_SETTINGS = encodeURIComponent(JSON.stringify({
|
|
location: { name:'Duisburg', uniqueName:'duisburg', longitude:6.7625, latitude:51.4332, zipCodes:['47259'] },
|
|
locationSource: 'manual', hidePageflipMenu: false, cookieAccept: true, userGroup: 35,
|
|
}));
|
|
|
|
// ── Gewünschte Händler ────────────────────────────────────────────────────────
|
|
const TARGET_RETAILERS = ['edeka','e center','rewe','netto','kaufland','penny','trinkgut'];
|
|
const RETAILER_ADDRESSES = {
|
|
'edeka': 'Düsseldorfer Landstr. 361',
|
|
'e center': 'Düsseldorfer Landstr. 361',
|
|
'rewe': 'Mündelheimer Str. 132 / Keniastr. 39',
|
|
'netto': 'Im Bonnefeld 19-21',
|
|
'kaufland': 'Auf der Höhe 20',
|
|
'penny': 'Sittardsberger Allee 10',
|
|
'trinkgut': 'Keniastraße 39',
|
|
};
|
|
function isTargetRetailer(name) {
|
|
if (!name) return false;
|
|
const n = name.toLowerCase();
|
|
return TARGET_RETAILERS.some(r => n.includes(r));
|
|
}
|
|
function getAddress(retailer) {
|
|
if (!retailer) return '';
|
|
const n = retailer.toLowerCase();
|
|
for (const [key, addr] of Object.entries(RETAILER_ADDRESSES)) {
|
|
if (n.includes(key)) return addr;
|
|
}
|
|
return '';
|
|
}
|
|
|
|
// ── Angebote via Puppeteer von marktguru ─────────────────────────────────────
|
|
async function scrapeMarktguru() {
|
|
const cacheKey = 'koepi:mg';
|
|
const cached = cacheGet(cacheKey);
|
|
if (cached) return cached;
|
|
|
|
let puppeteer;
|
|
try { puppeteer = require('puppeteer-core'); }
|
|
catch(e) { return { offers:[], error:'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() }; }
|
|
|
|
const browser = await puppeteer.launch({
|
|
executablePath: '/usr/bin/chromium-browser',
|
|
args: ['--no-sandbox','--disable-setuid-sandbox','--disable-dev-shm-usage','--disable-gpu','--single-process'],
|
|
headless: true,
|
|
});
|
|
|
|
try {
|
|
const page = await browser.newPage();
|
|
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
|
|
await page.setViewport({ width:1280, height:800 });
|
|
await page.setCookie({ name:'mg_user-settings', value:MG_SETTINGS, domain:'.marktguru.de', path:'/' });
|
|
|
|
await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', {
|
|
waitUntil: 'networkidle2', timeout: 30000,
|
|
});
|
|
await new Promise(r => setTimeout(r, 4000));
|
|
|
|
const offers = await page.evaluate(() => {
|
|
// Exakte Selektoren basierend auf marktguru HTML-Struktur
|
|
const cards = document.querySelectorAll('li.offer-list-item');
|
|
return Array.from(cards).map(card => {
|
|
const name = card.querySelector('h3')?.textContent?.trim() || '';
|
|
const brand = card.querySelector('dd.brand a')?.textContent?.trim() || '';
|
|
const price = card.querySelector('span.price')?.textContent?.trim() || '';
|
|
const oldPrice = card.querySelector('.price-bubble .old-price, .crossed')?.textContent?.trim() || '';
|
|
const retailer = card.querySelector('dd.retailer-name a')?.textContent?.trim() || '';
|
|
const dateRange= card.querySelector('dd.valid')?.textContent?.trim() || '';
|
|
const daysLeft = card.querySelector('dd.time-left span')?.textContent?.trim() || '';
|
|
const daysText = card.querySelector('dd.time-left')?.textContent?.trim() || '';
|
|
const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` :
|
|
card.querySelector('dt.time-left')?.textContent?.includes('Brandneu') ? 'Neu' : '';
|
|
const descEl = card.querySelector('.info div, .info');
|
|
const description = descEl?.textContent?.trim() || '';
|
|
const img = card.querySelector('img.offer-list-item-img');
|
|
const badge = card.querySelector('.badge, .discount-badge')?.textContent?.trim() || '';
|
|
const imgSrc = img?.src || null;
|
|
return { name, brand, price, oldPrice, retailer, dateRange, validity, description, badge, address: '', image: imgSrc };
|
|
});
|
|
});
|
|
|
|
// Bilder via Puppeteer screenshot der img-Elemente
|
|
const imgElements = await page.$$('img.offer-list-item-img');
|
|
for (let i = 0; i < offers.length; i++) {
|
|
if (!offers[i].image || !imgElements[i]) continue;
|
|
try {
|
|
// Warte bis Bild geladen
|
|
await page.evaluate(el => new Promise(res => {
|
|
if (el.complete) return res();
|
|
el.onload = res; el.onerror = res;
|
|
setTimeout(res, 3000);
|
|
}), imgElements[i]);
|
|
const b64 = await imgElements[i].screenshot({ encoding: 'base64' });
|
|
offers[i].image = b64 ? `data:image/png;base64,${b64}` : null;
|
|
} catch { offers[i].image = null; }
|
|
}
|
|
|
|
// Filtern auf Ziel-Händler + nur König Pilsener
|
|
const filtered = offers.filter(o => {
|
|
const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase();
|
|
const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener');
|
|
return hasKP && isTargetRetailer(o.retailer);
|
|
});
|
|
|
|
// Händler normalisieren: E center = EDEKA
|
|
for (const o of filtered) {
|
|
if (o.retailer?.toLowerCase().includes('e center')) o.retailerDisplay = o.retailer;
|
|
else o.retailerDisplay = o.retailer;
|
|
}
|
|
|
|
// Deduplizieren: gleicher normalisierter Händler + gleicher Preis → ein Eintrag
|
|
// Kürzeste Beschreibung nehmen (sauberste)
|
|
const merged = new Map();
|
|
for (const o of filtered) {
|
|
const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || '';
|
|
const key = `${normalizedRetailer}|${o.price}`;
|
|
if (merged.has(key)) {
|
|
const existing = merged.get(key);
|
|
// Kürzere Beschreibung bevorzugen
|
|
if (o.description && (!existing.description || o.description.length < existing.description.length)) {
|
|
existing.description = o.description;
|
|
}
|
|
// Besseres Bild (erstes das nicht null ist behalten)
|
|
if (!existing.image && o.image) existing.image = o.image;
|
|
} else {
|
|
merged.set(key, { ...o });
|
|
}
|
|
}
|
|
const unique = [...merged.values()].map(o => ({ ...o, address: getAddress(o.retailer) }));
|
|
|
|
const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source: 'marktguru.de' };
|
|
cacheSet(cacheKey, result);
|
|
return result;
|
|
} finally {
|
|
await browser.close();
|
|
}
|
|
}
|
|
|
|
// ── Prospekte von kaufda ──────────────────────────────────────────────────────
|
|
function fetchUrl(url) {
|
|
return new Promise((resolve, reject) => {
|
|
const req = https.get(url, {
|
|
headers: {
|
|
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
|
'Accept': 'text/html', 'Accept-Language': 'de-DE,de;q=0.9',
|
|
}
|
|
}, res => {
|
|
let d = '';
|
|
res.on('data', c => d += c);
|
|
res.on('end', () => resolve({ status: res.statusCode, body: d }));
|
|
});
|
|
req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); });
|
|
req.on('error', reject);
|
|
});
|
|
}
|
|
|
|
function extractNextData(html) {
|
|
const marker = 'application/json">';
|
|
const ni = html.indexOf('NEXT_DATA');
|
|
if (ni === -1) return null;
|
|
const st = html.indexOf(marker, ni) + marker.length;
|
|
const en = html.indexOf('</script>', st);
|
|
try { return JSON.parse(html.slice(st, en)); } catch { return null; }
|
|
}
|
|
|
|
async function scrapeProspekte() {
|
|
const cacheKey = 'koepi:prospekte';
|
|
const cached = cacheGet(cacheKey);
|
|
if (cached) return cached;
|
|
|
|
const { body } = await fetchUrl(
|
|
'https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=51.3750&lng=6.7680&zip=47259&city=Duisburg'
|
|
);
|
|
const nd = extractNextData(body);
|
|
const contents = nd?.props?.pageProps?.pageInformation?.shelfContents?.contents || [];
|
|
|
|
const seen = new Map();
|
|
for (const item of contents) {
|
|
const c = item.content;
|
|
if (!c || item.contentType !== 'brochure') continue;
|
|
const id = c.contentId;
|
|
if (!id || seen.has(id)) continue;
|
|
const pub = c.publisher?.name || '';
|
|
const store = c.closestStore;
|
|
seen.set(id, {
|
|
id, publisher: pub,
|
|
title: c.title || '',
|
|
store: store?.name || pub,
|
|
city: store?.city || 'Duisburg',
|
|
street: store ? `${store.street||''} ${store.streetNumber||''}`.trim() : '',
|
|
zip: store?.zip || '47259',
|
|
image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null,
|
|
validFrom: c.validFrom || null,
|
|
validTo: c.validUntil || null,
|
|
pageCount: c.pageCount || null,
|
|
url: `https://www.kaufda.de/Geschaefte/${encodeURIComponent(pub.replace(/\s+/g,'-'))}`,
|
|
isTarget: isTargetRetailer(pub),
|
|
badges: c.contentBadges?.map(b => b.name) || [],
|
|
});
|
|
}
|
|
|
|
const all = [...seen.values()];
|
|
const result = {
|
|
targeted: all.filter(b => b.isTarget),
|
|
others: all.filter(b => !b.isTarget),
|
|
scrapedAt: new Date().toISOString(),
|
|
};
|
|
cacheSet(cacheKey, result);
|
|
return result;
|
|
}
|
|
|
|
// ── Bild-Proxy (für kaufda Bilder) ───────────────────────────────────────────
|
|
router.get('/img', authenticate, (req, res) => {
|
|
const url = req.query.url;
|
|
if (!url || !url.startsWith('https://')) return res.status(400).end();
|
|
try {
|
|
const parsed = new URL(url);
|
|
const allowed = ['content-media.bonial.biz','publisher-media.bonial.biz'];
|
|
if (!allowed.some(h => parsed.hostname === h)) return res.status(403).end();
|
|
const r = https.get(url, { headers: {'User-Agent':'Mozilla/5.0'} }, imgRes => {
|
|
res.setHeader('Content-Type', imgRes.headers['content-type'] || 'image/jpeg');
|
|
res.setHeader('Cache-Control', 'public, max-age=3600');
|
|
imgRes.pipe(res);
|
|
});
|
|
r.on('error', () => res.status(502).end());
|
|
} catch { res.status(400).end(); }
|
|
});
|
|
|
|
// ── Routes ────────────────────────────────────────────────────────────────────
|
|
router.get('/offers', authenticate, async (req, res) => {
|
|
try { res.json(await scrapeMarktguru()); }
|
|
catch(e) { console.error('KöPi offers:', e.message); res.status(500).json({ error: e.message }); }
|
|
});
|
|
|
|
router.get('/prospekte', authenticate, async (req, res) => {
|
|
try { res.json(await scrapeProspekte()); }
|
|
catch(e) { res.status(500).json({ error: e.message }); }
|
|
});
|
|
|
|
router.post('/clear-cache', authenticate, (req, res) => {
|
|
if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' });
|
|
['koepi:mg','koepi:prospekte'].forEach(k => delete cache[k]);
|
|
res.json({ ok: true });
|
|
});
|
|
|
|
module.exports = router;
|