Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: KöPi - Puppeteer/Chromium für echte Produktangebote

Browse Source
This commit is contained in:
Dicken
2026-07-08 12:58:15 +02:00
parent 8324e0e3fe
commit ec872eb76d
3 changed files with 126 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

268
backend/src/tools/koepi/routes.js
View File

@@ -1,7 +1,7 @@
const express = require('express');
const https = require('https');
const express = require('express');
const https = require('https');
const { authenticate } = require('../../middleware/auth');
const router = express.Router();
const router = express.Router();
// ── Cache (in-memory, 1h) ─────────────────────────────────────────────────────
const cache = {};
@@ -15,7 +15,7 @@ function cacheSet(key, data, ttlMs = 60 * 60 * 1000) {
cache[key] = { data, expires: Date.now() + ttlMs };
}
// ── HTTP-Helper mit Redirect-Follow ──────────────────────────────────────────
// ── HTTP-Helper ───────────────────────────────────────────────────────────────
function fetchUrl(url, maxRedirects = 5) {
return new Promise((resolve, reject) => {
const doGet = (u, remaining) => {
@@ -24,7 +24,6 @@ function fetchUrl(url, maxRedirects = 5) {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml',
'Accept-Language': 'de-DE,de;q=0.9',
'Accept-Encoding': 'identity',
}
}, (res) => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && remaining > 0) {
@@ -39,149 +38,128 @@ function fetchUrl(url, maxRedirects = 5) {
res.on('data', c => d += c);
res.on('end', () => resolve({ status: res.statusCode, body: d }));
});
req.setTimeout(12000, () => { req.destroy(); reject(new Error('Timeout')); });
req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); });
req.on('error', reject);
};
doGet(url, maxRedirects);
});
}
// ── NEXT_DATA aus HTML extrahieren ────────────────────────────────────────────
// ── NEXT_DATA extrahieren ─────────────────────────────────────────────────────
function extractNextData(html) {
const marker = 'application/json">';
const nidx = html.indexOf('NEXT_DATA');
const marker = 'application/json">';
const nidx = html.indexOf('NEXT_DATA');
if (nidx === -1) return null;
const start = html.indexOf(marker, nidx) + marker.length;
const end = html.indexOf('</script>', start);
const start = html.indexOf(marker, nidx) + marker.length;
const end = html.indexOf('</script>', start);
if (start <= marker.length || end === -1) return null;
try { return JSON.parse(html.slice(start, end)); } catch { return null; }
}
// ── Händler-Filter (Prospekte) ────────────────────────────────────────────────
const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl','real'];
// ── Händler-Filter ────────────────────────────────────────────────────────────
const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl'];
function isTargetPublisher(name) {
if (!name) return false;
const n = name.toLowerCase();
return TARGET_PUBLISHERS.some(p => n.includes(p));
}
// ── Angebote scrapen (Produktseite) ──────────────────────────────────────────
async function scrapeOffers() {
// ── Angebote via Puppeteer ────────────────────────────────────────────────────
async function scrapeOffersWithPuppeteer() {
const cacheKey = 'koepi:offers';
const cached = cacheGet(cacheKey);
if (cached) return cached;
const { body } = await fetchUrl(
'https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg'
);
const offers = [];
// Methode 1: Produktkacheln aus HTML parsen
// Suche nach offer-card Patterns im HTML
const offerPatterns = [
// Schema.org JSON-LD
/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g,
// Meta-Tags mit Produktinfos
/data-offer-id="([^"]+)"[^>]*data-price="([^"]+)"[^>]*data-publisher="([^"]+)"/g,
];
// Schema.org JSON-LD extrahieren
let match;
const ldReg = /<script type="application\/ld\+json">([\s\S]*?)<\/script>/g;
while ((match = ldReg.exec(body)) !== null) {
try {
const ld = JSON.parse(match[1]);
const items = Array.isArray(ld) ? ld : [ld];
for (const item of items) {
if (item['@type'] === 'ItemList' && item.itemListElement) {
for (const el of item.itemListElement) {
const thing = el.item || el;
if (thing.name?.toLowerCase().includes('könig') || thing.name?.toLowerCase().includes('pilsener') || thing.name?.toLowerCase().includes('pilsen')) {
offers.push({
name: thing.name,
price: thing.offers?.price || thing.offers?.lowPrice || null,
currency: 'EUR',
store: thing.offers?.seller?.name || null,
url: thing.url || 'https://www.kaufda.de/Angebote/Koenig-Pilsener',
image: thing.image || null,
validFrom: thing.offers?.validFrom || null,
validTo: thing.offers?.validThrough || null,
});
}
}
}
if (item['@type'] === 'Product' && item.name) {
if (item.name.toLowerCase().includes('könig') || item.name.toLowerCase().includes('pilsen')) {
const offer = Array.isArray(item.offers) ? item.offers[0] : item.offers;
offers.push({
name: item.name,
price: offer?.price || offer?.lowPrice || null,
currency: 'EUR',
store: offer?.seller?.name || null,
url: item.url || 'https://www.kaufda.de/Angebote/Koenig-Pilsener',
image: Array.isArray(item.image) ? item.image[0] : item.image || null,
validFrom: offer?.validFrom || null,
validTo: offer?.validThrough || null,
});
}
}
}
} catch {}
let puppeteer;
try { puppeteer = require('puppeteer-core'); } catch(e) {
return { offers: [], error: 'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() };
}
// Methode 2: NEXT_DATA
const nextData = extractNextData(body);
if (nextData) {
const pp = nextData.props?.pageProps;
// Suche nach Angeboten in allen möglichen Keys
const str = JSON.stringify(pp || {});
// Extrahiere strukturierte Angebotsdaten wenn vorhanden
const offerData = pp?.offers || pp?.pageInformation?.template?.content?.offers || [];
if (Array.isArray(offerData)) {
for (const o of offerData) {
if (o.name || o.title) {
offers.push({
name: o.name || o.title,
price: o.price || o.minPrice || null,
currency: 'EUR',
store: o.publisher?.name || o.retailer || null,
image: o.image?.url || o.imageUrl || null,
validFrom: o.validFrom || null,
validTo: o.validUntil || null,
url: `https://www.kaufda.de/Angebote/Koenig-Pilsener`,
});
}
}
}
}
// Deduplizieren
const seen = new Set();
const unique = offers.filter(o => {
const key = `${o.name}|${o.store}|${o.price}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
const browser = await puppeteer.launch({
executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium-browser',
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'],
headless: true,
});
const result = { offers: unique, scrapedAt: new Date().toISOString(), source: 'kaufda.de' };
cacheSet(cacheKey, result);
return result;
try {
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
await page.setExtraHTTPHeaders({ 'Accept-Language': 'de-DE,de;q=0.9' });
await page.goto('https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg', {
waitUntil: 'networkidle2',
timeout: 30000,
});
// Warte auf Produktkacheln
await page.waitForSelector('[data-testid="offer-card"], .offer-card, article, [class*="offer"]', {
timeout: 10000,
}).catch(() => {});
// Alle Angebotsdaten extrahieren
const offers = await page.evaluate(() => {
const results = [];
// Methode 1: Schema.org
document.querySelectorAll('script[type="application/ld+json"]').forEach(s => {
try {
const data = JSON.parse(s.textContent);
const items = Array.isArray(data) ? data : [data];
items.forEach(item => {
if (item['@type'] === 'Product') {
const offer = Array.isArray(item.offers) ? item.offers[0] : item.offers;
results.push({
name: item.name,
price: offer?.price || null,
store: offer?.seller?.name || null,
image: Array.isArray(item.image) ? item.image[0] : item.image || null,
validFrom: offer?.validFrom || null,
validTo: offer?.validThrough || null,
url: item.url || window.location.href,
});
}
});
} catch {}
});
// Methode 2: DOM-Kacheln
const cards = document.querySelectorAll('[class*="offer-card"], [class*="OfferCard"], [data-testid*="offer"], article[class*="offer"]');
cards.forEach(card => {
const name = card.querySelector('[class*="title"], [class*="name"], h2, h3')?.textContent?.trim();
const price = card.querySelector('[class*="price"], [class*="Price"]')?.textContent?.trim();
const store = card.querySelector('[class*="retailer"], [class*="publisher"], [class*="store"]')?.textContent?.trim();
const img = card.querySelector('img')?.src;
if (name) results.push({ name, price, store, image: img, url: window.location.href });
});
return results;
});
const result = {
offers: offers.filter((o, i, arr) =>
arr.findIndex(x => x.name === o.name && x.store === o.store) === i
),
scrapedAt: new Date().toISOString(),
source: 'kaufda.de (puppeteer)',
};
cacheSet(cacheKey, result);
return result;
} finally {
await browser.close();
}
}
// ── Prospekte scrapen (shelf-Seite) ──────────────────────────────────────────
// ── Prospekte scrapen ─────────────────────────────────────────────────────────
async function scrapeProspekte() {
const cacheKey = 'koepi:prospekte';
const cached = cacheGet(cacheKey);
if (cached) return cached;
// Mehrere PLZ/Koordinaten für Umkreis
const locations = [
{ lat: 51.4332, lng: 6.7625, city: 'Duisburg', zip: '47051' },
{ lat: 51.5167, lng: 6.9833, city: 'Essen', zip: '45127' },
{ lat: 51.2217, lng: 6.7762, city: 'Düsseldorf', zip: '40213' },
{ lat: 51.4332, lng: 6.7625, city: 'Duisburg', zip: '47051' },
{ lat: 51.5167, lng: 6.9833, city: 'Essen', zip: '45127' },
{ lat: 51.2217, lng: 6.7762, city: 'Düsseldorf', zip: '40213' },
];
const allBrochures = new Map();
@@ -191,56 +169,48 @@ async function scrapeProspekte() {
const { body } = await fetchUrl(
`https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=${loc.lat}&lng=${loc.lng}&zip=${loc.zip}&city=${encodeURIComponent(loc.city)}`
);
const nextData = extractNextData(body);
if (!nextData) continue;
const pp = nextData.props?.pageProps;
const contents = pp?.pageInformation?.shelfContents?.contents
|| pp?.pageInformation?.template?.shelfContents?.contents
|| pp?.pageInformation?.template?.content?.shelfContents?.contents
|| [];
const pp = nextData.props?.pageProps;
const contents = pp?.pageInformation?.shelfContents?.contents || [];
for (const item of contents) {
const c = item.content;
if (!c || item.contentType !== 'brochure') continue;
const id = c.contentId;
if (!id || allBrochures.has(id)) continue;
const publisher = c.publisher?.name || '';
const store = c.closestStore;
const id = c.contentId;
if (!allBrochures.has(id)) {
allBrochures.set(id, {
id,
publisher,
title: c.title || '',
store: store?.name || publisher,
city: store?.city || loc.city,
street: store ? `${store.street || ''} ${store.streetNumber || ''}`.trim() : '',
zip: store?.zip || loc.zip,
image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null,
validFrom: c.validFrom || null,
validTo: c.validUntil || null,
pageCount: c.pageCount || null,
url: `https://www.kaufda.de/webapp/brochure/${id}`,
isTarget: isTargetPublisher(publisher),
badges: c.contentBadges?.map(b => b.name) || [],
});
}
allBrochures.set(id, {
id,
publisher,
title: c.title || '',
store: store?.name || publisher,
city: store?.city || loc.city,
street: store ? `${store.street || ''} ${store.streetNumber || ''}`.trim() : '',
zip: store?.zip || loc.zip,
image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null,
validFrom: c.validFrom || null,
validTo: c.validUntil || null,
pageCount: c.pageCount || null,
url: `https://www.kaufda.de/webapp/brochure/${id}`,
isTarget: isTargetPublisher(publisher),
badges: c.contentBadges?.map(b => b.name) || [],
});
}
} catch(e) {
console.error('KöPi scrape error:', loc.city, e.message);
console.error('KöPi prospekte error:', loc.city, e.message);
}
}
const brochures = [...allBrochures.values()];
const targeted = brochures.filter(b => b.isTarget);
const others = brochures.filter(b => !b.isTarget);
const brochures = [...allBrochures.values()];
const result = {
targeted,
others,
total: brochures.length,
targeted: brochures.filter(b => b.isTarget),
others: brochures.filter(b => !b.isTarget),
total: brochures.length,
scrapedAt: new Date().toISOString(),
};
@@ -251,9 +221,10 @@ async function scrapeProspekte() {
// ── Routes ────────────────────────────────────────────────────────────────────
router.get('/offers', authenticate, async (req, res) => {
try {
const data = await scrapeOffers();
const data = await scrapeOffersWithPuppeteer();
res.json(data);
} catch(e) {
console.error('KöPi offers error:', e.message);
res.status(500).json({ error: e.message });
}
});
@@ -263,6 +234,7 @@ router.get('/prospekte', authenticate, async (req, res) => {
const data = await scrapeProspekte();
res.json(data);
} catch(e) {
console.error('KöPi prospekte error:', e.message);
res.status(500).json({ error: e.message });
}
});