Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: KöPi - Puppeteer/Chromium für echte Produktangebote

Browse Source
This commit is contained in:
Dicken
2026-07-08 12:58:15 +02:00
parent 8324e0e3fe
commit ec872eb76d
3 changed files with 126 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
Dockerfile
View File

@@ -14,7 +14,10 @@ RUN npm run build
# ── Stage 2: Express liefert API + React in einem Container ──────────────────
FROM node:20-alpine
RUN apk add --no-cache tzdata python3 make g++
# Puppeteer/Chromium für KöPi Web-Scraping
RUN apk add --no-cache tzdata python3 make g++ chromium nss freetype harfbuzz ca-certificates ttf-freefont
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
# Zeitzone fest auf Europe/Berlin setzen, damit Server und Browser (DE) immer
# dieselbe lokale Zeit berechnen – inkl. automatischer Sommer-/Winterzeit-Umstellung
ENV TZ=Europe/Berlin

3
backend/package.json
View File

@@ -8,6 +8,7 @@
"express": "^4.18.2",
"jsonwebtoken": "^9.0.2",
"multer": "^1.4.5-lts.1",
"ws": "^8.17.1"
"ws": "^8.17.1",
"puppeteer-core": "^22.0.0"
}
}

188
backend/src/tools/koepi/routes.js
View File

@@ -15,7 +15,7 @@ function cacheSet(key, data, ttlMs = 60 * 60 * 1000) {
cache[key] = { data, expires: Date.now() + ttlMs };
}
// ── HTTP-Helper mit Redirect-Follow ──────────────────────────────────────────
// ── HTTP-Helper ───────────────────────────────────────────────────────────────
function fetchUrl(url, maxRedirects = 5) {
return new Promise((resolve, reject) => {
const doGet = (u, remaining) => {
@@ -24,7 +24,6 @@ function fetchUrl(url, maxRedirects = 5) {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml',
'Accept-Language': 'de-DE,de;q=0.9',
'Accept-Encoding': 'identity',
}
}, (res) => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && remaining > 0) {
@@ -39,14 +38,14 @@ function fetchUrl(url, maxRedirects = 5) {
res.on('data', c => d += c);
res.on('end', () => resolve({ status: res.statusCode, body: d }));
});
req.setTimeout(12000, () => { req.destroy(); reject(new Error('Timeout')); });
req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); });
req.on('error', reject);
};
doGet(url, maxRedirects);
});
}
// ── NEXT_DATA aus HTML extrahieren ────────────────────────────────────────────
// ── NEXT_DATA extrahieren ─────────────────────────────────────────────────────
function extractNextData(html) {
const marker = 'application/json">';
const nidx = html.indexOf('NEXT_DATA');
@@ -57,127 +56,106 @@ function extractNextData(html) {
try { return JSON.parse(html.slice(start, end)); } catch { return null; }
}
// ── Händler-Filter (Prospekte) ────────────────────────────────────────────────
const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl','real'];
// ── Händler-Filter ────────────────────────────────────────────────────────────
const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl'];
function isTargetPublisher(name) {
if (!name) return false;
const n = name.toLowerCase();
return TARGET_PUBLISHERS.some(p => n.includes(p));
}
// ── Angebote scrapen (Produktseite) ──────────────────────────────────────────
async function scrapeOffers() {
// ── Angebote via Puppeteer ────────────────────────────────────────────────────
async function scrapeOffersWithPuppeteer() {
const cacheKey = 'koepi:offers';
const cached = cacheGet(cacheKey);
if (cached) return cached;
const { body } = await fetchUrl(
'https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg'
);
let puppeteer;
try { puppeteer = require('puppeteer-core'); } catch(e) {
return { offers: [], error: 'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() };
}
const offers = [];
// Methode 1: Produktkacheln aus HTML parsen
// Suche nach offer-card Patterns im HTML
const offerPatterns = [
// Schema.org JSON-LD
/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g,
// Meta-Tags mit Produktinfos
/data-offer-id="([^"]+)"[^>]*data-price="([^"]+)"[^>]*data-publisher="([^"]+)"/g,
];
// Schema.org JSON-LD extrahieren
let match;
const ldReg = /<script type="application\/ld\+json">([\s\S]*?)<\/script>/g;
while ((match = ldReg.exec(body)) !== null) {
try {
const ld = JSON.parse(match[1]);
const items = Array.isArray(ld) ? ld : [ld];
for (const item of items) {
if (item['@type'] === 'ItemList' && item.itemListElement) {
for (const el of item.itemListElement) {
const thing = el.item || el;
if (thing.name?.toLowerCase().includes('könig') || thing.name?.toLowerCase().includes('pilsener') || thing.name?.toLowerCase().includes('pilsen')) {
offers.push({
name: thing.name,
price: thing.offers?.price || thing.offers?.lowPrice || null,
currency: 'EUR',
store: thing.offers?.seller?.name || null,
url: thing.url || 'https://www.kaufda.de/Angebote/Koenig-Pilsener',
image: thing.image || null,
validFrom: thing.offers?.validFrom || null,
validTo: thing.offers?.validThrough || null,
const browser = await puppeteer.launch({
executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium-browser',
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'],
headless: true,
});
}
}
}
if (item['@type'] === 'Product' && item.name) {
if (item.name.toLowerCase().includes('könig') || item.name.toLowerCase().includes('pilsen')) {
try {
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
await page.setExtraHTTPHeaders({ 'Accept-Language': 'de-DE,de;q=0.9' });
await page.goto('https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg', {
waitUntil: 'networkidle2',
timeout: 30000,
});
// Warte auf Produktkacheln
await page.waitForSelector('[data-testid="offer-card"], .offer-card, article, [class*="offer"]', {
timeout: 10000,
}).catch(() => {});
// Alle Angebotsdaten extrahieren
const offers = await page.evaluate(() => {
const results = [];
// Methode 1: Schema.org
document.querySelectorAll('script[type="application/ld+json"]').forEach(s => {
try {
const data = JSON.parse(s.textContent);
const items = Array.isArray(data) ? data : [data];
items.forEach(item => {
if (item['@type'] === 'Product') {
const offer = Array.isArray(item.offers) ? item.offers[0] : item.offers;
offers.push({
results.push({
name: item.name,
price: offer?.price || offer?.lowPrice || null,
currency: 'EUR',
price: offer?.price || null,
store: offer?.seller?.name || null,
url: item.url || 'https://www.kaufda.de/Angebote/Koenig-Pilsener',
image: Array.isArray(item.image) ? item.image[0] : item.image || null,
validFrom: offer?.validFrom || null,
validTo: offer?.validThrough || null,
url: item.url || window.location.href,
});
}
}
}
});
} catch {}
}
// Methode 2: NEXT_DATA
const nextData = extractNextData(body);
if (nextData) {
const pp = nextData.props?.pageProps;
// Suche nach Angeboten in allen möglichen Keys
const str = JSON.stringify(pp || {});
// Extrahiere strukturierte Angebotsdaten wenn vorhanden
const offerData = pp?.offers || pp?.pageInformation?.template?.content?.offers || [];
if (Array.isArray(offerData)) {
for (const o of offerData) {
if (o.name || o.title) {
offers.push({
name: o.name || o.title,
price: o.price || o.minPrice || null,
currency: 'EUR',
store: o.publisher?.name || o.retailer || null,
image: o.image?.url || o.imageUrl || null,
validFrom: o.validFrom || null,
validTo: o.validUntil || null,
url: `https://www.kaufda.de/Angebote/Koenig-Pilsener`,
});
}
}
}
}
// Deduplizieren
const seen = new Set();
const unique = offers.filter(o => {
const key = `${o.name}|${o.store}|${o.price}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
});
const result = { offers: unique, scrapedAt: new Date().toISOString(), source: 'kaufda.de' };
// Methode 2: DOM-Kacheln
const cards = document.querySelectorAll('[class*="offer-card"], [class*="OfferCard"], [data-testid*="offer"], article[class*="offer"]');
cards.forEach(card => {
const name = card.querySelector('[class*="title"], [class*="name"], h2, h3')?.textContent?.trim();
const price = card.querySelector('[class*="price"], [class*="Price"]')?.textContent?.trim();
const store = card.querySelector('[class*="retailer"], [class*="publisher"], [class*="store"]')?.textContent?.trim();
const img = card.querySelector('img')?.src;
if (name) results.push({ name, price, store, image: img, url: window.location.href });
});
return results;
});
const result = {
offers: offers.filter((o, i, arr) =>
arr.findIndex(x => x.name === o.name && x.store === o.store) === i
),
scrapedAt: new Date().toISOString(),
source: 'kaufda.de (puppeteer)',
};
cacheSet(cacheKey, result);
return result;
} finally {
await browser.close();
}
}
// ── Prospekte scrapen (shelf-Seite) ──────────────────────────────────────────
// ── Prospekte scrapen ─────────────────────────────────────────────────────────
async function scrapeProspekte() {
const cacheKey = 'koepi:prospekte';
const cached = cacheGet(cacheKey);
if (cached) return cached;
// Mehrere PLZ/Koordinaten für Umkreis
const locations = [
{ lat: 51.4332, lng: 6.7625, city: 'Duisburg', zip: '47051' },
{ lat: 51.5167, lng: 6.9833, city: 'Essen', zip: '45127' },
@@ -191,25 +169,21 @@ async function scrapeProspekte() {
const { body } = await fetchUrl(
`https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=${loc.lat}&lng=${loc.lng}&zip=${loc.zip}&city=${encodeURIComponent(loc.city)}`
);
const nextData = extractNextData(body);
if (!nextData) continue;
const pp = nextData.props?.pageProps;
const contents = pp?.pageInformation?.shelfContents?.contents
|| pp?.pageInformation?.template?.shelfContents?.contents
|| pp?.pageInformation?.template?.content?.shelfContents?.contents
|| [];
const contents = pp?.pageInformation?.shelfContents?.contents || [];
for (const item of contents) {
const c = item.content;
if (!c || item.contentType !== 'brochure') continue;
const id = c.contentId;
if (!id || allBrochures.has(id)) continue;
const publisher = c.publisher?.name || '';
const store = c.closestStore;
const id = c.contentId;
if (!allBrochures.has(id)) {
allBrochures.set(id, {
id,
publisher,
@@ -227,19 +201,15 @@ async function scrapeProspekte() {
badges: c.contentBadges?.map(b => b.name) || [],
});
}
}
} catch(e) {
console.error('KöPi scrape error:', loc.city, e.message);
console.error('KöPi prospekte error:', loc.city, e.message);
}
}
const brochures = [...allBrochures.values()];
const targeted = brochures.filter(b => b.isTarget);
const others = brochures.filter(b => !b.isTarget);
const result = {
targeted,
others,
targeted: brochures.filter(b => b.isTarget),
others: brochures.filter(b => !b.isTarget),
total: brochures.length,
scrapedAt: new Date().toISOString(),
};
@@ -251,9 +221,10 @@ async function scrapeProspekte() {
// ── Routes ────────────────────────────────────────────────────────────────────
router.get('/offers', authenticate, async (req, res) => {
try {
const data = await scrapeOffers();
const data = await scrapeOffersWithPuppeteer();
res.json(data);
} catch(e) {
console.error('KöPi offers error:', e.message);
res.status(500).json({ error: e.message });
}
});
@@ -263,6 +234,7 @@ router.get('/prospekte', authenticate, async (req, res) => {
const data = await scrapeProspekte();
res.json(data);
} catch(e) {
console.error('KöPi prospekte error:', e.message);
res.status(500).json({ error: e.message });
}
});