Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: KöPi - Puppeteer/Chromium für echte Produktangebote

Browse Source
This commit is contained in:
Dicken
2026-07-08 12:58:15 +02:00
parent 8324e0e3fe
commit ec872eb76d
3 changed files with 126 additions and 150 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
Dockerfile
View File

@@ -14,7 +14,10 @@ RUN npm run build
# ── Stage 2: Express liefert API + React in einem Container ────────────────── # ── Stage 2: Express liefert API + React in einem Container ──────────────────
FROM node:20-alpine FROM node:20-alpine
RUN apk add --no-cache tzdata python3 make g++ # Puppeteer/Chromium für KöPi Web-Scraping
RUN apk add --no-cache tzdata python3 make g++ chromium nss freetype harfbuzz ca-certificates ttf-freefont
ENV PUPPETEER_SKIP_CHROMIUM_DOWNLOAD=true
ENV PUPPETEER_EXECUTABLE_PATH=/usr/bin/chromium-browser
# Zeitzone fest auf Europe/Berlin setzen, damit Server und Browser (DE) immer # Zeitzone fest auf Europe/Berlin setzen, damit Server und Browser (DE) immer
# dieselbe lokale Zeit berechnen – inkl. automatischer Sommer-/Winterzeit-Umstellung # dieselbe lokale Zeit berechnen – inkl. automatischer Sommer-/Winterzeit-Umstellung
ENV TZ=Europe/Berlin ENV TZ=Europe/Berlin

3
backend/package.json
View File

@@ -8,6 +8,7 @@
"express": "^4.18.2", "express": "^4.18.2",
"jsonwebtoken": "^9.0.2", "jsonwebtoken": "^9.0.2",
"multer": "^1.4.5-lts.1", "multer": "^1.4.5-lts.1",
"ws": "^8.17.1" "ws": "^8.17.1",
"puppeteer-core": "^22.0.0"
} }
} }

268
backend/src/tools/koepi/routes.js
View File

@@ -1,7 +1,7 @@
const express = require('express'); const express = require('express');
const https = require('https'); const https = require('https');
const { authenticate } = require('../../middleware/auth'); const { authenticate } = require('../../middleware/auth');
const router = express.Router(); const router = express.Router();
// ── Cache (in-memory, 1h) ───────────────────────────────────────────────────── // ── Cache (in-memory, 1h) ─────────────────────────────────────────────────────
const cache = {}; const cache = {};
@@ -15,7 +15,7 @@ function cacheSet(key, data, ttlMs = 60 * 60 * 1000) {
cache[key] = { data, expires: Date.now() + ttlMs }; cache[key] = { data, expires: Date.now() + ttlMs };
} }
// ── HTTP-Helper mit Redirect-Follow ────────────────────────────────────────── // ── HTTP-Helper ───────────────────────────────────────────────────────────────
function fetchUrl(url, maxRedirects = 5) { function fetchUrl(url, maxRedirects = 5) {
return new Promise((resolve, reject) => { return new Promise((resolve, reject) => {
const doGet = (u, remaining) => { const doGet = (u, remaining) => {
@@ -24,7 +24,6 @@ function fetchUrl(url, maxRedirects = 5) {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
'Accept': 'text/html,application/xhtml+xml', 'Accept': 'text/html,application/xhtml+xml',
'Accept-Language': 'de-DE,de;q=0.9', 'Accept-Language': 'de-DE,de;q=0.9',
'Accept-Encoding': 'identity',
} }
}, (res) => { }, (res) => {
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && remaining > 0) { if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && remaining > 0) {
@@ -39,149 +38,128 @@ function fetchUrl(url, maxRedirects = 5) {
res.on('data', c => d += c); res.on('data', c => d += c);
res.on('end', () => resolve({ status: res.statusCode, body: d })); res.on('end', () => resolve({ status: res.statusCode, body: d }));
}); });
req.setTimeout(12000, () => { req.destroy(); reject(new Error('Timeout')); }); req.setTimeout(15000, () => { req.destroy(); reject(new Error('Timeout')); });
req.on('error', reject); req.on('error', reject);
}; };
doGet(url, maxRedirects); doGet(url, maxRedirects);
}); });
} }
// ── NEXT_DATA aus HTML extrahieren ──────────────────────────────────────────── // ── NEXT_DATA extrahieren ─────────────────────────────────────────────────────
function extractNextData(html) { function extractNextData(html) {
const marker = 'application/json">'; const marker = 'application/json">';
const nidx = html.indexOf('NEXT_DATA'); const nidx = html.indexOf('NEXT_DATA');
if (nidx === -1) return null; if (nidx === -1) return null;
const start = html.indexOf(marker, nidx) + marker.length; const start = html.indexOf(marker, nidx) + marker.length;
const end = html.indexOf('</script>', start); const end = html.indexOf('</script>', start);
if (start <= marker.length || end === -1) return null; if (start <= marker.length || end === -1) return null;
try { return JSON.parse(html.slice(start, end)); } catch { return null; } try { return JSON.parse(html.slice(start, end)); } catch { return null; }
} }
// ── Händler-Filter (Prospekte) ──────────────────────────────────────────────── // ── Händler-Filter ────────────────────────────────────────────────────────────
const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl','real']; const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl'];
function isTargetPublisher(name) { function isTargetPublisher(name) {
if (!name) return false; if (!name) return false;
const n = name.toLowerCase(); const n = name.toLowerCase();
return TARGET_PUBLISHERS.some(p => n.includes(p)); return TARGET_PUBLISHERS.some(p => n.includes(p));
} }
// ── Angebote scrapen (Produktseite) ────────────────────────────────────────── // ── Angebote via Puppeteer ────────────────────────────────────────────────────
async function scrapeOffers() { async function scrapeOffersWithPuppeteer() {
const cacheKey = 'koepi:offers'; const cacheKey = 'koepi:offers';
const cached = cacheGet(cacheKey); const cached = cacheGet(cacheKey);
if (cached) return cached; if (cached) return cached;
const { body } = await fetchUrl( let puppeteer;
'https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg' try { puppeteer = require('puppeteer-core'); } catch(e) {
); return { offers: [], error: 'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() };
const offers = [];
// Methode 1: Produktkacheln aus HTML parsen
// Suche nach offer-card Patterns im HTML
const offerPatterns = [
// Schema.org JSON-LD
/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g,
// Meta-Tags mit Produktinfos
/data-offer-id="([^"]+)"[^>]*data-price="([^"]+)"[^>]*data-publisher="([^"]+)"/g,
];
// Schema.org JSON-LD extrahieren
let match;
const ldReg = /<script type="application\/ld\+json">([\s\S]*?)<\/script>/g;
while ((match = ldReg.exec(body)) !== null) {
try {
const ld = JSON.parse(match[1]);
const items = Array.isArray(ld) ? ld : [ld];
for (const item of items) {
if (item['@type'] === 'ItemList' && item.itemListElement) {
for (const el of item.itemListElement) {
const thing = el.item || el;
if (thing.name?.toLowerCase().includes('könig') || thing.name?.toLowerCase().includes('pilsener') || thing.name?.toLowerCase().includes('pilsen')) {
offers.push({
name: thing.name,
price: thing.offers?.price || thing.offers?.lowPrice || null,
currency: 'EUR',
store: thing.offers?.seller?.name || null,
url: thing.url || 'https://www.kaufda.de/Angebote/Koenig-Pilsener',
image: thing.image || null,
validFrom: thing.offers?.validFrom || null,
validTo: thing.offers?.validThrough || null,
});
}
}
}
if (item['@type'] === 'Product' && item.name) {
if (item.name.toLowerCase().includes('könig') || item.name.toLowerCase().includes('pilsen')) {
const offer = Array.isArray(item.offers) ? item.offers[0] : item.offers;
offers.push({
name: item.name,
price: offer?.price || offer?.lowPrice || null,
currency: 'EUR',
store: offer?.seller?.name || null,
url: item.url || 'https://www.kaufda.de/Angebote/Koenig-Pilsener',
image: Array.isArray(item.image) ? item.image[0] : item.image || null,
validFrom: offer?.validFrom || null,
validTo: offer?.validThrough || null,
});
}
}
}
} catch {}
} }
// Methode 2: NEXT_DATA const browser = await puppeteer.launch({
const nextData = extractNextData(body); executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium-browser',
if (nextData) { args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'],
const pp = nextData.props?.pageProps; headless: true,
// Suche nach Angeboten in allen möglichen Keys
const str = JSON.stringify(pp || {});
// Extrahiere strukturierte Angebotsdaten wenn vorhanden
const offerData = pp?.offers || pp?.pageInformation?.template?.content?.offers || [];
if (Array.isArray(offerData)) {
for (const o of offerData) {
if (o.name || o.title) {
offers.push({
name: o.name || o.title,
price: o.price || o.minPrice || null,
currency: 'EUR',
store: o.publisher?.name || o.retailer || null,
image: o.image?.url || o.imageUrl || null,
validFrom: o.validFrom || null,
validTo: o.validUntil || null,
url: `https://www.kaufda.de/Angebote/Koenig-Pilsener`,
});
}
}
}
}
// Deduplizieren
const seen = new Set();
const unique = offers.filter(o => {
const key = `${o.name}|${o.store}|${o.price}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
}); });
const result = { offers: unique, scrapedAt: new Date().toISOString(), source: 'kaufda.de' }; try {
cacheSet(cacheKey, result); const page = await browser.newPage();
return result; await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
await page.setExtraHTTPHeaders({ 'Accept-Language': 'de-DE,de;q=0.9' });
await page.goto('https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg', {
waitUntil: 'networkidle2',
timeout: 30000,
});
// Warte auf Produktkacheln
await page.waitForSelector('[data-testid="offer-card"], .offer-card, article, [class*="offer"]', {
timeout: 10000,
}).catch(() => {});
// Alle Angebotsdaten extrahieren
const offers = await page.evaluate(() => {
const results = [];
// Methode 1: Schema.org
document.querySelectorAll('script[type="application/ld+json"]').forEach(s => {
try {
const data = JSON.parse(s.textContent);
const items = Array.isArray(data) ? data : [data];
items.forEach(item => {
if (item['@type'] === 'Product') {
const offer = Array.isArray(item.offers) ? item.offers[0] : item.offers;
results.push({
name: item.name,
price: offer?.price || null,
store: offer?.seller?.name || null,
image: Array.isArray(item.image) ? item.image[0] : item.image || null,
validFrom: offer?.validFrom || null,
validTo: offer?.validThrough || null,
url: item.url || window.location.href,
});
}
});
} catch {}
});
// Methode 2: DOM-Kacheln
const cards = document.querySelectorAll('[class*="offer-card"], [class*="OfferCard"], [data-testid*="offer"], article[class*="offer"]');
cards.forEach(card => {
const name = card.querySelector('[class*="title"], [class*="name"], h2, h3')?.textContent?.trim();
const price = card.querySelector('[class*="price"], [class*="Price"]')?.textContent?.trim();
const store = card.querySelector('[class*="retailer"], [class*="publisher"], [class*="store"]')?.textContent?.trim();
const img = card.querySelector('img')?.src;
if (name) results.push({ name, price, store, image: img, url: window.location.href });
});
return results;
});
const result = {
offers: offers.filter((o, i, arr) =>
arr.findIndex(x => x.name === o.name && x.store === o.store) === i
),
scrapedAt: new Date().toISOString(),
source: 'kaufda.de (puppeteer)',
};
cacheSet(cacheKey, result);
return result;
} finally {
await browser.close();
}
} }
// ── Prospekte scrapen (shelf-Seite) ────────────────────────────────────────── // ── Prospekte scrapen ─────────────────────────────────────────────────────────
async function scrapeProspekte() { async function scrapeProspekte() {
const cacheKey = 'koepi:prospekte'; const cacheKey = 'koepi:prospekte';
const cached = cacheGet(cacheKey); const cached = cacheGet(cacheKey);
if (cached) return cached; if (cached) return cached;
// Mehrere PLZ/Koordinaten für Umkreis
const locations = [ const locations = [
{ lat: 51.4332, lng: 6.7625, city: 'Duisburg', zip: '47051' }, { lat: 51.4332, lng: 6.7625, city: 'Duisburg', zip: '47051' },
{ lat: 51.5167, lng: 6.9833, city: 'Essen', zip: '45127' }, { lat: 51.5167, lng: 6.9833, city: 'Essen', zip: '45127' },
{ lat: 51.2217, lng: 6.7762, city: 'Düsseldorf', zip: '40213' }, { lat: 51.2217, lng: 6.7762, city: 'Düsseldorf', zip: '40213' },
]; ];
const allBrochures = new Map(); const allBrochures = new Map();
@@ -191,56 +169,48 @@ async function scrapeProspekte() {
const { body } = await fetchUrl( const { body } = await fetchUrl(
`https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=${loc.lat}&lng=${loc.lng}&zip=${loc.zip}&city=${encodeURIComponent(loc.city)}` `https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=${loc.lat}&lng=${loc.lng}&zip=${loc.zip}&city=${encodeURIComponent(loc.city)}`
); );
const nextData = extractNextData(body); const nextData = extractNextData(body);
if (!nextData) continue; if (!nextData) continue;
const pp = nextData.props?.pageProps; const pp = nextData.props?.pageProps;
const contents = pp?.pageInformation?.shelfContents?.contents const contents = pp?.pageInformation?.shelfContents?.contents || [];
|| pp?.pageInformation?.template?.shelfContents?.contents
|| pp?.pageInformation?.template?.content?.shelfContents?.contents
|| [];
for (const item of contents) { for (const item of contents) {
const c = item.content; const c = item.content;
if (!c || item.contentType !== 'brochure') continue; if (!c || item.contentType !== 'brochure') continue;
const id = c.contentId;
if (!id || allBrochures.has(id)) continue;
const publisher = c.publisher?.name || ''; const publisher = c.publisher?.name || '';
const store = c.closestStore; const store = c.closestStore;
const id = c.contentId;
if (!allBrochures.has(id)) { allBrochures.set(id, {
allBrochures.set(id, { id,
id, publisher,
publisher, title: c.title || '',
title: c.title || '', store: store?.name || publisher,
store: store?.name || publisher, city: store?.city || loc.city,
city: store?.city || loc.city, street: store ? `${store.street || ''} ${store.streetNumber || ''}`.trim() : '',
street: store ? `${store.street || ''} ${store.streetNumber || ''}`.trim() : '', zip: store?.zip || loc.zip,
zip: store?.zip || loc.zip, image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null,
image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null, validFrom: c.validFrom || null,
validFrom: c.validFrom || null, validTo: c.validUntil || null,
validTo: c.validUntil || null, pageCount: c.pageCount || null,
pageCount: c.pageCount || null, url: `https://www.kaufda.de/webapp/brochure/${id}`,
url: `https://www.kaufda.de/webapp/brochure/${id}`, isTarget: isTargetPublisher(publisher),
isTarget: isTargetPublisher(publisher), badges: c.contentBadges?.map(b => b.name) || [],
badges: c.contentBadges?.map(b => b.name) || [], });
});
}
} }
} catch(e) { } catch(e) {
console.error('KöPi scrape error:', loc.city, e.message); console.error('KöPi prospekte error:', loc.city, e.message);
} }
} }
const brochures = [...allBrochures.values()]; const brochures = [...allBrochures.values()];
const targeted = brochures.filter(b => b.isTarget);
const others = brochures.filter(b => !b.isTarget);
const result = { const result = {
targeted, targeted: brochures.filter(b => b.isTarget),
others, others: brochures.filter(b => !b.isTarget),
total: brochures.length, total: brochures.length,
scrapedAt: new Date().toISOString(), scrapedAt: new Date().toISOString(),
}; };
@@ -251,9 +221,10 @@ async function scrapeProspekte() {
// ── Routes ──────────────────────────────────────────────────────────────────── // ── Routes ────────────────────────────────────────────────────────────────────
router.get('/offers', authenticate, async (req, res) => { router.get('/offers', authenticate, async (req, res) => {
try { try {
const data = await scrapeOffers(); const data = await scrapeOffersWithPuppeteer();
res.json(data); res.json(data);
} catch(e) { } catch(e) {
console.error('KöPi offers error:', e.message);
res.status(500).json({ error: e.message }); res.status(500).json({ error: e.message });
} }
}); });
@@ -263,6 +234,7 @@ router.get('/prospekte', authenticate, async (req, res) => {
const data = await scrapeProspekte(); const data = await scrapeProspekte();
res.json(data); res.json(data);
} catch(e) { } catch(e) {
console.error('KöPi prospekte error:', e.message);
res.status(500).json({ error: e.message }); res.status(500).json({ error: e.message });
} }
}); });