Dicken/dickendock
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 1 Wiki Activity

feat: KöPi - echte Angebote mit Bild, Preis, Markt direkt in der App

Browse Source
This commit is contained in:
Dicken
2026-07-08 17:09:09 +02:00
parent ec872eb76d
commit 017decf59f
2 changed files with 270 additions and 242 deletions
Download Patch File Download Diff File Expand all files Collapse all files

99
backend/src/tools/koepi/routes.js
View File

@@ -64,7 +64,7 @@ function isTargetPublisher(name) {
return TARGET_PUBLISHERS.some(p => n.includes(p));
}
// ── Angebote via Puppeteer ────────────────────────────────────────────────────
// ── Angebote via Puppeteer (BrochureBox_Basic + OfferGrid) ──────────────────
async function scrapeOffersWithPuppeteer() {
const cacheKey = 'koepi:offers';
const cached = cacheGet(cacheKey);
@@ -77,70 +77,69 @@ async function scrapeOffersWithPuppeteer() {
const browser = await puppeteer.launch({
executablePath: process.env.PUPPETEER_EXECUTABLE_PATH || '/usr/bin/chromium-browser',
args: ['--no-sandbox', '--disable-setuid-sandbox', '--disable-dev-shm-usage', '--disable-gpu'],
args: ['--no-sandbox','--disable-setuid-sandbox','--disable-dev-shm-usage','--disable-gpu','--single-process'],
headless: true,
});
try {
const page = await browser.newPage();
await page.setUserAgent('Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36');
await page.setExtraHTTPHeaders({ 'Accept-Language': 'de-DE,de;q=0.9' });
await page.goto(
'https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg',
{ waitUntil: 'domcontentloaded', timeout: 25000 }
);
await new Promise(r => setTimeout(r, 3000));
await page.goto('https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg', {
waitUntil: 'networkidle2',
timeout: 30000,
});
const data = await page.evaluate(() => {
// ── BrochureBox: Prospekte wo König Pilsener drin ist ──────────────────
const brochureOffers = Array.from(document.querySelectorAll('[data-testid="BrochureBox_Basic"]')).map(b => {
const text = b.textContent.trim();
const imgs = Array.from(b.querySelectorAll('img'));
const productImg = imgs[0]?.src || null;
const publisherImg = imgs[1]?.src || null;
// Warte auf Produktkacheln
await page.waitForSelector('[data-testid="offer-card"], .offer-card, article, [class*="offer"]', {
timeout: 10000,
}).catch(() => {});
// Text parsen: "neuKönig Pilsener bei Trinkgut"Aktuelle Angebote", Seite 6Mo. 6.7. - Sa. 11.7.2026..."
const storeMatch = text.match(/bei\s+(.+?)\s*[""„]/);
const titleMatch = text.match(/[""„](.+?)[""„]/);
const pageMatch = text.match(/Seite\s+(\d+)/);
const dateMatch = text.match(/(\w+\.\s+\d+\.\d+\.)\s*-\s*(\w+\.\s+\d+\.\d+\.\d+)/);
const validMatch = text.match(/(Noch \d+ Tage? gültig|Gültig bis \S+)/);
const isNew = text.startsWith('neu');
// Alle Angebotsdaten extrahieren
const offers = await page.evaluate(() => {
const results = [];
// Methode 1: Schema.org
document.querySelectorAll('script[type="application/ld+json"]').forEach(s => {
try {
const data = JSON.parse(s.textContent);
const items = Array.isArray(data) ? data : [data];
items.forEach(item => {
if (item['@type'] === 'Product') {
const offer = Array.isArray(item.offers) ? item.offers[0] : item.offers;
results.push({
name: item.name,
price: offer?.price || null,
store: offer?.seller?.name || null,
image: Array.isArray(item.image) ? item.image[0] : item.image || null,
validFrom: offer?.validFrom || null,
validTo: offer?.validThrough || null,
url: item.url || window.location.href,
});
}
});
} catch {}
return {
type: 'brochure',
store: storeMatch?.[1]?.trim() || null,
title: titleMatch?.[1]?.trim() || null,
page: pageMatch?.[1] ? parseInt(pageMatch[1]) : null,
dateRange: dateMatch ? `${dateMatch[1]} - ${dateMatch[2]}` : null,
validity: validMatch?.[1] || null,
isNew,
productImage: productImg,
publisherLogo: publisherImg,
};
});
// Methode 2: DOM-Kacheln
const cards = document.querySelectorAll('[class*="offer-card"], [class*="OfferCard"], [data-testid*="offer"], article[class*="offer"]');
cards.forEach(card => {
const name = card.querySelector('[class*="title"], [class*="name"], h2, h3')?.textContent?.trim();
const price = card.querySelector('[class*="price"], [class*="Price"]')?.textContent?.trim();
const store = card.querySelector('[class*="retailer"], [class*="publisher"], [class*="store"]')?.textContent?.trim();
const img = card.querySelector('img')?.src;
if (name) results.push({ name, price, store, image: img, url: window.location.href });
});
// ── OfferGrid: einzelne Produktkacheln mit Preisen ──────────────────────
const grid = document.querySelector('[data-testid="OfferGrid"]');
const gridOffers = grid ? Array.from(grid.querySelectorAll('[role="listitem"]')).map(item => {
const img = item.querySelector('img');
const ps = item.querySelectorAll('p');
const brand = ps[0]?.textContent?.trim() || null;
const name = ps[1]?.textContent?.trim() || null;
const store = ps[2]?.textContent?.trim() || null;
const price = item.querySelector('.text-primary, [class*="text-primary"]')?.textContent?.trim() || null;
const baseUnit = ps[3]?.textContent?.trim() || null;
return { type:'offer', brand, name, store, price, baseUnit, image: img?.src || null };
}).filter(o => o.name) : [];
return results;
return { brochureOffers, gridOffers };
});
const result = {
offers: offers.filter((o, i, arr) =>
arr.findIndex(x => x.name === o.name && x.store === o.store) === i
),
scrapedAt: new Date().toISOString(),
source: 'kaufda.de (puppeteer)',
brochureOffers: data.brochureOffers,
gridOffers: data.gridOffers,
scrapedAt: new Date().toISOString(),
source: 'kaufda.de',
};
cacheSet(cacheKey, result);