feat: KoePi nutzt jetzt primaer die marktguru-JSON-API statt Puppeteer, mit automatischem Fallback
This commit is contained in:
@@ -81,17 +81,76 @@ function isLocalBranch(retailer, realAddress) {
|
||||
return expectedParts.some(p => real.includes(p) || (p && p.includes(real)));
|
||||
}
|
||||
|
||||
// ── Angebote via Puppeteer von marktguru ─────────────────────────────────────
|
||||
async function scrapeMarktguru(forceRefresh = false) {
|
||||
const cacheKey = 'koepi:mg';
|
||||
if (!forceRefresh) {
|
||||
const cached = cacheGet(cacheKey);
|
||||
if (cached) return cached;
|
||||
// ── Angebote von marktguru ────────────────────────────────────────────────────
|
||||
// Primär über die öffentlich bekannte marktguru-JSON-API (schnell, kein Browser
|
||||
// nötig, kein Cloudflare-Risiko). Schlägt das fehl (z.B. weil die Keys mal
|
||||
// ungültig werden), fällt der Code automatisch auf das alte Puppeteer/HTML-
|
||||
// Scraping zurück, damit KöPi nicht komplett ausfällt.
|
||||
const MARKTGURU_API_URL = 'https://api.marktguru.de/api/v1/offers/search';
|
||||
const MARKTGURU_CLIENT_KEY = 'WU/RH+PMGDi+gkZer3WbMelt6zcYHSTytNB7VpTia90=';
|
||||
const MARKTGURU_API_KEY = '8Kk+pmbf7TgJ9nVj2cXeA7P5zBGv8iuutVVMRfOfvNE=';
|
||||
const MG_ZIP = '47259';
|
||||
const MG_QUERY = 'könig pilsener';
|
||||
|
||||
function formatIsoDate(iso) {
|
||||
if (!iso) return '';
|
||||
const d = new Date(iso);
|
||||
if (Number.isNaN(d.getTime())) return '';
|
||||
const pad = n => String(n).padStart(2, '0');
|
||||
return `${pad(d.getDate())}.${pad(d.getMonth() + 1)}.`;
|
||||
}
|
||||
|
||||
async function fetchOffersViaApi() {
|
||||
const url = `${MARKTGURU_API_URL}?as=web&limit=24&offset=0&q=${encodeURIComponent(MG_QUERY)}&zipCode=${MG_ZIP}`;
|
||||
const res = await fetch(url, {
|
||||
headers: {
|
||||
'x-clientkey': MARKTGURU_CLIENT_KEY,
|
||||
'x-apikey': MARKTGURU_API_KEY,
|
||||
'Accept': 'application/json',
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
},
|
||||
});
|
||||
if (!res.ok) {
|
||||
const text = await res.text().catch(() => '');
|
||||
throw new Error(`API HTTP ${res.status}: ${text.slice(0, 200)}`);
|
||||
}
|
||||
const json = await res.json();
|
||||
|
||||
// Diagnose: rohe Struktur des ersten Treffers loggen, damit sich Feldnamen
|
||||
// (z.B. für eine echte Store-Adresse) im Zweifel nachträglich anpassen lassen
|
||||
console.log('🍺 KöPi API: totalResults=', json.totalResults, '| results.length=', json.results?.length);
|
||||
if (json.results?.[0]) {
|
||||
console.log('🍺 KöPi API Rohdaten (erster Treffer):', JSON.stringify(json.results[0]).slice(0, 1500));
|
||||
}
|
||||
|
||||
return (json.results || []).map(r => {
|
||||
const advertiser = r.advertisers?.[0] || {};
|
||||
const store = r.closestStore || r.store || {};
|
||||
const priceNum = typeof r.price === 'number' ? r.price : parseFloat(r.price);
|
||||
const oldPriceNum = typeof r.oldPrice === 'number' ? r.oldPrice : parseFloat(r.oldPrice);
|
||||
const realAddress = [store.street, store.streetNumber].filter(Boolean).join(' ').trim()
|
||||
|| store.address || advertiser.address || '';
|
||||
return {
|
||||
name: r.description || r.title || advertiser.name || '',
|
||||
brand: r.brand?.name || '',
|
||||
price: Number.isFinite(priceNum) ? `€ ${priceNum.toFixed(2).replace('.', ',')}` : (r.price || ''),
|
||||
oldPrice: Number.isFinite(oldPriceNum) ? `€ ${oldPriceNum.toFixed(2).replace('.', ',')}` : (r.oldPrice || ''),
|
||||
retailer: advertiser.name || '',
|
||||
dateRange: r.validFrom && r.validUntil ? `${formatIsoDate(r.validFrom)} - ${formatIsoDate(r.validUntil)}` : formatIsoDate(r.validUntil),
|
||||
validity: '',
|
||||
description: r.description || '',
|
||||
badge: '',
|
||||
address: '',
|
||||
realAddress,
|
||||
image: r.image?.url || r.images?.[0]?.url || null,
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchOffersViaPuppeteer() {
|
||||
let puppeteer;
|
||||
try { puppeteer = require('puppeteer-core'); }
|
||||
catch(e) { return { offers:[], error:'puppeteer-core nicht installiert', scrapedAt: new Date().toISOString() }; }
|
||||
catch(e) { throw new Error('puppeteer-core nicht installiert'); }
|
||||
|
||||
const browser = await puppeteer.launch({
|
||||
executablePath: '/usr/bin/chromium-browser',
|
||||
@@ -105,7 +164,7 @@ async function scrapeMarktguru(forceRefresh = false) {
|
||||
await page.setViewport({ width:1280, height:800 });
|
||||
await page.setCookie({ name:'mg_user-settings', value:MG_SETTINGS, domain:'.marktguru.de', path:'/' });
|
||||
|
||||
await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', {
|
||||
await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=47259', {
|
||||
waitUntil: 'networkidle2', timeout: 30000,
|
||||
});
|
||||
// Seite nach unten scrollen damit lazy-loaded Bilder geladen werden
|
||||
@@ -136,18 +195,13 @@ await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=4
|
||||
const retailer = card.querySelector('dd.retailer-name a')?.textContent?.trim() || '';
|
||||
const dateRange= card.querySelector('dd.valid')?.textContent?.trim() || '';
|
||||
const daysLeft = card.querySelector('dd.time-left span')?.textContent?.trim() || '';
|
||||
const daysText = card.querySelector('dd.time-left')?.textContent?.trim() || '';
|
||||
const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` :
|
||||
const validity = daysLeft ? `Noch ${daysLeft} Tag${daysLeft==='1'?'':'e'}` :
|
||||
card.querySelector('dt.time-left')?.textContent?.includes('Brandneu') ? 'Neu' : '';
|
||||
const descEl = card.querySelector('.info div, .info');
|
||||
const description = descEl?.textContent?.trim() || '';
|
||||
const img = card.querySelector('img.offer-list-item-img');
|
||||
const badge = card.querySelector('.badge, .discount-badge')?.textContent?.trim() || '';
|
||||
const imgSrc = img?.src?.replace('/medium.webp', '/large.webp') || img?.src || null;
|
||||
// Adresse steht bei marktguru direkt nach dem Händlernamen, aber nicht
|
||||
// zwingend als direktes DOM-Geschwisterelement (z.B. wenn dazwischen
|
||||
// noch ein <dt>-Label liegt) — daher über die Position in der Liste
|
||||
// ALLER <dd>-Elemente der Karte suchen statt über nextElementSibling
|
||||
const allDds = Array.from(card.querySelectorAll('dd'));
|
||||
const retailerIdx = allDds.findIndex(d => d.classList.contains('retailer-name'));
|
||||
let realAddress = '';
|
||||
@@ -159,84 +213,103 @@ await page.goto('https://www.marktguru.de/search/k%C3%B6nig%20pilsener?zipCode=4
|
||||
return { name, brand, price, oldPrice, retailer, dateRange, validity, description, badge, address: '', realAddress, image: imgSrc };
|
||||
});
|
||||
});
|
||||
|
||||
// Lokale Bilder auswählen basierend auf Beschreibung
|
||||
for (const offer of offers) {
|
||||
const desc = (offer.description || '').toLowerCase();
|
||||
console.log('DESC:', JSON.stringify(desc), '| RETAILER:', offer.retailer);
|
||||
if (desc.includes('steini')) {
|
||||
offer.imageLocal = '/koepi/kasten_steini.png';
|
||||
} else if (desc.includes('11 x 0,5') || desc.includes('11x0,5')) {
|
||||
offer.imageLocal = '/koepi/kasten_lang.png';
|
||||
} else if (desc.includes('6 x') || desc.includes('6x') || (desc.includes('flasche') && !desc.includes('kasten'))) {
|
||||
offer.imageLocal = '/koepi/traeger.png';
|
||||
} else if (desc.includes('24 x 0,33') || desc.includes('24x0,33') || desc.includes('24 x 0,33')) {
|
||||
offer.imageLocal = '/koepi/kasten_033.png';
|
||||
} else if (desc.includes('0,5')) {
|
||||
offer.imageLocal = '/koepi/kasten_050.png';
|
||||
} else if (desc.includes('0,33')) {
|
||||
offer.imageLocal = '/koepi/kasten_033.png';
|
||||
} else {
|
||||
offer.imageLocal = '/koepi/kasten_050.png';
|
||||
}
|
||||
offer.image = null; // CDN nicht mehr nötig
|
||||
console.log('IMAGE:', offer.imageLocal, '| RETAILER:', offer.retailer);
|
||||
}
|
||||
|
||||
// Filtern auf Ziel-Händler + nur König Pilsener
|
||||
const filtered = offers.filter(o => {
|
||||
const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase();
|
||||
const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener');
|
||||
return hasKP && isTargetRetailer(o.retailer);
|
||||
});
|
||||
|
||||
// Nur Angebote der tatsächlich bekannten, lokalen Filialen behalten. Die
|
||||
// Adress-basierte Prüfung (isLocalBranch) greift nur, wenn marktguru für
|
||||
// ein Angebot eine echte Adresse liefert — auf der aktuell genutzten
|
||||
// Such-URL ist das nicht der Fall, daher bleibt sie hier wirkungslos
|
||||
// (siehe Kommentar an isLocalBranch). Verlässlich funktioniert aktuell nur
|
||||
// der explizite Namens-Ausschluss (z.B. "REWE Dortmund").
|
||||
const excludedNow = filtered.filter(o => isExcludedRetailer(o.retailer));
|
||||
if (excludedNow.length) {
|
||||
console.log('🍺 KöPi: Angebote per Namens-Filter ausgeschlossen:', excludedNow.map(o => o.retailer).join(', '));
|
||||
}
|
||||
|
||||
const localOnly = filtered
|
||||
.filter(o => !isExcludedRetailer(o.retailer))
|
||||
.filter(o => isLocalBranch(o.retailer, o.realAddress));
|
||||
|
||||
// Händler normalisieren: E center = EDEKA
|
||||
for (const o of localOnly) {
|
||||
if (o.retailer?.toLowerCase().includes('e center')) o.retailerDisplay = o.retailer;
|
||||
else o.retailerDisplay = o.retailer;
|
||||
}
|
||||
|
||||
// Deduplizieren: gleicher normalisierter Händler + gleicher Preis → ein Eintrag
|
||||
// Kürzeste Beschreibung nehmen (sauberste)
|
||||
const merged = new Map();
|
||||
for (const o of localOnly) {
|
||||
const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || '';
|
||||
const key = `${normalizedRetailer}|${o.price}`;
|
||||
if (merged.has(key)) {
|
||||
const existing = merged.get(key);
|
||||
// Kürzere Beschreibung bevorzugen
|
||||
if (o.description && (!existing.description || o.description.length < existing.description.length)) {
|
||||
existing.description = o.description;
|
||||
}
|
||||
} else {
|
||||
merged.set(key, { ...o });
|
||||
}
|
||||
}
|
||||
const unique = [...merged.values()].map(o => ({ ...o, address: getAddress(o.retailer) }));
|
||||
|
||||
const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source: 'marktguru.de' };
|
||||
cacheSet(cacheKey, result);
|
||||
return result;
|
||||
return offers;
|
||||
} finally {
|
||||
await browser.close();
|
||||
}
|
||||
}
|
||||
|
||||
// Gemeinsame Weiterverarbeitung, egal ob die Rohdaten über die API oder per
|
||||
// Puppeteer/HTML-Scraping ermittelt wurden
|
||||
function processOffers(offers) {
|
||||
// Lokale Bilder auswählen basierend auf Beschreibung
|
||||
for (const offer of offers) {
|
||||
const desc = (offer.description || '').toLowerCase();
|
||||
if (desc.includes('steini')) {
|
||||
offer.imageLocal = '/koepi/kasten_steini.png';
|
||||
} else if (desc.includes('11 x 0,5') || desc.includes('11x0,5')) {
|
||||
offer.imageLocal = '/koepi/kasten_lang.png';
|
||||
} else if (desc.includes('6 x') || desc.includes('6x') || (desc.includes('flasche') && !desc.includes('kasten'))) {
|
||||
offer.imageLocal = '/koepi/traeger.png';
|
||||
} else if (desc.includes('24 x 0,33') || desc.includes('24x0,33')) {
|
||||
offer.imageLocal = '/koepi/kasten_033.png';
|
||||
} else if (desc.includes('0,5')) {
|
||||
offer.imageLocal = '/koepi/kasten_050.png';
|
||||
} else if (desc.includes('0,33')) {
|
||||
offer.imageLocal = '/koepi/kasten_033.png';
|
||||
} else {
|
||||
offer.imageLocal = '/koepi/kasten_050.png';
|
||||
}
|
||||
offer.image = null; // CDN nicht mehr nötig
|
||||
}
|
||||
|
||||
// Filtern auf Ziel-Händler + nur König Pilsener
|
||||
const filtered = offers.filter(o => {
|
||||
const isKP = (o.name + ' ' + o.brand + ' ' + o.description).toLowerCase();
|
||||
const hasKP = isKP.includes('pilsen') || isKP.includes('pilsener');
|
||||
return hasKP && isTargetRetailer(o.retailer);
|
||||
});
|
||||
|
||||
const excludedNow = filtered.filter(o => isExcludedRetailer(o.retailer));
|
||||
if (excludedNow.length) {
|
||||
console.log('🍺 KöPi: Angebote per Namens-Filter ausgeschlossen:', excludedNow.map(o => o.retailer).join(', '));
|
||||
}
|
||||
|
||||
const localOnly = filtered
|
||||
.filter(o => !isExcludedRetailer(o.retailer))
|
||||
.filter(o => isLocalBranch(o.retailer, o.realAddress));
|
||||
|
||||
for (const o of localOnly) {
|
||||
o.retailerDisplay = o.retailer;
|
||||
}
|
||||
|
||||
// Deduplizieren: gleicher normalisierter Händler + gleicher Preis → ein Eintrag
|
||||
const merged = new Map();
|
||||
for (const o of localOnly) {
|
||||
const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || '';
|
||||
const key = `${normalizedRetailer}|${o.price}`;
|
||||
if (merged.has(key)) {
|
||||
const existing = merged.get(key);
|
||||
if (o.description && (!existing.description || o.description.length < existing.description.length)) {
|
||||
existing.description = o.description;
|
||||
}
|
||||
} else {
|
||||
merged.set(key, { ...o });
|
||||
}
|
||||
}
|
||||
return [...merged.values()].map(o => ({ ...o, address: getAddress(o.retailer) }));
|
||||
}
|
||||
|
||||
async function scrapeMarktguru(forceRefresh = false) {
|
||||
const cacheKey = 'koepi:mg';
|
||||
if (!forceRefresh) {
|
||||
const cached = cacheGet(cacheKey);
|
||||
if (cached) return cached;
|
||||
}
|
||||
|
||||
let offers, source;
|
||||
try {
|
||||
offers = await fetchOffersViaApi();
|
||||
source = 'marktguru.de (API)';
|
||||
console.log(`🍺 KöPi: ${offers.length} Rohtreffer über die marktguru-API`);
|
||||
} catch (apiErr) {
|
||||
console.error('🍺 KöPi: API-Weg fehlgeschlagen, falle zurück auf Puppeteer:', apiErr.message);
|
||||
try {
|
||||
offers = await fetchOffersViaPuppeteer();
|
||||
source = 'marktguru.de (Puppeteer-Fallback)';
|
||||
console.log(`🍺 KöPi: ${offers.length} Rohtreffer per Puppeteer (Fallback)`);
|
||||
} catch (ppErr) {
|
||||
console.error('🍺 KöPi: auch Puppeteer-Fallback fehlgeschlagen:', ppErr.message);
|
||||
return { offers: [], error: `API: ${apiErr.message} | Puppeteer: ${ppErr.message}`, scrapedAt: new Date().toISOString() };
|
||||
}
|
||||
}
|
||||
|
||||
const unique = processOffers(offers);
|
||||
const result = { offers: unique, total: unique.length, scrapedAt: new Date().toISOString(), source };
|
||||
cacheSet(cacheKey, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Prospekte von kaufda ──────────────────────────────────────────────────────
|
||||
function fetchUrl(url) {
|
||||
return new Promise((resolve, reject) => {
|
||||
|
||||
Reference in New Issue
Block a user