diff --git a/backend/src/tools/koepi/routes.js b/backend/src/tools/koepi/routes.js index c04b754..74511ae 100644 --- a/backend/src/tools/koepi/routes.js +++ b/backend/src/tools/koepi/routes.js @@ -94,15 +94,20 @@ async function scrapeMarktguru() { if (!offer.image) continue; try { const imgData = await page.evaluate(async (url) => { - const resp = await fetch(url); - const blob = await resp.blob(); - return new Promise(resolve => { - const reader = new FileReader(); - reader.onloadend = () => resolve(reader.result); - reader.readAsDataURL(blob); - }); + try { + const resp = await fetch(url, {credentials:'include'}); + if (!resp.ok) return null; + const blob = await resp.blob(); + return new Promise(resolve => { + const reader = new FileReader(); + reader.onloadend = () => resolve(reader.result); + reader.onerror = () => resolve(null); + reader.readAsDataURL(blob); + }); + } catch { return null; } }, offer.image); - offer.image = imgData; // base64 data URL + if (imgData) offer.image = imgData; + else offer.image = null; } catch { offer.image = null; } } @@ -113,18 +118,26 @@ async function scrapeMarktguru() { return hasKP && isTargetRetailer(o.retailer); }); - // Deduplizieren: gleicher Händler + gleicher Preis → zusammenfassen + // Händler normalisieren: E center = EDEKA + for (const o of filtered) { + if (o.retailer?.toLowerCase().includes('e center')) o.retailerDisplay = o.retailer; + else o.retailerDisplay = o.retailer; + } + + // Deduplizieren: gleicher normalisierter Händler + gleicher Preis → ein Eintrag + // Kürzeste Beschreibung nehmen (sauberste) const merged = new Map(); for (const o of filtered) { - const key = `${o.retailer}|${o.price}`; + const normalizedRetailer = o.retailer?.toLowerCase().replace('e center','edeka') || ''; + const key = `${normalizedRetailer}|${o.price}`; if (merged.has(key)) { - // Beschreibungen zusammenführen falls unterschiedlich const existing = merged.get(key); - if (o.description && o.description !== existing.description) { - existing.description = existing.description - ? `${existing.description} / ${o.description}` - : o.description; + // Kürzere Beschreibung bevorzugen + if (o.description && (!existing.description || o.description.length < existing.description.length)) { + existing.description = o.description; } + // Besseres Bild (erstes das nicht null ist behalten) + if (!existing.image && o.image) existing.image = o.image; } else { merged.set(key, { ...o }); }