fix: KöPi - Bilder via Node+Cookies, NEU Badge weg, Prospekt URLs
This commit is contained in:
@@ -89,27 +89,32 @@ async function scrapeMarktguru() {
|
||||
});
|
||||
});
|
||||
|
||||
// Bilder als Base64 laden (cdn.marktguru.de blockt externe Requests)
|
||||
for (const offer of offers) {
|
||||
if (!offer.image) continue;
|
||||
try {
|
||||
const imgData = await page.evaluate(async (url) => {
|
||||
try {
|
||||
const resp = await fetch(url, {credentials:'include'});
|
||||
if (!resp.ok) return null;
|
||||
const blob = await resp.blob();
|
||||
return new Promise(resolve => {
|
||||
const reader = new FileReader();
|
||||
reader.onloadend = () => resolve(reader.result);
|
||||
reader.onerror = () => resolve(null);
|
||||
reader.readAsDataURL(blob);
|
||||
});
|
||||
} catch { return null; }
|
||||
}, offer.image);
|
||||
if (imgData) offer.image = imgData;
|
||||
else offer.image = null;
|
||||
} catch { offer.image = null; }
|
||||
}
|
||||
// Bilder über Node https laden mit Cookies aus dem Browser
|
||||
const cookies = await page.cookies('https://cdn.marktguru.de');
|
||||
const cookieStr = cookies.map(c => `${c.name}=${c.value}`).join('; ');
|
||||
|
||||
await Promise.all(offers.map(offer => new Promise(resolve => {
|
||||
if (!offer.image) return resolve();
|
||||
const req = require('https').get(offer.image, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
|
||||
'Referer': 'https://www.marktguru.de/',
|
||||
'Cookie': cookieStr,
|
||||
}
|
||||
}, res => {
|
||||
if (res.statusCode !== 200) { offer.image = null; return resolve(); }
|
||||
const chunks = [];
|
||||
res.on('data', c => chunks.push(c));
|
||||
res.on('end', () => {
|
||||
const b64 = Buffer.concat(chunks).toString('base64');
|
||||
const mime = res.headers['content-type'] || 'image/webp';
|
||||
offer.image = `data:${mime};base64,${b64}`;
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
req.on('error', () => { offer.image = null; resolve(); });
|
||||
req.setTimeout(8000, () => { req.destroy(); offer.image = null; resolve(); });
|
||||
})));
|
||||
|
||||
// Filtern auf Ziel-Händler + nur König Pilsener
|
||||
const filtered = offers.filter(o => {
|
||||
@@ -209,7 +214,7 @@ async function scrapeProspekte() {
|
||||
validFrom: c.validFrom || null,
|
||||
validTo: c.validUntil || null,
|
||||
pageCount: c.pageCount || null,
|
||||
url: `https://www.kaufda.de/webapp/brochure/${id}`,
|
||||
url: `https://www.kaufda.de/Prospekte/${encodeURIComponent(pub.replace(/\s+/g,'-'))}?brochureId=${id}`,
|
||||
isTarget: isTargetRetailer(pub),
|
||||
badges: c.contentBadges?.map(b => b.name) || [],
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user