fix: KöPi - Bilder mit korrekten sec-fetch Headers laden
This commit is contained in:
@@ -95,19 +95,37 @@ async function scrapeMarktguru() {
|
||||
});
|
||||
});
|
||||
|
||||
// Bilder: separaten Tab öffnen pro Bild-URL und screenshotten
|
||||
const imgPage = await browser.newPage();
|
||||
await imgPage.setViewport({ width: 300, height: 300 });
|
||||
// Bilder via Node https mit exakten Browser-Headers laden
|
||||
const nodeHttps = require('https');
|
||||
for (let i = 0; i < offers.length; i++) {
|
||||
const url = offers[i].image;
|
||||
if (!url || !url.startsWith('https://')) continue;
|
||||
try {
|
||||
await imgPage.goto(url, { waitUntil: 'networkidle0', timeout: 10000 });
|
||||
const b64 = await imgPage.screenshot({ encoding: 'base64' });
|
||||
offers[i].image = b64 ? `data:image/png;base64,${b64}` : null;
|
||||
} catch { offers[i].image = null; }
|
||||
offers[i].image = await new Promise(resolve => {
|
||||
const req = nodeHttps.get(url, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'image/avif,image/webp,image/apng,image/svg+xml,image/*,*/*;q=0.8',
|
||||
'Accept-Language': 'de-DE,de;q=0.9',
|
||||
'Referer': 'https://www.marktguru.de/',
|
||||
'Origin': 'https://www.marktguru.de',
|
||||
'sec-fetch-dest': 'image',
|
||||
'sec-fetch-mode': 'no-cors',
|
||||
'sec-fetch-site': 'cross-site',
|
||||
}
|
||||
}, res => {
|
||||
if (res.statusCode !== 200) return resolve(null);
|
||||
const chunks = [];
|
||||
res.on('data', c => chunks.push(c));
|
||||
res.on('end', () => {
|
||||
const b64 = Buffer.concat(chunks).toString('base64');
|
||||
const mime = res.headers['content-type'] || 'image/webp';
|
||||
resolve(`data:${mime};base64,${b64}`);
|
||||
});
|
||||
});
|
||||
req.on('error', () => resolve(null));
|
||||
req.setTimeout(8000, () => { req.destroy(); resolve(null); });
|
||||
});
|
||||
}
|
||||
await imgPage.close();
|
||||
|
||||
// Filtern auf Ziel-Händler + nur König Pilsener
|
||||
const filtered = offers.filter(o => {
|
||||
|
||||
Reference in New Issue
Block a user