feat: KöPi - König Pilsener Angebote & Prospekte via kaufDA.de
This commit is contained in:
275
backend/src/tools/koepi/routes.js
Normal file
275
backend/src/tools/koepi/routes.js
Normal file
@@ -0,0 +1,275 @@
|
||||
const express = require('express');
|
||||
const https = require('https');
|
||||
const { authenticate } = require('../../middleware/auth');
|
||||
const router = express.Router();
|
||||
|
||||
// ── Cache (in-memory, 1h) ─────────────────────────────────────────────────────
|
||||
const cache = {};
|
||||
function cacheGet(key) {
|
||||
const e = cache[key];
|
||||
if (!e) return null;
|
||||
if (Date.now() > e.expires) { delete cache[key]; return null; }
|
||||
return e.data;
|
||||
}
|
||||
function cacheSet(key, data, ttlMs = 60 * 60 * 1000) {
|
||||
cache[key] = { data, expires: Date.now() + ttlMs };
|
||||
}
|
||||
|
||||
// ── HTTP-Helper mit Redirect-Follow ──────────────────────────────────────────
|
||||
function fetchUrl(url, maxRedirects = 5) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const doGet = (u, remaining) => {
|
||||
const req = https.get(u, {
|
||||
headers: {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
|
||||
'Accept': 'text/html,application/xhtml+xml',
|
||||
'Accept-Language': 'de-DE,de;q=0.9',
|
||||
'Accept-Encoding': 'identity',
|
||||
}
|
||||
}, (res) => {
|
||||
if (res.statusCode >= 300 && res.statusCode < 400 && res.headers.location && remaining > 0) {
|
||||
const next = res.headers.location.startsWith('http')
|
||||
? res.headers.location
|
||||
: `https://www.kaufda.de${res.headers.location}`;
|
||||
res.resume();
|
||||
doGet(next, remaining - 1);
|
||||
return;
|
||||
}
|
||||
let d = '';
|
||||
res.on('data', c => d += c);
|
||||
res.on('end', () => resolve({ status: res.statusCode, body: d }));
|
||||
});
|
||||
req.setTimeout(12000, () => { req.destroy(); reject(new Error('Timeout')); });
|
||||
req.on('error', reject);
|
||||
};
|
||||
doGet(url, maxRedirects);
|
||||
});
|
||||
}
|
||||
|
||||
// ── NEXT_DATA aus HTML extrahieren ────────────────────────────────────────────
|
||||
function extractNextData(html) {
|
||||
const marker = 'application/json">';
|
||||
const nidx = html.indexOf('NEXT_DATA');
|
||||
if (nidx === -1) return null;
|
||||
const start = html.indexOf(marker, nidx) + marker.length;
|
||||
const end = html.indexOf('</script>', start);
|
||||
if (start <= marker.length || end === -1) return null;
|
||||
try { return JSON.parse(html.slice(start, end)); } catch { return null; }
|
||||
}
|
||||
|
||||
// ── Händler-Filter (Prospekte) ────────────────────────────────────────────────
|
||||
const TARGET_PUBLISHERS = ['rewe','edeka','netto','trinkgut','getränke','penny','aldi','lidl','real'];
|
||||
|
||||
function isTargetPublisher(name) {
|
||||
if (!name) return false;
|
||||
const n = name.toLowerCase();
|
||||
return TARGET_PUBLISHERS.some(p => n.includes(p));
|
||||
}
|
||||
|
||||
// ── Angebote scrapen (Produktseite) ──────────────────────────────────────────
|
||||
async function scrapeOffers() {
|
||||
const cacheKey = 'koepi:offers';
|
||||
const cached = cacheGet(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
const { body } = await fetchUrl(
|
||||
'https://www.kaufda.de/Angebote/Koenig-Pilsener?lat=51.4332&lng=6.7625&zip=47051&city=Duisburg'
|
||||
);
|
||||
|
||||
const offers = [];
|
||||
|
||||
// Methode 1: Produktkacheln aus HTML parsen
|
||||
// Suche nach offer-card Patterns im HTML
|
||||
const offerPatterns = [
|
||||
// Schema.org JSON-LD
|
||||
/<script type="application\/ld\+json">([\s\S]*?)<\/script>/g,
|
||||
// Meta-Tags mit Produktinfos
|
||||
/data-offer-id="([^"]+)"[^>]*data-price="([^"]+)"[^>]*data-publisher="([^"]+)"/g,
|
||||
];
|
||||
|
||||
// Schema.org JSON-LD extrahieren
|
||||
let match;
|
||||
const ldReg = /<script type="application\/ld\+json">([\s\S]*?)<\/script>/g;
|
||||
while ((match = ldReg.exec(body)) !== null) {
|
||||
try {
|
||||
const ld = JSON.parse(match[1]);
|
||||
const items = Array.isArray(ld) ? ld : [ld];
|
||||
for (const item of items) {
|
||||
if (item['@type'] === 'ItemList' && item.itemListElement) {
|
||||
for (const el of item.itemListElement) {
|
||||
const thing = el.item || el;
|
||||
if (thing.name?.toLowerCase().includes('könig') || thing.name?.toLowerCase().includes('pilsener') || thing.name?.toLowerCase().includes('pilsen')) {
|
||||
offers.push({
|
||||
name: thing.name,
|
||||
price: thing.offers?.price || thing.offers?.lowPrice || null,
|
||||
currency: 'EUR',
|
||||
store: thing.offers?.seller?.name || null,
|
||||
url: thing.url || 'https://www.kaufda.de/Angebote/Koenig-Pilsener',
|
||||
image: thing.image || null,
|
||||
validFrom: thing.offers?.validFrom || null,
|
||||
validTo: thing.offers?.validThrough || null,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
if (item['@type'] === 'Product' && item.name) {
|
||||
if (item.name.toLowerCase().includes('könig') || item.name.toLowerCase().includes('pilsen')) {
|
||||
const offer = Array.isArray(item.offers) ? item.offers[0] : item.offers;
|
||||
offers.push({
|
||||
name: item.name,
|
||||
price: offer?.price || offer?.lowPrice || null,
|
||||
currency: 'EUR',
|
||||
store: offer?.seller?.name || null,
|
||||
url: item.url || 'https://www.kaufda.de/Angebote/Koenig-Pilsener',
|
||||
image: Array.isArray(item.image) ? item.image[0] : item.image || null,
|
||||
validFrom: offer?.validFrom || null,
|
||||
validTo: offer?.validThrough || null,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
// Methode 2: NEXT_DATA
|
||||
const nextData = extractNextData(body);
|
||||
if (nextData) {
|
||||
const pp = nextData.props?.pageProps;
|
||||
// Suche nach Angeboten in allen möglichen Keys
|
||||
const str = JSON.stringify(pp || {});
|
||||
// Extrahiere strukturierte Angebotsdaten wenn vorhanden
|
||||
const offerData = pp?.offers || pp?.pageInformation?.template?.content?.offers || [];
|
||||
if (Array.isArray(offerData)) {
|
||||
for (const o of offerData) {
|
||||
if (o.name || o.title) {
|
||||
offers.push({
|
||||
name: o.name || o.title,
|
||||
price: o.price || o.minPrice || null,
|
||||
currency: 'EUR',
|
||||
store: o.publisher?.name || o.retailer || null,
|
||||
image: o.image?.url || o.imageUrl || null,
|
||||
validFrom: o.validFrom || null,
|
||||
validTo: o.validUntil || null,
|
||||
url: `https://www.kaufda.de/Angebote/Koenig-Pilsener`,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplizieren
|
||||
const seen = new Set();
|
||||
const unique = offers.filter(o => {
|
||||
const key = `${o.name}|${o.store}|${o.price}`;
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
|
||||
const result = { offers: unique, scrapedAt: new Date().toISOString(), source: 'kaufda.de' };
|
||||
cacheSet(cacheKey, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Prospekte scrapen (shelf-Seite) ──────────────────────────────────────────
|
||||
async function scrapeProspekte() {
|
||||
const cacheKey = 'koepi:prospekte';
|
||||
const cached = cacheGet(cacheKey);
|
||||
if (cached) return cached;
|
||||
|
||||
// Mehrere PLZ/Koordinaten für Umkreis
|
||||
const locations = [
|
||||
{ lat: 51.4332, lng: 6.7625, city: 'Duisburg', zip: '47051' },
|
||||
{ lat: 51.5167, lng: 6.9833, city: 'Essen', zip: '45127' },
|
||||
{ lat: 51.2217, lng: 6.7762, city: 'Düsseldorf', zip: '40213' },
|
||||
];
|
||||
|
||||
const allBrochures = new Map();
|
||||
|
||||
for (const loc of locations) {
|
||||
try {
|
||||
const { body } = await fetchUrl(
|
||||
`https://www.kaufda.de/shelf?query=K%C3%B6nig+Pilsener&lat=${loc.lat}&lng=${loc.lng}&zip=${loc.zip}&city=${encodeURIComponent(loc.city)}`
|
||||
);
|
||||
|
||||
const nextData = extractNextData(body);
|
||||
if (!nextData) continue;
|
||||
|
||||
const contents = nextData.props?.pageProps?.pageInformation?.template?.content?.shelfContents?.contents
|
||||
|| nextData.props?.pageProps?.shelfContents?.contents
|
||||
|| [];
|
||||
|
||||
for (const item of contents) {
|
||||
const c = item.content;
|
||||
if (!c || item.contentType !== 'brochure') continue;
|
||||
|
||||
const publisher = c.publisher?.name || '';
|
||||
const store = c.closestStore;
|
||||
const id = c.contentId;
|
||||
|
||||
if (!allBrochures.has(id)) {
|
||||
allBrochures.set(id, {
|
||||
id,
|
||||
publisher,
|
||||
title: c.title || '',
|
||||
store: store?.name || publisher,
|
||||
city: store?.city || loc.city,
|
||||
street: store ? `${store.street || ''} ${store.streetNumber || ''}`.trim() : '',
|
||||
zip: store?.zip || loc.zip,
|
||||
image: c.brochureImages?.find(i => i.size === '260x270')?.url || c.brochureImage?.url || null,
|
||||
validFrom: c.validFrom || null,
|
||||
validTo: c.validUntil || null,
|
||||
pageCount: c.pageCount || null,
|
||||
url: `https://www.kaufda.de/webapp/brochure/${id}`,
|
||||
isTarget: isTargetPublisher(publisher),
|
||||
badges: c.contentBadges?.map(b => b.name) || [],
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch(e) {
|
||||
console.error('KöPi scrape error:', loc.city, e.message);
|
||||
}
|
||||
}
|
||||
|
||||
const brochures = [...allBrochures.values()];
|
||||
const targeted = brochures.filter(b => b.isTarget);
|
||||
const others = brochures.filter(b => !b.isTarget);
|
||||
|
||||
const result = {
|
||||
targeted,
|
||||
others,
|
||||
total: brochures.length,
|
||||
scrapedAt: new Date().toISOString(),
|
||||
};
|
||||
|
||||
cacheSet(cacheKey, result);
|
||||
return result;
|
||||
}
|
||||
|
||||
// ── Routes ────────────────────────────────────────────────────────────────────
|
||||
router.get('/offers', authenticate, async (req, res) => {
|
||||
try {
|
||||
const data = await scrapeOffers();
|
||||
res.json(data);
|
||||
} catch(e) {
|
||||
res.status(500).json({ error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.get('/prospekte', authenticate, async (req, res) => {
|
||||
try {
|
||||
const data = await scrapeProspekte();
|
||||
res.json(data);
|
||||
} catch(e) {
|
||||
res.status(500).json({ error: e.message });
|
||||
}
|
||||
});
|
||||
|
||||
router.post('/clear-cache', authenticate, (req, res) => {
|
||||
if (req.user?.role !== 'admin') return res.status(403).json({ error: 'Kein Zugriff' });
|
||||
delete cache['koepi:offers'];
|
||||
delete cache['koepi:prospekte'];
|
||||
res.json({ ok: true });
|
||||
});
|
||||
|
||||
module.exports = router;
|
||||
Reference in New Issue
Block a user