From 0fc063b2a2601a5b79ac54b538ebe8da1a32f51f Mon Sep 17 00:00:00 2001 From: Dicken Date: Fri, 24 Jul 2026 02:21:58 +0200 Subject: [PATCH] round26: Ping vor Portscan, Liste neuer Dienste, Suchmaschinen-Integration (OpenSearch), API-Scanner --- apps/backend/src/db/client.ts | 9 ++ apps/backend/src/db/repositories/apis.ts | 60 +++++++++ apps/backend/src/db/schema.ts | 17 +++ apps/backend/src/index.ts | 2 + apps/backend/src/routes/apis.ts | 57 +++++++++ apps/backend/src/routes/scan.ts | 1 + apps/backend/src/scanner/apiDetector.ts | 97 ++++++++++++++ apps/backend/src/scanner/networkScanner.ts | 13 ++ apps/backend/src/scanner/ping.ts | 24 +++- apps/frontend/index.html | 6 + apps/frontend/public/opensearch.xml | 8 ++ apps/frontend/src/hooks/useDetectedApis.ts | 25 ++++ apps/frontend/src/router.tsx | 8 ++ apps/frontend/src/routes/HomePage.tsx | 8 +- .../src/routes/SearchRedirectPage.tsx | 54 ++++++++ .../frontend/src/routes/admin/ScannerPage.tsx | 120 ++++++++++++++++++ packages/shared/src/index.ts | 2 +- 17 files changed, 507 insertions(+), 4 deletions(-) create mode 100644 apps/backend/src/db/repositories/apis.ts create mode 100644 apps/backend/src/routes/apis.ts create mode 100644 apps/backend/src/scanner/apiDetector.ts create mode 100644 apps/frontend/public/opensearch.xml create mode 100644 apps/frontend/src/hooks/useDetectedApis.ts create mode 100644 apps/frontend/src/routes/SearchRedirectPage.tsx diff --git a/apps/backend/src/db/client.ts b/apps/backend/src/db/client.ts index d94487e..7c0278a 100644 --- a/apps/backend/src/db/client.ts +++ b/apps/backend/src/db/client.ts @@ -119,6 +119,15 @@ export function ensureSchema(): void { saved_at TEXT NOT NULL, updated_at TEXT NOT NULL ); + + CREATE TABLE IF NOT EXISTS detected_apis ( + id TEXT PRIMARY KEY, + service_id TEXT NOT NULL REFERENCES services(id) ON DELETE CASCADE, + path TEXT NOT NULL, + type TEXT NOT NULL, + status INTEGER NOT NULL, + detected_at TEXT NOT NULL + ); `); // Leichte Migration für Datenbanken, die vor Einführung von "visible"/ diff --git a/apps/backend/src/db/repositories/apis.ts b/apps/backend/src/db/repositories/apis.ts new file mode 100644 index 0000000..d92916c --- /dev/null +++ b/apps/backend/src/db/repositories/apis.ts @@ -0,0 +1,60 @@ +import { randomUUID } from "node:crypto"; +import { eq } from "drizzle-orm"; +import { db } from "../client.js"; +import { detectedApis } from "../schema.js"; + +export interface DetectedApiEntry { + id: string; + serviceId: string; + path: string; + type: string; + status: number; + detectedAt: string; +} + +function mapRow(row: typeof detectedApis.$inferSelect): DetectedApiEntry { + return { + id: row.id, + serviceId: row.serviceId, + path: row.path, + type: row.type, + status: row.status, + detectedAt: row.detectedAt, + }; +} + +export function listDetectedApis(): DetectedApiEntry[] { + return db.select().from(detectedApis).all().map(mapRow); +} + +/** + * Ersetzt die gespeicherten API-Funde eines Dienstes komplett durch die + * neuen Ergebnisse eines Scan-Laufs - kein Anhäufen von veralteten + * Einträgen, wenn sich z. B. der API-Pfad einer Software mal ändert. + */ +export function replaceApisForService( + serviceId: string, + found: { path: string; type: string; status: number }[] +): DetectedApiEntry[] { + db.delete(detectedApis).where(eq(detectedApis.serviceId, serviceId)).run(); + + const timestamp = new Date().toISOString(); + const rows = found.map((f) => ({ + id: randomUUID(), + serviceId, + path: f.path, + type: f.type, + status: f.status, + detectedAt: timestamp, + })); + + if (rows.length > 0) { + db.insert(detectedApis).values(rows).run(); + } + + return rows.map(mapRow); +} + +export function deleteApisForService(serviceId: string): void { + db.delete(detectedApis).where(eq(detectedApis.serviceId, serviceId)).run(); +} diff --git a/apps/backend/src/db/schema.ts b/apps/backend/src/db/schema.ts index 47be10b..9208940 100644 --- a/apps/backend/src/db/schema.ts +++ b/apps/backend/src/db/schema.ts @@ -153,3 +153,20 @@ export const readLater = sqliteTable("read_later", { savedAt: text("saved_at").notNull(), updatedAt: text("updated_at").notNull(), }); + +/** + * Von einem eigenen, separaten Scanner ("API-Scanner", siehe + * scanner/apiDetector.ts) gefundene API-Endpunkte bereits bekannter Dienste. + * Kein automatischer Teil des normalen Geräte-/Dienste-Scans - läuft nur auf + * ausdrücklichen Knopfdruck, genau wie die anderen Scanner. + */ +export const detectedApis = sqliteTable("detected_apis", { + id: text("id").primaryKey(), + serviceId: text("service_id") + .notNull() + .references(() => services.id, { onDelete: "cascade" }), + path: text("path").notNull(), + type: text("type").notNull(), + status: integer("status").notNull(), + detectedAt: text("detected_at").notNull(), +}); diff --git a/apps/backend/src/index.ts b/apps/backend/src/index.ts index 705319f..c6e0cc0 100644 --- a/apps/backend/src/index.ts +++ b/apps/backend/src/index.ts @@ -17,6 +17,7 @@ import { settingsRoutes } from "./routes/settings.js"; import { readLaterRoutes } from "./routes/readLater.js"; import { faviconProxyRoutes } from "./routes/faviconProxy.js"; import { iconsRoutes } from "./routes/icons.js"; +import { apiRoutes } from "./routes/apis.js"; import { loadPlugins } from "./plugins/loader.js"; import { startLiveStatusHeartbeat } from "./liveStatus.js"; import * as serviceRepo from "./db/repositories/services.js"; @@ -78,6 +79,7 @@ async function main() { await app.register(readLaterRoutes); await app.register(faviconProxyRoutes); await app.register(iconsRoutes); + await app.register(apiRoutes); app.get("/", async () => { return { name: "LaunchPad API", status: "running" }; diff --git a/apps/backend/src/routes/apis.ts b/apps/backend/src/routes/apis.ts new file mode 100644 index 0000000..337955a --- /dev/null +++ b/apps/backend/src/routes/apis.ts @@ -0,0 +1,57 @@ +import type { FastifyInstance } from "fastify"; +import * as serviceRepo from "../db/repositories/services.js"; +import * as apiRepo from "../db/repositories/apis.js"; +import * as logRepo from "../db/repositories/logs.js"; +import { detectApis } from "../scanner/apiDetector.js"; + +/** + * API-Scanner: eigener, von den Geräte-/FritzBox-Scannern komplett + * unabhängiger manueller Scan (siehe Scanner-Seite) - durchsucht bereits + * bekannte Dienste nach üblichen API-Pfaden (siehe scanner/apiDetector.ts) + * und speichert die Funde. Läuft NIE automatisch, nur auf Knopfdruck. + */ +export async function apiRoutes(app: FastifyInstance): Promise { + app.get("/api/detected-apis", async () => { + return apiRepo.listDetectedApis(); + }); + + app.post("/api/scan/apis", async () => { + const services = serviceRepo.listServices(); + let servicesWithApi = 0; + let totalFound = 0; + + for (const service of services) { + const found = await detectApis(service.url); + if (found.length > 0) { + apiRepo.replaceApisForService(service.id, found); + servicesWithApi++; + totalFound += found.length; + } else { + apiRepo.deleteApisForService(service.id); + } + } + + logRepo.logScan({ + type: "api", + targetId: null, + level: "info", + message: `API-Scan: ${services.length} Dienst(e) geprüft, bei ${servicesWithApi} Dienst(en) ${totalFound} API-Endpunkt(e) gefunden.`, + }); + + return { checked: services.length, servicesWithApi, totalFound }; + }); + + app.post("/api/scan/apis/:serviceId", async (request, reply) => { + const { serviceId } = request.params as { serviceId: string }; + const service = serviceRepo.getService(serviceId); + if (!service) { + return reply.code(404).send({ error: "Dienst nicht gefunden" }); + } + + const found = await detectApis(service.url); + const saved = found.length > 0 ? apiRepo.replaceApisForService(service.id, found) : []; + if (found.length === 0) apiRepo.deleteApisForService(service.id); + + return { serviceId, apis: saved }; + }); +} diff --git a/apps/backend/src/routes/scan.ts b/apps/backend/src/routes/scan.ts index a4b1972..bae9c96 100644 --- a/apps/backend/src/routes/scan.ts +++ b/apps/backend/src/routes/scan.ts @@ -147,6 +147,7 @@ export async function scanRoutes(app: FastifyInstance): Promise { created, updated, services: results.map((r) => r.service), + newServices: results.filter((r) => r.created).map((r) => r.service), staleServices, nameChanges, deviceNameSuggestion, diff --git a/apps/backend/src/scanner/apiDetector.ts b/apps/backend/src/scanner/apiDetector.ts new file mode 100644 index 0000000..7068c22 --- /dev/null +++ b/apps/backend/src/scanner/apiDetector.ts @@ -0,0 +1,97 @@ +import http from "node:http"; +import https from "node:https"; + +export interface DetectedApi { + path: string; + type: string; + status: number; +} + +interface RawProbeResult { + status: number; + contentType: string | null; + bodySnippet: string; +} + +const MAX_BODY_BYTES = 8_192; + +function fetchRaw(url: string, timeoutMs = 2500): Promise { + return new Promise((resolve) => { + const isHttps = url.startsWith("https://"); + const client = isHttps ? https : http; + + const req = client.get( + url, + { timeout: timeoutMs, rejectUnauthorized: false, headers: { Accept: "application/json, */*" } }, + (res) => { + let body = ""; + let received = 0; + res.on("data", (chunk: Buffer) => { + received += chunk.length; + if (received <= MAX_BODY_BYTES) body += chunk.toString("utf-8"); + }); + res.on("end", () => { + const contentType = res.headers["content-type"] ?? null; + resolve({ status: res.statusCode ?? 0, contentType, bodySnippet: body }); + }); + res.on("error", () => resolve(null)); + } + ); + req.on("timeout", () => { + req.destroy(); + resolve(null); + }); + req.on("error", () => resolve(null)); + }); +} + +/** + * Wohlbekannte Pfade, unter denen selbstgehostete Software üblicherweise + * ihre API bzw. deren Dokumentation/Schema anbietet. Bewusst eine + * kuratierte, kurze Liste statt eines vollständigen Wortlisten-Bruteforce - + * das hier ist ein Hinweis-Scanner, kein Sicherheits-/Pentesting-Werkzeug. + */ +const CANDIDATE_PATHS: { path: string; type: string }[] = [ + { path: "/openapi.json", type: "OpenAPI" }, + { path: "/swagger.json", type: "OpenAPI (Swagger)" }, + { path: "/api-docs", type: "OpenAPI (Swagger)" }, + { path: "/swagger/index.html", type: "Swagger-UI" }, + { path: "/docs", type: "API-Dokumentation" }, + { path: "/graphql", type: "GraphQL" }, + { path: "/api/v1", type: "REST-API" }, + { path: "/api", type: "REST-API" }, + { path: "/.well-known/openapi.json", type: "OpenAPI" }, +]; + +function looksLikeJson(body: string): boolean { + const trimmed = body.trim(); + return trimmed.startsWith("{") || trimmed.startsWith("["); +} + +/** + * Prüft die kuratierten Kandidaten-Pfade unter einer Basis-URL parallel und + * liefert alle, die auf eine tatsächlich vorhandene API hindeuten: eine + * JSON-Antwort (egal ob 200 oder z. B. 401 "unauthorized" - eine + * JSON-Fehlermeldung zeigt trotzdem "hier läuft eine API"), oder ein + * Content-Type, der explizit auf JSON/GraphQL hindeutet. Reine HTML-Seiten + * (z. B. eine 404-Fehlerseite des Frontends) zählen nicht. + */ +export async function detectApis(baseUrl: string): Promise { + const checks = await Promise.all( + CANDIDATE_PATHS.map(async ({ path, type }) => { + const result = await fetchRaw(`${baseUrl}${path}`); + if (!result || result.status === 0 || result.status === 404) return null; + + const contentTypeIsApi = + result.contentType?.includes("json") || result.contentType?.includes("graphql"); + const bodyIsJson = looksLikeJson(result.bodySnippet); + + if (!contentTypeIsApi && !bodyIsJson) return null; + + const detected: DetectedApi = { path, type, status: result.status }; + return detected; + }) + ); + + return checks.filter((c): c is DetectedApi => c !== null); +} diff --git a/apps/backend/src/scanner/networkScanner.ts b/apps/backend/src/scanner/networkScanner.ts index 767ce1e..7a9533d 100644 --- a/apps/backend/src/scanner/networkScanner.ts +++ b/apps/backend/src/scanner/networkScanner.ts @@ -3,6 +3,7 @@ import { isPortOpen, TYPICAL_PORTS } from "./ports.js"; import { probeHttp } from "./http.js"; import { detectSoftware } from "./softwareDetection.js"; import { findBestIconMatch } from "./iconDb.js"; +import { pingHost, isPingBinaryConfirmedMissing } from "./ping.js"; export interface ScanTarget { hostname: string; @@ -90,6 +91,18 @@ export async function scanDeviceServices( // gemeldeten Namen ein. const suggestedHostname = dnsResult ? null : await reverseLookup(device.ip); + // Vor dem eigentlichen Portscan erst ein einzelner Ping (ICMP) - ist das + // Gerät gar nicht erreichbar (aus, im Standby, vom Netz getrennt), spart + // das den kompletten Portscan (auch parallel noch ~800ms) UND alle + // nachfolgenden DNS/HTTP-Versuche. Nur wenn KEIN Ping ankommt wird + // übersprungen - manche Geräte blocken ICMP, antworten aber auf TCP, dafür + // bleibt genau deswegen bewusst KEIN weiterer früher Abbruch bestehen, + // sondern nur dieser eine zusätzliche, sehr schnelle Vorab-Check. + const reachable = await pingHost(device.ip); + if (!reachable && !isPingBinaryConfirmedMissing()) { + return { services: [], suggestedHostname }; + } + const candidatePorts = Array.from(new Set([80, 443, ...extraPorts])); // Die offenen Ports werden PARALLEL geprüft, nicht nacheinander - bei diff --git a/apps/backend/src/scanner/ping.ts b/apps/backend/src/scanner/ping.ts index f4471af..9d1ac0d 100644 --- a/apps/backend/src/scanner/ping.ts +++ b/apps/backend/src/scanner/ping.ts @@ -1,6 +1,8 @@ import { exec } from "node:child_process"; import { platform } from "node:os"; +let pingBinaryConfirmedMissing = false; + /** * Prüft per System-Ping (ICMP), ob eine IP erreichbar ist. Bewusst NICHT über * einen Port-Connect (wie isPortOpen in ports.ts) - ein Gerät kann online @@ -16,8 +18,28 @@ export function pingHost(ip: string): Promise { const command = isWindows ? `ping -n 1 -w 1000 ${ip}` : `ping -c 1 -W 1 ${ip}`; return new Promise((resolve) => { - exec(command, { timeout: 2000 }, (error) => { + exec(command, { timeout: 2000 }, (error, _stdout, stderr) => { + // Fehlt der ping-Befehl im Container, meldet die Shell das über exec + // NICHT als Node-ENOENT, sondern als regulären Fehlschlag mit + // Exit-Code 127 und "not found" in stderr (z. B. "/bin/sh: 1: ping: + // not found") - das wird hier separat erkannt (siehe + // isPingBinaryConfirmedMissing), damit ein fehlendes ping-Programm + // nicht stillschweigend JEDEN Scan leerlaufen lässt. + if (error && (error.code === 127 || /not found/i.test(stderr))) { + pingBinaryConfirmedMissing = true; + } resolve(!error); }); }); } + +/** + * true, wenn ein vorheriger pingHost()-Aufruf festgestellt hat, dass der + * ping-Befehl im Container gar nicht existiert (z. B. iputils-ping fehlt im + * Image). Wird von scanDeviceServices genutzt, um den Ping-Vorab-Check in + * dem Fall zu überspringen, statt fälschlich jedes Gerät als nicht + * erreichbar zu melden. + */ +export function isPingBinaryConfirmedMissing(): boolean { + return pingBinaryConfirmedMissing; +} diff --git a/apps/frontend/index.html b/apps/frontend/index.html index 034dfda..c61d088 100644 --- a/apps/frontend/index.html +++ b/apps/frontend/index.html @@ -7,6 +7,12 @@ + LaunchPad