// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0 // Records low-cardinality Cloud API backend performance metrics and Server-Timing phases. import { AsyncLocalStorage } from "node:async_hooks"; const REQUEST_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]; const PHASE_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30]; const UPSTREAM_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120]; const MAX_SERVER_TIMING_ENTRIES = 8; const backendPerformanceStorage = new AsyncLocalStorage(); interface HistogramSeries { labels: Record; buckets: number[]; sum: number; count: number; } interface CounterSeries { labels: Record; value: number; } interface BackendPerformanceStoreOptions { env?: Record; maxSeries?: number; } interface BackendPerformanceContextInput { route: string; method: string; } interface BackendPhaseInput { phase: string; durationMs: number; outcome?: string; } interface BackendUpstreamInput { component: string; operation?: string; route: string; method: string; statusClass?: string; outcome?: string; durationMs: number; } export function createBackendPerformanceStore(options: BackendPerformanceStoreOptions = {}) { const env = options.env ?? process.env; const maxSeries = parsePositiveInteger(env.HWLAB_BACKEND_PERFORMANCE_MAX_SERIES, options.maxSeries ?? 320); const requestDurationSeries = new Map(); const requestTotalSeries = new Map(); const requestPhaseSeries = new Map(); const upstreamSeries = new Map(); const droppedSeries = new Map(); const baseLabels = { service: "hwlab-cloud-api", namespace: sanitizeLabelValue(env.POD_NAMESPACE ?? env.HWLAB_METRICS_NAMESPACE ?? "hwlab-v02"), gitops_target: sanitizeLabelValue(env.HWLAB_GITOPS_TARGET ?? env.HWLAB_GITOPS_PROFILE ?? env.HWLAB_RUNTIME_LANE ?? "v02") }; function beginHttpRequest(request: any, response: any) { const url = safeUrl(request?.url || "/"); const context = createContext({ route: backendPerformanceRouteTemplate(url.pathname), method: normalizeMethod(request?.method) }); const originalWriteHead = typeof response?.writeHead === "function" ? response.writeHead : null; if (originalWriteHead) { response.writeHead = function writeHeadWithServerTiming(statusCode: number, ...args: unknown[]) { context.setStatus(statusCode); context.applyServerTimingHeader(response); return originalWriteHead.call(this, statusCode, ...args); }; } if (typeof response?.once === "function") { response.once("finish", () => context.finish(Number(response.statusCode ?? context.statusCode ?? 0))); } return context; } function createManualContext(input: BackendPerformanceContextInput) { return createContext({ route: backendPerformanceRouteTemplate(input.route), method: normalizeMethod(input.method) }); } function createContext(input: BackendPerformanceContextInput) { const startedAt = nowMs(); const phases: Array<{ name: string; durationMs: number; outcome: string }> = []; let statusCode = 200; let finished = false; const route = input.route; const method = input.method; function recordPhase(inputPhase: BackendPhaseInput) { const durationMs = boundedDurationMs(inputPhase.durationMs, 30_000); const phase = sanitizeMetricName(inputPhase.phase, "unknown"); const outcome = normalizeOutcome(inputPhase.outcome ?? "ok"); phases.push({ name: phase, durationMs, outcome }); const labels = { ...baseLabels, route, method, phase, outcome }; if (!recordHistogram(requestPhaseSeries, labels, durationMs / 1000, PHASE_BUCKETS_SECONDS, maxSeries)) incrementDropped("phase_series_limit"); } async function measure(phase: string, fn: () => Promise | T): Promise { const phaseStartedAt = nowMs(); try { const result = await fn(); recordPhase({ phase, durationMs: nowMs() - phaseStartedAt, outcome: "ok" }); return result; } catch (error) { recordPhase({ phase, durationMs: nowMs() - phaseStartedAt, outcome: "error" }); throw error; } } function recordUpstream(inputUpstream: BackendUpstreamInput) { const labels = { ...baseLabels, component: sanitizeMetricName(inputUpstream.component, "unknown"), operation: sanitizeMetricName(inputUpstream.operation ?? inputUpstream.route, "unknown"), route: backendPerformanceRouteTemplate(inputUpstream.route), method: normalizeMethod(inputUpstream.method), status_class: normalizeStatusClass(inputUpstream.statusClass ?? "unknown"), outcome: normalizeOutcome(inputUpstream.outcome ?? "ok") }; if (!recordHistogram(upstreamSeries, labels, boundedDurationMs(inputUpstream.durationMs, 120_000) / 1000, UPSTREAM_BUCKETS_SECONDS, maxSeries)) incrementDropped("upstream_series_limit"); } function setStatus(nextStatusCode: number) { if (Number.isInteger(nextStatusCode) && nextStatusCode > 0) statusCode = nextStatusCode; } function applyServerTimingHeader(response: any) { if (!response || typeof response.setHeader !== "function" || response.headersSent) return; const header = serverTimingHeader(nowMs() - startedAt, phases); if (header) response.setHeader("server-timing", header); } function finish(finalStatusCode = statusCode) { if (finished) return; finished = true; const statusClassValue = statusClass(finalStatusCode); const outcome = statusClassValue === "5xx" ? "error" : statusClassValue === "4xx" ? "denied" : "ok"; const labels = { ...baseLabels, route, method, status_class: statusClassValue, outcome }; if (!recordHistogram(requestDurationSeries, labels, boundedDurationMs(nowMs() - startedAt, 120_000) / 1000, REQUEST_BUCKETS_SECONDS, maxSeries)) incrementDropped("request_series_limit"); incrementCounter(requestTotalSeries, labels, maxSeries) || incrementDropped("request_counter_series_limit"); } return { route, method, startedAt, recordPhase, measure, recordUpstream, setStatus, applyServerTimingHeader, finish, get statusCode() { return statusCode; } }; } function incrementDropped(reason: string) { incrementCounter(droppedSeries, { ...baseLabels, reason }, Math.max(maxSeries, 16)); } function metricsText() { return [ "# HELP hwlab_cloud_api_requests_total Cloud API HTTP requests by low-cardinality route template.", "# TYPE hwlab_cloud_api_requests_total counter", ...renderCounters("hwlab_cloud_api_requests_total", requestTotalSeries), "# HELP hwlab_cloud_api_request_duration_seconds Cloud API HTTP request duration in seconds.", "# TYPE hwlab_cloud_api_request_duration_seconds histogram", ...renderHistogram("hwlab_cloud_api_request_duration_seconds", requestDurationSeries, REQUEST_BUCKETS_SECONDS), "# HELP hwlab_cloud_api_request_phase_duration_seconds Cloud API handler phase duration in seconds.", "# TYPE hwlab_cloud_api_request_phase_duration_seconds histogram", ...renderHistogram("hwlab_cloud_api_request_phase_duration_seconds", requestPhaseSeries, PHASE_BUCKETS_SECONDS), "# HELP hwlab_cloud_api_upstream_duration_seconds Cloud API upstream dependency call duration in seconds.", "# TYPE hwlab_cloud_api_upstream_duration_seconds histogram", ...renderHistogram("hwlab_cloud_api_upstream_duration_seconds", upstreamSeries, UPSTREAM_BUCKETS_SECONDS), "# HELP hwlab_cloud_api_performance_dropped_total Backend performance samples dropped before Prometheus export.", "# TYPE hwlab_cloud_api_performance_dropped_total counter", ...renderCounters("hwlab_cloud_api_performance_dropped_total", droppedSeries), "" ].join("\n"); } function snapshot() { return { requestSeries: requestDurationSeries.size, requestCount: [...requestTotalSeries.values()].reduce((sum, entry) => sum + entry.value, 0), requestPhaseSeries: requestPhaseSeries.size, requestPhaseCount: histogramSampleCount(requestPhaseSeries), upstreamSeries: upstreamSeries.size, upstreamCount: histogramSampleCount(upstreamSeries), droppedSeries: droppedSeries.size }; } return { beginHttpRequest, createManualContext, metricsText, snapshot }; } export function withBackendPerformanceContext(context: unknown, callback: () => T): T { return backendPerformanceStorage.run(context, callback); } export function currentBackendPerformanceContext(): any { return backendPerformanceStorage.getStore() ?? null; } export function backendPerformanceRouteTemplate(input: unknown): string { const raw = String(input ?? "").trim() || "/"; const pathOnly = raw.split(/[?#]/u, 1)[0] || "/"; let pathname = pathOnly; try { pathname = new URL(raw, "http://hwlab.local").pathname || pathOnly; } catch { // Keep pathOnly. } const parts = pathname.split("/").filter(Boolean).map((part) => routeSegmentTemplate(safeDecode(part))); const path = `/${parts.join("/")}`; return path.length > 140 ? `${path.slice(0, 137)}...` : path; } function serverTimingHeader(totalMs: number, phases: Array<{ name: string; durationMs: number }>) { const entries = phases.slice(-MAX_SERVER_TIMING_ENTRIES).map((phase) => `${serverTimingToken(phase.name)};dur=${roundDurationMs(phase.durationMs)}`); entries.push(`total;dur=${roundDurationMs(totalMs)}`); return entries.join(", "); } function recordHistogram(series: Map, labels: Record, value: number, buckets: number[], maxSeries: number) { const key = stableLabelKey(labels); let entry = series.get(key); if (!entry) { if (series.size >= maxSeries) return false; entry = { labels, buckets: buckets.map(() => 0), sum: 0, count: 0 }; series.set(key, entry); } for (let index = 0; index < buckets.length; index += 1) { if (value <= buckets[index]) entry.buckets[index] += 1; } entry.sum += value; entry.count += 1; return true; } function incrementCounter(series: Map, labels: Record, maxSeries: number) { const key = stableLabelKey(labels); let entry = series.get(key); if (!entry) { if (series.size >= maxSeries) return false; entry = { labels, value: 0 }; series.set(key, entry); } entry.value += 1; return true; } function renderCounters(metricName: string, series: Map) { return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${entry.value}`); } function renderHistogram(metricName: string, series: Map, buckets: number[]) { const lines: string[] = []; for (const entry of series.values()) { for (let index = 0; index < buckets.length; index += 1) { lines.push(`${metricName}_bucket{${renderLabels({ ...entry.labels, le: String(buckets[index]) })}} ${entry.buckets[index]}`); } lines.push(`${metricName}_bucket{${renderLabels({ ...entry.labels, le: "+Inf" })}} ${entry.count}`); lines.push(`${metricName}_sum{${renderLabels(entry.labels)}} ${entry.sum.toFixed(6)}`); lines.push(`${metricName}_count{${renderLabels(entry.labels)}} ${entry.count}`); } return lines; } function histogramSampleCount(series: Map) { return [...series.values()].reduce((sum, entry) => sum + entry.count, 0); } function renderLabels(labels: Record) { return Object.entries(labels).map(([key, value]) => `${key}="${escapeLabelValue(value)}"`).join(","); } function stableLabelKey(labels: Record) { return Object.keys(labels).sort().map((key) => `${key}=${labels[key]}`).join("\u0000"); } function routeSegmentTemplate(segment: string) { if (/^\d+$/u.test(segment)) return ":id"; if (/^[a-z][a-z0-9]{1,12}_[A-Za-z0-9_.:-]+$/iu.test(segment)) return ":id"; if (/^[0-9a-f]{12,}$/iu.test(segment)) return ":id"; if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu.test(segment)) return ":id"; if (segment.length > 48) return ":id"; return sanitizePathSegment(segment); } function safeUrl(value: string) { try { return new URL(value, "http://hwlab-cloud-api.local"); } catch { return new URL("/", "http://hwlab-cloud-api.local"); } } function safeDecode(value: string) { try { return decodeURIComponent(value); } catch { return value; } } function sanitizePathSegment(value: string) { const sanitized = value.replace(/[^A-Za-z0-9_.:-]/gu, "_"); return sanitized.slice(0, 64) || "_"; } function normalizeMethod(value: unknown) { const method = String(value ?? "GET").trim().toUpperCase(); return /^(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)$/u.test(method) ? method : "OTHER"; } function statusClass(value: unknown) { const status = Number(value); if (!Number.isFinite(status) || status <= 0) return "unknown"; return `${Math.floor(status / 100)}xx`; } function normalizeStatusClass(value: unknown) { const text = String(value ?? "").trim().toLowerCase(); return /^(?:1xx|2xx|3xx|4xx|5xx|network|timeout|unknown)$/u.test(text) ? text : "unknown"; } function normalizeOutcome(value: unknown) { const text = sanitizeMetricName(value, "unknown"); return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown)$/u.test(text) ? text : "unknown"; } function sanitizeMetricName(value: unknown, fallback = "unknown") { const text = String(value ?? "").trim().toLowerCase().replace(/[^a-z0-9_:-]+/gu, "_").replace(/[-:]+/gu, "_").replace(/^_+|_+$/gu, ""); return text || fallback; } function sanitizeLabelValue(value: unknown, fallback = "unknown") { const text = String(value ?? "").trim(); if (!text) return fallback; return text.replace(/[\n\r\t]/gu, " ").slice(0, 120); } function escapeLabelValue(value: string) { return String(value).replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"").replace(/\n/gu, "\\n"); } function parsePositiveInteger(value: unknown, fallback: number) { const parsed = Number.parseInt(String(value ?? ""), 10); return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; } function boundedDurationMs(value: unknown, maxMs: number) { const parsed = Number(value); if (!Number.isFinite(parsed) || parsed < 0) return 0; return Math.min(parsed, maxMs); } function roundDurationMs(value: number) { return Math.max(0, value).toFixed(1); } function serverTimingToken(value: string) { const token = sanitizeMetricName(value, "phase").replace(/_/gu, "-"); return token || "phase"; } function nowMs() { if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); return Date.now(); }