lyon
358 lines
15 KiB
TypeScript
358 lines
15 KiB
TypeScript
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0
|
|
// Records low-cardinality Cloud API backend performance metrics and Server-Timing phases.
|
|
|
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
|
|
const REQUEST_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60];
|
|
const PHASE_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30];
|
|
const UPSTREAM_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120];
|
|
const MAX_SERVER_TIMING_ENTRIES = 8;
|
|
|
|
const backendPerformanceStorage = new AsyncLocalStorage();
|
|
|
|
interface HistogramSeries {
|
|
labels: Record<string, string>;
|
|
buckets: number[];
|
|
sum: number;
|
|
count: number;
|
|
}
|
|
|
|
interface CounterSeries {
|
|
labels: Record<string, string>;
|
|
value: number;
|
|
}
|
|
|
|
interface BackendPerformanceStoreOptions {
|
|
env?: Record<string, unknown>;
|
|
maxSeries?: number;
|
|
}
|
|
|
|
interface BackendPerformanceContextInput {
|
|
route: string;
|
|
method: string;
|
|
}
|
|
|
|
interface BackendPhaseInput {
|
|
phase: string;
|
|
durationMs: number;
|
|
outcome?: string;
|
|
}
|
|
|
|
interface BackendUpstreamInput {
|
|
component: string;
|
|
operation?: string;
|
|
route: string;
|
|
method: string;
|
|
statusClass?: string;
|
|
outcome?: string;
|
|
durationMs: number;
|
|
}
|
|
|
|
export function createBackendPerformanceStore(options: BackendPerformanceStoreOptions = {}) {
|
|
const env = options.env ?? process.env;
|
|
const maxSeries = parsePositiveInteger(env.HWLAB_BACKEND_PERFORMANCE_MAX_SERIES, options.maxSeries ?? 320);
|
|
const requestDurationSeries = new Map<string, HistogramSeries>();
|
|
const requestTotalSeries = new Map<string, CounterSeries>();
|
|
const requestPhaseSeries = new Map<string, HistogramSeries>();
|
|
const upstreamSeries = new Map<string, HistogramSeries>();
|
|
const droppedSeries = new Map<string, CounterSeries>();
|
|
const baseLabels = {
|
|
service: "hwlab-cloud-api",
|
|
namespace: sanitizeLabelValue(env.POD_NAMESPACE ?? env.HWLAB_METRICS_NAMESPACE ?? "hwlab-v02"),
|
|
gitops_target: sanitizeLabelValue(env.HWLAB_GITOPS_TARGET ?? env.HWLAB_GITOPS_PROFILE ?? env.HWLAB_RUNTIME_LANE ?? "v02")
|
|
};
|
|
|
|
function beginHttpRequest(request: any, response: any) {
|
|
const url = safeUrl(request?.url || "/");
|
|
const context = createContext({ route: backendPerformanceRouteTemplate(url.pathname), method: normalizeMethod(request?.method) });
|
|
const originalWriteHead = typeof response?.writeHead === "function" ? response.writeHead : null;
|
|
if (originalWriteHead) {
|
|
response.writeHead = function writeHeadWithServerTiming(statusCode: number, ...args: unknown[]) {
|
|
context.setStatus(statusCode);
|
|
context.applyServerTimingHeader(response);
|
|
return originalWriteHead.call(this, statusCode, ...args);
|
|
};
|
|
}
|
|
if (typeof response?.once === "function") {
|
|
response.once("finish", () => context.finish(Number(response.statusCode ?? context.statusCode ?? 0)));
|
|
}
|
|
return context;
|
|
}
|
|
|
|
function createManualContext(input: BackendPerformanceContextInput) {
|
|
return createContext({ route: backendPerformanceRouteTemplate(input.route), method: normalizeMethod(input.method) });
|
|
}
|
|
|
|
function createContext(input: BackendPerformanceContextInput) {
|
|
const startedAt = nowMs();
|
|
const phases: Array<{ name: string; durationMs: number; outcome: string }> = [];
|
|
let statusCode = 200;
|
|
let finished = false;
|
|
const route = input.route;
|
|
const method = input.method;
|
|
|
|
function recordPhase(inputPhase: BackendPhaseInput) {
|
|
const durationMs = boundedDurationMs(inputPhase.durationMs, 30_000);
|
|
const phase = sanitizeMetricName(inputPhase.phase, "unknown");
|
|
const outcome = normalizeOutcome(inputPhase.outcome ?? "ok");
|
|
phases.push({ name: phase, durationMs, outcome });
|
|
const labels = { ...baseLabels, route, method, phase, outcome };
|
|
if (!recordHistogram(requestPhaseSeries, labels, durationMs / 1000, PHASE_BUCKETS_SECONDS, maxSeries)) incrementDropped("phase_series_limit");
|
|
}
|
|
|
|
async function measure<T>(phase: string, fn: () => Promise<T> | T): Promise<T> {
|
|
const phaseStartedAt = nowMs();
|
|
try {
|
|
const result = await fn();
|
|
recordPhase({ phase, durationMs: nowMs() - phaseStartedAt, outcome: "ok" });
|
|
return result;
|
|
} catch (error) {
|
|
recordPhase({ phase, durationMs: nowMs() - phaseStartedAt, outcome: "error" });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function recordUpstream(inputUpstream: BackendUpstreamInput) {
|
|
const labels = {
|
|
...baseLabels,
|
|
component: sanitizeMetricName(inputUpstream.component, "unknown"),
|
|
operation: sanitizeMetricName(inputUpstream.operation ?? inputUpstream.route, "unknown"),
|
|
route: backendPerformanceRouteTemplate(inputUpstream.route),
|
|
method: normalizeMethod(inputUpstream.method),
|
|
status_class: normalizeStatusClass(inputUpstream.statusClass ?? "unknown"),
|
|
outcome: normalizeOutcome(inputUpstream.outcome ?? "ok")
|
|
};
|
|
if (!recordHistogram(upstreamSeries, labels, boundedDurationMs(inputUpstream.durationMs, 120_000) / 1000, UPSTREAM_BUCKETS_SECONDS, maxSeries)) incrementDropped("upstream_series_limit");
|
|
}
|
|
|
|
function setStatus(nextStatusCode: number) {
|
|
if (Number.isInteger(nextStatusCode) && nextStatusCode > 0) statusCode = nextStatusCode;
|
|
}
|
|
|
|
function applyServerTimingHeader(response: any) {
|
|
if (!response || typeof response.setHeader !== "function" || response.headersSent) return;
|
|
const header = serverTimingHeader(nowMs() - startedAt, phases);
|
|
if (header) response.setHeader("server-timing", header);
|
|
}
|
|
|
|
function finish(finalStatusCode = statusCode) {
|
|
if (finished) return;
|
|
finished = true;
|
|
const statusClassValue = statusClass(finalStatusCode);
|
|
const outcome = statusClassValue === "5xx" ? "error" : statusClassValue === "4xx" ? "denied" : "ok";
|
|
const labels = { ...baseLabels, route, method, status_class: statusClassValue, outcome };
|
|
if (!recordHistogram(requestDurationSeries, labels, boundedDurationMs(nowMs() - startedAt, 120_000) / 1000, REQUEST_BUCKETS_SECONDS, maxSeries)) incrementDropped("request_series_limit");
|
|
incrementCounter(requestTotalSeries, labels, maxSeries) || incrementDropped("request_counter_series_limit");
|
|
}
|
|
|
|
return { route, method, startedAt, recordPhase, measure, recordUpstream, setStatus, applyServerTimingHeader, finish, get statusCode() { return statusCode; } };
|
|
}
|
|
|
|
function incrementDropped(reason: string) {
|
|
incrementCounter(droppedSeries, { ...baseLabels, reason }, Math.max(maxSeries, 16));
|
|
}
|
|
|
|
function metricsText() {
|
|
return [
|
|
"# HELP hwlab_cloud_api_requests_total Cloud API HTTP requests by low-cardinality route template.",
|
|
"# TYPE hwlab_cloud_api_requests_total counter",
|
|
...renderCounters("hwlab_cloud_api_requests_total", requestTotalSeries),
|
|
"# HELP hwlab_cloud_api_request_duration_seconds Cloud API HTTP request duration in seconds.",
|
|
"# TYPE hwlab_cloud_api_request_duration_seconds histogram",
|
|
...renderHistogram("hwlab_cloud_api_request_duration_seconds", requestDurationSeries, REQUEST_BUCKETS_SECONDS),
|
|
"# HELP hwlab_cloud_api_request_phase_duration_seconds Cloud API handler phase duration in seconds.",
|
|
"# TYPE hwlab_cloud_api_request_phase_duration_seconds histogram",
|
|
...renderHistogram("hwlab_cloud_api_request_phase_duration_seconds", requestPhaseSeries, PHASE_BUCKETS_SECONDS),
|
|
"# HELP hwlab_cloud_api_upstream_duration_seconds Cloud API upstream dependency call duration in seconds.",
|
|
"# TYPE hwlab_cloud_api_upstream_duration_seconds histogram",
|
|
...renderHistogram("hwlab_cloud_api_upstream_duration_seconds", upstreamSeries, UPSTREAM_BUCKETS_SECONDS),
|
|
"# HELP hwlab_cloud_api_performance_dropped_total Backend performance samples dropped before Prometheus export.",
|
|
"# TYPE hwlab_cloud_api_performance_dropped_total counter",
|
|
...renderCounters("hwlab_cloud_api_performance_dropped_total", droppedSeries),
|
|
""
|
|
].join("\n");
|
|
}
|
|
|
|
function snapshot() {
|
|
return {
|
|
requestSeries: requestDurationSeries.size,
|
|
requestCount: [...requestTotalSeries.values()].reduce((sum, entry) => sum + entry.value, 0),
|
|
requestPhaseSeries: requestPhaseSeries.size,
|
|
requestPhaseCount: histogramSampleCount(requestPhaseSeries),
|
|
upstreamSeries: upstreamSeries.size,
|
|
upstreamCount: histogramSampleCount(upstreamSeries),
|
|
droppedSeries: droppedSeries.size
|
|
};
|
|
}
|
|
|
|
return { beginHttpRequest, createManualContext, metricsText, snapshot };
|
|
}
|
|
|
|
export function withBackendPerformanceContext<T>(context: unknown, callback: () => T): T {
|
|
return backendPerformanceStorage.run(context, callback);
|
|
}
|
|
|
|
export function currentBackendPerformanceContext(): any {
|
|
return backendPerformanceStorage.getStore() ?? null;
|
|
}
|
|
|
|
export function backendPerformanceRouteTemplate(input: unknown): string {
|
|
const raw = String(input ?? "").trim() || "/";
|
|
const pathOnly = raw.split(/[?#]/u, 1)[0] || "/";
|
|
let pathname = pathOnly;
|
|
try {
|
|
pathname = new URL(raw, "http://hwlab.local").pathname || pathOnly;
|
|
} catch {
|
|
// Keep pathOnly.
|
|
}
|
|
const parts = pathname.split("/").filter(Boolean).map((part) => routeSegmentTemplate(safeDecode(part)));
|
|
const path = `/${parts.join("/")}`;
|
|
return path.length > 140 ? `${path.slice(0, 137)}...` : path;
|
|
}
|
|
|
|
function serverTimingHeader(totalMs: number, phases: Array<{ name: string; durationMs: number }>) {
|
|
const entries = phases.slice(-MAX_SERVER_TIMING_ENTRIES).map((phase) => `${serverTimingToken(phase.name)};dur=${roundDurationMs(phase.durationMs)}`);
|
|
entries.push(`total;dur=${roundDurationMs(totalMs)}`);
|
|
return entries.join(", ");
|
|
}
|
|
|
|
function recordHistogram(series: Map<string, HistogramSeries>, labels: Record<string, string>, value: number, buckets: number[], maxSeries: number) {
|
|
const key = stableLabelKey(labels);
|
|
let entry = series.get(key);
|
|
if (!entry) {
|
|
if (series.size >= maxSeries) return false;
|
|
entry = { labels, buckets: buckets.map(() => 0), sum: 0, count: 0 };
|
|
series.set(key, entry);
|
|
}
|
|
for (let index = 0; index < buckets.length; index += 1) {
|
|
if (value <= buckets[index]) entry.buckets[index] += 1;
|
|
}
|
|
entry.sum += value;
|
|
entry.count += 1;
|
|
return true;
|
|
}
|
|
|
|
function incrementCounter(series: Map<string, CounterSeries>, labels: Record<string, string>, maxSeries: number) {
|
|
const key = stableLabelKey(labels);
|
|
let entry = series.get(key);
|
|
if (!entry) {
|
|
if (series.size >= maxSeries) return false;
|
|
entry = { labels, value: 0 };
|
|
series.set(key, entry);
|
|
}
|
|
entry.value += 1;
|
|
return true;
|
|
}
|
|
|
|
function renderCounters(metricName: string, series: Map<string, CounterSeries>) {
|
|
return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${entry.value}`);
|
|
}
|
|
|
|
function renderHistogram(metricName: string, series: Map<string, HistogramSeries>, buckets: number[]) {
|
|
const lines: string[] = [];
|
|
for (const entry of series.values()) {
|
|
for (let index = 0; index < buckets.length; index += 1) {
|
|
lines.push(`${metricName}_bucket{${renderLabels({ ...entry.labels, le: String(buckets[index]) })}} ${entry.buckets[index]}`);
|
|
}
|
|
lines.push(`${metricName}_bucket{${renderLabels({ ...entry.labels, le: "+Inf" })}} ${entry.count}`);
|
|
lines.push(`${metricName}_sum{${renderLabels(entry.labels)}} ${entry.sum.toFixed(6)}`);
|
|
lines.push(`${metricName}_count{${renderLabels(entry.labels)}} ${entry.count}`);
|
|
}
|
|
return lines;
|
|
}
|
|
|
|
function histogramSampleCount(series: Map<string, HistogramSeries>) {
|
|
return [...series.values()].reduce((sum, entry) => sum + entry.count, 0);
|
|
}
|
|
|
|
function renderLabels(labels: Record<string, string>) {
|
|
return Object.entries(labels).map(([key, value]) => `${key}="${escapeLabelValue(value)}"`).join(",");
|
|
}
|
|
|
|
function stableLabelKey(labels: Record<string, string>) {
|
|
return Object.keys(labels).sort().map((key) => `${key}=${labels[key]}`).join("\u0000");
|
|
}
|
|
|
|
function routeSegmentTemplate(segment: string) {
|
|
if (/^\d+$/u.test(segment)) return ":id";
|
|
if (/^[a-z][a-z0-9]{1,12}_[A-Za-z0-9_.:-]+$/iu.test(segment)) return ":id";
|
|
if (/^[0-9a-f]{12,}$/iu.test(segment)) return ":id";
|
|
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu.test(segment)) return ":id";
|
|
if (segment.length > 48) return ":id";
|
|
return sanitizePathSegment(segment);
|
|
}
|
|
|
|
function safeUrl(value: string) {
|
|
try { return new URL(value, "http://hwlab-cloud-api.local"); } catch { return new URL("/", "http://hwlab-cloud-api.local"); }
|
|
}
|
|
|
|
function safeDecode(value: string) {
|
|
try { return decodeURIComponent(value); } catch { return value; }
|
|
}
|
|
|
|
function sanitizePathSegment(value: string) {
|
|
const sanitized = value.replace(/[^A-Za-z0-9_.:-]/gu, "_");
|
|
return sanitized.slice(0, 64) || "_";
|
|
}
|
|
|
|
function normalizeMethod(value: unknown) {
|
|
const method = String(value ?? "GET").trim().toUpperCase();
|
|
return /^(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)$/u.test(method) ? method : "OTHER";
|
|
}
|
|
|
|
function statusClass(value: unknown) {
|
|
const status = Number(value);
|
|
if (!Number.isFinite(status) || status <= 0) return "unknown";
|
|
return `${Math.floor(status / 100)}xx`;
|
|
}
|
|
|
|
function normalizeStatusClass(value: unknown) {
|
|
const text = String(value ?? "").trim().toLowerCase();
|
|
return /^(?:1xx|2xx|3xx|4xx|5xx|network|timeout|unknown)$/u.test(text) ? text : "unknown";
|
|
}
|
|
|
|
function normalizeOutcome(value: unknown) {
|
|
const text = sanitizeMetricName(value, "unknown");
|
|
return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown)$/u.test(text) ? text : "unknown";
|
|
}
|
|
|
|
function sanitizeMetricName(value: unknown, fallback = "unknown") {
|
|
const text = String(value ?? "").trim().toLowerCase().replace(/[^a-z0-9_:-]+/gu, "_").replace(/[-:]+/gu, "_").replace(/^_+|_+$/gu, "");
|
|
return text || fallback;
|
|
}
|
|
|
|
function sanitizeLabelValue(value: unknown, fallback = "unknown") {
|
|
const text = String(value ?? "").trim();
|
|
if (!text) return fallback;
|
|
return text.replace(/[\n\r\t]/gu, " ").slice(0, 120);
|
|
}
|
|
|
|
function escapeLabelValue(value: string) {
|
|
return String(value).replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"").replace(/\n/gu, "\\n");
|
|
}
|
|
|
|
function parsePositiveInteger(value: unknown, fallback: number) {
|
|
const parsed = Number.parseInt(String(value ?? ""), 10);
|
|
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
|
}
|
|
|
|
function boundedDurationMs(value: unknown, maxMs: number) {
|
|
const parsed = Number(value);
|
|
if (!Number.isFinite(parsed) || parsed < 0) return 0;
|
|
return Math.min(parsed, maxMs);
|
|
}
|
|
|
|
function roundDurationMs(value: number) {
|
|
return Math.max(0, value).toFixed(1);
|
|
}
|
|
|
|
function serverTimingToken(value: string) {
|
|
const token = sanitizeMetricName(value, "phase").replace(/_/gu, "-");
|
|
return token || "phase";
|
|
}
|
|
|
|
function nowMs() {
|
|
if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now();
|
|
return Date.now();
|
|
}
|