lyon
744 lines
35 KiB
TypeScript
744 lines
35 KiB
TypeScript
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0
|
|
// Records low-cardinality Cloud API backend performance metrics and Server-Timing phases.
|
|
|
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
|
|
const REQUEST_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60];
|
|
const PHASE_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30];
|
|
const UPSTREAM_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120];
|
|
const PROJECTION_LAG_EVENT_BUCKETS = [0, 1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 10000];
|
|
const PROJECTION_LAG_SECONDS_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 900, 1800, 3600];
|
|
const WORKBENCH_TURN_DURATION_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60];
|
|
const WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304];
|
|
const AGENTRUN_RESULT_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120];
|
|
const AGENTRUN_RESULT_EVENTS_BUCKETS = [0, 50, 100, 500, 1000, 2000, 5000, 10000];
|
|
const AGENTRUN_RESULT_PAGES_BUCKETS = [0, 1, 2, 5, 10, 20, 50, 100];
|
|
const PROJECTOR_BATCH_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60];
|
|
const MAX_SERVER_TIMING_ENTRIES = 8;
|
|
|
|
const backendPerformanceStorage = new AsyncLocalStorage();
|
|
|
|
interface HistogramSeries {
|
|
labels: Record<string, string>;
|
|
buckets: number[];
|
|
sum: number;
|
|
count: number;
|
|
}
|
|
|
|
interface CounterSeries {
|
|
labels: Record<string, string>;
|
|
value: number;
|
|
}
|
|
|
|
interface GaugeSeries {
|
|
labels: Record<string, string>;
|
|
value: number;
|
|
}
|
|
|
|
interface BackendPerformanceStoreOptions {
|
|
env?: Record<string, unknown>;
|
|
maxSeries?: number;
|
|
}
|
|
|
|
interface BackendPerformanceContextInput {
|
|
route: string;
|
|
method: string;
|
|
}
|
|
|
|
interface BackendPhaseInput {
|
|
phase: string;
|
|
durationMs: number;
|
|
outcome?: string;
|
|
}
|
|
|
|
interface BackendUpstreamInput {
|
|
component: string;
|
|
operation?: string;
|
|
route: string;
|
|
method: string;
|
|
statusClass?: string;
|
|
outcome?: string;
|
|
durationMs: number;
|
|
}
|
|
|
|
interface WorkbenchProjectionInput {
|
|
sourceLatestSeq?: unknown;
|
|
lastProjectedSeq?: unknown;
|
|
sourceLatestEventAt?: unknown;
|
|
projectedLatestEventAt?: unknown;
|
|
terminalSourceAt?: unknown;
|
|
terminalProjectedAt?: unknown;
|
|
status?: unknown;
|
|
reason?: unknown;
|
|
projectionStatus?: unknown;
|
|
source?: unknown;
|
|
}
|
|
|
|
interface WorkbenchTurnReadInput {
|
|
route: string;
|
|
status?: unknown;
|
|
degradedReason?: unknown;
|
|
durationMs: number;
|
|
responseBytes?: unknown;
|
|
timedOut?: boolean;
|
|
}
|
|
|
|
interface AgentRunResultInput {
|
|
durationMs?: unknown;
|
|
eventsScanned?: unknown;
|
|
pagesScanned?: unknown;
|
|
eventCount?: unknown;
|
|
status?: unknown;
|
|
budget?: unknown;
|
|
timedOut?: boolean;
|
|
}
|
|
|
|
interface WorkbenchProjectorBatchInput {
|
|
phase: string;
|
|
status?: unknown;
|
|
durationMs: number;
|
|
}
|
|
|
|
interface WorkbenchProjectorCounterInput {
|
|
status?: unknown;
|
|
reason?: unknown;
|
|
eventKind?: unknown;
|
|
count?: unknown;
|
|
}
|
|
|
|
export function createBackendPerformanceStore(options: BackendPerformanceStoreOptions = {}) {
|
|
const env = options.env ?? process.env;
|
|
const maxSeries = parsePositiveInteger(env.HWLAB_BACKEND_PERFORMANCE_MAX_SERIES, options.maxSeries ?? 320);
|
|
const requestDurationSeries = new Map<string, HistogramSeries>();
|
|
const requestTotalSeries = new Map<string, CounterSeries>();
|
|
const requestPhaseSeries = new Map<string, HistogramSeries>();
|
|
const upstreamSeries = new Map<string, HistogramSeries>();
|
|
const droppedSeries = new Map<string, CounterSeries>();
|
|
const projectionLagEventsSeries = new Map<string, HistogramSeries>();
|
|
const projectionLagSecondsSeries = new Map<string, HistogramSeries>();
|
|
const projectionStuckGaugeSeries = new Map<string, GaugeSeries>();
|
|
const projectionCursorAdvanceSeries = new Map<string, CounterSeries>();
|
|
const terminalProjectionDelaySeries = new Map<string, HistogramSeries>();
|
|
const turnGetDurationSeries = new Map<string, HistogramSeries>();
|
|
const turnGetTimeoutSeries = new Map<string, CounterSeries>();
|
|
const turnGetResponseBytesSeries = new Map<string, HistogramSeries>();
|
|
const agentRunResultDurationSeries = new Map<string, HistogramSeries>();
|
|
const agentRunResultPagesSeries = new Map<string, HistogramSeries>();
|
|
const agentRunResultEventsSeries = new Map<string, HistogramSeries>();
|
|
const agentRunResultTimeoutSeries = new Map<string, CounterSeries>();
|
|
const projectorBatchDurationSeries = new Map<string, HistogramSeries>();
|
|
const projectorCandidatesSeries = new Map<string, CounterSeries>();
|
|
const projectorEventsProcessedSeries = new Map<string, CounterSeries>();
|
|
const projectorErrorsSeries = new Map<string, CounterSeries>();
|
|
const projectorLastSuccessSeries = new Map<string, GaugeSeries>();
|
|
const baseLabels = {
|
|
service: "hwlab-cloud-api",
|
|
namespace: sanitizeLabelValue(env.POD_NAMESPACE ?? env.HWLAB_METRICS_NAMESPACE ?? "hwlab-v02"),
|
|
gitops_target: sanitizeLabelValue(env.HWLAB_GITOPS_TARGET ?? env.HWLAB_GITOPS_PROFILE ?? env.HWLAB_RUNTIME_LANE ?? "v02")
|
|
};
|
|
const workbenchBaseLabels = {
|
|
...baseLabels,
|
|
node: sanitizeLabelValue(env.HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID ?? env.AGENTRUN_PROVIDER_ID ?? env.HWLAB_NODE_ID ?? "unknown"),
|
|
lane: sanitizeLabelValue(env.HWLAB_RUNTIME_LANE ?? env.HWLAB_GITOPS_TARGET ?? env.HWLAB_GITOPS_PROFILE ?? "unknown")
|
|
};
|
|
|
|
function beginHttpRequest(request: any, response: any) {
|
|
const url = safeUrl(request?.url || "/");
|
|
const context = createContext({ route: backendPerformanceRouteTemplate(url.pathname), method: normalizeMethod(request?.method) });
|
|
const originalWriteHead = typeof response?.writeHead === "function" ? response.writeHead : null;
|
|
if (originalWriteHead) {
|
|
response.writeHead = function writeHeadWithServerTiming(statusCode: number, ...args: unknown[]) {
|
|
context.setStatus(statusCode);
|
|
context.applyServerTimingHeader(response);
|
|
return originalWriteHead.call(this, statusCode, ...args);
|
|
};
|
|
}
|
|
if (typeof response?.once === "function") {
|
|
response.once("finish", () => context.finish(Number(response.statusCode ?? context.statusCode ?? 0)));
|
|
}
|
|
return context;
|
|
}
|
|
|
|
function createManualContext(input: BackendPerformanceContextInput) {
|
|
return createContext({ route: backendPerformanceRouteTemplate(input.route), method: normalizeMethod(input.method) });
|
|
}
|
|
|
|
function createContext(input: BackendPerformanceContextInput) {
|
|
const startedAt = nowMs();
|
|
const phases: Array<{ name: string; durationMs: number; outcome: string }> = [];
|
|
let statusCode = 200;
|
|
let finished = false;
|
|
const route = input.route;
|
|
const method = input.method;
|
|
|
|
function recordPhase(inputPhase: BackendPhaseInput) {
|
|
const durationMs = boundedDurationMs(inputPhase.durationMs, 30_000);
|
|
const phase = sanitizeMetricName(inputPhase.phase, "unknown");
|
|
const outcome = normalizeOutcome(inputPhase.outcome ?? "ok");
|
|
phases.push({ name: phase, durationMs, outcome });
|
|
const labels = { ...baseLabels, route, method, phase, outcome };
|
|
if (!recordHistogram(requestPhaseSeries, labels, durationMs / 1000, PHASE_BUCKETS_SECONDS, maxSeries)) incrementDropped("phase_series_limit");
|
|
}
|
|
|
|
async function measure<T>(phase: string, fn: () => Promise<T> | T): Promise<T> {
|
|
const phaseStartedAt = nowMs();
|
|
try {
|
|
const result = await fn();
|
|
recordPhase({ phase, durationMs: nowMs() - phaseStartedAt, outcome: "ok" });
|
|
return result;
|
|
} catch (error) {
|
|
recordPhase({ phase, durationMs: nowMs() - phaseStartedAt, outcome: "error" });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function recordUpstream(inputUpstream: BackendUpstreamInput) {
|
|
const labels = {
|
|
...baseLabels,
|
|
component: sanitizeMetricName(inputUpstream.component, "unknown"),
|
|
operation: sanitizeMetricName(inputUpstream.operation ?? inputUpstream.route, "unknown"),
|
|
route: backendPerformanceRouteTemplate(inputUpstream.route),
|
|
method: normalizeMethod(inputUpstream.method),
|
|
status_class: normalizeStatusClass(inputUpstream.statusClass ?? "unknown"),
|
|
outcome: normalizeOutcome(inputUpstream.outcome ?? "ok")
|
|
};
|
|
if (!recordHistogram(upstreamSeries, labels, boundedDurationMs(inputUpstream.durationMs, 120_000) / 1000, UPSTREAM_BUCKETS_SECONDS, maxSeries)) incrementDropped("upstream_series_limit");
|
|
}
|
|
|
|
function setStatus(nextStatusCode: number) {
|
|
if (Number.isInteger(nextStatusCode) && nextStatusCode > 0) statusCode = nextStatusCode;
|
|
}
|
|
|
|
function applyServerTimingHeader(response: any) {
|
|
if (!response || typeof response.setHeader !== "function" || response.headersSent) return;
|
|
const header = serverTimingHeader(nowMs() - startedAt, phases);
|
|
if (header) response.setHeader("server-timing", header);
|
|
}
|
|
|
|
function finish(finalStatusCode = statusCode) {
|
|
if (finished) return;
|
|
finished = true;
|
|
const statusClassValue = statusClass(finalStatusCode);
|
|
const outcome = statusClassValue === "5xx" ? "error" : statusClassValue === "4xx" ? "denied" : "ok";
|
|
const labels = { ...baseLabels, route, method, status_class: statusClassValue, outcome };
|
|
if (!recordHistogram(requestDurationSeries, labels, boundedDurationMs(nowMs() - startedAt, 120_000) / 1000, REQUEST_BUCKETS_SECONDS, maxSeries)) incrementDropped("request_series_limit");
|
|
incrementCounter(requestTotalSeries, labels, maxSeries) || incrementDropped("request_counter_series_limit");
|
|
}
|
|
|
|
return { route, method, startedAt, recordPhase, measure, recordUpstream, setStatus, applyServerTimingHeader, finish, get statusCode() { return statusCode; } };
|
|
}
|
|
|
|
function recordWorkbenchProjection(input: WorkbenchProjectionInput = {}) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
status: normalizeStatusLabel(input.status ?? "unknown"),
|
|
reason: normalizeReason(input.reason ?? "none"),
|
|
projection_status: normalizeProjectionStatus(input.projectionStatus ?? "unknown"),
|
|
source: normalizeSourceLabel(input.source ?? "agentrun_v01")
|
|
};
|
|
const sourceSeq = finiteNonNegativeNumber(input.sourceLatestSeq);
|
|
const projectedSeq = finiteNonNegativeNumber(input.lastProjectedSeq);
|
|
if (sourceSeq !== null && projectedSeq !== null) {
|
|
const lagEvents = isTerminalProjectionStatus(labels.projection_status)
|
|
? 0
|
|
: Math.max(0, Math.floor(sourceSeq - projectedSeq));
|
|
if (!recordHistogram(projectionLagEventsSeries, labels, lagEvents, PROJECTION_LAG_EVENT_BUCKETS, maxSeries)) incrementDropped("projection_lag_events_series_limit");
|
|
setGauge(projectionStuckGaugeSeries, labels, lagEvents > 0 && labels.projection_status !== "caught_up" ? 1 : 0, maxSeries) || incrementDropped("projection_stuck_series_limit");
|
|
}
|
|
const sourceAt = unixMs(input.sourceLatestEventAt);
|
|
const projectedAt = unixMs(input.projectedLatestEventAt);
|
|
if (sourceAt !== null && projectedAt !== null) {
|
|
const lagSeconds = isTerminalProjectionStatus(labels.projection_status) ? 0 : Math.max(0, (sourceAt - projectedAt) / 1000);
|
|
if (!recordHistogram(projectionLagSecondsSeries, labels, lagSeconds, PROJECTION_LAG_SECONDS_BUCKETS, maxSeries)) incrementDropped("projection_lag_seconds_series_limit");
|
|
}
|
|
const terminalSourceAt = unixMs(input.terminalSourceAt);
|
|
const terminalProjectedAt = unixMs(input.terminalProjectedAt) ?? (isTerminalProjectionStatus(labels.projection_status) ? projectedAt : null);
|
|
if (terminalSourceAt !== null) {
|
|
const delaySeconds = Math.max(0, ((terminalProjectedAt ?? Date.now()) - terminalSourceAt) / 1000);
|
|
if (!recordHistogram(terminalProjectionDelaySeries, labels, delaySeconds, PROJECTION_LAG_SECONDS_BUCKETS, maxSeries)) incrementDropped("terminal_projection_delay_series_limit");
|
|
}
|
|
}
|
|
|
|
function recordWorkbenchProjectionCursorAdvance(input: WorkbenchProjectorCounterInput = {}) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
status: normalizeStatusLabel(input.status ?? "advanced"),
|
|
reason: normalizeReason(input.reason ?? "sync"),
|
|
projection_status: normalizeProjectionStatus((input as any).projectionStatus ?? "projecting"),
|
|
source: normalizeSourceLabel((input as any).source ?? "agentrun_v01")
|
|
};
|
|
incrementCounterBy(projectionCursorAdvanceSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projection_cursor_advance_series_limit");
|
|
}
|
|
|
|
function recordWorkbenchTurnRead(input: WorkbenchTurnReadInput) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
route: workbenchTurnRouteTemplate(input.route),
|
|
status: normalizeStatusLabel(input.status ?? "unknown"),
|
|
degraded_reason: normalizeReason(input.degradedReason ?? "none")
|
|
};
|
|
if (!recordHistogram(turnGetDurationSeries, labels, boundedDurationMs(input.durationMs, 120_000) / 1000, WORKBENCH_TURN_DURATION_BUCKETS_SECONDS, maxSeries)) incrementDropped("turn_get_duration_series_limit");
|
|
const bytes = finiteNonNegativeNumber(input.responseBytes);
|
|
if (bytes !== null) {
|
|
const byteLabels = { ...workbenchBaseLabels, route: labels.route, status: labels.status };
|
|
if (!recordHistogram(turnGetResponseBytesSeries, byteLabels, bytes, WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS, maxSeries)) incrementDropped("turn_get_response_bytes_series_limit");
|
|
}
|
|
if (input.timedOut === true || labels.status === "timeout") {
|
|
const timeoutLabels = { ...workbenchBaseLabels, route: labels.route, degraded_reason: labels.degraded_reason };
|
|
incrementCounter(turnGetTimeoutSeries, timeoutLabels, maxSeries) || incrementDropped("turn_get_timeout_series_limit");
|
|
}
|
|
}
|
|
|
|
function recordAgentRunResult(input: AgentRunResultInput = {}) {
|
|
const eventCount = finiteNonNegativeNumber(input.eventCount ?? input.eventsScanned) ?? 0;
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
event_count_bucket: agentRunEventCountBucket(eventCount),
|
|
status: normalizeStatusLabel(input.status ?? (input.timedOut ? "timeout" : "unknown"))
|
|
};
|
|
const durationMs = finiteNonNegativeNumber(input.durationMs);
|
|
if (durationMs !== null) {
|
|
if (!recordHistogram(agentRunResultDurationSeries, labels, boundedDurationMs(durationMs, 120_000) / 1000, AGENTRUN_RESULT_BUCKETS_SECONDS, maxSeries)) incrementDropped("agentrun_result_duration_series_limit");
|
|
}
|
|
const pagesScanned = finiteNonNegativeNumber(input.pagesScanned);
|
|
if (pagesScanned !== null) {
|
|
if (!recordHistogram(agentRunResultPagesSeries, workbenchBaseLabels, pagesScanned, AGENTRUN_RESULT_PAGES_BUCKETS, maxSeries)) incrementDropped("agentrun_result_pages_series_limit");
|
|
}
|
|
const eventsScanned = finiteNonNegativeNumber(input.eventsScanned);
|
|
if (eventsScanned !== null) {
|
|
if (!recordHistogram(agentRunResultEventsSeries, workbenchBaseLabels, eventsScanned, AGENTRUN_RESULT_EVENTS_BUCKETS, maxSeries)) incrementDropped("agentrun_result_events_series_limit");
|
|
}
|
|
if (input.timedOut === true || labels.status === "timeout") {
|
|
const timeoutLabels = {
|
|
...workbenchBaseLabels,
|
|
budget: normalizeBudgetLabel(input.budget),
|
|
status: labels.status
|
|
};
|
|
incrementCounter(agentRunResultTimeoutSeries, timeoutLabels, maxSeries) || incrementDropped("agentrun_result_timeout_series_limit");
|
|
}
|
|
}
|
|
|
|
function recordWorkbenchProjectorBatch(input: WorkbenchProjectorBatchInput) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
phase: normalizeProjectorPhase(input.phase),
|
|
status: normalizeStatusLabel(input.status ?? "ok")
|
|
};
|
|
if (!recordHistogram(projectorBatchDurationSeries, labels, boundedDurationMs(input.durationMs, 120_000) / 1000, PROJECTOR_BATCH_BUCKETS_SECONDS, maxSeries)) incrementDropped("projector_batch_series_limit");
|
|
if (labels.status === "ok") setGauge(projectorLastSuccessSeries, workbenchBaseLabels, Math.floor(Date.now() / 1000), maxSeries) || incrementDropped("projector_last_success_series_limit");
|
|
}
|
|
|
|
function recordWorkbenchProjectorCandidate(input: WorkbenchProjectorCounterInput = {}) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
status: normalizeStatusLabel(input.status ?? "seen"),
|
|
reason: normalizeReason(input.reason ?? "unknown")
|
|
};
|
|
incrementCounterBy(projectorCandidatesSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projector_candidates_series_limit");
|
|
}
|
|
|
|
function recordWorkbenchProjectorEvents(input: WorkbenchProjectorCounterInput = {}) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
event_kind: normalizeReason(input.eventKind ?? "unknown"),
|
|
status: normalizeStatusLabel(input.status ?? "ok")
|
|
};
|
|
incrementCounterBy(projectorEventsProcessedSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projector_events_series_limit");
|
|
}
|
|
|
|
function recordWorkbenchProjectorError(input: WorkbenchProjectorCounterInput = {}) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
reason: normalizeReason(input.reason ?? "unknown")
|
|
};
|
|
incrementCounterBy(projectorErrorsSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projector_errors_series_limit");
|
|
}
|
|
|
|
function incrementDropped(reason: string) {
|
|
incrementCounter(droppedSeries, { ...baseLabels, reason }, Math.max(maxSeries, 16));
|
|
}
|
|
|
|
function metricsText() {
|
|
return [
|
|
"# HELP hwlab_cloud_api_requests_total Cloud API HTTP requests by low-cardinality route template.",
|
|
"# TYPE hwlab_cloud_api_requests_total counter",
|
|
...renderCounters("hwlab_cloud_api_requests_total", requestTotalSeries),
|
|
"# HELP hwlab_cloud_api_request_duration_seconds Cloud API HTTP request duration in seconds.",
|
|
"# TYPE hwlab_cloud_api_request_duration_seconds histogram",
|
|
...renderHistogram("hwlab_cloud_api_request_duration_seconds", requestDurationSeries, REQUEST_BUCKETS_SECONDS),
|
|
"# HELP hwlab_cloud_api_request_phase_duration_seconds Cloud API handler phase duration in seconds.",
|
|
"# TYPE hwlab_cloud_api_request_phase_duration_seconds histogram",
|
|
...renderHistogram("hwlab_cloud_api_request_phase_duration_seconds", requestPhaseSeries, PHASE_BUCKETS_SECONDS),
|
|
"# HELP hwlab_cloud_api_upstream_duration_seconds Cloud API upstream dependency call duration in seconds.",
|
|
"# TYPE hwlab_cloud_api_upstream_duration_seconds histogram",
|
|
...renderHistogram("hwlab_cloud_api_upstream_duration_seconds", upstreamSeries, UPSTREAM_BUCKETS_SECONDS),
|
|
"# HELP hwlab_cloud_api_performance_dropped_total Backend performance samples dropped before Prometheus export.",
|
|
"# TYPE hwlab_cloud_api_performance_dropped_total counter",
|
|
...renderCounters("hwlab_cloud_api_performance_dropped_total", droppedSeries),
|
|
"# HELP hwlab_workbench_projection_lag_events Workbench projection source seq lag in AgentRun events.",
|
|
"# TYPE hwlab_workbench_projection_lag_events histogram",
|
|
...renderHistogram("hwlab_workbench_projection_lag_events", projectionLagEventsSeries, PROJECTION_LAG_EVENT_BUCKETS),
|
|
"# HELP hwlab_workbench_projection_lag_seconds Workbench projection source time lag in seconds.",
|
|
"# TYPE hwlab_workbench_projection_lag_seconds histogram",
|
|
...renderHistogram("hwlab_workbench_projection_lag_seconds", projectionLagSecondsSeries, PROJECTION_LAG_SECONDS_BUCKETS),
|
|
"# HELP hwlab_workbench_projection_stuck_traces Low-cardinality gauge set to 1 when the observed projection is behind its source.",
|
|
"# TYPE hwlab_workbench_projection_stuck_traces gauge",
|
|
...renderGauges("hwlab_workbench_projection_stuck_traces", projectionStuckGaugeSeries),
|
|
"# HELP hwlab_workbench_projection_cursor_advance_total Workbench projection cursor advance events.",
|
|
"# TYPE hwlab_workbench_projection_cursor_advance_total counter",
|
|
...renderCounters("hwlab_workbench_projection_cursor_advance_total", projectionCursorAdvanceSeries),
|
|
"# HELP hwlab_workbench_terminal_projection_delay_seconds Delay from AgentRun terminal source to Workbench terminal projection.",
|
|
"# TYPE hwlab_workbench_terminal_projection_delay_seconds histogram",
|
|
...renderHistogram("hwlab_workbench_terminal_projection_delay_seconds", terminalProjectionDelaySeries, PROJECTION_LAG_SECONDS_BUCKETS),
|
|
"# HELP hwlab_workbench_turn_get_duration_seconds Workbench turn/read API duration in seconds.",
|
|
"# TYPE hwlab_workbench_turn_get_duration_seconds histogram",
|
|
...renderHistogram("hwlab_workbench_turn_get_duration_seconds", turnGetDurationSeries, WORKBENCH_TURN_DURATION_BUCKETS_SECONDS),
|
|
"# HELP hwlab_workbench_turn_get_timeout_total Workbench turn/read API timeout count.",
|
|
"# TYPE hwlab_workbench_turn_get_timeout_total counter",
|
|
...renderCounters("hwlab_workbench_turn_get_timeout_total", turnGetTimeoutSeries),
|
|
"# HELP hwlab_workbench_turn_get_response_bytes Workbench turn/read API response bytes.",
|
|
"# TYPE hwlab_workbench_turn_get_response_bytes histogram",
|
|
...renderHistogram("hwlab_workbench_turn_get_response_bytes", turnGetResponseBytesSeries, WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS),
|
|
"# HELP hwlab_agentrun_result_duration_seconds AgentRun command result aggregation duration in seconds.",
|
|
"# TYPE hwlab_agentrun_result_duration_seconds histogram",
|
|
...renderHistogram("hwlab_agentrun_result_duration_seconds", agentRunResultDurationSeries, AGENTRUN_RESULT_BUCKETS_SECONDS),
|
|
"# HELP hwlab_agentrun_result_pages_scanned AgentRun result estimated pages scanned.",
|
|
"# TYPE hwlab_agentrun_result_pages_scanned histogram",
|
|
...renderHistogram("hwlab_agentrun_result_pages_scanned", agentRunResultPagesSeries, AGENTRUN_RESULT_PAGES_BUCKETS),
|
|
"# HELP hwlab_agentrun_result_events_scanned AgentRun result estimated events scanned.",
|
|
"# TYPE hwlab_agentrun_result_events_scanned histogram",
|
|
...renderHistogram("hwlab_agentrun_result_events_scanned", agentRunResultEventsSeries, AGENTRUN_RESULT_EVENTS_BUCKETS),
|
|
"# HELP hwlab_agentrun_result_timeout_total AgentRun result timeout count by refresh budget.",
|
|
"# TYPE hwlab_agentrun_result_timeout_total counter",
|
|
...renderCounters("hwlab_agentrun_result_timeout_total", agentRunResultTimeoutSeries),
|
|
"# HELP hwlab_workbench_projector_batch_duration_seconds Workbench projector/finalizer batch phase duration in seconds.",
|
|
"# TYPE hwlab_workbench_projector_batch_duration_seconds histogram",
|
|
...renderHistogram("hwlab_workbench_projector_batch_duration_seconds", projectorBatchDurationSeries, PROJECTOR_BATCH_BUCKETS_SECONDS),
|
|
"# HELP hwlab_workbench_projector_candidates_total Workbench projector/finalizer candidate count.",
|
|
"# TYPE hwlab_workbench_projector_candidates_total counter",
|
|
...renderCounters("hwlab_workbench_projector_candidates_total", projectorCandidatesSeries),
|
|
"# HELP hwlab_workbench_projector_events_processed_total Workbench projector/finalizer processed event count.",
|
|
"# TYPE hwlab_workbench_projector_events_processed_total counter",
|
|
...renderCounters("hwlab_workbench_projector_events_processed_total", projectorEventsProcessedSeries),
|
|
"# HELP hwlab_workbench_projector_errors_total Workbench projector/finalizer error count.",
|
|
"# TYPE hwlab_workbench_projector_errors_total counter",
|
|
...renderCounters("hwlab_workbench_projector_errors_total", projectorErrorsSeries),
|
|
"# HELP hwlab_workbench_projector_last_success_unixtime Last successful Workbench projector/finalizer sync unix time.",
|
|
"# TYPE hwlab_workbench_projector_last_success_unixtime gauge",
|
|
...renderGauges("hwlab_workbench_projector_last_success_unixtime", projectorLastSuccessSeries),
|
|
""
|
|
].join("\n");
|
|
}
|
|
|
|
function snapshot() {
|
|
return {
|
|
requestSeries: requestDurationSeries.size,
|
|
requestCount: [...requestTotalSeries.values()].reduce((sum, entry) => sum + entry.value, 0),
|
|
requestPhaseSeries: requestPhaseSeries.size,
|
|
requestPhaseCount: histogramSampleCount(requestPhaseSeries),
|
|
upstreamSeries: upstreamSeries.size,
|
|
upstreamCount: histogramSampleCount(upstreamSeries),
|
|
projectionLagSeries: projectionLagEventsSeries.size + projectionLagSecondsSeries.size,
|
|
turnReadSeries: turnGetDurationSeries.size,
|
|
agentRunResultSeries: agentRunResultDurationSeries.size,
|
|
projectorSeries: projectorBatchDurationSeries.size + projectorCandidatesSeries.size + projectorEventsProcessedSeries.size + projectorErrorsSeries.size,
|
|
droppedSeries: droppedSeries.size
|
|
};
|
|
}
|
|
|
|
return {
|
|
beginHttpRequest,
|
|
createManualContext,
|
|
recordWorkbenchProjection,
|
|
recordWorkbenchProjectionCursorAdvance,
|
|
recordWorkbenchTurnRead,
|
|
recordAgentRunResult,
|
|
recordWorkbenchProjectorBatch,
|
|
recordWorkbenchProjectorCandidate,
|
|
recordWorkbenchProjectorEvents,
|
|
recordWorkbenchProjectorError,
|
|
metricsText,
|
|
snapshot
|
|
};
|
|
}
|
|
|
|
export function withBackendPerformanceContext<T>(context: unknown, callback: () => T): T {
|
|
return backendPerformanceStorage.run(context, callback);
|
|
}
|
|
|
|
export function currentBackendPerformanceContext(): any {
|
|
return backendPerformanceStorage.getStore() ?? null;
|
|
}
|
|
|
|
export function backendPerformanceRouteTemplate(input: unknown): string {
|
|
const raw = String(input ?? "").trim() || "/";
|
|
const pathOnly = raw.split(/[?#]/u, 1)[0] || "/";
|
|
let pathname = pathOnly;
|
|
try {
|
|
pathname = new URL(raw, "http://hwlab.local").pathname || pathOnly;
|
|
} catch {
|
|
// Keep pathOnly.
|
|
}
|
|
const parts = pathname.split("/").filter(Boolean).map((part) => routeSegmentTemplate(safeDecode(part)));
|
|
const path = `/${parts.join("/")}`;
|
|
return path.length > 140 ? `${path.slice(0, 137)}...` : path;
|
|
}
|
|
|
|
function workbenchTurnRouteTemplate(input: unknown): string {
|
|
const route = backendPerformanceRouteTemplate(input);
|
|
if (route === "/v1/workbench/turns/:id") return "/v1/workbench/turns/:traceId";
|
|
if (route === "/v1/agent/turns/:id") return "/v1/agent/turns/:traceId";
|
|
return route;
|
|
}
|
|
|
|
function serverTimingHeader(totalMs: number, phases: Array<{ name: string; durationMs: number }>) {
|
|
const entries = phases.slice(-MAX_SERVER_TIMING_ENTRIES).map((phase) => `${serverTimingToken(phase.name)};dur=${roundDurationMs(phase.durationMs)}`);
|
|
entries.push(`total;dur=${roundDurationMs(totalMs)}`);
|
|
return entries.join(", ");
|
|
}
|
|
|
|
function recordHistogram(series: Map<string, HistogramSeries>, labels: Record<string, string>, value: number, buckets: number[], maxSeries: number) {
|
|
const key = stableLabelKey(labels);
|
|
let entry = series.get(key);
|
|
if (!entry) {
|
|
if (series.size >= maxSeries) return false;
|
|
entry = { labels, buckets: buckets.map(() => 0), sum: 0, count: 0 };
|
|
series.set(key, entry);
|
|
}
|
|
for (let index = 0; index < buckets.length; index += 1) {
|
|
if (value <= buckets[index]) entry.buckets[index] += 1;
|
|
}
|
|
entry.sum += value;
|
|
entry.count += 1;
|
|
return true;
|
|
}
|
|
|
|
function incrementCounter(series: Map<string, CounterSeries>, labels: Record<string, string>, maxSeries: number) {
|
|
const key = stableLabelKey(labels);
|
|
let entry = series.get(key);
|
|
if (!entry) {
|
|
if (series.size >= maxSeries) return false;
|
|
entry = { labels, value: 0 };
|
|
series.set(key, entry);
|
|
}
|
|
entry.value += 1;
|
|
return true;
|
|
}
|
|
|
|
function incrementCounterBy(series: Map<string, CounterSeries>, labels: Record<string, string>, value: number, maxSeries: number) {
|
|
const key = stableLabelKey(labels);
|
|
let entry = series.get(key);
|
|
if (!entry) {
|
|
if (series.size >= maxSeries) return false;
|
|
entry = { labels, value: 0 };
|
|
series.set(key, entry);
|
|
}
|
|
entry.value += value;
|
|
return true;
|
|
}
|
|
|
|
function setGauge(series: Map<string, GaugeSeries>, labels: Record<string, string>, value: number, maxSeries: number) {
|
|
const key = stableLabelKey(labels);
|
|
let entry = series.get(key);
|
|
if (!entry) {
|
|
if (series.size >= maxSeries) return false;
|
|
entry = { labels, value: 0 };
|
|
series.set(key, entry);
|
|
}
|
|
entry.value = Number.isFinite(value) ? value : 0;
|
|
return true;
|
|
}
|
|
|
|
function renderCounters(metricName: string, series: Map<string, CounterSeries>) {
|
|
return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${entry.value}`);
|
|
}
|
|
|
|
function renderGauges(metricName: string, series: Map<string, GaugeSeries>) {
|
|
return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${formatPrometheusValue(entry.value)}`);
|
|
}
|
|
|
|
function renderHistogram(metricName: string, series: Map<string, HistogramSeries>, buckets: number[]) {
|
|
const lines: string[] = [];
|
|
for (const entry of series.values()) {
|
|
for (let index = 0; index < buckets.length; index += 1) {
|
|
lines.push(`${metricName}_bucket{${renderLabels({ ...entry.labels, le: String(buckets[index]) })}} ${entry.buckets[index]}`);
|
|
}
|
|
lines.push(`${metricName}_bucket{${renderLabels({ ...entry.labels, le: "+Inf" })}} ${entry.count}`);
|
|
lines.push(`${metricName}_sum{${renderLabels(entry.labels)}} ${entry.sum.toFixed(6)}`);
|
|
lines.push(`${metricName}_count{${renderLabels(entry.labels)}} ${entry.count}`);
|
|
}
|
|
return lines;
|
|
}
|
|
|
|
function histogramSampleCount(series: Map<string, HistogramSeries>) {
|
|
return [...series.values()].reduce((sum, entry) => sum + entry.count, 0);
|
|
}
|
|
|
|
function renderLabels(labels: Record<string, string>) {
|
|
return Object.entries(labels).map(([key, value]) => `${key}="${escapeLabelValue(value)}"`).join(",");
|
|
}
|
|
|
|
function stableLabelKey(labels: Record<string, string>) {
|
|
return Object.keys(labels).sort().map((key) => `${key}=${labels[key]}`).join("\u0000");
|
|
}
|
|
|
|
function routeSegmentTemplate(segment: string) {
|
|
if (/^\d+$/u.test(segment)) return ":id";
|
|
if (/^[a-z][a-z0-9]{1,12}_[A-Za-z0-9_.:-]+$/iu.test(segment)) return ":id";
|
|
if (/^[0-9a-f]{12,}$/iu.test(segment)) return ":id";
|
|
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu.test(segment)) return ":id";
|
|
if (segment.length > 48) return ":id";
|
|
return sanitizePathSegment(segment);
|
|
}
|
|
|
|
function safeUrl(value: string) {
|
|
try { return new URL(value, "http://hwlab-cloud-api.local"); } catch { return new URL("/", "http://hwlab-cloud-api.local"); }
|
|
}
|
|
|
|
function safeDecode(value: string) {
|
|
try { return decodeURIComponent(value); } catch { return value; }
|
|
}
|
|
|
|
function sanitizePathSegment(value: string) {
|
|
const sanitized = value.replace(/[^A-Za-z0-9_.:-]/gu, "_");
|
|
return sanitized.slice(0, 64) || "_";
|
|
}
|
|
|
|
function normalizeMethod(value: unknown) {
|
|
const method = String(value ?? "GET").trim().toUpperCase();
|
|
return /^(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)$/u.test(method) ? method : "OTHER";
|
|
}
|
|
|
|
function statusClass(value: unknown) {
|
|
const status = Number(value);
|
|
if (!Number.isFinite(status) || status <= 0) return "unknown";
|
|
return `${Math.floor(status / 100)}xx`;
|
|
}
|
|
|
|
function normalizeStatusClass(value: unknown) {
|
|
const text = String(value ?? "").trim().toLowerCase();
|
|
return /^(?:1xx|2xx|3xx|4xx|5xx|network|timeout|unknown)$/u.test(text) ? text : "unknown";
|
|
}
|
|
|
|
function normalizeStatusLabel(value: unknown) {
|
|
const text = sanitizeMetricName(value, "unknown");
|
|
return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown|running|pending|completed|failed|blocked|degraded|advanced|unchanged|scheduled|seen|skipped|caught_up|projecting|stale|missing)$/u.test(text) ? text : "unknown";
|
|
}
|
|
|
|
function normalizeOutcome(value: unknown) {
|
|
const text = sanitizeMetricName(value, "unknown");
|
|
return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown)$/u.test(text) ? text : "unknown";
|
|
}
|
|
|
|
function normalizeProjectionStatus(value: unknown) {
|
|
const text = sanitizeMetricName(value, "unknown");
|
|
if (text === "caught_up" || text === "caughtup") return "caught_up";
|
|
return /^(?:projecting|caught_up|blocked|unknown|missing|stale|degraded|terminal|running)$/u.test(text) ? text : "unknown";
|
|
}
|
|
|
|
function isTerminalProjectionStatus(value: unknown) {
|
|
const status = normalizeProjectionStatus(value);
|
|
return status === "caught_up" || status === "terminal";
|
|
}
|
|
|
|
function normalizeReason(value: unknown) {
|
|
return sanitizeMetricName(value, "unknown").slice(0, 64) || "unknown";
|
|
}
|
|
|
|
function normalizeSourceLabel(value: unknown) {
|
|
const text = sanitizeMetricName(value, "unknown");
|
|
if (text === "agentrun_v0_1" || text === "agentrun_v01") return "agentrun_v01";
|
|
return text.slice(0, 64) || "unknown";
|
|
}
|
|
|
|
function normalizeProjectorPhase(value: unknown) {
|
|
const phase = sanitizeMetricName(value, "unknown");
|
|
return /^(?:candidate_scan|agentrun_events_fetch|event_map|trace_append|turn_finalize|session_write|status_effects|unknown)$/u.test(phase) ? phase : "unknown";
|
|
}
|
|
|
|
function normalizeBudgetLabel(value: unknown) {
|
|
const parsed = Number(value);
|
|
if (Number.isFinite(parsed) && parsed > 0) {
|
|
if (parsed <= 2500) return "turn_2_5s";
|
|
if (parsed <= 10000) return "short_10s";
|
|
if (parsed <= 30000) return "medium_30s";
|
|
if (parsed <= 120000) return "long_120s";
|
|
return "long_over_120s";
|
|
}
|
|
return normalizeReason(value ?? "unknown");
|
|
}
|
|
|
|
function agentRunEventCountBucket(value: number) {
|
|
if (value <= 500) return "0_500";
|
|
if (value <= 2000) return "501_2000";
|
|
if (value <= 5000) return "2001_5000";
|
|
return "5000_plus";
|
|
}
|
|
|
|
function sanitizeMetricName(value: unknown, fallback = "unknown") {
|
|
const text = String(value ?? "").trim().toLowerCase().replace(/[^a-z0-9_:-]+/gu, "_").replace(/[-:]+/gu, "_").replace(/^_+|_+$/gu, "");
|
|
return text || fallback;
|
|
}
|
|
|
|
function sanitizeLabelValue(value: unknown, fallback = "unknown") {
|
|
const text = String(value ?? "").trim();
|
|
if (!text) return fallback;
|
|
return text.replace(/[\n\r\t]/gu, " ").slice(0, 120);
|
|
}
|
|
|
|
function escapeLabelValue(value: string) {
|
|
return String(value).replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"").replace(/\n/gu, "\\n");
|
|
}
|
|
|
|
function parsePositiveInteger(value: unknown, fallback: number) {
|
|
const parsed = Number.parseInt(String(value ?? ""), 10);
|
|
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
|
}
|
|
|
|
function boundedDurationMs(value: unknown, maxMs: number) {
|
|
const parsed = Number(value);
|
|
if (!Number.isFinite(parsed) || parsed < 0) return 0;
|
|
return Math.min(parsed, maxMs);
|
|
}
|
|
|
|
function boundedCounterValue(value: unknown) {
|
|
const parsed = Number(value ?? 1);
|
|
if (!Number.isFinite(parsed) || parsed <= 0) return 1;
|
|
return Math.min(Math.floor(parsed), 1_000_000);
|
|
}
|
|
|
|
function finiteNonNegativeNumber(value: unknown): number | null {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
|
|
}
|
|
|
|
function unixMs(value: unknown): number | null {
|
|
if (value instanceof Date) {
|
|
const time = value.getTime();
|
|
return Number.isFinite(time) ? time : null;
|
|
}
|
|
const numeric = Number(value);
|
|
if (Number.isFinite(numeric) && numeric > 0) return numeric > 10_000_000_000 ? numeric : numeric * 1000;
|
|
const parsed = Date.parse(String(value ?? ""));
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function formatPrometheusValue(value: number) {
|
|
if (!Number.isFinite(value)) return "0";
|
|
return Number.isInteger(value) ? String(value) : value.toFixed(6);
|
|
}
|
|
|
|
function roundDurationMs(value: number) {
|
|
return Math.max(0, value).toFixed(1);
|
|
}
|
|
|
|
function serverTimingToken(value: string) {
|
|
const token = sanitizeMetricName(value, "phase").replace(/_/gu, "-");
|
|
return token || "phase";
|
|
}
|
|
|
|
function nowMs() {
|
|
if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now();
|
|
return Date.now();
|
|
}
|