// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0 // Records low-cardinality Cloud API backend performance metrics and Server-Timing phases. import { AsyncLocalStorage } from "node:async_hooks"; const REQUEST_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]; const PHASE_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30]; const UPSTREAM_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120]; const PROJECTION_LAG_EVENT_BUCKETS = [0, 1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 10000]; const PROJECTION_LAG_SECONDS_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 900, 1800, 3600]; const WORKBENCH_TURN_DURATION_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]; const WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304]; const WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]; const WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304]; const AGENTRUN_RESULT_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120]; const AGENTRUN_RESULT_EVENTS_BUCKETS = [0, 50, 100, 500, 1000, 2000, 5000, 10000]; const AGENTRUN_RESULT_PAGES_BUCKETS = [0, 1, 2, 5, 10, 20, 50, 100]; const PROJECTOR_BATCH_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]; const MAX_SERVER_TIMING_ENTRIES = 8; const DEFAULT_BACKEND_EVIDENCE_WINDOW = "15m"; const BACKEND_EVIDENCE_WINDOWS = { "5m": { id: "5m", label: "最近 5 分钟", milliseconds: 5 * 60 * 1000 }, "15m": { id: "15m", label: "最近 15 分钟", milliseconds: 15 * 60 * 1000 }, "1h": { id: "1h", label: "最近 1 小时", milliseconds: 60 * 60 * 1000 }, "6h": { id: "6h", label: "最近 6 小时", milliseconds: 6 * 60 * 60 * 1000 }, "24h": { id: "24h", label: "最近 24 小时", milliseconds: 24 * 60 * 60 * 1000 }, all: { id: "all", label: "全部累计", milliseconds: null } } as const; const backendPerformanceStorage = new AsyncLocalStorage(); interface HistogramSeries { labels: Record; buckets: number[]; sum: number; count: number; } interface CounterSeries { labels: Record; value: number; } interface GaugeSeries { labels: Record; value: number; } interface BackendPerformanceStoreOptions { env?: Record; maxSeries?: number; maxEvidenceSamples?: number; } interface BackendPerformanceContextInput { route: string; method: string; } interface BackendPhaseInput { phase: string; durationMs: number; outcome?: string; } interface BackendUpstreamInput { component: string; operation?: string; route: string; method: string; statusClass?: string; outcome?: string; durationMs: number; } interface WorkbenchProjectionInput { sourceLatestSeq?: unknown; lastProjectedSeq?: unknown; sourceLatestEventAt?: unknown; projectedLatestEventAt?: unknown; terminalSourceAt?: unknown; terminalProjectedAt?: unknown; status?: unknown; reason?: unknown; projectionStatus?: unknown; source?: unknown; } interface WorkbenchTurnReadInput { route: string; status?: unknown; degradedReason?: unknown; durationMs: number; responseBytes?: unknown; timedOut?: boolean; } interface WorkbenchCacheInput { route: string; cacheKeyClass?: unknown; cacheStatus?: unknown; operation?: unknown; durationMs?: unknown; payloadBytes?: unknown; cacheAgeMs?: unknown; dbQueryAvoided?: unknown; freshnessSloMs?: unknown; projectionSeq?: unknown; } interface AgentRunResultInput { durationMs?: unknown; eventsScanned?: unknown; pagesScanned?: unknown; eventCount?: unknown; status?: unknown; budget?: unknown; timedOut?: boolean; } interface WorkbenchProjectorBatchInput { phase: string; status?: unknown; durationMs: number; } interface WorkbenchProjectorCounterInput { status?: unknown; reason?: unknown; eventKind?: unknown; count?: unknown; } interface BackendRequestSample { atMs: number; route: string; method: string; statusClass: string; outcome: string; durationSeconds: number; responseBytes: number | null; phases: Array<{ phase: string; outcome: string; durationSeconds: number }>; } interface WorkbenchCacheEvidenceSample { atMs: number; route: string; cacheKeyClass: string; cacheStatus: string; operation: string; durationSeconds: number | null; payloadBytes: number | null; cacheAgeMs: number | null; dbQueryAvoided: boolean; freshnessSloMs: number | null; projectionSeq: number | null; } export function createBackendPerformanceStore(options: BackendPerformanceStoreOptions = {}) { const env = options.env ?? process.env; const maxSeries = parsePositiveInteger(env.HWLAB_BACKEND_PERFORMANCE_MAX_SERIES, options.maxSeries ?? 320); const maxEvidenceSamples = parsePositiveInteger(env.HWLAB_BACKEND_PERFORMANCE_MAX_EVIDENCE_SAMPLES, options.maxEvidenceSamples ?? 5000); const requestDurationSeries = new Map(); const requestTotalSeries = new Map(); const requestPhaseSeries = new Map(); const upstreamSeries = new Map(); const droppedSeries = new Map(); const projectionLagEventsSeries = new Map(); const projectionLagSecondsSeries = new Map(); const projectionStuckGaugeSeries = new Map(); const projectionCursorAdvanceSeries = new Map(); const terminalProjectionDelaySeries = new Map(); const turnGetDurationSeries = new Map(); const turnGetTimeoutSeries = new Map(); const turnGetResponseBytesSeries = new Map(); const cacheRequestSeries = new Map(); const cacheOperationDurationSeries = new Map(); const cachePayloadBytesSeries = new Map(); const cacheStaleSeries = new Map(); const cacheUnavailableSeries = new Map(); const dbQueryAvoidedSeries = new Map(); const agentRunResultDurationSeries = new Map(); const agentRunResultPagesSeries = new Map(); const agentRunResultEventsSeries = new Map(); const agentRunResultTimeoutSeries = new Map(); const projectorBatchDurationSeries = new Map(); const projectorCandidatesSeries = new Map(); const projectorEventsProcessedSeries = new Map(); const projectorErrorsSeries = new Map(); const projectorLastSuccessSeries = new Map(); const requestSamples: BackendRequestSample[] = []; const cacheSamples: WorkbenchCacheEvidenceSample[] = []; const baseLabels = { service: "hwlab-cloud-api", namespace: sanitizeLabelValue(env.POD_NAMESPACE ?? env.HWLAB_METRICS_NAMESPACE ?? "hwlab-v02"), gitops_target: sanitizeLabelValue(env.HWLAB_GITOPS_TARGET ?? env.HWLAB_GITOPS_PROFILE ?? env.HWLAB_RUNTIME_LANE ?? "v02") }; const workbenchBaseLabels = { ...baseLabels, node: sanitizeLabelValue(env.HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID ?? env.AGENTRUN_PROVIDER_ID ?? env.HWLAB_NODE_ID ?? "unknown"), lane: sanitizeLabelValue(env.HWLAB_RUNTIME_LANE ?? env.HWLAB_GITOPS_TARGET ?? env.HWLAB_GITOPS_PROFILE ?? "unknown") }; function beginHttpRequest(request: any, response: any) { const url = safeUrl(request?.url || "/"); const context = createContext({ route: backendPerformanceRouteTemplate(url.pathname), method: normalizeMethod(request?.method) }); const originalWriteHead = typeof response?.writeHead === "function" ? response.writeHead : null; if (originalWriteHead) { response.writeHead = function writeHeadWithServerTiming(statusCode: number, ...args: unknown[]) { context.setStatus(statusCode); context.applyServerTimingHeader(response); return originalWriteHead.call(this, statusCode, ...args); }; } const originalEnd = typeof response?.end === "function" ? response.end : null; if (originalEnd) { response.end = function endWithBackendEvidenceBytes(chunk?: unknown, encoding?: unknown, callback?: unknown) { context.setResponseBytes(responseEndChunkBytes(chunk, encoding)); return originalEnd.apply(this, arguments as any); }; } if (typeof response?.once === "function") { response.once("finish", () => context.finish(Number(response.statusCode ?? context.statusCode ?? 0))); } return context; } function createManualContext(input: BackendPerformanceContextInput) { return createContext({ route: backendPerformanceRouteTemplate(input.route), method: normalizeMethod(input.method) }); } function createContext(input: BackendPerformanceContextInput) { const startedAt = nowMs(); const phases: Array<{ name: string; durationMs: number; outcome: string }> = []; let statusCode = 200; let responseBytes: number | null = null; let finished = false; const route = input.route; const method = input.method; function recordPhase(inputPhase: BackendPhaseInput) { const durationMs = boundedDurationMs(inputPhase.durationMs, 30_000); const phase = sanitizeMetricName(inputPhase.phase, "unknown"); const outcome = normalizeOutcome(inputPhase.outcome ?? "ok"); phases.push({ name: phase, durationMs, outcome }); const labels = { ...baseLabels, route, method, phase, outcome }; if (!recordHistogram(requestPhaseSeries, labels, durationMs / 1000, PHASE_BUCKETS_SECONDS, maxSeries)) incrementDropped("phase_series_limit"); } async function measure(phase: string, fn: () => Promise | T): Promise { const phaseStartedAt = nowMs(); try { const result = await fn(); recordPhase({ phase, durationMs: nowMs() - phaseStartedAt, outcome: "ok" }); return result; } catch (error) { recordPhase({ phase, durationMs: nowMs() - phaseStartedAt, outcome: "error" }); throw error; } } function recordUpstream(inputUpstream: BackendUpstreamInput) { const labels = { ...baseLabels, component: sanitizeMetricName(inputUpstream.component, "unknown"), operation: sanitizeMetricName(inputUpstream.operation ?? inputUpstream.route, "unknown"), route: backendPerformanceRouteTemplate(inputUpstream.route), method: normalizeMethod(inputUpstream.method), status_class: normalizeStatusClass(inputUpstream.statusClass ?? "unknown"), outcome: normalizeOutcome(inputUpstream.outcome ?? "ok") }; if (!recordHistogram(upstreamSeries, labels, boundedDurationMs(inputUpstream.durationMs, 120_000) / 1000, UPSTREAM_BUCKETS_SECONDS, maxSeries)) incrementDropped("upstream_series_limit"); } function setStatus(nextStatusCode: number) { if (Number.isInteger(nextStatusCode) && nextStatusCode > 0) statusCode = nextStatusCode; } function setResponseBytes(bytes: number | null) { if (bytes === null) return; responseBytes = Math.min(64 * 1024 * 1024, Math.max(0, Math.trunc(bytes))); } function applyServerTimingHeader(response: any) { if (!response || typeof response.setHeader !== "function" || response.headersSent) return; const header = serverTimingHeader(nowMs() - startedAt, phases); if (header) response.setHeader("server-timing", header); } function finish(finalStatusCode = statusCode) { if (finished) return; finished = true; const statusClassValue = statusClass(finalStatusCode); const outcome = statusClassValue === "5xx" ? "error" : statusClassValue === "4xx" ? "denied" : "ok"; const labels = { ...baseLabels, route, method, status_class: statusClassValue, outcome }; if (!recordHistogram(requestDurationSeries, labels, boundedDurationMs(nowMs() - startedAt, 120_000) / 1000, REQUEST_BUCKETS_SECONDS, maxSeries)) incrementDropped("request_series_limit"); incrementCounter(requestTotalSeries, labels, maxSeries) || incrementDropped("request_counter_series_limit"); recordBackendRequestSample({ atMs: Date.now(), route, method, statusClass: statusClassValue, outcome, durationSeconds: boundedDurationMs(nowMs() - startedAt, 120_000) / 1000, responseBytes, phases: phases.map((phase) => ({ phase: phase.name, outcome: phase.outcome, durationSeconds: boundedDurationMs(phase.durationMs, 30_000) / 1000 })) }); } return { route, method, startedAt, recordPhase, measure, recordUpstream, setStatus, setResponseBytes, applyServerTimingHeader, finish, get statusCode() { return statusCode; } }; } function recordBackendRequestSample(sample: BackendRequestSample) { requestSamples.push(sample); if (requestSamples.length > maxEvidenceSamples) requestSamples.splice(0, requestSamples.length - maxEvidenceSamples); } function recordWorkbenchProjection(input: WorkbenchProjectionInput = {}) { const labels = { ...workbenchBaseLabels, status: normalizeStatusLabel(input.status ?? "unknown"), reason: normalizeReason(input.reason ?? "none"), projection_status: normalizeProjectionStatus(input.projectionStatus ?? "unknown"), source: normalizeSourceLabel(input.source ?? "agentrun_v01") }; const sourceSeq = finiteNonNegativeNumber(input.sourceLatestSeq); const projectedSeq = finiteNonNegativeNumber(input.lastProjectedSeq); if (sourceSeq !== null && projectedSeq !== null) { const lagEvents = isTerminalProjectionStatus(labels.projection_status) ? 0 : Math.max(0, Math.floor(sourceSeq - projectedSeq)); if (!recordHistogram(projectionLagEventsSeries, labels, lagEvents, PROJECTION_LAG_EVENT_BUCKETS, maxSeries)) incrementDropped("projection_lag_events_series_limit"); setGauge(projectionStuckGaugeSeries, labels, lagEvents > 0 && labels.projection_status !== "caught_up" ? 1 : 0, maxSeries) || incrementDropped("projection_stuck_series_limit"); } const sourceAt = unixMs(input.sourceLatestEventAt); const projectedAt = unixMs(input.projectedLatestEventAt); if (sourceAt !== null && projectedAt !== null) { const lagSeconds = isTerminalProjectionStatus(labels.projection_status) ? 0 : Math.max(0, (sourceAt - projectedAt) / 1000); if (!recordHistogram(projectionLagSecondsSeries, labels, lagSeconds, PROJECTION_LAG_SECONDS_BUCKETS, maxSeries)) incrementDropped("projection_lag_seconds_series_limit"); } const terminalSourceAt = unixMs(input.terminalSourceAt); const terminalProjectedAt = unixMs(input.terminalProjectedAt) ?? (isTerminalProjectionStatus(labels.projection_status) ? projectedAt : null); if (terminalSourceAt !== null) { const delaySeconds = Math.max(0, ((terminalProjectedAt ?? Date.now()) - terminalSourceAt) / 1000); if (!recordHistogram(terminalProjectionDelaySeries, labels, delaySeconds, PROJECTION_LAG_SECONDS_BUCKETS, maxSeries)) incrementDropped("terminal_projection_delay_series_limit"); } } function recordWorkbenchProjectionCursorAdvance(input: WorkbenchProjectorCounterInput = {}) { const labels = { ...workbenchBaseLabels, status: normalizeStatusLabel(input.status ?? "advanced"), reason: normalizeReason(input.reason ?? "sync"), projection_status: normalizeProjectionStatus((input as any).projectionStatus ?? "projecting"), source: normalizeSourceLabel((input as any).source ?? "agentrun_v01") }; incrementCounterBy(projectionCursorAdvanceSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projection_cursor_advance_series_limit"); } function recordWorkbenchTurnRead(input: WorkbenchTurnReadInput) { const labels = { ...workbenchBaseLabels, route: workbenchTurnRouteTemplate(input.route), status: normalizeStatusLabel(input.status ?? "unknown"), degraded_reason: normalizeReason(input.degradedReason ?? "none") }; if (!recordHistogram(turnGetDurationSeries, labels, boundedDurationMs(input.durationMs, 120_000) / 1000, WORKBENCH_TURN_DURATION_BUCKETS_SECONDS, maxSeries)) incrementDropped("turn_get_duration_series_limit"); const bytes = finiteNonNegativeNumber(input.responseBytes); if (bytes !== null) { const byteLabels = { ...workbenchBaseLabels, route: labels.route, status: labels.status }; if (!recordHistogram(turnGetResponseBytesSeries, byteLabels, bytes, WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS, maxSeries)) incrementDropped("turn_get_response_bytes_series_limit"); } if (input.timedOut === true || labels.status === "timeout") { const timeoutLabels = { ...workbenchBaseLabels, route: labels.route, degraded_reason: labels.degraded_reason }; incrementCounter(turnGetTimeoutSeries, timeoutLabels, maxSeries) || incrementDropped("turn_get_timeout_series_limit"); } } function recordWorkbenchCache(input: WorkbenchCacheInput) { const route = workbenchCacheRouteTemplate(input.route); const cacheKeyClass = normalizeCacheKeyClass(input.cacheKeyClass); const cacheStatus = normalizeCacheStatus(input.cacheStatus); const operation = normalizeCacheOperation(input.operation ?? "read"); const labels = { ...workbenchBaseLabels, route, cache_key_class: cacheKeyClass, operation, cache_status: cacheStatus }; incrementCounter(cacheRequestSeries, labels, maxSeries) || incrementDropped("workbench_cache_request_series_limit"); const durationMs = finiteNonNegativeNumber(input.durationMs); const durationSeconds = durationMs === null ? null : boundedDurationMs(durationMs, 120_000) / 1000; if (durationSeconds !== null) { if (!recordHistogram(cacheOperationDurationSeries, labels, durationSeconds, WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS, maxSeries)) incrementDropped("workbench_cache_operation_series_limit"); } const payloadBytes = finiteNonNegativeNumber(input.payloadBytes); if (payloadBytes !== null) { if (!recordHistogram(cachePayloadBytesSeries, labels, payloadBytes, WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS, maxSeries)) incrementDropped("workbench_cache_payload_series_limit"); } if (cacheStatus === "stale") { incrementCounter(cacheStaleSeries, labels, maxSeries) || incrementDropped("workbench_cache_stale_series_limit"); } if (cacheStatus === "unavailable" || cacheStatus === "set_failed" || cacheStatus === "error") { incrementCounter(cacheUnavailableSeries, labels, maxSeries) || incrementDropped("workbench_cache_unavailable_series_limit"); } const dbQueryAvoided = booleanValue(input.dbQueryAvoided); if (dbQueryAvoided) { const avoidedLabels = { ...workbenchBaseLabels, route, cache_key_class: cacheKeyClass }; incrementCounter(dbQueryAvoidedSeries, avoidedLabels, maxSeries) || incrementDropped("workbench_db_query_avoided_series_limit"); } cacheSamples.push({ atMs: Date.now(), route, cacheKeyClass, cacheStatus, operation, durationSeconds, payloadBytes, cacheAgeMs: finiteNonNegativeNumber(input.cacheAgeMs), dbQueryAvoided, freshnessSloMs: finiteNonNegativeNumber(input.freshnessSloMs), projectionSeq: finiteNonNegativeNumber(input.projectionSeq) }); while (cacheSamples.length > maxEvidenceSamples) cacheSamples.shift(); } function recordAgentRunResult(input: AgentRunResultInput = {}) { const eventCount = finiteNonNegativeNumber(input.eventCount ?? input.eventsScanned) ?? 0; const labels = { ...workbenchBaseLabels, event_count_bucket: agentRunEventCountBucket(eventCount), status: normalizeStatusLabel(input.status ?? (input.timedOut ? "timeout" : "unknown")) }; const durationMs = finiteNonNegativeNumber(input.durationMs); if (durationMs !== null) { if (!recordHistogram(agentRunResultDurationSeries, labels, boundedDurationMs(durationMs, 120_000) / 1000, AGENTRUN_RESULT_BUCKETS_SECONDS, maxSeries)) incrementDropped("agentrun_result_duration_series_limit"); } const pagesScanned = finiteNonNegativeNumber(input.pagesScanned); if (pagesScanned !== null) { if (!recordHistogram(agentRunResultPagesSeries, workbenchBaseLabels, pagesScanned, AGENTRUN_RESULT_PAGES_BUCKETS, maxSeries)) incrementDropped("agentrun_result_pages_series_limit"); } const eventsScanned = finiteNonNegativeNumber(input.eventsScanned); if (eventsScanned !== null) { if (!recordHistogram(agentRunResultEventsSeries, workbenchBaseLabels, eventsScanned, AGENTRUN_RESULT_EVENTS_BUCKETS, maxSeries)) incrementDropped("agentrun_result_events_series_limit"); } if (input.timedOut === true || labels.status === "timeout") { const timeoutLabels = { ...workbenchBaseLabels, budget: normalizeBudgetLabel(input.budget), status: labels.status }; incrementCounter(agentRunResultTimeoutSeries, timeoutLabels, maxSeries) || incrementDropped("agentrun_result_timeout_series_limit"); } } function recordWorkbenchProjectorBatch(input: WorkbenchProjectorBatchInput) { const labels = { ...workbenchBaseLabels, phase: normalizeProjectorPhase(input.phase), status: normalizeStatusLabel(input.status ?? "ok") }; if (!recordHistogram(projectorBatchDurationSeries, labels, boundedDurationMs(input.durationMs, 120_000) / 1000, PROJECTOR_BATCH_BUCKETS_SECONDS, maxSeries)) incrementDropped("projector_batch_series_limit"); if (labels.status === "ok") setGauge(projectorLastSuccessSeries, workbenchBaseLabels, Math.floor(Date.now() / 1000), maxSeries) || incrementDropped("projector_last_success_series_limit"); } function recordWorkbenchProjectorCandidate(input: WorkbenchProjectorCounterInput = {}) { const labels = { ...workbenchBaseLabels, status: normalizeStatusLabel(input.status ?? "seen"), reason: normalizeReason(input.reason ?? "unknown") }; incrementCounterBy(projectorCandidatesSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projector_candidates_series_limit"); } function recordWorkbenchProjectorEvents(input: WorkbenchProjectorCounterInput = {}) { const labels = { ...workbenchBaseLabels, event_kind: normalizeReason(input.eventKind ?? "unknown"), status: normalizeStatusLabel(input.status ?? "ok") }; incrementCounterBy(projectorEventsProcessedSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projector_events_series_limit"); } function recordWorkbenchProjectorError(input: WorkbenchProjectorCounterInput = {}) { const labels = { ...workbenchBaseLabels, reason: normalizeReason(input.reason ?? "unknown") }; incrementCounterBy(projectorErrorsSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projector_errors_series_limit"); } function incrementDropped(reason: string) { incrementCounter(droppedSeries, { ...baseLabels, reason }, Math.max(maxSeries, 16)); } function metricsText() { return [ "# HELP hwlab_cloud_api_requests_total Cloud API HTTP requests by low-cardinality route template.", "# TYPE hwlab_cloud_api_requests_total counter", ...renderCounters("hwlab_cloud_api_requests_total", requestTotalSeries), "# HELP hwlab_cloud_api_request_duration_seconds Cloud API HTTP request duration in seconds.", "# TYPE hwlab_cloud_api_request_duration_seconds histogram", ...renderHistogram("hwlab_cloud_api_request_duration_seconds", requestDurationSeries, REQUEST_BUCKETS_SECONDS), "# HELP hwlab_cloud_api_request_phase_duration_seconds Cloud API handler phase duration in seconds.", "# TYPE hwlab_cloud_api_request_phase_duration_seconds histogram", ...renderHistogram("hwlab_cloud_api_request_phase_duration_seconds", requestPhaseSeries, PHASE_BUCKETS_SECONDS), "# HELP hwlab_cloud_api_upstream_duration_seconds Cloud API upstream dependency call duration in seconds.", "# TYPE hwlab_cloud_api_upstream_duration_seconds histogram", ...renderHistogram("hwlab_cloud_api_upstream_duration_seconds", upstreamSeries, UPSTREAM_BUCKETS_SECONDS), "# HELP hwlab_cloud_api_performance_dropped_total Backend performance samples dropped before Prometheus export.", "# TYPE hwlab_cloud_api_performance_dropped_total counter", ...renderCounters("hwlab_cloud_api_performance_dropped_total", droppedSeries), "# HELP hwlab_workbench_projection_lag_events Workbench projection source seq lag in AgentRun events.", "# TYPE hwlab_workbench_projection_lag_events histogram", ...renderHistogram("hwlab_workbench_projection_lag_events", projectionLagEventsSeries, PROJECTION_LAG_EVENT_BUCKETS), "# HELP hwlab_workbench_projection_lag_seconds Workbench projection source time lag in seconds.", "# TYPE hwlab_workbench_projection_lag_seconds histogram", ...renderHistogram("hwlab_workbench_projection_lag_seconds", projectionLagSecondsSeries, PROJECTION_LAG_SECONDS_BUCKETS), "# HELP hwlab_workbench_projection_stuck_traces Low-cardinality gauge set to 1 when the observed projection is behind its source.", "# TYPE hwlab_workbench_projection_stuck_traces gauge", ...renderGauges("hwlab_workbench_projection_stuck_traces", projectionStuckGaugeSeries), "# HELP hwlab_workbench_projection_cursor_advance_total Workbench projection cursor advance events.", "# TYPE hwlab_workbench_projection_cursor_advance_total counter", ...renderCounters("hwlab_workbench_projection_cursor_advance_total", projectionCursorAdvanceSeries), "# HELP hwlab_workbench_terminal_projection_delay_seconds Delay from AgentRun terminal source to Workbench terminal projection.", "# TYPE hwlab_workbench_terminal_projection_delay_seconds histogram", ...renderHistogram("hwlab_workbench_terminal_projection_delay_seconds", terminalProjectionDelaySeries, PROJECTION_LAG_SECONDS_BUCKETS), "# HELP hwlab_workbench_turn_get_duration_seconds Workbench turn/read API duration in seconds.", "# TYPE hwlab_workbench_turn_get_duration_seconds histogram", ...renderHistogram("hwlab_workbench_turn_get_duration_seconds", turnGetDurationSeries, WORKBENCH_TURN_DURATION_BUCKETS_SECONDS), "# HELP hwlab_workbench_turn_get_timeout_total Workbench turn/read API timeout count.", "# TYPE hwlab_workbench_turn_get_timeout_total counter", ...renderCounters("hwlab_workbench_turn_get_timeout_total", turnGetTimeoutSeries), "# HELP hwlab_workbench_turn_get_response_bytes Workbench turn/read API response bytes.", "# TYPE hwlab_workbench_turn_get_response_bytes histogram", ...renderHistogram("hwlab_workbench_turn_get_response_bytes", turnGetResponseBytesSeries, WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS), "# HELP workbench_cache_requests_total Workbench derived cache requests observed by Cloud API read path.", "# TYPE workbench_cache_requests_total counter", ...renderCounters("workbench_cache_requests_total", cacheRequestSeries), "# HELP workbench_cache_hit_ratio Workbench derived cache hit ratio by route and cache class.", "# TYPE workbench_cache_hit_ratio gauge", ...renderWorkbenchCacheHitRatio(cacheRequestSeries), "# HELP workbench_cache_operation_duration_seconds Workbench derived cache operation observation duration in seconds.", "# TYPE workbench_cache_operation_duration_seconds histogram", ...renderHistogram("workbench_cache_operation_duration_seconds", cacheOperationDurationSeries, WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS), "# HELP workbench_cache_payload_bytes Workbench derived cache payload size in bytes.", "# TYPE workbench_cache_payload_bytes histogram", ...renderHistogram("workbench_cache_payload_bytes", cachePayloadBytesSeries, WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS), "# HELP workbench_cache_stale_total Workbench derived cache stale observations.", "# TYPE workbench_cache_stale_total counter", ...renderCounters("workbench_cache_stale_total", cacheStaleSeries), "# HELP workbench_cache_unavailable_total Workbench derived cache unavailable observations.", "# TYPE workbench_cache_unavailable_total counter", ...renderCounters("workbench_cache_unavailable_total", cacheUnavailableSeries), "# HELP workbench_db_query_avoided_total Workbench DB reads avoided by derived cache hits.", "# TYPE workbench_db_query_avoided_total counter", ...renderCounters("workbench_db_query_avoided_total", dbQueryAvoidedSeries), "# HELP hwlab_agentrun_result_duration_seconds AgentRun command result aggregation duration in seconds.", "# TYPE hwlab_agentrun_result_duration_seconds histogram", ...renderHistogram("hwlab_agentrun_result_duration_seconds", agentRunResultDurationSeries, AGENTRUN_RESULT_BUCKETS_SECONDS), "# HELP hwlab_agentrun_result_pages_scanned AgentRun result estimated pages scanned.", "# TYPE hwlab_agentrun_result_pages_scanned histogram", ...renderHistogram("hwlab_agentrun_result_pages_scanned", agentRunResultPagesSeries, AGENTRUN_RESULT_PAGES_BUCKETS), "# HELP hwlab_agentrun_result_events_scanned AgentRun result estimated events scanned.", "# TYPE hwlab_agentrun_result_events_scanned histogram", ...renderHistogram("hwlab_agentrun_result_events_scanned", agentRunResultEventsSeries, AGENTRUN_RESULT_EVENTS_BUCKETS), "# HELP hwlab_agentrun_result_timeout_total AgentRun result timeout count by refresh budget.", "# TYPE hwlab_agentrun_result_timeout_total counter", ...renderCounters("hwlab_agentrun_result_timeout_total", agentRunResultTimeoutSeries), "# HELP hwlab_workbench_projector_batch_duration_seconds Workbench projector/finalizer batch phase duration in seconds.", "# TYPE hwlab_workbench_projector_batch_duration_seconds histogram", ...renderHistogram("hwlab_workbench_projector_batch_duration_seconds", projectorBatchDurationSeries, PROJECTOR_BATCH_BUCKETS_SECONDS), "# HELP hwlab_workbench_projector_candidates_total Workbench projector/finalizer candidate count.", "# TYPE hwlab_workbench_projector_candidates_total counter", ...renderCounters("hwlab_workbench_projector_candidates_total", projectorCandidatesSeries), "# HELP hwlab_workbench_projector_events_processed_total Workbench projector/finalizer processed event count.", "# TYPE hwlab_workbench_projector_events_processed_total counter", ...renderCounters("hwlab_workbench_projector_events_processed_total", projectorEventsProcessedSeries), "# HELP hwlab_workbench_projector_errors_total Workbench projector/finalizer error count.", "# TYPE hwlab_workbench_projector_errors_total counter", ...renderCounters("hwlab_workbench_projector_errors_total", projectorErrorsSeries), "# HELP hwlab_workbench_projector_last_success_unixtime Last successful Workbench projector/finalizer sync unix time.", "# TYPE hwlab_workbench_projector_last_success_unixtime gauge", ...renderGauges("hwlab_workbench_projector_last_success_unixtime", projectorLastSuccessSeries), "" ].join("\n"); } function snapshot() { return { requestSeries: requestDurationSeries.size, requestCount: [...requestTotalSeries.values()].reduce((sum, entry) => sum + entry.value, 0), requestPhaseSeries: requestPhaseSeries.size, requestPhaseCount: histogramSampleCount(requestPhaseSeries), upstreamSeries: upstreamSeries.size, upstreamCount: histogramSampleCount(upstreamSeries), projectionLagSeries: projectionLagEventsSeries.size + projectionLagSecondsSeries.size, turnReadSeries: turnGetDurationSeries.size, cacheSeries: cacheRequestSeries.size + cacheOperationDurationSeries.size + cachePayloadBytesSeries.size + cacheStaleSeries.size + cacheUnavailableSeries.size + dbQueryAvoidedSeries.size, agentRunResultSeries: agentRunResultDurationSeries.size, projectorSeries: projectorBatchDurationSeries.size + projectorCandidatesSeries.size + projectorEventsProcessedSeries.size + projectorErrorsSeries.size, droppedSeries: droppedSeries.size, evidenceSamples: requestSamples.length, cacheEvidenceSamples: cacheSamples.length }; } function evidence(input: { window?: unknown } = {}) { const sampleWindow = resolveBackendEvidenceWindow(input.window); const generatedAtMs = Date.now(); const observedAt = new Date(generatedAtMs).toISOString(); const cutoff = sampleWindow.milliseconds === null ? null : generatedAtMs - sampleWindow.milliseconds; const windowSamples = requestSamples.filter((sample) => cutoff === null || (sample.atMs >= cutoff && sample.atMs <= generatedAtMs)); const windowCacheSamples = cacheSamples.filter((sample) => cutoff === null || (sample.atMs >= cutoff && sample.atMs <= generatedAtMs)); const rows = backendEvidenceRows(windowSamples); const cache = backendCacheEvidenceRows(windowCacheSamples); return { schemaVersion: "hwlab-backend-performance-evidence-v1", source: "cloud-api-backend-performance-store", observedAt, sampleWindow: { id: sampleWindow.id, label: sampleWindow.label, seconds: sampleWindow.milliseconds === null ? null : Math.round(sampleWindow.milliseconds / 1000), windowFrom: cutoff === null ? null : new Date(Math.max(0, cutoff)).toISOString(), windowTo: observedAt }, sampleCount: windowSamples.length, retainedSampleCount: requestSamples.length, cacheSampleCount: windowCacheSamples.length, retainedCacheSampleCount: cacheSamples.length, routeCount: new Set(rows.map((row) => row.route)).size, rows, cache, valuesRedacted: true }; } return { beginHttpRequest, createManualContext, recordWorkbenchProjection, recordWorkbenchProjectionCursorAdvance, recordWorkbenchTurnRead, recordWorkbenchCache, recordAgentRunResult, recordWorkbenchProjectorBatch, recordWorkbenchProjectorCandidate, recordWorkbenchProjectorEvents, recordWorkbenchProjectorError, metricsText, snapshot, evidence }; } export function withBackendPerformanceContext(context: unknown, callback: () => T): T { return backendPerformanceStorage.run(context, callback); } export function currentBackendPerformanceContext(): any { return backendPerformanceStorage.getStore() ?? null; } export function backendPerformanceRouteTemplate(input: unknown): string { const raw = String(input ?? "").trim() || "/"; const pathOnly = raw.split(/[?#]/u, 1)[0] || "/"; let pathname = pathOnly; try { pathname = new URL(raw, "http://hwlab.local").pathname || pathOnly; } catch { // Keep pathOnly. } const parts = pathname.split("/").filter(Boolean).map((part) => routeSegmentTemplate(safeDecode(part))); const path = `/${parts.join("/")}`; return path.length > 140 ? `${path.slice(0, 137)}...` : path; } function workbenchTurnRouteTemplate(input: unknown): string { const route = backendPerformanceRouteTemplate(input); if (route === "/v1/workbench/turns/:id") return "/v1/workbench/turns/:traceId"; if (route === "/v1/agent/turns/:id") return "/v1/agent/turns/:traceId"; return route; } function workbenchCacheRouteTemplate(input: unknown): string { const route = workbenchTurnRouteTemplate(input); if (route === "/v1/workbench/traces/:id/events") return "/v1/workbench/traces/:traceId/events"; return route; } function resolveBackendEvidenceWindow(input: unknown) { const key = String(input ?? DEFAULT_BACKEND_EVIDENCE_WINDOW).trim().toLowerCase(); if (Object.prototype.hasOwnProperty.call(BACKEND_EVIDENCE_WINDOWS, key)) return BACKEND_EVIDENCE_WINDOWS[key as keyof typeof BACKEND_EVIDENCE_WINDOWS]; return BACKEND_EVIDENCE_WINDOWS[DEFAULT_BACKEND_EVIDENCE_WINDOW]; } function backendEvidenceRows(samples: BackendRequestSample[]) { const groups = new Map; firstAt: number; lastAt: number }>(); for (const sample of samples) { const key = stableLabelKey({ route: sample.route, method: sample.method, statusClass: sample.statusClass, outcome: sample.outcome }); let group = groups.get(key); if (!group) { group = { sample, durations: [], bytes: [], phases: new Map(), firstAt: sample.atMs, lastAt: sample.atMs }; groups.set(key, group); } group.durations.push(sample.durationSeconds); if (sample.responseBytes !== null) group.bytes.push(sample.responseBytes); group.firstAt = Math.min(group.firstAt, sample.atMs); group.lastAt = Math.max(group.lastAt, sample.atMs); for (const phase of sample.phases) { const phaseKey = stableLabelKey({ phase: phase.phase, outcome: phase.outcome }); let phaseGroup = group.phases.get(phaseKey); if (!phaseGroup) { phaseGroup = { phase: phase.phase, outcome: phase.outcome, durations: [] }; group.phases.set(phaseKey, phaseGroup); } phaseGroup.durations.push(phase.durationSeconds); } } return [...groups.values()].map((group) => { const duration = exactEvidenceStats(group.durations); const responseBytes = group.bytes.length ? exactEvidenceStats(group.bytes) : null; const phases = [...group.phases.values()].map((phaseGroup) => ({ phase: phaseGroup.phase, outcome: phaseGroup.outcome, ...exactEvidenceStats(phaseGroup.durations) })).sort((left, right) => (right.p95 - left.p95) || (right.count - left.count)).slice(0, 8); return { route: group.sample.route, method: group.sample.method, statusClass: group.sample.statusClass, outcome: group.sample.outcome, count: duration.count, average: duration.average, p50: duration.p50, p75: duration.p75, p95: duration.p95, unit: "seconds", firstObservedAt: new Date(group.firstAt).toISOString(), lastObservedAt: new Date(group.lastAt).toISOString(), responseBytes: responseBytes ? { count: responseBytes.count, average: Math.round(responseBytes.average), p50: Math.round(responseBytes.p50), p75: Math.round(responseBytes.p75), p95: Math.round(responseBytes.p95), unit: "bytes" } : null, phases, valuesRedacted: true }; }).sort((left, right) => (right.p95 - left.p95) || (right.count - left.count)); } function backendCacheEvidenceRows(samples: WorkbenchCacheEvidenceSample[]) { const groups = new Map(); for (const sample of samples) { const key = stableLabelKey({ route: sample.route, cache_key_class: sample.cacheKeyClass }); let group = groups.get(key); if (!group) { group = { route: sample.route, cacheKeyClass: sample.cacheKeyClass, count: 0, hitCount: 0, missCount: 0, staleCount: 0, unavailableCount: 0, dbQueryAvoidedCount: 0, durations: [], payloads: [], ages: [], freshness: [], projectionSeqMax: null, firstAt: sample.atMs, lastAt: sample.atMs }; groups.set(key, group); } group.count += 1; if (sample.cacheStatus === "hit") group.hitCount += 1; if (sample.cacheStatus === "miss") group.missCount += 1; if (sample.cacheStatus === "stale") group.staleCount += 1; if (sample.cacheStatus === "unavailable" || sample.cacheStatus === "set_failed" || sample.cacheStatus === "error") group.unavailableCount += 1; if (sample.dbQueryAvoided) group.dbQueryAvoidedCount += 1; if (sample.durationSeconds !== null) group.durations.push(sample.durationSeconds); if (sample.payloadBytes !== null) group.payloads.push(sample.payloadBytes); if (sample.cacheAgeMs !== null) group.ages.push(sample.cacheAgeMs); if (sample.freshnessSloMs !== null) group.freshness.push(sample.freshnessSloMs); if (sample.projectionSeq !== null) group.projectionSeqMax = Math.max(group.projectionSeqMax ?? 0, sample.projectionSeq); group.firstAt = Math.min(group.firstAt, sample.atMs); group.lastAt = Math.max(group.lastAt, sample.atMs); } return [...groups.values()].map((group) => { const duration = exactEvidenceStats(group.durations); const payloadBytes = group.payloads.length ? exactEvidenceStats(group.payloads) : null; const cacheAgeMs = group.ages.length ? exactEvidenceStats(group.ages) : null; const freshnessSloMs = group.freshness.length ? exactEvidenceStats(group.freshness) : null; return { route: group.route, cacheKeyClass: group.cacheKeyClass, count: group.count, hitCount: group.hitCount, missCount: group.missCount, staleCount: group.staleCount, unavailableCount: group.unavailableCount, dbQueryAvoidedCount: group.dbQueryAvoidedCount, hitRatio: group.count > 0 ? roundEvidenceValue(group.hitCount / group.count) : 0, dbQueryAvoidedRatio: group.count > 0 ? roundEvidenceValue(group.dbQueryAvoidedCount / group.count) : 0, unavailableRatio: group.count > 0 ? roundEvidenceValue(group.unavailableCount / group.count) : 0, operationDuration: { ...duration, unit: "seconds" }, payloadBytes: payloadBytes ? { count: payloadBytes.count, average: Math.round(payloadBytes.average), p50: Math.round(payloadBytes.p50), p75: Math.round(payloadBytes.p75), p95: Math.round(payloadBytes.p95), unit: "bytes" } : null, cacheAgeMs: cacheAgeMs ? { ...cacheAgeMs, unit: "milliseconds" } : null, freshnessSloMs: freshnessSloMs ? { ...freshnessSloMs, unit: "milliseconds" } : null, projectionSeqMax: group.projectionSeqMax, firstObservedAt: new Date(group.firstAt).toISOString(), lastObservedAt: new Date(group.lastAt).toISOString(), valuesRedacted: true }; }).sort((left, right) => (right.unavailableRatio - left.unavailableRatio) || (right.count - left.count) || left.route.localeCompare(right.route)); } function exactEvidenceStats(values: number[]) { const sorted = values.filter(Number.isFinite).sort((left, right) => left - right); const count = sorted.length; const average = count > 0 ? sorted.reduce((sum, value) => sum + value, 0) / count : 0; return { count, average: roundEvidenceValue(average), p50: roundEvidenceValue(exactEvidenceQuantile(sorted, 0.5)), p75: roundEvidenceValue(exactEvidenceQuantile(sorted, 0.75)), p95: roundEvidenceValue(exactEvidenceQuantile(sorted, 0.95)) }; } function exactEvidenceQuantile(sortedValues: number[], quantile: number) { if (sortedValues.length <= 0) return 0; const index = Math.min(sortedValues.length - 1, Math.max(0, Math.ceil(sortedValues.length * quantile) - 1)); return sortedValues[index] ?? 0; } function roundEvidenceValue(value: number) { if (!Number.isFinite(value)) return 0; return Math.round(value * 1000) / 1000; } function responseEndChunkBytes(chunk: unknown, encoding: unknown): number | null { if (chunk === undefined || chunk === null) return null; if (Buffer.isBuffer(chunk)) return chunk.length; if (typeof chunk === "string") return Buffer.byteLength(chunk, typeof encoding === "string" ? encoding as BufferEncoding : "utf8"); if (chunk instanceof Uint8Array) return chunk.byteLength; return null; } function serverTimingHeader(totalMs: number, phases: Array<{ name: string; durationMs: number }>) { const entries = phases.slice(-MAX_SERVER_TIMING_ENTRIES).map((phase) => `${serverTimingToken(phase.name)};dur=${roundDurationMs(phase.durationMs)}`); entries.push(`total;dur=${roundDurationMs(totalMs)}`); return entries.join(", "); } function recordHistogram(series: Map, labels: Record, value: number, buckets: number[], maxSeries: number) { const key = stableLabelKey(labels); let entry = series.get(key); if (!entry) { if (series.size >= maxSeries) return false; entry = { labels, buckets: buckets.map(() => 0), sum: 0, count: 0 }; series.set(key, entry); } for (let index = 0; index < buckets.length; index += 1) { if (value <= buckets[index]) entry.buckets[index] += 1; } entry.sum += value; entry.count += 1; return true; } function incrementCounter(series: Map, labels: Record, maxSeries: number) { const key = stableLabelKey(labels); let entry = series.get(key); if (!entry) { if (series.size >= maxSeries) return false; entry = { labels, value: 0 }; series.set(key, entry); } entry.value += 1; return true; } function incrementCounterBy(series: Map, labels: Record, value: number, maxSeries: number) { const key = stableLabelKey(labels); let entry = series.get(key); if (!entry) { if (series.size >= maxSeries) return false; entry = { labels, value: 0 }; series.set(key, entry); } entry.value += value; return true; } function setGauge(series: Map, labels: Record, value: number, maxSeries: number) { const key = stableLabelKey(labels); let entry = series.get(key); if (!entry) { if (series.size >= maxSeries) return false; entry = { labels, value: 0 }; series.set(key, entry); } entry.value = Number.isFinite(value) ? value : 0; return true; } function renderCounters(metricName: string, series: Map) { return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${entry.value}`); } function renderGauges(metricName: string, series: Map) { return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${formatPrometheusValue(entry.value)}`); } function renderWorkbenchCacheHitRatio(series: Map) { const groups = new Map; hit: number; total: number }>(); for (const entry of series.values()) { const labels = { ...entry.labels }; const cacheStatus = labels.cache_status; delete labels.cache_status; const key = stableLabelKey(labels); const group = groups.get(key) ?? { labels, hit: 0, total: 0 }; if (cacheStatus === "hit") group.hit += entry.value; group.total += entry.value; groups.set(key, group); } return [...groups.values()].map((group) => `${"workbench_cache_hit_ratio"}{${renderLabels(group.labels)}} ${formatPrometheusValue(group.total > 0 ? group.hit / group.total : 0)}`); } function renderHistogram(metricName: string, series: Map, buckets: number[]) { const lines: string[] = []; for (const entry of series.values()) { for (let index = 0; index < buckets.length; index += 1) { lines.push(`${metricName}_bucket{${renderLabels({ ...entry.labels, le: String(buckets[index]) })}} ${entry.buckets[index]}`); } lines.push(`${metricName}_bucket{${renderLabels({ ...entry.labels, le: "+Inf" })}} ${entry.count}`); lines.push(`${metricName}_sum{${renderLabels(entry.labels)}} ${entry.sum.toFixed(6)}`); lines.push(`${metricName}_count{${renderLabels(entry.labels)}} ${entry.count}`); } return lines; } function histogramSampleCount(series: Map) { return [...series.values()].reduce((sum, entry) => sum + entry.count, 0); } function renderLabels(labels: Record) { return Object.entries(labels).map(([key, value]) => `${key}="${escapeLabelValue(value)}"`).join(","); } function stableLabelKey(labels: Record) { return Object.keys(labels).sort().map((key) => `${key}=${labels[key]}`).join("\u0000"); } function routeSegmentTemplate(segment: string) { if (/^\d+$/u.test(segment)) return ":id"; if (/^[a-z][a-z0-9]{1,12}_[A-Za-z0-9_.:-]+$/iu.test(segment)) return ":id"; if (/^[0-9a-f]{12,}$/iu.test(segment)) return ":id"; if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu.test(segment)) return ":id"; if (segment.length > 48) return ":id"; return sanitizePathSegment(segment); } function safeUrl(value: string) { try { return new URL(value, "http://hwlab-cloud-api.local"); } catch { return new URL("/", "http://hwlab-cloud-api.local"); } } function safeDecode(value: string) { try { return decodeURIComponent(value); } catch { return value; } } function sanitizePathSegment(value: string) { const sanitized = value.replace(/[^A-Za-z0-9_.:-]/gu, "_"); return sanitized.slice(0, 64) || "_"; } function normalizeMethod(value: unknown) { const method = String(value ?? "GET").trim().toUpperCase(); return /^(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)$/u.test(method) ? method : "OTHER"; } function statusClass(value: unknown) { const status = Number(value); if (!Number.isFinite(status) || status <= 0) return "unknown"; return `${Math.floor(status / 100)}xx`; } function normalizeStatusClass(value: unknown) { const text = String(value ?? "").trim().toLowerCase(); return /^(?:1xx|2xx|3xx|4xx|5xx|network|timeout|unknown)$/u.test(text) ? text : "unknown"; } function normalizeStatusLabel(value: unknown) { const text = sanitizeMetricName(value, "unknown"); return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown|running|pending|completed|failed|blocked|degraded|advanced|unchanged|scheduled|seen|skipped|caught_up|projecting|stale|missing|hit|miss|stored|set_failed|unavailable|disabled)$/u.test(text) ? text : "unknown"; } function normalizeCacheStatus(value: unknown) { const text = normalizeStatusLabel(value); return /^(?:hit|miss|stale|skipped|stored|set_failed|unavailable|disabled|error|unknown)$/u.test(text) ? text : "unknown"; } function normalizeCacheOperation(value: unknown) { const text = sanitizeMetricName(value, "read"); return /^(?:read|get|set|write|delete|ttl|ping)$/u.test(text) ? text : "read"; } function normalizeCacheKeyClass(value: unknown) { const text = sanitizeMetricName(value, "unknown").slice(0, 64); return text || "unknown"; } function normalizeOutcome(value: unknown) { const text = sanitizeMetricName(value, "unknown"); return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown)$/u.test(text) ? text : "unknown"; } function normalizeProjectionStatus(value: unknown) { const text = sanitizeMetricName(value, "unknown"); if (text === "caught_up" || text === "caughtup") return "caught_up"; return /^(?:projecting|caught_up|blocked|unknown|missing|stale|degraded|terminal|running)$/u.test(text) ? text : "unknown"; } function isTerminalProjectionStatus(value: unknown) { const status = normalizeProjectionStatus(value); return status === "caught_up" || status === "terminal"; } function normalizeReason(value: unknown) { return sanitizeMetricName(value, "unknown").slice(0, 64) || "unknown"; } function normalizeSourceLabel(value: unknown) { const text = sanitizeMetricName(value, "unknown"); if (text === "agentrun_v0_1" || text === "agentrun_v01") return "agentrun_v01"; return text.slice(0, 64) || "unknown"; } function normalizeProjectorPhase(value: unknown) { const phase = sanitizeMetricName(value, "unknown"); return /^(?:candidate_scan|agentrun_events_fetch|event_map|trace_append|turn_finalize|session_write|status_effects|unknown)$/u.test(phase) ? phase : "unknown"; } function normalizeBudgetLabel(value: unknown) { const parsed = Number(value); if (Number.isFinite(parsed) && parsed > 0) { if (parsed <= 2500) return "turn_2_5s"; if (parsed <= 10000) return "short_10s"; if (parsed <= 30000) return "medium_30s"; if (parsed <= 120000) return "long_120s"; return "long_over_120s"; } return normalizeReason(value ?? "unknown"); } function agentRunEventCountBucket(value: number) { if (value <= 500) return "0_500"; if (value <= 2000) return "501_2000"; if (value <= 5000) return "2001_5000"; return "5000_plus"; } function sanitizeMetricName(value: unknown, fallback = "unknown") { const text = String(value ?? "").trim().toLowerCase().replace(/[^a-z0-9_:-]+/gu, "_").replace(/[-:]+/gu, "_").replace(/^_+|_+$/gu, ""); return text || fallback; } function sanitizeLabelValue(value: unknown, fallback = "unknown") { const text = String(value ?? "").trim(); if (!text) return fallback; return text.replace(/[\n\r\t]/gu, " ").slice(0, 120); } function escapeLabelValue(value: string) { return String(value).replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"").replace(/\n/gu, "\\n"); } function parsePositiveInteger(value: unknown, fallback: number) { const parsed = Number.parseInt(String(value ?? ""), 10); return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback; } function boundedDurationMs(value: unknown, maxMs: number) { const parsed = Number(value); if (!Number.isFinite(parsed) || parsed < 0) return 0; return Math.min(parsed, maxMs); } function boundedCounterValue(value: unknown) { const parsed = Number(value ?? 1); if (!Number.isFinite(parsed) || parsed <= 0) return 1; return Math.min(Math.floor(parsed), 1_000_000); } function finiteNonNegativeNumber(value: unknown): number | null { const parsed = Number(value); return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; } function booleanValue(value: unknown) { if (value === true) return true; const text = String(value ?? "").trim().toLowerCase(); return text === "true" || text === "1" || text === "yes"; } function unixMs(value: unknown): number | null { if (value instanceof Date) { const time = value.getTime(); return Number.isFinite(time) ? time : null; } const numeric = Number(value); if (Number.isFinite(numeric) && numeric > 0) return numeric > 10_000_000_000 ? numeric : numeric * 1000; const parsed = Date.parse(String(value ?? "")); return Number.isFinite(parsed) ? parsed : null; } function formatPrometheusValue(value: number) { if (!Number.isFinite(value)) return "0"; return Number.isInteger(value) ? String(value) : value.toFixed(6); } function roundDurationMs(value: number) { return Math.max(0, value).toFixed(1); } function serverTimingToken(value: string) { const token = sanitizeMetricName(value, "phase").replace(/_/gu, "-"); return token || "phase"; } function nowMs() { if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now(); return Date.now(); }