root
1158 lines
55 KiB
TypeScript
1158 lines
55 KiB
TypeScript
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0
|
|
// Records low-cardinality Cloud API backend performance metrics and Server-Timing phases.
|
|
|
|
import { AsyncLocalStorage } from "node:async_hooks";
|
|
|
|
const REQUEST_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60];
|
|
const PHASE_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30];
|
|
const UPSTREAM_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120];
|
|
const PROJECTION_LAG_EVENT_BUCKETS = [0, 1, 5, 10, 50, 100, 500, 1000, 2000, 5000, 10000];
|
|
const PROJECTION_LAG_SECONDS_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 900, 1800, 3600];
|
|
const WORKBENCH_TURN_DURATION_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60];
|
|
const WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304];
|
|
const WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10];
|
|
const WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304];
|
|
const AGENTRUN_RESULT_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120];
|
|
const AGENTRUN_RESULT_EVENTS_BUCKETS = [0, 50, 100, 500, 1000, 2000, 5000, 10000];
|
|
const AGENTRUN_RESULT_PAGES_BUCKETS = [0, 1, 2, 5, 10, 20, 50, 100];
|
|
const PROJECTOR_BATCH_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60];
|
|
const MAX_SERVER_TIMING_ENTRIES = 8;
|
|
const DEFAULT_BACKEND_EVIDENCE_WINDOW = "15m";
|
|
const BACKEND_EVIDENCE_WINDOWS = {
|
|
"5m": { id: "5m", label: "最近 5 分钟", milliseconds: 5 * 60 * 1000 },
|
|
"15m": { id: "15m", label: "最近 15 分钟", milliseconds: 15 * 60 * 1000 },
|
|
"1h": { id: "1h", label: "最近 1 小时", milliseconds: 60 * 60 * 1000 },
|
|
"6h": { id: "6h", label: "最近 6 小时", milliseconds: 6 * 60 * 60 * 1000 },
|
|
"24h": { id: "24h", label: "最近 24 小时", milliseconds: 24 * 60 * 60 * 1000 },
|
|
all: { id: "all", label: "全部累计", milliseconds: null }
|
|
} as const;
|
|
|
|
const backendPerformanceStorage = new AsyncLocalStorage();
|
|
|
|
interface HistogramSeries {
|
|
labels: Record<string, string>;
|
|
buckets: number[];
|
|
sum: number;
|
|
count: number;
|
|
}
|
|
|
|
interface CounterSeries {
|
|
labels: Record<string, string>;
|
|
value: number;
|
|
}
|
|
|
|
interface GaugeSeries {
|
|
labels: Record<string, string>;
|
|
value: number;
|
|
}
|
|
|
|
interface BackendPerformanceStoreOptions {
|
|
env?: Record<string, unknown>;
|
|
maxSeries?: number;
|
|
maxEvidenceSamples?: number;
|
|
}
|
|
|
|
interface BackendPerformanceContextInput {
|
|
route: string;
|
|
method: string;
|
|
}
|
|
|
|
interface BackendPhaseInput {
|
|
phase: string;
|
|
durationMs: number;
|
|
outcome?: string;
|
|
}
|
|
|
|
interface BackendUpstreamInput {
|
|
component: string;
|
|
operation?: string;
|
|
route: string;
|
|
method: string;
|
|
statusClass?: string;
|
|
outcome?: string;
|
|
durationMs: number;
|
|
}
|
|
|
|
interface WorkbenchProjectionInput {
|
|
sourceLatestSeq?: unknown;
|
|
lastProjectedSeq?: unknown;
|
|
sourceLatestEventAt?: unknown;
|
|
projectedLatestEventAt?: unknown;
|
|
terminalSourceAt?: unknown;
|
|
terminalProjectedAt?: unknown;
|
|
status?: unknown;
|
|
reason?: unknown;
|
|
projectionStatus?: unknown;
|
|
source?: unknown;
|
|
}
|
|
|
|
interface WorkbenchTurnReadInput {
|
|
route: string;
|
|
status?: unknown;
|
|
degradedReason?: unknown;
|
|
durationMs: number;
|
|
responseBytes?: unknown;
|
|
timedOut?: boolean;
|
|
}
|
|
|
|
interface WorkbenchCacheInput {
|
|
route: string;
|
|
cacheKeyClass?: unknown;
|
|
cacheStatus?: unknown;
|
|
operation?: unknown;
|
|
durationMs?: unknown;
|
|
payloadBytes?: unknown;
|
|
cacheAgeMs?: unknown;
|
|
dbQueryAvoided?: unknown;
|
|
freshnessSloMs?: unknown;
|
|
projectionSeq?: unknown;
|
|
}
|
|
|
|
interface AgentRunResultInput {
|
|
durationMs?: unknown;
|
|
eventsScanned?: unknown;
|
|
pagesScanned?: unknown;
|
|
eventCount?: unknown;
|
|
status?: unknown;
|
|
budget?: unknown;
|
|
timedOut?: boolean;
|
|
}
|
|
|
|
interface WorkbenchProjectorBatchInput {
|
|
phase: string;
|
|
status?: unknown;
|
|
durationMs: number;
|
|
}
|
|
|
|
interface WorkbenchProjectorCounterInput {
|
|
status?: unknown;
|
|
reason?: unknown;
|
|
eventKind?: unknown;
|
|
count?: unknown;
|
|
}
|
|
|
|
interface BackendRequestSample {
|
|
atMs: number;
|
|
route: string;
|
|
method: string;
|
|
statusClass: string;
|
|
outcome: string;
|
|
durationSeconds: number;
|
|
responseBytes: number | null;
|
|
phases: Array<{ phase: string; outcome: string; durationSeconds: number }>;
|
|
}
|
|
|
|
interface WorkbenchCacheEvidenceSample {
|
|
atMs: number;
|
|
route: string;
|
|
cacheKeyClass: string;
|
|
cacheStatus: string;
|
|
operation: string;
|
|
durationSeconds: number | null;
|
|
payloadBytes: number | null;
|
|
cacheAgeMs: number | null;
|
|
dbQueryAvoided: boolean;
|
|
freshnessSloMs: number | null;
|
|
projectionSeq: number | null;
|
|
}
|
|
|
|
export function createBackendPerformanceStore(options: BackendPerformanceStoreOptions = {}) {
|
|
const env = options.env ?? process.env;
|
|
const maxSeries = parsePositiveInteger(env.HWLAB_BACKEND_PERFORMANCE_MAX_SERIES, options.maxSeries ?? 320);
|
|
const maxEvidenceSamples = parsePositiveInteger(env.HWLAB_BACKEND_PERFORMANCE_MAX_EVIDENCE_SAMPLES, options.maxEvidenceSamples ?? 5000);
|
|
const requestDurationSeries = new Map<string, HistogramSeries>();
|
|
const requestTotalSeries = new Map<string, CounterSeries>();
|
|
const requestPhaseSeries = new Map<string, HistogramSeries>();
|
|
const upstreamSeries = new Map<string, HistogramSeries>();
|
|
const droppedSeries = new Map<string, CounterSeries>();
|
|
const projectionLagEventsSeries = new Map<string, HistogramSeries>();
|
|
const projectionLagSecondsSeries = new Map<string, HistogramSeries>();
|
|
const projectionStuckGaugeSeries = new Map<string, GaugeSeries>();
|
|
const projectionCursorAdvanceSeries = new Map<string, CounterSeries>();
|
|
const terminalProjectionDelaySeries = new Map<string, HistogramSeries>();
|
|
const turnGetDurationSeries = new Map<string, HistogramSeries>();
|
|
const turnGetTimeoutSeries = new Map<string, CounterSeries>();
|
|
const turnGetResponseBytesSeries = new Map<string, HistogramSeries>();
|
|
const cacheRequestSeries = new Map<string, CounterSeries>();
|
|
const cacheOperationDurationSeries = new Map<string, HistogramSeries>();
|
|
const cachePayloadBytesSeries = new Map<string, HistogramSeries>();
|
|
const cacheStaleSeries = new Map<string, CounterSeries>();
|
|
const cacheUnavailableSeries = new Map<string, CounterSeries>();
|
|
const dbQueryAvoidedSeries = new Map<string, CounterSeries>();
|
|
const agentRunResultDurationSeries = new Map<string, HistogramSeries>();
|
|
const agentRunResultPagesSeries = new Map<string, HistogramSeries>();
|
|
const agentRunResultEventsSeries = new Map<string, HistogramSeries>();
|
|
const agentRunResultTimeoutSeries = new Map<string, CounterSeries>();
|
|
const projectorBatchDurationSeries = new Map<string, HistogramSeries>();
|
|
const projectorCandidatesSeries = new Map<string, CounterSeries>();
|
|
const projectorEventsProcessedSeries = new Map<string, CounterSeries>();
|
|
const projectorErrorsSeries = new Map<string, CounterSeries>();
|
|
const projectorLastSuccessSeries = new Map<string, GaugeSeries>();
|
|
const requestSamples: BackendRequestSample[] = [];
|
|
const cacheSamples: WorkbenchCacheEvidenceSample[] = [];
|
|
const baseLabels = {
|
|
service: "hwlab-cloud-api",
|
|
namespace: sanitizeLabelValue(env.POD_NAMESPACE ?? env.HWLAB_METRICS_NAMESPACE ?? "hwlab-v02"),
|
|
gitops_target: sanitizeLabelValue(env.HWLAB_GITOPS_TARGET ?? env.HWLAB_GITOPS_PROFILE ?? env.HWLAB_RUNTIME_LANE ?? "v02")
|
|
};
|
|
const workbenchBaseLabels = {
|
|
...baseLabels,
|
|
node: sanitizeLabelValue(env.HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID ?? env.AGENTRUN_PROVIDER_ID ?? env.HWLAB_NODE_ID ?? "unknown"),
|
|
lane: sanitizeLabelValue(env.HWLAB_RUNTIME_LANE ?? env.HWLAB_GITOPS_TARGET ?? env.HWLAB_GITOPS_PROFILE ?? "unknown")
|
|
};
|
|
|
|
function beginHttpRequest(request: any, response: any) {
|
|
const url = safeUrl(request?.url || "/");
|
|
const context = createContext({ route: backendPerformanceRouteTemplate(url.pathname), method: normalizeMethod(request?.method) });
|
|
const originalWriteHead = typeof response?.writeHead === "function" ? response.writeHead : null;
|
|
if (originalWriteHead) {
|
|
response.writeHead = function writeHeadWithServerTiming(statusCode: number, ...args: unknown[]) {
|
|
context.setStatus(statusCode);
|
|
context.applyServerTimingHeader(response);
|
|
return originalWriteHead.call(this, statusCode, ...args);
|
|
};
|
|
}
|
|
const originalEnd = typeof response?.end === "function" ? response.end : null;
|
|
if (originalEnd) {
|
|
response.end = function endWithBackendEvidenceBytes(chunk?: unknown, encoding?: unknown, callback?: unknown) {
|
|
context.setResponseBytes(responseEndChunkBytes(chunk, encoding));
|
|
return originalEnd.apply(this, arguments as any);
|
|
};
|
|
}
|
|
if (typeof response?.once === "function") {
|
|
response.once("finish", () => context.finish(Number(response.statusCode ?? context.statusCode ?? 0)));
|
|
}
|
|
return context;
|
|
}
|
|
|
|
function createManualContext(input: BackendPerformanceContextInput) {
|
|
return createContext({ route: backendPerformanceRouteTemplate(input.route), method: normalizeMethod(input.method) });
|
|
}
|
|
|
|
function createContext(input: BackendPerformanceContextInput) {
|
|
const startedAt = nowMs();
|
|
const phases: Array<{ name: string; durationMs: number; outcome: string }> = [];
|
|
let statusCode = 200;
|
|
let responseBytes: number | null = null;
|
|
let finished = false;
|
|
const route = input.route;
|
|
const method = input.method;
|
|
|
|
function recordPhase(inputPhase: BackendPhaseInput) {
|
|
const durationMs = boundedDurationMs(inputPhase.durationMs, 30_000);
|
|
const phase = sanitizeMetricName(inputPhase.phase, "unknown");
|
|
const outcome = normalizeOutcome(inputPhase.outcome ?? "ok");
|
|
phases.push({ name: phase, durationMs, outcome });
|
|
const labels = { ...baseLabels, route, method, phase, outcome };
|
|
if (!recordHistogram(requestPhaseSeries, labels, durationMs / 1000, PHASE_BUCKETS_SECONDS, maxSeries)) incrementDropped("phase_series_limit");
|
|
}
|
|
|
|
async function measure<T>(phase: string, fn: () => Promise<T> | T): Promise<T> {
|
|
const phaseStartedAt = nowMs();
|
|
try {
|
|
const result = await fn();
|
|
recordPhase({ phase, durationMs: nowMs() - phaseStartedAt, outcome: "ok" });
|
|
return result;
|
|
} catch (error) {
|
|
recordPhase({ phase, durationMs: nowMs() - phaseStartedAt, outcome: "error" });
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function recordUpstream(inputUpstream: BackendUpstreamInput) {
|
|
const labels = {
|
|
...baseLabels,
|
|
component: sanitizeMetricName(inputUpstream.component, "unknown"),
|
|
operation: sanitizeMetricName(inputUpstream.operation ?? inputUpstream.route, "unknown"),
|
|
route: backendPerformanceRouteTemplate(inputUpstream.route),
|
|
method: normalizeMethod(inputUpstream.method),
|
|
status_class: normalizeStatusClass(inputUpstream.statusClass ?? "unknown"),
|
|
outcome: normalizeOutcome(inputUpstream.outcome ?? "ok")
|
|
};
|
|
if (!recordHistogram(upstreamSeries, labels, boundedDurationMs(inputUpstream.durationMs, 120_000) / 1000, UPSTREAM_BUCKETS_SECONDS, maxSeries)) incrementDropped("upstream_series_limit");
|
|
}
|
|
|
|
function setStatus(nextStatusCode: number) {
|
|
if (Number.isInteger(nextStatusCode) && nextStatusCode > 0) statusCode = nextStatusCode;
|
|
}
|
|
|
|
function setResponseBytes(bytes: number | null) {
|
|
if (bytes === null) return;
|
|
responseBytes = Math.min(64 * 1024 * 1024, Math.max(0, Math.trunc(bytes)));
|
|
}
|
|
|
|
function applyServerTimingHeader(response: any) {
|
|
if (!response || typeof response.setHeader !== "function" || response.headersSent) return;
|
|
const header = serverTimingHeader(nowMs() - startedAt, phases);
|
|
if (header) response.setHeader("server-timing", header);
|
|
}
|
|
|
|
function finish(finalStatusCode = statusCode) {
|
|
if (finished) return;
|
|
finished = true;
|
|
const statusClassValue = statusClass(finalStatusCode);
|
|
const outcome = statusClassValue === "5xx" ? "error" : statusClassValue === "4xx" ? "denied" : "ok";
|
|
const labels = { ...baseLabels, route, method, status_class: statusClassValue, outcome };
|
|
if (!recordHistogram(requestDurationSeries, labels, boundedDurationMs(nowMs() - startedAt, 120_000) / 1000, REQUEST_BUCKETS_SECONDS, maxSeries)) incrementDropped("request_series_limit");
|
|
incrementCounter(requestTotalSeries, labels, maxSeries) || incrementDropped("request_counter_series_limit");
|
|
recordBackendRequestSample({
|
|
atMs: Date.now(),
|
|
route,
|
|
method,
|
|
statusClass: statusClassValue,
|
|
outcome,
|
|
durationSeconds: boundedDurationMs(nowMs() - startedAt, 120_000) / 1000,
|
|
responseBytes,
|
|
phases: phases.map((phase) => ({ phase: phase.name, outcome: phase.outcome, durationSeconds: boundedDurationMs(phase.durationMs, 30_000) / 1000 }))
|
|
});
|
|
}
|
|
|
|
return {
|
|
route,
|
|
method,
|
|
startedAt,
|
|
recordPhase,
|
|
measure,
|
|
recordUpstream,
|
|
setStatus,
|
|
setResponseBytes,
|
|
applyServerTimingHeader,
|
|
finish,
|
|
get statusCode() { return statusCode; },
|
|
get responseBytes() { return responseBytes; },
|
|
get phases() { return phases.map((phase) => ({ phase: phase.name, outcome: phase.outcome, durationSeconds: boundedDurationMs(phase.durationMs, 30_000) / 1000 })); }
|
|
};
|
|
}
|
|
|
|
function recordBackendRequestSample(sample: BackendRequestSample) {
|
|
requestSamples.push(sample);
|
|
if (requestSamples.length > maxEvidenceSamples) requestSamples.splice(0, requestSamples.length - maxEvidenceSamples);
|
|
}
|
|
|
|
function recordWorkbenchProjection(input: WorkbenchProjectionInput = {}) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
status: normalizeStatusLabel(input.status ?? "unknown"),
|
|
reason: normalizeReason(input.reason ?? "none"),
|
|
projection_status: normalizeProjectionStatus(input.projectionStatus ?? "unknown"),
|
|
source: normalizeSourceLabel(input.source ?? "agentrun_v01")
|
|
};
|
|
const sourceSeq = finiteNonNegativeNumber(input.sourceLatestSeq);
|
|
const projectedSeq = finiteNonNegativeNumber(input.lastProjectedSeq);
|
|
if (sourceSeq !== null && projectedSeq !== null) {
|
|
const lagEvents = isTerminalProjectionStatus(labels.projection_status)
|
|
? 0
|
|
: Math.max(0, Math.floor(sourceSeq - projectedSeq));
|
|
if (!recordHistogram(projectionLagEventsSeries, labels, lagEvents, PROJECTION_LAG_EVENT_BUCKETS, maxSeries)) incrementDropped("projection_lag_events_series_limit");
|
|
setGauge(projectionStuckGaugeSeries, labels, lagEvents > 0 && labels.projection_status !== "caught_up" ? 1 : 0, maxSeries) || incrementDropped("projection_stuck_series_limit");
|
|
}
|
|
const sourceAt = unixMs(input.sourceLatestEventAt);
|
|
const projectedAt = unixMs(input.projectedLatestEventAt);
|
|
if (sourceAt !== null && projectedAt !== null) {
|
|
const lagSeconds = isTerminalProjectionStatus(labels.projection_status) ? 0 : Math.max(0, (sourceAt - projectedAt) / 1000);
|
|
if (!recordHistogram(projectionLagSecondsSeries, labels, lagSeconds, PROJECTION_LAG_SECONDS_BUCKETS, maxSeries)) incrementDropped("projection_lag_seconds_series_limit");
|
|
}
|
|
const terminalSourceAt = unixMs(input.terminalSourceAt);
|
|
const terminalProjectedAt = unixMs(input.terminalProjectedAt) ?? (isTerminalProjectionStatus(labels.projection_status) ? projectedAt : null);
|
|
if (terminalSourceAt !== null) {
|
|
const delaySeconds = Math.max(0, ((terminalProjectedAt ?? Date.now()) - terminalSourceAt) / 1000);
|
|
if (!recordHistogram(terminalProjectionDelaySeries, labels, delaySeconds, PROJECTION_LAG_SECONDS_BUCKETS, maxSeries)) incrementDropped("terminal_projection_delay_series_limit");
|
|
}
|
|
}
|
|
|
|
function recordWorkbenchProjectionCursorAdvance(input: WorkbenchProjectorCounterInput = {}) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
status: normalizeStatusLabel(input.status ?? "advanced"),
|
|
reason: normalizeReason(input.reason ?? "sync"),
|
|
projection_status: normalizeProjectionStatus((input as any).projectionStatus ?? "projecting"),
|
|
source: normalizeSourceLabel((input as any).source ?? "agentrun_v01")
|
|
};
|
|
incrementCounterBy(projectionCursorAdvanceSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projection_cursor_advance_series_limit");
|
|
}
|
|
|
|
function recordWorkbenchTurnRead(input: WorkbenchTurnReadInput) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
route: workbenchTurnRouteTemplate(input.route),
|
|
status: normalizeStatusLabel(input.status ?? "unknown"),
|
|
degraded_reason: normalizeReason(input.degradedReason ?? "none")
|
|
};
|
|
if (!recordHistogram(turnGetDurationSeries, labels, boundedDurationMs(input.durationMs, 120_000) / 1000, WORKBENCH_TURN_DURATION_BUCKETS_SECONDS, maxSeries)) incrementDropped("turn_get_duration_series_limit");
|
|
const bytes = finiteNonNegativeNumber(input.responseBytes);
|
|
if (bytes !== null) {
|
|
const byteLabels = { ...workbenchBaseLabels, route: labels.route, status: labels.status };
|
|
if (!recordHistogram(turnGetResponseBytesSeries, byteLabels, bytes, WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS, maxSeries)) incrementDropped("turn_get_response_bytes_series_limit");
|
|
}
|
|
if (input.timedOut === true || labels.status === "timeout") {
|
|
const timeoutLabels = { ...workbenchBaseLabels, route: labels.route, degraded_reason: labels.degraded_reason };
|
|
incrementCounter(turnGetTimeoutSeries, timeoutLabels, maxSeries) || incrementDropped("turn_get_timeout_series_limit");
|
|
}
|
|
}
|
|
|
|
function recordWorkbenchCache(input: WorkbenchCacheInput) {
|
|
const route = workbenchCacheRouteTemplate(input.route);
|
|
const cacheKeyClass = normalizeCacheKeyClass(input.cacheKeyClass);
|
|
const cacheStatus = normalizeCacheStatus(input.cacheStatus);
|
|
const operation = normalizeCacheOperation(input.operation ?? "read");
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
route,
|
|
cache_key_class: cacheKeyClass,
|
|
operation,
|
|
cache_status: cacheStatus
|
|
};
|
|
incrementCounter(cacheRequestSeries, labels, maxSeries) || incrementDropped("workbench_cache_request_series_limit");
|
|
const durationMs = finiteNonNegativeNumber(input.durationMs);
|
|
const durationSeconds = durationMs === null ? null : boundedDurationMs(durationMs, 120_000) / 1000;
|
|
if (durationSeconds !== null) {
|
|
if (!recordHistogram(cacheOperationDurationSeries, labels, durationSeconds, WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS, maxSeries)) incrementDropped("workbench_cache_operation_series_limit");
|
|
}
|
|
const payloadBytes = finiteNonNegativeNumber(input.payloadBytes);
|
|
if (payloadBytes !== null) {
|
|
if (!recordHistogram(cachePayloadBytesSeries, labels, payloadBytes, WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS, maxSeries)) incrementDropped("workbench_cache_payload_series_limit");
|
|
}
|
|
if (cacheStatus === "stale") {
|
|
incrementCounter(cacheStaleSeries, labels, maxSeries) || incrementDropped("workbench_cache_stale_series_limit");
|
|
}
|
|
if (cacheStatus === "unavailable" || cacheStatus === "set_failed" || cacheStatus === "error") {
|
|
incrementCounter(cacheUnavailableSeries, labels, maxSeries) || incrementDropped("workbench_cache_unavailable_series_limit");
|
|
}
|
|
const dbQueryAvoided = booleanValue(input.dbQueryAvoided);
|
|
if (dbQueryAvoided) {
|
|
const avoidedLabels = { ...workbenchBaseLabels, route, cache_key_class: cacheKeyClass };
|
|
incrementCounter(dbQueryAvoidedSeries, avoidedLabels, maxSeries) || incrementDropped("workbench_db_query_avoided_series_limit");
|
|
}
|
|
cacheSamples.push({
|
|
atMs: Date.now(),
|
|
route,
|
|
cacheKeyClass,
|
|
cacheStatus,
|
|
operation,
|
|
durationSeconds,
|
|
payloadBytes,
|
|
cacheAgeMs: finiteNonNegativeNumber(input.cacheAgeMs),
|
|
dbQueryAvoided,
|
|
freshnessSloMs: finiteNonNegativeNumber(input.freshnessSloMs),
|
|
projectionSeq: finiteNonNegativeNumber(input.projectionSeq)
|
|
});
|
|
while (cacheSamples.length > maxEvidenceSamples) cacheSamples.shift();
|
|
}
|
|
|
|
function recordAgentRunResult(input: AgentRunResultInput = {}) {
|
|
const eventCount = finiteNonNegativeNumber(input.eventCount ?? input.eventsScanned) ?? 0;
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
event_count_bucket: agentRunEventCountBucket(eventCount),
|
|
status: normalizeStatusLabel(input.status ?? (input.timedOut ? "timeout" : "unknown"))
|
|
};
|
|
const durationMs = finiteNonNegativeNumber(input.durationMs);
|
|
if (durationMs !== null) {
|
|
if (!recordHistogram(agentRunResultDurationSeries, labels, boundedDurationMs(durationMs, 120_000) / 1000, AGENTRUN_RESULT_BUCKETS_SECONDS, maxSeries)) incrementDropped("agentrun_result_duration_series_limit");
|
|
}
|
|
const pagesScanned = finiteNonNegativeNumber(input.pagesScanned);
|
|
if (pagesScanned !== null) {
|
|
if (!recordHistogram(agentRunResultPagesSeries, workbenchBaseLabels, pagesScanned, AGENTRUN_RESULT_PAGES_BUCKETS, maxSeries)) incrementDropped("agentrun_result_pages_series_limit");
|
|
}
|
|
const eventsScanned = finiteNonNegativeNumber(input.eventsScanned);
|
|
if (eventsScanned !== null) {
|
|
if (!recordHistogram(agentRunResultEventsSeries, workbenchBaseLabels, eventsScanned, AGENTRUN_RESULT_EVENTS_BUCKETS, maxSeries)) incrementDropped("agentrun_result_events_series_limit");
|
|
}
|
|
if (input.timedOut === true || labels.status === "timeout") {
|
|
const timeoutLabels = {
|
|
...workbenchBaseLabels,
|
|
budget: normalizeBudgetLabel(input.budget),
|
|
status: labels.status
|
|
};
|
|
incrementCounter(agentRunResultTimeoutSeries, timeoutLabels, maxSeries) || incrementDropped("agentrun_result_timeout_series_limit");
|
|
}
|
|
}
|
|
|
|
function recordWorkbenchProjectorBatch(input: WorkbenchProjectorBatchInput) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
phase: normalizeProjectorPhase(input.phase),
|
|
status: normalizeStatusLabel(input.status ?? "ok")
|
|
};
|
|
if (!recordHistogram(projectorBatchDurationSeries, labels, boundedDurationMs(input.durationMs, 120_000) / 1000, PROJECTOR_BATCH_BUCKETS_SECONDS, maxSeries)) incrementDropped("projector_batch_series_limit");
|
|
if (labels.status === "ok") setGauge(projectorLastSuccessSeries, workbenchBaseLabels, Math.floor(Date.now() / 1000), maxSeries) || incrementDropped("projector_last_success_series_limit");
|
|
}
|
|
|
|
function recordWorkbenchProjectorCandidate(input: WorkbenchProjectorCounterInput = {}) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
status: normalizeStatusLabel(input.status ?? "seen"),
|
|
reason: normalizeReason(input.reason ?? "unknown")
|
|
};
|
|
incrementCounterBy(projectorCandidatesSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projector_candidates_series_limit");
|
|
}
|
|
|
|
function recordWorkbenchProjectorEvents(input: WorkbenchProjectorCounterInput = {}) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
event_kind: normalizeReason(input.eventKind ?? "unknown"),
|
|
status: normalizeStatusLabel(input.status ?? "ok")
|
|
};
|
|
incrementCounterBy(projectorEventsProcessedSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projector_events_series_limit");
|
|
}
|
|
|
|
function recordWorkbenchProjectorError(input: WorkbenchProjectorCounterInput = {}) {
|
|
const labels = {
|
|
...workbenchBaseLabels,
|
|
reason: normalizeReason(input.reason ?? "unknown")
|
|
};
|
|
incrementCounterBy(projectorErrorsSeries, labels, boundedCounterValue(input.count), maxSeries) || incrementDropped("projector_errors_series_limit");
|
|
}
|
|
|
|
function incrementDropped(reason: string) {
|
|
incrementCounter(droppedSeries, { ...baseLabels, reason }, Math.max(maxSeries, 16));
|
|
}
|
|
|
|
function metricsText() {
|
|
return [
|
|
"# HELP hwlab_cloud_api_requests_total Cloud API HTTP requests by low-cardinality route template.",
|
|
"# TYPE hwlab_cloud_api_requests_total counter",
|
|
...renderCounters("hwlab_cloud_api_requests_total", requestTotalSeries),
|
|
"# HELP hwlab_cloud_api_request_duration_seconds Cloud API HTTP request duration in seconds.",
|
|
"# TYPE hwlab_cloud_api_request_duration_seconds histogram",
|
|
...renderHistogram("hwlab_cloud_api_request_duration_seconds", requestDurationSeries, REQUEST_BUCKETS_SECONDS),
|
|
"# HELP hwlab_cloud_api_request_phase_duration_seconds Cloud API handler phase duration in seconds.",
|
|
"# TYPE hwlab_cloud_api_request_phase_duration_seconds histogram",
|
|
...renderHistogram("hwlab_cloud_api_request_phase_duration_seconds", requestPhaseSeries, PHASE_BUCKETS_SECONDS),
|
|
"# HELP hwlab_cloud_api_upstream_duration_seconds Cloud API upstream dependency call duration in seconds.",
|
|
"# TYPE hwlab_cloud_api_upstream_duration_seconds histogram",
|
|
...renderHistogram("hwlab_cloud_api_upstream_duration_seconds", upstreamSeries, UPSTREAM_BUCKETS_SECONDS),
|
|
"# HELP hwlab_cloud_api_performance_dropped_total Backend performance samples dropped before Prometheus export.",
|
|
"# TYPE hwlab_cloud_api_performance_dropped_total counter",
|
|
...renderCounters("hwlab_cloud_api_performance_dropped_total", droppedSeries),
|
|
"# HELP hwlab_workbench_projection_lag_events Workbench projection source seq lag in AgentRun events.",
|
|
"# TYPE hwlab_workbench_projection_lag_events histogram",
|
|
...renderHistogram("hwlab_workbench_projection_lag_events", projectionLagEventsSeries, PROJECTION_LAG_EVENT_BUCKETS),
|
|
"# HELP hwlab_workbench_projection_lag_seconds Workbench projection source time lag in seconds.",
|
|
"# TYPE hwlab_workbench_projection_lag_seconds histogram",
|
|
...renderHistogram("hwlab_workbench_projection_lag_seconds", projectionLagSecondsSeries, PROJECTION_LAG_SECONDS_BUCKETS),
|
|
"# HELP hwlab_workbench_projection_stuck_traces Low-cardinality gauge set to 1 when the observed projection is behind its source.",
|
|
"# TYPE hwlab_workbench_projection_stuck_traces gauge",
|
|
...renderGauges("hwlab_workbench_projection_stuck_traces", projectionStuckGaugeSeries),
|
|
"# HELP hwlab_workbench_projection_cursor_advance_total Workbench projection cursor advance events.",
|
|
"# TYPE hwlab_workbench_projection_cursor_advance_total counter",
|
|
...renderCounters("hwlab_workbench_projection_cursor_advance_total", projectionCursorAdvanceSeries),
|
|
"# HELP hwlab_workbench_terminal_projection_delay_seconds Delay from AgentRun terminal source to Workbench terminal projection.",
|
|
"# TYPE hwlab_workbench_terminal_projection_delay_seconds histogram",
|
|
...renderHistogram("hwlab_workbench_terminal_projection_delay_seconds", terminalProjectionDelaySeries, PROJECTION_LAG_SECONDS_BUCKETS),
|
|
"# HELP hwlab_workbench_turn_get_duration_seconds Workbench turn/read API duration in seconds.",
|
|
"# TYPE hwlab_workbench_turn_get_duration_seconds histogram",
|
|
...renderHistogram("hwlab_workbench_turn_get_duration_seconds", turnGetDurationSeries, WORKBENCH_TURN_DURATION_BUCKETS_SECONDS),
|
|
"# HELP hwlab_workbench_turn_get_timeout_total Workbench turn/read API timeout count.",
|
|
"# TYPE hwlab_workbench_turn_get_timeout_total counter",
|
|
...renderCounters("hwlab_workbench_turn_get_timeout_total", turnGetTimeoutSeries),
|
|
"# HELP hwlab_workbench_turn_get_response_bytes Workbench turn/read API response bytes.",
|
|
"# TYPE hwlab_workbench_turn_get_response_bytes histogram",
|
|
...renderHistogram("hwlab_workbench_turn_get_response_bytes", turnGetResponseBytesSeries, WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS),
|
|
"# HELP workbench_cache_requests_total Workbench derived cache requests observed by Cloud API read path.",
|
|
"# TYPE workbench_cache_requests_total counter",
|
|
...renderCounters("workbench_cache_requests_total", cacheRequestSeries),
|
|
"# HELP workbench_cache_hit_ratio Workbench derived cache hit ratio by route and cache class.",
|
|
"# TYPE workbench_cache_hit_ratio gauge",
|
|
...renderWorkbenchCacheHitRatio(cacheRequestSeries),
|
|
"# HELP workbench_cache_operation_duration_seconds Workbench derived cache operation observation duration in seconds.",
|
|
"# TYPE workbench_cache_operation_duration_seconds histogram",
|
|
...renderHistogram("workbench_cache_operation_duration_seconds", cacheOperationDurationSeries, WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS),
|
|
"# HELP workbench_cache_payload_bytes Workbench derived cache payload size in bytes.",
|
|
"# TYPE workbench_cache_payload_bytes histogram",
|
|
...renderHistogram("workbench_cache_payload_bytes", cachePayloadBytesSeries, WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS),
|
|
"# HELP workbench_cache_stale_total Workbench derived cache stale observations.",
|
|
"# TYPE workbench_cache_stale_total counter",
|
|
...renderCounters("workbench_cache_stale_total", cacheStaleSeries),
|
|
"# HELP workbench_cache_unavailable_total Workbench derived cache unavailable observations.",
|
|
"# TYPE workbench_cache_unavailable_total counter",
|
|
...renderCounters("workbench_cache_unavailable_total", cacheUnavailableSeries),
|
|
"# HELP workbench_db_query_avoided_total Workbench DB reads avoided by derived cache hits.",
|
|
"# TYPE workbench_db_query_avoided_total counter",
|
|
...renderCounters("workbench_db_query_avoided_total", dbQueryAvoidedSeries),
|
|
"# HELP hwlab_agentrun_result_duration_seconds AgentRun command result aggregation duration in seconds.",
|
|
"# TYPE hwlab_agentrun_result_duration_seconds histogram",
|
|
...renderHistogram("hwlab_agentrun_result_duration_seconds", agentRunResultDurationSeries, AGENTRUN_RESULT_BUCKETS_SECONDS),
|
|
"# HELP hwlab_agentrun_result_pages_scanned AgentRun result estimated pages scanned.",
|
|
"# TYPE hwlab_agentrun_result_pages_scanned histogram",
|
|
...renderHistogram("hwlab_agentrun_result_pages_scanned", agentRunResultPagesSeries, AGENTRUN_RESULT_PAGES_BUCKETS),
|
|
"# HELP hwlab_agentrun_result_events_scanned AgentRun result estimated events scanned.",
|
|
"# TYPE hwlab_agentrun_result_events_scanned histogram",
|
|
...renderHistogram("hwlab_agentrun_result_events_scanned", agentRunResultEventsSeries, AGENTRUN_RESULT_EVENTS_BUCKETS),
|
|
"# HELP hwlab_agentrun_result_timeout_total AgentRun result timeout count by refresh budget.",
|
|
"# TYPE hwlab_agentrun_result_timeout_total counter",
|
|
...renderCounters("hwlab_agentrun_result_timeout_total", agentRunResultTimeoutSeries),
|
|
"# HELP hwlab_workbench_projector_batch_duration_seconds Workbench Kafka projector batch phase duration in seconds.",
|
|
"# TYPE hwlab_workbench_projector_batch_duration_seconds histogram",
|
|
...renderHistogram("hwlab_workbench_projector_batch_duration_seconds", projectorBatchDurationSeries, PROJECTOR_BATCH_BUCKETS_SECONDS),
|
|
"# HELP hwlab_workbench_projector_candidates_total Workbench Kafka projector candidate count.",
|
|
"# TYPE hwlab_workbench_projector_candidates_total counter",
|
|
...renderCounters("hwlab_workbench_projector_candidates_total", projectorCandidatesSeries),
|
|
"# HELP hwlab_workbench_projector_events_processed_total Workbench Kafka projector processed event count.",
|
|
"# TYPE hwlab_workbench_projector_events_processed_total counter",
|
|
...renderCounters("hwlab_workbench_projector_events_processed_total", projectorEventsProcessedSeries),
|
|
"# HELP hwlab_workbench_projector_errors_total Workbench Kafka projector error count.",
|
|
"# TYPE hwlab_workbench_projector_errors_total counter",
|
|
...renderCounters("hwlab_workbench_projector_errors_total", projectorErrorsSeries),
|
|
"# HELP hwlab_workbench_projector_last_success_unixtime Last successful Workbench Kafka projector transaction unix time.",
|
|
"# TYPE hwlab_workbench_projector_last_success_unixtime gauge",
|
|
...renderGauges("hwlab_workbench_projector_last_success_unixtime", projectorLastSuccessSeries),
|
|
""
|
|
].join("\n");
|
|
}
|
|
|
|
function snapshot() {
|
|
return {
|
|
requestSeries: requestDurationSeries.size,
|
|
requestCount: [...requestTotalSeries.values()].reduce((sum, entry) => sum + entry.value, 0),
|
|
requestPhaseSeries: requestPhaseSeries.size,
|
|
requestPhaseCount: histogramSampleCount(requestPhaseSeries),
|
|
upstreamSeries: upstreamSeries.size,
|
|
upstreamCount: histogramSampleCount(upstreamSeries),
|
|
projectionLagSeries: projectionLagEventsSeries.size + projectionLagSecondsSeries.size,
|
|
turnReadSeries: turnGetDurationSeries.size,
|
|
cacheSeries: cacheRequestSeries.size + cacheOperationDurationSeries.size + cachePayloadBytesSeries.size + cacheStaleSeries.size + cacheUnavailableSeries.size + dbQueryAvoidedSeries.size,
|
|
agentRunResultSeries: agentRunResultDurationSeries.size,
|
|
projectorSeries: projectorBatchDurationSeries.size + projectorCandidatesSeries.size + projectorEventsProcessedSeries.size + projectorErrorsSeries.size,
|
|
droppedSeries: droppedSeries.size,
|
|
evidenceSamples: requestSamples.length,
|
|
cacheEvidenceSamples: cacheSamples.length
|
|
};
|
|
}
|
|
|
|
function evidence(input: { window?: unknown } = {}) {
|
|
const sampleWindow = resolveBackendEvidenceWindow(input.window);
|
|
const generatedAtMs = Date.now();
|
|
const observedAt = new Date(generatedAtMs).toISOString();
|
|
const cutoff = sampleWindow.milliseconds === null ? null : generatedAtMs - sampleWindow.milliseconds;
|
|
const windowSamples = requestSamples.filter((sample) => cutoff === null || (sample.atMs >= cutoff && sample.atMs <= generatedAtMs));
|
|
const windowCacheSamples = cacheSamples.filter((sample) => cutoff === null || (sample.atMs >= cutoff && sample.atMs <= generatedAtMs));
|
|
const rows = backendEvidenceRows(windowSamples);
|
|
const cache = backendCacheEvidenceRows(windowCacheSamples);
|
|
return {
|
|
schemaVersion: "hwlab-backend-performance-evidence-v1",
|
|
source: "cloud-api-backend-performance-store",
|
|
observedAt,
|
|
sampleWindow: {
|
|
id: sampleWindow.id,
|
|
label: sampleWindow.label,
|
|
seconds: sampleWindow.milliseconds === null ? null : Math.round(sampleWindow.milliseconds / 1000),
|
|
windowFrom: cutoff === null ? null : new Date(Math.max(0, cutoff)).toISOString(),
|
|
windowTo: observedAt
|
|
},
|
|
sampleCount: windowSamples.length,
|
|
retainedSampleCount: requestSamples.length,
|
|
cacheSampleCount: windowCacheSamples.length,
|
|
retainedCacheSampleCount: cacheSamples.length,
|
|
routeCount: new Set(rows.map((row) => row.route)).size,
|
|
rows,
|
|
cache,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
return {
|
|
beginHttpRequest,
|
|
createManualContext,
|
|
recordWorkbenchProjection,
|
|
recordWorkbenchProjectionCursorAdvance,
|
|
recordWorkbenchTurnRead,
|
|
recordWorkbenchCache,
|
|
recordAgentRunResult,
|
|
recordWorkbenchProjectorBatch,
|
|
recordWorkbenchProjectorCandidate,
|
|
recordWorkbenchProjectorEvents,
|
|
recordWorkbenchProjectorError,
|
|
metricsText,
|
|
snapshot,
|
|
evidence
|
|
};
|
|
}
|
|
|
|
export function withBackendPerformanceContext<T>(context: unknown, callback: () => T): T {
|
|
return backendPerformanceStorage.run(context, callback);
|
|
}
|
|
|
|
export function currentBackendPerformanceContext(): any {
|
|
return backendPerformanceStorage.getStore() ?? null;
|
|
}
|
|
|
|
export function backendPerformanceRouteTemplate(input: unknown): string {
|
|
const raw = String(input ?? "").trim() || "/";
|
|
const pathOnly = raw.split(/[?#]/u, 1)[0] || "/";
|
|
let pathname = pathOnly;
|
|
try {
|
|
pathname = new URL(raw, "http://hwlab.local").pathname || pathOnly;
|
|
} catch {
|
|
// Keep pathOnly.
|
|
}
|
|
const parts = pathname.split("/").filter(Boolean).map((part) => routeSegmentTemplate(safeDecode(part)));
|
|
const path = normalizeBackendRouteTemplate(`/${parts.join("/")}`);
|
|
return path.length > 140 ? `${path.slice(0, 137)}...` : path;
|
|
}
|
|
|
|
function normalizeBackendRouteTemplate(path: string): string {
|
|
if (path === "/v1/workbench/traces/:id/events") return "/v1/workbench/traces/:traceId/events";
|
|
if (path === "/v1/workbench/turns/:id") return "/v1/workbench/turns/:traceId";
|
|
if (path === "/v1/agent/turns/:id") return "/v1/agent/turns/:traceId";
|
|
if (path === "/v1/agent/traces/:id") return "/v1/agent/traces/:traceId";
|
|
return path;
|
|
}
|
|
|
|
function workbenchTurnRouteTemplate(input: unknown): string {
|
|
const route = backendPerformanceRouteTemplate(input);
|
|
if (route === "/v1/workbench/turns/:id") return "/v1/workbench/turns/:traceId";
|
|
if (route === "/v1/agent/turns/:id") return "/v1/agent/turns/:traceId";
|
|
return route;
|
|
}
|
|
|
|
function workbenchCacheRouteTemplate(input: unknown): string {
|
|
const route = workbenchTurnRouteTemplate(input);
|
|
if (route === "/v1/workbench/traces/:id/events") return "/v1/workbench/traces/:traceId/events";
|
|
return route;
|
|
}
|
|
|
|
function resolveBackendEvidenceWindow(input: unknown) {
|
|
const key = String(input ?? DEFAULT_BACKEND_EVIDENCE_WINDOW).trim().toLowerCase();
|
|
if (Object.prototype.hasOwnProperty.call(BACKEND_EVIDENCE_WINDOWS, key)) return BACKEND_EVIDENCE_WINDOWS[key as keyof typeof BACKEND_EVIDENCE_WINDOWS];
|
|
return BACKEND_EVIDENCE_WINDOWS[DEFAULT_BACKEND_EVIDENCE_WINDOW];
|
|
}
|
|
|
|
function backendEvidenceRows(samples: BackendRequestSample[]) {
|
|
const groups = new Map<string, { sample: BackendRequestSample; durations: number[]; bytes: number[]; phases: Map<string, { phase: string; outcome: string; durations: number[] }>; firstAt: number; lastAt: number }>();
|
|
for (const sample of samples) {
|
|
const key = stableLabelKey({ route: sample.route, method: sample.method, statusClass: sample.statusClass, outcome: sample.outcome });
|
|
let group = groups.get(key);
|
|
if (!group) {
|
|
group = { sample, durations: [], bytes: [], phases: new Map(), firstAt: sample.atMs, lastAt: sample.atMs };
|
|
groups.set(key, group);
|
|
}
|
|
group.durations.push(sample.durationSeconds);
|
|
if (sample.responseBytes !== null) group.bytes.push(sample.responseBytes);
|
|
group.firstAt = Math.min(group.firstAt, sample.atMs);
|
|
group.lastAt = Math.max(group.lastAt, sample.atMs);
|
|
for (const phase of sample.phases) {
|
|
const phaseKey = stableLabelKey({ phase: phase.phase, outcome: phase.outcome });
|
|
let phaseGroup = group.phases.get(phaseKey);
|
|
if (!phaseGroup) {
|
|
phaseGroup = { phase: phase.phase, outcome: phase.outcome, durations: [] };
|
|
group.phases.set(phaseKey, phaseGroup);
|
|
}
|
|
phaseGroup.durations.push(phase.durationSeconds);
|
|
}
|
|
}
|
|
return [...groups.values()].map((group) => {
|
|
const duration = exactEvidenceStats(group.durations);
|
|
const responseBytes = group.bytes.length ? exactEvidenceStats(group.bytes) : null;
|
|
const phases = [...group.phases.values()].map((phaseGroup) => ({
|
|
phase: phaseGroup.phase,
|
|
outcome: phaseGroup.outcome,
|
|
...exactEvidenceStats(phaseGroup.durations)
|
|
})).sort((left, right) => (right.p95 - left.p95) || (right.count - left.count)).slice(0, 8);
|
|
return {
|
|
route: group.sample.route,
|
|
method: group.sample.method,
|
|
statusClass: group.sample.statusClass,
|
|
outcome: group.sample.outcome,
|
|
count: duration.count,
|
|
average: duration.average,
|
|
p50: duration.p50,
|
|
p75: duration.p75,
|
|
p95: duration.p95,
|
|
unit: "seconds",
|
|
firstObservedAt: new Date(group.firstAt).toISOString(),
|
|
lastObservedAt: new Date(group.lastAt).toISOString(),
|
|
responseBytes: responseBytes ? { count: responseBytes.count, average: Math.round(responseBytes.average), p50: Math.round(responseBytes.p50), p75: Math.round(responseBytes.p75), p95: Math.round(responseBytes.p95), unit: "bytes" } : null,
|
|
phases,
|
|
valuesRedacted: true
|
|
};
|
|
}).sort((left, right) => (right.p95 - left.p95) || (right.count - left.count));
|
|
}
|
|
|
|
function backendCacheEvidenceRows(samples: WorkbenchCacheEvidenceSample[]) {
|
|
const groups = new Map<string, {
|
|
route: string;
|
|
cacheKeyClass: string;
|
|
count: number;
|
|
hitCount: number;
|
|
missCount: number;
|
|
staleCount: number;
|
|
unavailableCount: number;
|
|
dbQueryAvoidedCount: number;
|
|
durations: number[];
|
|
payloads: number[];
|
|
ages: number[];
|
|
freshness: number[];
|
|
projectionSeqMax: number | null;
|
|
firstAt: number;
|
|
lastAt: number;
|
|
}>();
|
|
for (const sample of samples) {
|
|
const key = stableLabelKey({ route: sample.route, cache_key_class: sample.cacheKeyClass });
|
|
let group = groups.get(key);
|
|
if (!group) {
|
|
group = { route: sample.route, cacheKeyClass: sample.cacheKeyClass, count: 0, hitCount: 0, missCount: 0, staleCount: 0, unavailableCount: 0, dbQueryAvoidedCount: 0, durations: [], payloads: [], ages: [], freshness: [], projectionSeqMax: null, firstAt: sample.atMs, lastAt: sample.atMs };
|
|
groups.set(key, group);
|
|
}
|
|
group.count += 1;
|
|
if (sample.cacheStatus === "hit") group.hitCount += 1;
|
|
if (sample.cacheStatus === "miss") group.missCount += 1;
|
|
if (sample.cacheStatus === "stale") group.staleCount += 1;
|
|
if (sample.cacheStatus === "unavailable" || sample.cacheStatus === "set_failed" || sample.cacheStatus === "error") group.unavailableCount += 1;
|
|
if (sample.dbQueryAvoided) group.dbQueryAvoidedCount += 1;
|
|
if (sample.durationSeconds !== null) group.durations.push(sample.durationSeconds);
|
|
if (sample.payloadBytes !== null) group.payloads.push(sample.payloadBytes);
|
|
if (sample.cacheAgeMs !== null) group.ages.push(sample.cacheAgeMs);
|
|
if (sample.freshnessSloMs !== null) group.freshness.push(sample.freshnessSloMs);
|
|
if (sample.projectionSeq !== null) group.projectionSeqMax = Math.max(group.projectionSeqMax ?? 0, sample.projectionSeq);
|
|
group.firstAt = Math.min(group.firstAt, sample.atMs);
|
|
group.lastAt = Math.max(group.lastAt, sample.atMs);
|
|
}
|
|
return [...groups.values()].map((group) => {
|
|
const duration = exactEvidenceStats(group.durations);
|
|
const payloadBytes = group.payloads.length ? exactEvidenceStats(group.payloads) : null;
|
|
const cacheAgeMs = group.ages.length ? exactEvidenceStats(group.ages) : null;
|
|
const freshnessSloMs = group.freshness.length ? exactEvidenceStats(group.freshness) : null;
|
|
return {
|
|
route: group.route,
|
|
cacheKeyClass: group.cacheKeyClass,
|
|
count: group.count,
|
|
hitCount: group.hitCount,
|
|
missCount: group.missCount,
|
|
staleCount: group.staleCount,
|
|
unavailableCount: group.unavailableCount,
|
|
dbQueryAvoidedCount: group.dbQueryAvoidedCount,
|
|
hitRatio: group.count > 0 ? roundEvidenceValue(group.hitCount / group.count) : 0,
|
|
dbQueryAvoidedRatio: group.count > 0 ? roundEvidenceValue(group.dbQueryAvoidedCount / group.count) : 0,
|
|
unavailableRatio: group.count > 0 ? roundEvidenceValue(group.unavailableCount / group.count) : 0,
|
|
operationDuration: { ...duration, unit: "seconds" },
|
|
payloadBytes: payloadBytes ? { count: payloadBytes.count, average: Math.round(payloadBytes.average), p50: Math.round(payloadBytes.p50), p75: Math.round(payloadBytes.p75), p95: Math.round(payloadBytes.p95), unit: "bytes" } : null,
|
|
cacheAgeMs: cacheAgeMs ? { ...cacheAgeMs, unit: "milliseconds" } : null,
|
|
freshnessSloMs: freshnessSloMs ? { ...freshnessSloMs, unit: "milliseconds" } : null,
|
|
projectionSeqMax: group.projectionSeqMax,
|
|
firstObservedAt: new Date(group.firstAt).toISOString(),
|
|
lastObservedAt: new Date(group.lastAt).toISOString(),
|
|
valuesRedacted: true
|
|
};
|
|
}).sort((left, right) => (right.unavailableRatio - left.unavailableRatio) || (right.count - left.count) || left.route.localeCompare(right.route));
|
|
}
|
|
|
|
function exactEvidenceStats(values: number[]) {
|
|
const sorted = values.filter(Number.isFinite).sort((left, right) => left - right);
|
|
const count = sorted.length;
|
|
const average = count > 0 ? sorted.reduce((sum, value) => sum + value, 0) / count : 0;
|
|
return {
|
|
count,
|
|
average: roundEvidenceValue(average),
|
|
p50: roundEvidenceValue(exactEvidenceQuantile(sorted, 0.5)),
|
|
p75: roundEvidenceValue(exactEvidenceQuantile(sorted, 0.75)),
|
|
p95: roundEvidenceValue(exactEvidenceQuantile(sorted, 0.95))
|
|
};
|
|
}
|
|
|
|
function exactEvidenceQuantile(sortedValues: number[], quantile: number) {
|
|
if (sortedValues.length <= 0) return 0;
|
|
const index = Math.min(sortedValues.length - 1, Math.max(0, Math.ceil(sortedValues.length * quantile) - 1));
|
|
return sortedValues[index] ?? 0;
|
|
}
|
|
|
|
function roundEvidenceValue(value: number) {
|
|
if (!Number.isFinite(value)) return 0;
|
|
return Math.round(value * 1000) / 1000;
|
|
}
|
|
|
|
function responseEndChunkBytes(chunk: unknown, encoding: unknown): number | null {
|
|
if (chunk === undefined || chunk === null) return null;
|
|
if (Buffer.isBuffer(chunk)) return chunk.length;
|
|
if (typeof chunk === "string") return Buffer.byteLength(chunk, typeof encoding === "string" ? encoding as BufferEncoding : "utf8");
|
|
if (chunk instanceof Uint8Array) return chunk.byteLength;
|
|
return null;
|
|
}
|
|
|
|
function serverTimingHeader(totalMs: number, phases: Array<{ name: string; durationMs: number }>) {
|
|
const entries = phases.slice(-MAX_SERVER_TIMING_ENTRIES).map((phase) => `${serverTimingToken(phase.name)};dur=${roundDurationMs(phase.durationMs)}`);
|
|
entries.push(`total;dur=${roundDurationMs(totalMs)}`);
|
|
return entries.join(", ");
|
|
}
|
|
|
|
function recordHistogram(series: Map<string, HistogramSeries>, labels: Record<string, string>, value: number, buckets: number[], maxSeries: number) {
|
|
const key = stableLabelKey(labels);
|
|
let entry = series.get(key);
|
|
if (!entry) {
|
|
if (series.size >= maxSeries) return false;
|
|
entry = { labels, buckets: buckets.map(() => 0), sum: 0, count: 0 };
|
|
series.set(key, entry);
|
|
}
|
|
for (let index = 0; index < buckets.length; index += 1) {
|
|
if (value <= buckets[index]) entry.buckets[index] += 1;
|
|
}
|
|
entry.sum += value;
|
|
entry.count += 1;
|
|
return true;
|
|
}
|
|
|
|
function incrementCounter(series: Map<string, CounterSeries>, labels: Record<string, string>, maxSeries: number) {
|
|
const key = stableLabelKey(labels);
|
|
let entry = series.get(key);
|
|
if (!entry) {
|
|
if (series.size >= maxSeries) return false;
|
|
entry = { labels, value: 0 };
|
|
series.set(key, entry);
|
|
}
|
|
entry.value += 1;
|
|
return true;
|
|
}
|
|
|
|
function incrementCounterBy(series: Map<string, CounterSeries>, labels: Record<string, string>, value: number, maxSeries: number) {
|
|
const key = stableLabelKey(labels);
|
|
let entry = series.get(key);
|
|
if (!entry) {
|
|
if (series.size >= maxSeries) return false;
|
|
entry = { labels, value: 0 };
|
|
series.set(key, entry);
|
|
}
|
|
entry.value += value;
|
|
return true;
|
|
}
|
|
|
|
function setGauge(series: Map<string, GaugeSeries>, labels: Record<string, string>, value: number, maxSeries: number) {
|
|
const key = stableLabelKey(labels);
|
|
let entry = series.get(key);
|
|
if (!entry) {
|
|
if (series.size >= maxSeries) return false;
|
|
entry = { labels, value: 0 };
|
|
series.set(key, entry);
|
|
}
|
|
entry.value = Number.isFinite(value) ? value : 0;
|
|
return true;
|
|
}
|
|
|
|
function renderCounters(metricName: string, series: Map<string, CounterSeries>) {
|
|
return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${entry.value}`);
|
|
}
|
|
|
|
function renderGauges(metricName: string, series: Map<string, GaugeSeries>) {
|
|
return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${formatPrometheusValue(entry.value)}`);
|
|
}
|
|
|
|
function renderWorkbenchCacheHitRatio(series: Map<string, CounterSeries>) {
|
|
const groups = new Map<string, { labels: Record<string, string>; hit: number; total: number }>();
|
|
for (const entry of series.values()) {
|
|
const labels = { ...entry.labels };
|
|
const cacheStatus = labels.cache_status;
|
|
delete labels.cache_status;
|
|
const key = stableLabelKey(labels);
|
|
const group = groups.get(key) ?? { labels, hit: 0, total: 0 };
|
|
if (cacheStatus === "hit") group.hit += entry.value;
|
|
group.total += entry.value;
|
|
groups.set(key, group);
|
|
}
|
|
return [...groups.values()].map((group) => `${"workbench_cache_hit_ratio"}{${renderLabels(group.labels)}} ${formatPrometheusValue(group.total > 0 ? group.hit / group.total : 0)}`);
|
|
}
|
|
|
|
function renderHistogram(metricName: string, series: Map<string, HistogramSeries>, buckets: number[]) {
|
|
const lines: string[] = [];
|
|
for (const entry of series.values()) {
|
|
for (let index = 0; index < buckets.length; index += 1) {
|
|
lines.push(`${metricName}_bucket{${renderLabels({ ...entry.labels, le: String(buckets[index]) })}} ${entry.buckets[index]}`);
|
|
}
|
|
lines.push(`${metricName}_bucket{${renderLabels({ ...entry.labels, le: "+Inf" })}} ${entry.count}`);
|
|
lines.push(`${metricName}_sum{${renderLabels(entry.labels)}} ${entry.sum.toFixed(6)}`);
|
|
lines.push(`${metricName}_count{${renderLabels(entry.labels)}} ${entry.count}`);
|
|
}
|
|
return lines;
|
|
}
|
|
|
|
function histogramSampleCount(series: Map<string, HistogramSeries>) {
|
|
return [...series.values()].reduce((sum, entry) => sum + entry.count, 0);
|
|
}
|
|
|
|
function renderLabels(labels: Record<string, string>) {
|
|
return Object.entries(labels).map(([key, value]) => `${key}="${escapeLabelValue(value)}"`).join(",");
|
|
}
|
|
|
|
function stableLabelKey(labels: Record<string, string>) {
|
|
return Object.keys(labels).sort().map((key) => `${key}=${labels[key]}`).join("\u0000");
|
|
}
|
|
|
|
function routeSegmentTemplate(segment: string) {
|
|
if (/^\d+$/u.test(segment)) return ":id";
|
|
if (/^[a-z][a-z0-9]{1,12}_[A-Za-z0-9_.:-]+$/iu.test(segment)) return ":id";
|
|
if (/^[0-9a-f]{12,}$/iu.test(segment)) return ":id";
|
|
if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/iu.test(segment)) return ":id";
|
|
if (segment.length > 48) return ":id";
|
|
return sanitizePathSegment(segment);
|
|
}
|
|
|
|
function safeUrl(value: string) {
|
|
try { return new URL(value, "http://hwlab-cloud-api.local"); } catch { return new URL("/", "http://hwlab-cloud-api.local"); }
|
|
}
|
|
|
|
function safeDecode(value: string) {
|
|
try { return decodeURIComponent(value); } catch { return value; }
|
|
}
|
|
|
|
function sanitizePathSegment(value: string) {
|
|
const sanitized = value.replace(/[^A-Za-z0-9_.:-]/gu, "_");
|
|
return sanitized.slice(0, 64) || "_";
|
|
}
|
|
|
|
function normalizeMethod(value: unknown) {
|
|
const method = String(value ?? "GET").trim().toUpperCase();
|
|
return /^(GET|POST|PUT|PATCH|DELETE|OPTIONS|HEAD)$/u.test(method) ? method : "OTHER";
|
|
}
|
|
|
|
function statusClass(value: unknown) {
|
|
const status = Number(value);
|
|
if (!Number.isFinite(status) || status <= 0) return "unknown";
|
|
return `${Math.floor(status / 100)}xx`;
|
|
}
|
|
|
|
function normalizeStatusClass(value: unknown) {
|
|
const text = String(value ?? "").trim().toLowerCase();
|
|
return /^(?:1xx|2xx|3xx|4xx|5xx|network|timeout|unknown)$/u.test(text) ? text : "unknown";
|
|
}
|
|
|
|
function normalizeStatusLabel(value: unknown) {
|
|
const text = sanitizeMetricName(value, "unknown");
|
|
return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown|running|pending|completed|failed|blocked|degraded|advanced|unchanged|scheduled|seen|skipped|caught_up|projecting|stale|missing|hit|miss|stored|set_failed|unavailable|disabled)$/u.test(text) ? text : "unknown";
|
|
}
|
|
|
|
function normalizeCacheStatus(value: unknown) {
|
|
const text = normalizeStatusLabel(value);
|
|
return /^(?:hit|miss|stale|skipped|stored|set_failed|unavailable|disabled|error|unknown)$/u.test(text) ? text : "unknown";
|
|
}
|
|
|
|
function normalizeCacheOperation(value: unknown) {
|
|
const text = sanitizeMetricName(value, "read");
|
|
return /^(?:read|get|set|write|delete|ttl|ping)$/u.test(text) ? text : "read";
|
|
}
|
|
|
|
function normalizeCacheKeyClass(value: unknown) {
|
|
const text = sanitizeMetricName(value, "unknown").slice(0, 64);
|
|
return text || "unknown";
|
|
}
|
|
|
|
function normalizeOutcome(value: unknown) {
|
|
const text = sanitizeMetricName(value, "unknown");
|
|
return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown)$/u.test(text) ? text : "unknown";
|
|
}
|
|
|
|
function normalizeProjectionStatus(value: unknown) {
|
|
const text = sanitizeMetricName(value, "unknown");
|
|
if (text === "caught_up" || text === "caughtup") return "caught_up";
|
|
return /^(?:projecting|caught_up|blocked|unknown|missing|stale|degraded|terminal|running)$/u.test(text) ? text : "unknown";
|
|
}
|
|
|
|
function isTerminalProjectionStatus(value: unknown) {
|
|
const status = normalizeProjectionStatus(value);
|
|
return status === "caught_up" || status === "terminal";
|
|
}
|
|
|
|
function normalizeReason(value: unknown) {
|
|
return sanitizeMetricName(value, "unknown").slice(0, 64) || "unknown";
|
|
}
|
|
|
|
function normalizeSourceLabel(value: unknown) {
|
|
const text = sanitizeMetricName(value, "unknown");
|
|
if (text === "agentrun_v0_1" || text === "agentrun_v01") return "agentrun_v01";
|
|
return text.slice(0, 64) || "unknown";
|
|
}
|
|
|
|
function normalizeProjectorPhase(value: unknown) {
|
|
const phase = sanitizeMetricName(value, "unknown");
|
|
return /^(?:candidate_scan|agentrun_events_fetch|event_map|trace_append|turn_finalize|session_write|status_effects|unknown)$/u.test(phase) ? phase : "unknown";
|
|
}
|
|
|
|
function normalizeBudgetLabel(value: unknown) {
|
|
const parsed = Number(value);
|
|
if (Number.isFinite(parsed) && parsed > 0) {
|
|
if (parsed <= 2500) return "turn_2_5s";
|
|
if (parsed <= 10000) return "short_10s";
|
|
if (parsed <= 30000) return "medium_30s";
|
|
if (parsed <= 120000) return "long_120s";
|
|
return "long_over_120s";
|
|
}
|
|
return normalizeReason(value ?? "unknown");
|
|
}
|
|
|
|
function agentRunEventCountBucket(value: number) {
|
|
if (value <= 500) return "0_500";
|
|
if (value <= 2000) return "501_2000";
|
|
if (value <= 5000) return "2001_5000";
|
|
return "5000_plus";
|
|
}
|
|
|
|
function sanitizeMetricName(value: unknown, fallback = "unknown") {
|
|
const text = String(value ?? "").trim().toLowerCase().replace(/[^a-z0-9_:-]+/gu, "_").replace(/[-:]+/gu, "_").replace(/^_+|_+$/gu, "");
|
|
return text || fallback;
|
|
}
|
|
|
|
function sanitizeLabelValue(value: unknown, fallback = "unknown") {
|
|
const text = String(value ?? "").trim();
|
|
if (!text) return fallback;
|
|
return text.replace(/[\n\r\t]/gu, " ").slice(0, 120);
|
|
}
|
|
|
|
function escapeLabelValue(value: string) {
|
|
return String(value).replace(/\\/gu, "\\\\").replace(/"/gu, "\\\"").replace(/\n/gu, "\\n");
|
|
}
|
|
|
|
function parsePositiveInteger(value: unknown, fallback: number) {
|
|
const parsed = Number.parseInt(String(value ?? ""), 10);
|
|
return Number.isInteger(parsed) && parsed > 0 ? parsed : fallback;
|
|
}
|
|
|
|
function boundedDurationMs(value: unknown, maxMs: number) {
|
|
const parsed = Number(value);
|
|
if (!Number.isFinite(parsed) || parsed < 0) return 0;
|
|
return Math.min(parsed, maxMs);
|
|
}
|
|
|
|
function boundedCounterValue(value: unknown) {
|
|
const parsed = Number(value ?? 1);
|
|
if (!Number.isFinite(parsed) || parsed <= 0) return 1;
|
|
return Math.min(Math.floor(parsed), 1_000_000);
|
|
}
|
|
|
|
function finiteNonNegativeNumber(value: unknown): number | null {
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
|
|
}
|
|
|
|
function booleanValue(value: unknown) {
|
|
if (value === true) return true;
|
|
const text = String(value ?? "").trim().toLowerCase();
|
|
return text === "true" || text === "1" || text === "yes";
|
|
}
|
|
|
|
function unixMs(value: unknown): number | null {
|
|
if (value instanceof Date) {
|
|
const time = value.getTime();
|
|
return Number.isFinite(time) ? time : null;
|
|
}
|
|
const numeric = Number(value);
|
|
if (Number.isFinite(numeric) && numeric > 0) return numeric > 10_000_000_000 ? numeric : numeric * 1000;
|
|
const parsed = Date.parse(String(value ?? ""));
|
|
return Number.isFinite(parsed) ? parsed : null;
|
|
}
|
|
|
|
function formatPrometheusValue(value: number) {
|
|
if (!Number.isFinite(value)) return "0";
|
|
return Number.isInteger(value) ? String(value) : value.toFixed(6);
|
|
}
|
|
|
|
function roundDurationMs(value: number) {
|
|
return Math.max(0, value).toFixed(1);
|
|
}
|
|
|
|
function serverTimingToken(value: string) {
|
|
const token = sanitizeMetricName(value, "phase").replace(/_/gu, "-");
|
|
return token || "phase";
|
|
}
|
|
|
|
function nowMs() {
|
|
if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now();
|
|
return Date.now();
|
|
}
|