fix: expose workbench cache observability
This commit is contained in:
lyon
@@ -56,6 +56,24 @@ test("backend performance store exposes low-cardinality request evidence for RUM
|
||||
assert.doesNotMatch(JSON.stringify(evidence), /ses_secret|traceId|sessionId|runId|prompt|api key/iu);
|
||||
});
|
||||
|
||||
test("backend performance store records Workbench cache metrics without cache keys or ids", () => {
|
||||
const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } });
|
||||
store.recordWorkbenchCache({ route: "/v1/workbench/turns/trc_secret_0123456789abcdef", cacheKeyClass: "turn.terminal.snapshot", cacheStatus: "hit", durationMs: 12, payloadBytes: 4096, cacheAgeMs: 5, dbQueryAvoided: true, freshnessSloMs: 300000, projectionSeq: 42 });
|
||||
store.recordWorkbenchCache({ route: "/v1/workbench/sessions", cacheKeyClass: "sessions.summary", cacheStatus: "unavailable", durationMs: 120, dbQueryAvoided: false });
|
||||
|
||||
const text = store.metricsText();
|
||||
assert.match(text, /workbench_cache_requests_total\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*cache_status="hit"[^}]*\} 1/u);
|
||||
assert.match(text, /workbench_cache_hit_ratio\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*\} 1(?:\.0+)?/u);
|
||||
assert.match(text, /workbench_cache_unavailable_total\{[^}]*route="\/v1\/workbench\/sessions"[^}]*cache_key_class="sessions_summary"[^}]*\} 1/u);
|
||||
assert.match(text, /workbench_db_query_avoided_total\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*\} 1/u);
|
||||
|
||||
const evidence = store.evidence({ window: "15m" });
|
||||
assert.equal(evidence.cacheSampleCount, 2);
|
||||
assert.ok(evidence.cache.some((row) => row.route === "/v1/workbench/turns/:traceId" && row.cacheKeyClass === "turn_terminal_snapshot" && row.hitRatio === 1));
|
||||
assert.ok(evidence.cache.some((row) => row.route === "/v1/workbench/sessions" && row.unavailableCount === 1));
|
||||
assert.doesNotMatch(`${text}\n${JSON.stringify(evidence)}`, /trc_secret|0123456789abcdef|ses_secret|prompt|api key|authorization|cookie/iu);
|
||||
});
|
||||
|
||||
test("backend performance records Code Agent trace handler phases without high-cardinality labels", async () => {
|
||||
const traceId = "trc_backend_phase_trace";
|
||||
const runId = "run_backend_phase";
|
||||
|
||||
@@ -10,6 +10,8 @@ const PROJECTION_LAG_EVENT_BUCKETS = [0, 1, 5, 10, 50, 100, 500, 1000, 2000, 500
|
||||
const PROJECTION_LAG_SECONDS_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 900, 1800, 3600];
|
||||
const WORKBENCH_TURN_DURATION_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60];
|
||||
const WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304];
|
||||
const WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10];
|
||||
const WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304];
|
||||
const AGENTRUN_RESULT_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120];
|
||||
const AGENTRUN_RESULT_EVENTS_BUCKETS = [0, 50, 100, 500, 1000, 2000, 5000, 10000];
|
||||
const AGENTRUN_RESULT_PAGES_BUCKETS = [0, 1, 2, 5, 10, 20, 50, 100];
|
||||
@@ -93,6 +95,19 @@ interface WorkbenchTurnReadInput {
|
||||
timedOut?: boolean;
|
||||
}
|
||||
|
||||
interface WorkbenchCacheInput {
|
||||
route: string;
|
||||
cacheKeyClass?: unknown;
|
||||
cacheStatus?: unknown;
|
||||
operation?: unknown;
|
||||
durationMs?: unknown;
|
||||
payloadBytes?: unknown;
|
||||
cacheAgeMs?: unknown;
|
||||
dbQueryAvoided?: unknown;
|
||||
freshnessSloMs?: unknown;
|
||||
projectionSeq?: unknown;
|
||||
}
|
||||
|
||||
interface AgentRunResultInput {
|
||||
durationMs?: unknown;
|
||||
eventsScanned?: unknown;
|
||||
@@ -127,6 +142,20 @@ interface BackendRequestSample {
|
||||
phases: Array<{ phase: string; outcome: string; durationSeconds: number }>;
|
||||
}
|
||||
|
||||
interface WorkbenchCacheEvidenceSample {
|
||||
atMs: number;
|
||||
route: string;
|
||||
cacheKeyClass: string;
|
||||
cacheStatus: string;
|
||||
operation: string;
|
||||
durationSeconds: number | null;
|
||||
payloadBytes: number | null;
|
||||
cacheAgeMs: number | null;
|
||||
dbQueryAvoided: boolean;
|
||||
freshnessSloMs: number | null;
|
||||
projectionSeq: number | null;
|
||||
}
|
||||
|
||||
export function createBackendPerformanceStore(options: BackendPerformanceStoreOptions = {}) {
|
||||
const env = options.env ?? process.env;
|
||||
const maxSeries = parsePositiveInteger(env.HWLAB_BACKEND_PERFORMANCE_MAX_SERIES, options.maxSeries ?? 320);
|
||||
@@ -144,6 +173,12 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
|
||||
const turnGetDurationSeries = new Map<string, HistogramSeries>();
|
||||
const turnGetTimeoutSeries = new Map<string, CounterSeries>();
|
||||
const turnGetResponseBytesSeries = new Map<string, HistogramSeries>();
|
||||
const cacheRequestSeries = new Map<string, CounterSeries>();
|
||||
const cacheOperationDurationSeries = new Map<string, HistogramSeries>();
|
||||
const cachePayloadBytesSeries = new Map<string, HistogramSeries>();
|
||||
const cacheStaleSeries = new Map<string, CounterSeries>();
|
||||
const cacheUnavailableSeries = new Map<string, CounterSeries>();
|
||||
const dbQueryAvoidedSeries = new Map<string, CounterSeries>();
|
||||
const agentRunResultDurationSeries = new Map<string, HistogramSeries>();
|
||||
const agentRunResultPagesSeries = new Map<string, HistogramSeries>();
|
||||
const agentRunResultEventsSeries = new Map<string, HistogramSeries>();
|
||||
@@ -154,6 +189,7 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
|
||||
const projectorErrorsSeries = new Map<string, CounterSeries>();
|
||||
const projectorLastSuccessSeries = new Map<string, GaugeSeries>();
|
||||
const requestSamples: BackendRequestSample[] = [];
|
||||
const cacheSamples: WorkbenchCacheEvidenceSample[] = [];
|
||||
const baseLabels = {
|
||||
service: "hwlab-cloud-api",
|
||||
namespace: sanitizeLabelValue(env.POD_NAMESPACE ?? env.HWLAB_METRICS_NAMESPACE ?? "hwlab-v02"),
|
||||
@@ -340,6 +376,55 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
|
||||
}
|
||||
}
|
||||
|
||||
function recordWorkbenchCache(input: WorkbenchCacheInput) {
|
||||
const route = workbenchCacheRouteTemplate(input.route);
|
||||
const cacheKeyClass = normalizeCacheKeyClass(input.cacheKeyClass);
|
||||
const cacheStatus = normalizeCacheStatus(input.cacheStatus);
|
||||
const operation = normalizeCacheOperation(input.operation ?? "read");
|
||||
const labels = {
|
||||
...workbenchBaseLabels,
|
||||
route,
|
||||
cache_key_class: cacheKeyClass,
|
||||
operation,
|
||||
cache_status: cacheStatus
|
||||
};
|
||||
incrementCounter(cacheRequestSeries, labels, maxSeries) || incrementDropped("workbench_cache_request_series_limit");
|
||||
const durationMs = finiteNonNegativeNumber(input.durationMs);
|
||||
const durationSeconds = durationMs === null ? null : boundedDurationMs(durationMs, 120_000) / 1000;
|
||||
if (durationSeconds !== null) {
|
||||
if (!recordHistogram(cacheOperationDurationSeries, labels, durationSeconds, WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS, maxSeries)) incrementDropped("workbench_cache_operation_series_limit");
|
||||
}
|
||||
const payloadBytes = finiteNonNegativeNumber(input.payloadBytes);
|
||||
if (payloadBytes !== null) {
|
||||
if (!recordHistogram(cachePayloadBytesSeries, labels, payloadBytes, WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS, maxSeries)) incrementDropped("workbench_cache_payload_series_limit");
|
||||
}
|
||||
if (cacheStatus === "stale") {
|
||||
incrementCounter(cacheStaleSeries, labels, maxSeries) || incrementDropped("workbench_cache_stale_series_limit");
|
||||
}
|
||||
if (cacheStatus === "unavailable" || cacheStatus === "set_failed" || cacheStatus === "error") {
|
||||
incrementCounter(cacheUnavailableSeries, labels, maxSeries) || incrementDropped("workbench_cache_unavailable_series_limit");
|
||||
}
|
||||
const dbQueryAvoided = booleanValue(input.dbQueryAvoided);
|
||||
if (dbQueryAvoided) {
|
||||
const avoidedLabels = { ...workbenchBaseLabels, route, cache_key_class: cacheKeyClass };
|
||||
incrementCounter(dbQueryAvoidedSeries, avoidedLabels, maxSeries) || incrementDropped("workbench_db_query_avoided_series_limit");
|
||||
}
|
||||
cacheSamples.push({
|
||||
atMs: Date.now(),
|
||||
route,
|
||||
cacheKeyClass,
|
||||
cacheStatus,
|
||||
operation,
|
||||
durationSeconds,
|
||||
payloadBytes,
|
||||
cacheAgeMs: finiteNonNegativeNumber(input.cacheAgeMs),
|
||||
dbQueryAvoided,
|
||||
freshnessSloMs: finiteNonNegativeNumber(input.freshnessSloMs),
|
||||
projectionSeq: finiteNonNegativeNumber(input.projectionSeq)
|
||||
});
|
||||
while (cacheSamples.length > maxEvidenceSamples) cacheSamples.shift();
|
||||
}
|
||||
|
||||
function recordAgentRunResult(input: AgentRunResultInput = {}) {
|
||||
const eventCount = finiteNonNegativeNumber(input.eventCount ?? input.eventsScanned) ?? 0;
|
||||
const labels = {
|
||||
@@ -450,6 +535,27 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
|
||||
"# HELP hwlab_workbench_turn_get_response_bytes Workbench turn/read API response bytes.",
|
||||
"# TYPE hwlab_workbench_turn_get_response_bytes histogram",
|
||||
...renderHistogram("hwlab_workbench_turn_get_response_bytes", turnGetResponseBytesSeries, WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS),
|
||||
"# HELP workbench_cache_requests_total Workbench derived cache requests observed by Cloud API read path.",
|
||||
"# TYPE workbench_cache_requests_total counter",
|
||||
...renderCounters("workbench_cache_requests_total", cacheRequestSeries),
|
||||
"# HELP workbench_cache_hit_ratio Workbench derived cache hit ratio by route and cache class.",
|
||||
"# TYPE workbench_cache_hit_ratio gauge",
|
||||
...renderWorkbenchCacheHitRatio(cacheRequestSeries),
|
||||
"# HELP workbench_cache_operation_duration_seconds Workbench derived cache operation observation duration in seconds.",
|
||||
"# TYPE workbench_cache_operation_duration_seconds histogram",
|
||||
...renderHistogram("workbench_cache_operation_duration_seconds", cacheOperationDurationSeries, WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS),
|
||||
"# HELP workbench_cache_payload_bytes Workbench derived cache payload size in bytes.",
|
||||
"# TYPE workbench_cache_payload_bytes histogram",
|
||||
...renderHistogram("workbench_cache_payload_bytes", cachePayloadBytesSeries, WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS),
|
||||
"# HELP workbench_cache_stale_total Workbench derived cache stale observations.",
|
||||
"# TYPE workbench_cache_stale_total counter",
|
||||
...renderCounters("workbench_cache_stale_total", cacheStaleSeries),
|
||||
"# HELP workbench_cache_unavailable_total Workbench derived cache unavailable observations.",
|
||||
"# TYPE workbench_cache_unavailable_total counter",
|
||||
...renderCounters("workbench_cache_unavailable_total", cacheUnavailableSeries),
|
||||
"# HELP workbench_db_query_avoided_total Workbench DB reads avoided by derived cache hits.",
|
||||
"# TYPE workbench_db_query_avoided_total counter",
|
||||
...renderCounters("workbench_db_query_avoided_total", dbQueryAvoidedSeries),
|
||||
"# HELP hwlab_agentrun_result_duration_seconds AgentRun command result aggregation duration in seconds.",
|
||||
"# TYPE hwlab_agentrun_result_duration_seconds histogram",
|
||||
...renderHistogram("hwlab_agentrun_result_duration_seconds", agentRunResultDurationSeries, AGENTRUN_RESULT_BUCKETS_SECONDS),
|
||||
@@ -491,10 +597,12 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
|
||||
upstreamCount: histogramSampleCount(upstreamSeries),
|
||||
projectionLagSeries: projectionLagEventsSeries.size + projectionLagSecondsSeries.size,
|
||||
turnReadSeries: turnGetDurationSeries.size,
|
||||
cacheSeries: cacheRequestSeries.size + cacheOperationDurationSeries.size + cachePayloadBytesSeries.size + cacheStaleSeries.size + cacheUnavailableSeries.size + dbQueryAvoidedSeries.size,
|
||||
agentRunResultSeries: agentRunResultDurationSeries.size,
|
||||
projectorSeries: projectorBatchDurationSeries.size + projectorCandidatesSeries.size + projectorEventsProcessedSeries.size + projectorErrorsSeries.size,
|
||||
droppedSeries: droppedSeries.size,
|
||||
evidenceSamples: requestSamples.length
|
||||
evidenceSamples: requestSamples.length,
|
||||
cacheEvidenceSamples: cacheSamples.length
|
||||
};
|
||||
}
|
||||
|
||||
@@ -504,7 +612,9 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
|
||||
const observedAt = new Date(generatedAtMs).toISOString();
|
||||
const cutoff = sampleWindow.milliseconds === null ? null : generatedAtMs - sampleWindow.milliseconds;
|
||||
const windowSamples = requestSamples.filter((sample) => cutoff === null || (sample.atMs >= cutoff && sample.atMs <= generatedAtMs));
|
||||
const windowCacheSamples = cacheSamples.filter((sample) => cutoff === null || (sample.atMs >= cutoff && sample.atMs <= generatedAtMs));
|
||||
const rows = backendEvidenceRows(windowSamples);
|
||||
const cache = backendCacheEvidenceRows(windowCacheSamples);
|
||||
return {
|
||||
schemaVersion: "hwlab-backend-performance-evidence-v1",
|
||||
source: "cloud-api-backend-performance-store",
|
||||
@@ -518,8 +628,11 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
|
||||
},
|
||||
sampleCount: windowSamples.length,
|
||||
retainedSampleCount: requestSamples.length,
|
||||
cacheSampleCount: windowCacheSamples.length,
|
||||
retainedCacheSampleCount: cacheSamples.length,
|
||||
routeCount: new Set(rows.map((row) => row.route)).size,
|
||||
rows,
|
||||
cache,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
@@ -530,6 +643,7 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
|
||||
recordWorkbenchProjection,
|
||||
recordWorkbenchProjectionCursorAdvance,
|
||||
recordWorkbenchTurnRead,
|
||||
recordWorkbenchCache,
|
||||
recordAgentRunResult,
|
||||
recordWorkbenchProjectorBatch,
|
||||
recordWorkbenchProjectorCandidate,
|
||||
@@ -570,6 +684,12 @@ function workbenchTurnRouteTemplate(input: unknown): string {
|
||||
return route;
|
||||
}
|
||||
|
||||
function workbenchCacheRouteTemplate(input: unknown): string {
|
||||
const route = workbenchTurnRouteTemplate(input);
|
||||
if (route === "/v1/workbench/traces/:id/events") return "/v1/workbench/traces/:traceId/events";
|
||||
return route;
|
||||
}
|
||||
|
||||
function resolveBackendEvidenceWindow(input: unknown) {
|
||||
const key = String(input ?? DEFAULT_BACKEND_EVIDENCE_WINDOW).trim().toLowerCase();
|
||||
if (Object.prototype.hasOwnProperty.call(BACKEND_EVIDENCE_WINDOWS, key)) return BACKEND_EVIDENCE_WINDOWS[key as keyof typeof BACKEND_EVIDENCE_WINDOWS];
|
||||
@@ -627,6 +747,74 @@ function backendEvidenceRows(samples: BackendRequestSample[]) {
|
||||
}).sort((left, right) => (right.p95 - left.p95) || (right.count - left.count));
|
||||
}
|
||||
|
||||
function backendCacheEvidenceRows(samples: WorkbenchCacheEvidenceSample[]) {
|
||||
const groups = new Map<string, {
|
||||
route: string;
|
||||
cacheKeyClass: string;
|
||||
count: number;
|
||||
hitCount: number;
|
||||
missCount: number;
|
||||
staleCount: number;
|
||||
unavailableCount: number;
|
||||
dbQueryAvoidedCount: number;
|
||||
durations: number[];
|
||||
payloads: number[];
|
||||
ages: number[];
|
||||
freshness: number[];
|
||||
projectionSeqMax: number | null;
|
||||
firstAt: number;
|
||||
lastAt: number;
|
||||
}>();
|
||||
for (const sample of samples) {
|
||||
const key = stableLabelKey({ route: sample.route, cache_key_class: sample.cacheKeyClass });
|
||||
let group = groups.get(key);
|
||||
if (!group) {
|
||||
group = { route: sample.route, cacheKeyClass: sample.cacheKeyClass, count: 0, hitCount: 0, missCount: 0, staleCount: 0, unavailableCount: 0, dbQueryAvoidedCount: 0, durations: [], payloads: [], ages: [], freshness: [], projectionSeqMax: null, firstAt: sample.atMs, lastAt: sample.atMs };
|
||||
groups.set(key, group);
|
||||
}
|
||||
group.count += 1;
|
||||
if (sample.cacheStatus === "hit") group.hitCount += 1;
|
||||
if (sample.cacheStatus === "miss") group.missCount += 1;
|
||||
if (sample.cacheStatus === "stale") group.staleCount += 1;
|
||||
if (sample.cacheStatus === "unavailable" || sample.cacheStatus === "set_failed" || sample.cacheStatus === "error") group.unavailableCount += 1;
|
||||
if (sample.dbQueryAvoided) group.dbQueryAvoidedCount += 1;
|
||||
if (sample.durationSeconds !== null) group.durations.push(sample.durationSeconds);
|
||||
if (sample.payloadBytes !== null) group.payloads.push(sample.payloadBytes);
|
||||
if (sample.cacheAgeMs !== null) group.ages.push(sample.cacheAgeMs);
|
||||
if (sample.freshnessSloMs !== null) group.freshness.push(sample.freshnessSloMs);
|
||||
if (sample.projectionSeq !== null) group.projectionSeqMax = Math.max(group.projectionSeqMax ?? 0, sample.projectionSeq);
|
||||
group.firstAt = Math.min(group.firstAt, sample.atMs);
|
||||
group.lastAt = Math.max(group.lastAt, sample.atMs);
|
||||
}
|
||||
return [...groups.values()].map((group) => {
|
||||
const duration = exactEvidenceStats(group.durations);
|
||||
const payloadBytes = group.payloads.length ? exactEvidenceStats(group.payloads) : null;
|
||||
const cacheAgeMs = group.ages.length ? exactEvidenceStats(group.ages) : null;
|
||||
const freshnessSloMs = group.freshness.length ? exactEvidenceStats(group.freshness) : null;
|
||||
return {
|
||||
route: group.route,
|
||||
cacheKeyClass: group.cacheKeyClass,
|
||||
count: group.count,
|
||||
hitCount: group.hitCount,
|
||||
missCount: group.missCount,
|
||||
staleCount: group.staleCount,
|
||||
unavailableCount: group.unavailableCount,
|
||||
dbQueryAvoidedCount: group.dbQueryAvoidedCount,
|
||||
hitRatio: group.count > 0 ? roundEvidenceValue(group.hitCount / group.count) : 0,
|
||||
dbQueryAvoidedRatio: group.count > 0 ? roundEvidenceValue(group.dbQueryAvoidedCount / group.count) : 0,
|
||||
unavailableRatio: group.count > 0 ? roundEvidenceValue(group.unavailableCount / group.count) : 0,
|
||||
operationDuration: { ...duration, unit: "seconds" },
|
||||
payloadBytes: payloadBytes ? { count: payloadBytes.count, average: Math.round(payloadBytes.average), p50: Math.round(payloadBytes.p50), p75: Math.round(payloadBytes.p75), p95: Math.round(payloadBytes.p95), unit: "bytes" } : null,
|
||||
cacheAgeMs: cacheAgeMs ? { ...cacheAgeMs, unit: "milliseconds" } : null,
|
||||
freshnessSloMs: freshnessSloMs ? { ...freshnessSloMs, unit: "milliseconds" } : null,
|
||||
projectionSeqMax: group.projectionSeqMax,
|
||||
firstObservedAt: new Date(group.firstAt).toISOString(),
|
||||
lastObservedAt: new Date(group.lastAt).toISOString(),
|
||||
valuesRedacted: true
|
||||
};
|
||||
}).sort((left, right) => (right.unavailableRatio - left.unavailableRatio) || (right.count - left.count) || left.route.localeCompare(right.route));
|
||||
}
|
||||
|
||||
function exactEvidenceStats(values: number[]) {
|
||||
const sorted = values.filter(Number.isFinite).sort((left, right) => left - right);
|
||||
const count = sorted.length;
|
||||
@@ -725,6 +913,21 @@ function renderGauges(metricName: string, series: Map<string, GaugeSeries>) {
|
||||
return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${formatPrometheusValue(entry.value)}`);
|
||||
}
|
||||
|
||||
function renderWorkbenchCacheHitRatio(series: Map<string, CounterSeries>) {
|
||||
const groups = new Map<string, { labels: Record<string, string>; hit: number; total: number }>();
|
||||
for (const entry of series.values()) {
|
||||
const labels = { ...entry.labels };
|
||||
const cacheStatus = labels.cache_status;
|
||||
delete labels.cache_status;
|
||||
const key = stableLabelKey(labels);
|
||||
const group = groups.get(key) ?? { labels, hit: 0, total: 0 };
|
||||
if (cacheStatus === "hit") group.hit += entry.value;
|
||||
group.total += entry.value;
|
||||
groups.set(key, group);
|
||||
}
|
||||
return [...groups.values()].map((group) => `${"workbench_cache_hit_ratio"}{${renderLabels(group.labels)}} ${formatPrometheusValue(group.total > 0 ? group.hit / group.total : 0)}`);
|
||||
}
|
||||
|
||||
function renderHistogram(metricName: string, series: Map<string, HistogramSeries>, buckets: number[]) {
|
||||
const lines: string[] = [];
|
||||
for (const entry of series.values()) {
|
||||
@@ -790,7 +993,22 @@ function normalizeStatusClass(value: unknown) {
|
||||
|
||||
function normalizeStatusLabel(value: unknown) {
|
||||
const text = sanitizeMetricName(value, "unknown");
|
||||
return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown|running|pending|completed|failed|blocked|degraded|advanced|unchanged|scheduled|seen|skipped|caught_up|projecting|stale|missing)$/u.test(text) ? text : "unknown";
|
||||
return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown|running|pending|completed|failed|blocked|degraded|advanced|unchanged|scheduled|seen|skipped|caught_up|projecting|stale|missing|hit|miss|stored|set_failed|unavailable|disabled)$/u.test(text) ? text : "unknown";
|
||||
}
|
||||
|
||||
function normalizeCacheStatus(value: unknown) {
|
||||
const text = normalizeStatusLabel(value);
|
||||
return /^(?:hit|miss|stale|skipped|stored|set_failed|unavailable|disabled|error|unknown)$/u.test(text) ? text : "unknown";
|
||||
}
|
||||
|
||||
function normalizeCacheOperation(value: unknown) {
|
||||
const text = sanitizeMetricName(value, "read");
|
||||
return /^(?:read|get|set|write|delete|ttl|ping)$/u.test(text) ? text : "read";
|
||||
}
|
||||
|
||||
function normalizeCacheKeyClass(value: unknown) {
|
||||
const text = sanitizeMetricName(value, "unknown").slice(0, 64);
|
||||
return text || "unknown";
|
||||
}
|
||||
|
||||
function normalizeOutcome(value: unknown) {
|
||||
@@ -880,6 +1098,12 @@ function finiteNonNegativeNumber(value: unknown): number | null {
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
|
||||
}
|
||||
|
||||
function booleanValue(value: unknown) {
|
||||
if (value === true) return true;
|
||||
const text = String(value ?? "").trim().toLowerCase();
|
||||
return text === "true" || text === "1" || text === "yes";
|
||||
}
|
||||
|
||||
function unixMs(value: unknown): number | null {
|
||||
if (value instanceof Date) {
|
||||
const time = value.getTime();
|
||||
|
||||
@@ -300,6 +300,25 @@ function recordWorkbenchTurnReadMetric(options = {}, url, statusCode, body = {},
|
||||
});
|
||||
}
|
||||
|
||||
function recordWorkbenchCacheMetric(options = {}, url, body = {}, startedAt = nowMs()) {
|
||||
const performanceStore = options.backendPerformanceStore ?? options.backendPerformance;
|
||||
if (!performanceStore?.recordWorkbenchCache) return;
|
||||
const cache = body?.cache && typeof body.cache === "object" ? body.cache : null;
|
||||
if (!cache) return;
|
||||
performanceStore.recordWorkbenchCache({
|
||||
route: url?.pathname ?? "/v1/workbench",
|
||||
cacheKeyClass: cache.class,
|
||||
cacheStatus: cache.status,
|
||||
operation: "read",
|
||||
durationMs: nowMs() - startedAt,
|
||||
payloadBytes: cache.payloadBytes,
|
||||
cacheAgeMs: cache.cacheAgeMs,
|
||||
dbQueryAvoided: cache.dbQueryAvoided,
|
||||
freshnessSloMs: cache.freshnessSloMs,
|
||||
projectionSeq: cache.projectionSeq
|
||||
});
|
||||
}
|
||||
|
||||
function recordWorkbenchProjectionMetric(options = {}, { result = null, trace = null, projection = null, diagnostic = null } = {}) {
|
||||
const performanceStore = options.backendPerformanceStore ?? options.backendPerformance;
|
||||
if (!performanceStore?.recordWorkbenchProjection || !projection) return;
|
||||
@@ -405,6 +424,7 @@ async function authenticateWorkbenchRead(request, response, options) {
|
||||
}
|
||||
|
||||
async function handleWorkbenchSessionList(request, response, url, options, actor) {
|
||||
const startedAt = nowMs();
|
||||
if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) return sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench session list is keyed by sessionId only."));
|
||||
const limit = boundedSessionListLimit(url.searchParams.get("limit"));
|
||||
const offset = cursorOffset(url.searchParams.get("cursor") ?? url.searchParams.get("after"));
|
||||
@@ -421,6 +441,7 @@ async function handleWorkbenchSessionList(request, response, url, options, actor
|
||||
includeSessionId: includeRouteId ?? null,
|
||||
actor: { id: actor.id, role: actor.role ?? "user", valuesRedacted: true }
|
||||
});
|
||||
recordWorkbenchCacheMetric(options, url, payload, startedAt);
|
||||
return sendJson(response, 200, payload);
|
||||
}
|
||||
|
||||
@@ -1147,6 +1168,7 @@ async function handleWorkbenchTurnSnapshot(response, url, options, actor, rawTur
|
||||
contractVersion: "workbench-turn-snapshot-v1",
|
||||
turn: snapshot,
|
||||
projection,
|
||||
cache: result.cache ?? null,
|
||||
projectionStatus: projection.projectionStatus,
|
||||
projectionHealth: projection.projectionHealth,
|
||||
lastProjectedSeq: projection.lastProjectedSeq,
|
||||
@@ -1162,10 +1184,12 @@ async function handleWorkbenchTurnSnapshot(response, url, options, actor, rawTur
|
||||
diagnostic: projection
|
||||
});
|
||||
recordWorkbenchTurnReadMetric(options, url, 200, body, startedAt);
|
||||
recordWorkbenchCacheMetric(options, url, body, startedAt);
|
||||
sendJson(response, 200, body);
|
||||
}
|
||||
|
||||
async function handleWorkbenchTraceEventPage(response, url, options, actor, rawTraceId) {
|
||||
const startedAt = nowMs();
|
||||
const traceId = safeTraceId(rawTraceId);
|
||||
if (!traceId) return sendJson(response, 400, workbenchError("invalid_trace_id", "traceId must start with trc_.", { traceId: rawTraceId }));
|
||||
const readModel = createWorkbenchReadModel(options, actor);
|
||||
@@ -1199,7 +1223,7 @@ async function handleWorkbenchTraceEventPage(response, url, options, actor, rawT
|
||||
page.blocker = blocker;
|
||||
}
|
||||
const responseProjection = page.blocker ? blockedTraceEventProjection(projection, page.blocker) : projection;
|
||||
sendJson(response, 200, {
|
||||
const body = {
|
||||
ok: true,
|
||||
status: page.blocker ? "blocked" : "succeeded",
|
||||
contractVersion: "workbench-trace-events-v1",
|
||||
@@ -1208,6 +1232,7 @@ async function handleWorkbenchTraceEventPage(response, url, options, actor, rawT
|
||||
threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null),
|
||||
...page,
|
||||
projection: responseProjection,
|
||||
cache: pageResult.cache ?? metadata.cache ?? null,
|
||||
projectionStatus: responseProjection.projectionStatus,
|
||||
projectionHealth: responseProjection.projectionHealth,
|
||||
lastProjectedSeq: responseProjection.lastProjectedSeq,
|
||||
@@ -1222,7 +1247,9 @@ async function handleWorkbenchTraceEventPage(response, url, options, actor, rawT
|
||||
durationMs: responseProjection.durationMs,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
});
|
||||
};
|
||||
recordWorkbenchCacheMetric(options, url, body, startedAt);
|
||||
sendJson(response, 200, body);
|
||||
}
|
||||
|
||||
function traceEventPageFromFacts(sourceEvents, options, metadata = {}) {
|
||||
|
||||
@@ -62,6 +62,21 @@ test("web performance summary exposes user-perceived route latency without high-
|
||||
responseBytes: { count: 2, average: 2048, p50: 1024, p75: 2048, p95: 4096 },
|
||||
phases: [{ phase: "agent_result_read", outcome: "ok", count: 2, average: 0.05, p50: 0.04, p75: 0.05, p95: 0.08 }]
|
||||
}],
|
||||
cache: [{
|
||||
route: "/v1/workbench/sessions",
|
||||
cacheKeyClass: "sessions_summary",
|
||||
count: 4,
|
||||
hitCount: 3,
|
||||
missCount: 1,
|
||||
staleCount: 0,
|
||||
unavailableCount: 0,
|
||||
dbQueryAvoidedCount: 3,
|
||||
hitRatio: 0.75,
|
||||
dbQueryAvoidedRatio: 0.75,
|
||||
unavailableRatio: 0,
|
||||
operationDuration: { count: 4, average: 0.12, p50: 0.08, p75: 0.11, p95: 0.18, unit: "seconds" },
|
||||
valuesRedacted: true
|
||||
}],
|
||||
valuesRedacted: true
|
||||
} });
|
||||
const json = JSON.stringify(summary);
|
||||
@@ -76,6 +91,9 @@ test("web performance summary exposes user-perceived route latency without high-
|
||||
assert.ok(summary.dashboard.cards.some((card) => card.id === "samples" && card.value === 4));
|
||||
assert.ok(summary.dashboard.trends.some((trend) => trend.id === "api-timing" && trend.points.length > 0));
|
||||
assert.ok(summary.dashboard.topN.some((group) => group.id === "top-api" && group.rows.length > 0));
|
||||
assert.ok(summary.workbenchCacheRows.some((row) => row.metric === "cache_hit_ratio" && row.cache === "sessions_summary" && row.p95 === 0.75));
|
||||
assert.ok(summary.workbenchCacheRows.some((row) => row.metric === "db_query_avoided" && row.p95 === 0.75));
|
||||
assert.ok(summary.workbenchCacheRows.some((row) => row.metric === "sessions_api_p95" && row.p95 === 0.18));
|
||||
const matchedApi = summary.apiRoutes.find((row) => row.route === "/v1/agent/chat/result/:id" && row.statusClass === "2xx");
|
||||
assert.equal(matchedApi?.backendEvidence?.status, "matched");
|
||||
assert.equal(matchedApi?.backendEvidence?.diagnostic, "backend_low_sample");
|
||||
|
||||
@@ -215,6 +215,27 @@ interface BackendPerformanceEvidenceRow {
|
||||
phases?: Array<{ phase?: unknown; outcome?: unknown; count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown }>;
|
||||
}
|
||||
|
||||
interface BackendPerformanceCacheEvidenceRow {
|
||||
route?: unknown;
|
||||
cacheKeyClass?: unknown;
|
||||
count?: unknown;
|
||||
hitCount?: unknown;
|
||||
missCount?: unknown;
|
||||
staleCount?: unknown;
|
||||
unavailableCount?: unknown;
|
||||
dbQueryAvoidedCount?: unknown;
|
||||
hitRatio?: unknown;
|
||||
dbQueryAvoidedRatio?: unknown;
|
||||
unavailableRatio?: unknown;
|
||||
operationDuration?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null;
|
||||
payloadBytes?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null;
|
||||
cacheAgeMs?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null;
|
||||
freshnessSloMs?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null;
|
||||
projectionSeqMax?: unknown;
|
||||
firstObservedAt?: unknown;
|
||||
lastObservedAt?: unknown;
|
||||
}
|
||||
|
||||
interface BackendPerformanceEvidenceSnapshot {
|
||||
schemaVersion?: unknown;
|
||||
source?: unknown;
|
||||
@@ -223,6 +244,7 @@ interface BackendPerformanceEvidenceSnapshot {
|
||||
retainedSampleCount?: unknown;
|
||||
routeCount?: unknown;
|
||||
rows?: BackendPerformanceEvidenceRow[];
|
||||
cache?: BackendPerformanceCacheEvidenceRow[];
|
||||
}
|
||||
|
||||
interface PerformanceRowBackendEvidence {
|
||||
@@ -538,7 +560,8 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions =
|
||||
const workbenchBackendEvents = applySummaryContract(workbenchBackendEventVisibleSummaryRows(aggregate.backendEventVisibleSeries, WORKBENCH_JOURNEY_BUCKETS_SECONDS), exactStats, windowBounds)
|
||||
.sort(sortByProblemThenP95)
|
||||
.slice(0, 24);
|
||||
const workbenchRows = [...workbenchJourneys, ...workbenchEventPhases, ...workbenchBackendEvents];
|
||||
const workbenchCacheRows = backendCacheEvidenceSummaryRows(backendEvidence, windowBounds);
|
||||
const workbenchRows = [...workbenchJourneys, ...workbenchEventPhases, ...workbenchBackendEvents, ...workbenchCacheRows];
|
||||
const apiRoutes = durationRows
|
||||
.filter((row) => row.metric === "api_request")
|
||||
.sort(sortByProblemThenP95)
|
||||
@@ -570,6 +593,7 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions =
|
||||
workbenchJourneySeries: aggregate.journeyDurationSeries.size,
|
||||
workbenchEventPhaseSeries: aggregate.eventPhaseSeries.size,
|
||||
workbenchBackendEventVisibleSeries: aggregate.backendEventVisibleSeries.size,
|
||||
workbenchCacheEvidenceRows: workbenchCacheRows.length,
|
||||
workbenchUiEventSeries: aggregate.workbenchUiEventSeries.size,
|
||||
routeCount: new Set([...rows, ...workbenchRows].map((row) => row.route)).size,
|
||||
problemCount: problems.length,
|
||||
@@ -639,6 +663,7 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions =
|
||||
workbenchJourneys,
|
||||
workbenchEventPhases,
|
||||
workbenchBackendEvents,
|
||||
workbenchCacheRows,
|
||||
workbenchRows: dashboard.source.workbenchRows,
|
||||
rows: dashboard.source.rows
|
||||
};
|
||||
@@ -1047,6 +1072,7 @@ function applySummaryContract(rows: PerformanceSummaryRow[], exactStatsByRow: Ma
|
||||
|
||||
function normalizeBackendEvidence(input?: BackendPerformanceEvidenceSnapshot | null) {
|
||||
const rows = Array.isArray(input?.rows) ? input.rows.map(normalizeBackendEvidenceRow).filter(Boolean) as BackendPerformanceEvidenceRow[] : [];
|
||||
const cache = Array.isArray(input?.cache) ? input.cache.map(normalizeBackendCacheEvidenceRow).filter(Boolean) as ReturnType<typeof normalizeBackendCacheEvidenceRow>[] : [];
|
||||
return {
|
||||
schemaVersion: "hwlab-backend-performance-evidence-v1",
|
||||
source: sanitizeLabelValue(input?.source ?? "cloud-api-backend-performance-store"),
|
||||
@@ -1055,6 +1081,7 @@ function normalizeBackendEvidence(input?: BackendPerformanceEvidenceSnapshot | n
|
||||
retainedSampleCount: finiteNonNegativeInteger(input?.retainedSampleCount),
|
||||
routeCount: finiteNonNegativeInteger(input?.routeCount ?? new Set(rows.map((row) => row.route)).size),
|
||||
rows: rows.slice(0, 80),
|
||||
cache: cache.slice(0, 80),
|
||||
available: Boolean(input && String(input.schemaVersion ?? "") === "hwlab-backend-performance-evidence-v1"),
|
||||
valuesRedacted: true
|
||||
};
|
||||
@@ -1100,6 +1127,44 @@ function normalizeBackendEvidencePhase(input: { phase?: unknown; outcome?: unkno
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeBackendCacheEvidenceRow(input: BackendPerformanceCacheEvidenceRow) {
|
||||
const route = webPerformanceRouteTemplate(input?.route ?? "");
|
||||
if (!route || route === "/") return null;
|
||||
return {
|
||||
route,
|
||||
cacheKeyClass: sanitizeMetricName(input.cacheKeyClass, "unknown").slice(0, 64) || "unknown",
|
||||
count: finiteNonNegativeInteger(input.count),
|
||||
hitCount: finiteNonNegativeInteger(input.hitCount),
|
||||
missCount: finiteNonNegativeInteger(input.missCount),
|
||||
staleCount: finiteNonNegativeInteger(input.staleCount),
|
||||
unavailableCount: finiteNonNegativeInteger(input.unavailableCount),
|
||||
dbQueryAvoidedCount: finiteNonNegativeInteger(input.dbQueryAvoidedCount),
|
||||
hitRatio: boundedRatio(input.hitRatio),
|
||||
dbQueryAvoidedRatio: boundedRatio(input.dbQueryAvoidedRatio),
|
||||
unavailableRatio: boundedRatio(input.unavailableRatio),
|
||||
operationDuration: normalizeMetricStats(input.operationDuration, "seconds"),
|
||||
payloadBytes: normalizeMetricStats(input.payloadBytes, "bytes"),
|
||||
cacheAgeMs: normalizeMetricStats(input.cacheAgeMs, "milliseconds"),
|
||||
freshnessSloMs: normalizeMetricStats(input.freshnessSloMs, "milliseconds"),
|
||||
projectionSeqMax: finiteNonNegativeInteger(input.projectionSeqMax),
|
||||
firstObservedAt: isoOrNull(input.firstObservedAt),
|
||||
lastObservedAt: isoOrNull(input.lastObservedAt),
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeMetricStats(input: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null | undefined, unit: string) {
|
||||
if (!input || typeof input !== "object") return null;
|
||||
return {
|
||||
count: finiteNonNegativeInteger(input.count),
|
||||
average: finiteNonNegativeMetric(input.average),
|
||||
p50: finiteNonNegativeMetric(input.p50),
|
||||
p75: finiteNonNegativeMetric(input.p75),
|
||||
p95: finiteNonNegativeMetric(input.p95),
|
||||
unit
|
||||
};
|
||||
}
|
||||
|
||||
function attachBackendEvidenceToRows(rows: PerformanceSummaryRow[], evidence: ReturnType<typeof normalizeBackendEvidence>) {
|
||||
return rows.map((row) => {
|
||||
if (row.kind !== "api" || row.metric !== "api_request") return row;
|
||||
@@ -1188,6 +1253,71 @@ function backendEvidenceDetail(input: { diagnostic: string; count: number; backe
|
||||
return `${input.count} 个后端样本已匹配，后端 P95 ${formatBackendSeconds(input.backendP95)}${phaseText}。`;
|
||||
}
|
||||
|
||||
function backendCacheEvidenceSummaryRows(evidence: ReturnType<typeof normalizeBackendEvidence>, windowBounds: { windowFrom: string | null; windowTo: string; generatedAt: string }): PerformanceSummaryRow[] {
|
||||
if (!evidence.available) return [];
|
||||
const rows: PerformanceSummaryRow[] = [];
|
||||
for (const item of evidence.cache) {
|
||||
rows.push(cacheEvidenceRow({ metric: "cache_hit_ratio", route: item.route, cache: item.cacheKeyClass, value: item.hitRatio, count: item.count, unit: "score", windowBounds, outcome: "ok" }));
|
||||
rows.push(cacheEvidenceRow({ metric: "db_query_avoided", route: item.route, cache: item.cacheKeyClass, value: item.dbQueryAvoidedRatio, count: item.count, unit: "score", windowBounds, outcome: "ok" }));
|
||||
rows.push(cacheEvidenceRow({ metric: "cache_unavailable", route: item.route, cache: item.cacheKeyClass, value: item.unavailableRatio, count: item.count, unit: "score", windowBounds, outcome: item.unavailableCount > 0 ? "error" : "ok" }));
|
||||
if (item.route === "/v1/workbench/sessions" && item.operationDuration) {
|
||||
rows.push(cacheEvidenceRow({
|
||||
metric: "sessions_api_p95",
|
||||
route: item.route,
|
||||
cache: item.cacheKeyClass,
|
||||
value: item.operationDuration.p95,
|
||||
p50: item.operationDuration.p50,
|
||||
p75: item.operationDuration.p75,
|
||||
p95: item.operationDuration.p95,
|
||||
count: item.operationDuration.count || item.count,
|
||||
unit: "seconds",
|
||||
windowBounds,
|
||||
outcome: "ok"
|
||||
}));
|
||||
}
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
function cacheEvidenceRow(input: { metric: string; route: string; cache: string; value: number; p50?: number; p75?: number; p95?: number; count: number; unit: "seconds" | "score"; outcome: string; windowBounds: { windowFrom: string | null; windowTo: string; generatedAt: string } }): PerformanceSummaryRow {
|
||||
const count = finiteNonNegativeInteger(input.count);
|
||||
const value = finiteNonNegativeMetric(input.value);
|
||||
const p50 = input.p50 === undefined ? value : finiteNonNegativeMetric(input.p50);
|
||||
const p75 = input.p75 === undefined ? value : finiteNonNegativeMetric(input.p75);
|
||||
const p95 = input.p95 === undefined ? value : finiteNonNegativeMetric(input.p95);
|
||||
const sample = sampleState(count);
|
||||
let tone: "ok" | "warn" | "blocked" | "source" = count < LOW_SAMPLE_THRESHOLD ? "source" : "ok";
|
||||
if (sample === "ok" && input.metric === "cache_unavailable" && p95 > 0) tone = "blocked";
|
||||
if (sample === "ok" && input.metric === "sessions_api_p95") tone = thresholdTone(p95, 1, 2.5);
|
||||
const problem = tone === "source" ? (count <= 0 ? "no-sample" : "low-sample") : tone === "ok" ? "ok" : input.metric === "cache_unavailable" ? "cache-unavailable" : `slow-${input.metric}`;
|
||||
return {
|
||||
kind: "workbench_cache",
|
||||
metric: input.metric,
|
||||
route: input.route,
|
||||
method: "-",
|
||||
statusClass: "-",
|
||||
outcome: input.outcome,
|
||||
count,
|
||||
average: value,
|
||||
p50,
|
||||
p75,
|
||||
p95,
|
||||
unit: input.unit,
|
||||
tone,
|
||||
problem,
|
||||
lowSample: count > 0 && count < LOW_SAMPLE_THRESHOLD,
|
||||
sampleState: sample,
|
||||
sourceFamily: "backend_cache_evidence",
|
||||
windowFrom: input.windowBounds.windowFrom,
|
||||
windowTo: input.windowBounds.windowTo,
|
||||
generatedAt: input.windowBounds.generatedAt,
|
||||
freshness: sampleFreshness(count),
|
||||
aggregationKind: "exact",
|
||||
approximation: "exact",
|
||||
cache: input.cache
|
||||
};
|
||||
}
|
||||
|
||||
function formatBackendSeconds(value: number) {
|
||||
if (!Number.isFinite(value)) return "未知";
|
||||
if (value < 1) return `${Math.round(value * 1000)}ms`;
|
||||
@@ -1199,6 +1329,12 @@ function finiteNonNegativeMetric(value: unknown) {
|
||||
return Number.isFinite(number) && number >= 0 ? roundMetric(number) : 0;
|
||||
}
|
||||
|
||||
function boundedRatio(value: unknown) {
|
||||
const number = Number(value);
|
||||
if (!Number.isFinite(number) || number < 0) return 0;
|
||||
return Math.min(1, roundMetric(number));
|
||||
}
|
||||
|
||||
function finiteNonNegativeInteger(value: unknown) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) && number >= 0 ? Math.trunc(number) : 0;
|
||||
@@ -1294,12 +1430,12 @@ function summaryToneForRow(row: PerformanceSummaryRow, p95: number, count: numbe
|
||||
|
||||
function summaryProblemForRow(row: PerformanceSummaryRow, p95: number, count: number, tone: string) {
|
||||
if (count <= 0) return "no-sample";
|
||||
if (row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible") return workbenchProblem(row.outcome, row.metric, count, tone);
|
||||
if (row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible" || row.kind === "workbench_cache") return workbenchProblem(row.outcome, row.metric, count, tone);
|
||||
return performanceProblem({ metric: row.metric, outcome: row.outcome, status_class: row.statusClass }, p95, count, row.unit === "score" ? "score" : "seconds", tone);
|
||||
}
|
||||
|
||||
function isPerformanceProblemRow(row: PerformanceSummaryRow) {
|
||||
if ((row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible") && row.sampleState !== "ok") return false;
|
||||
if ((row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible" || row.kind === "workbench_cache") && row.sampleState !== "ok") return false;
|
||||
return row.tone === "blocked" || row.tone === "warn" || row.outcome !== "ok" || row.statusClass === "network" || row.statusClass === "5xx" || row.statusClass === "4xx";
|
||||
}
|
||||
|
||||
@@ -1311,6 +1447,7 @@ function sourceFamilyForRow(row: PerformanceSummaryRow) {
|
||||
if (row.kind === "workbench_journey") return "workbench_journey";
|
||||
if (row.kind === "workbench_event_phase") return "workbench_event_phase";
|
||||
if (row.kind === "workbench_backend_event_visible") return "workbench_backend_event_visible";
|
||||
if (row.kind === "workbench_cache") return "backend_cache_evidence";
|
||||
return "unknown";
|
||||
}
|
||||
|
||||
@@ -1812,6 +1949,10 @@ function metricLabel(row: PerformanceSummaryRow) {
|
||||
submit_to_first_visible: "提交到首个可见结果",
|
||||
submit_to_failure: "提交失败可见",
|
||||
backend_event_to_visible: "后端事件到可见",
|
||||
cache_hit_ratio: "缓存命中率",
|
||||
db_query_avoided: "DB 查询避免率",
|
||||
cache_unavailable: "缓存不可用率",
|
||||
sessions_api_p95: "会话 API P95",
|
||||
session_switch_first_visible: "切换会话首个可见",
|
||||
session_switch_full_load: "切换会话完整加载",
|
||||
workbench_open_first_visible: "打开工作台首个可见",
|
||||
|
||||
@@ -34,6 +34,7 @@ export function createWorkbenchFactsStore(options = {}, actor = null) {
|
||||
count: Number.isFinite(Number(result?.count)) ? Number(result.count) : 0,
|
||||
durable: true,
|
||||
persistence: result?.persistence ?? null,
|
||||
cache: result?.cache ?? null,
|
||||
retry: attempt > 1 ? { attempts: attempt, maxAttempts: retryPolicy.maxAttempts, delaysMs: retryDelaysMs.slice(), valuesRedacted: true } : null
|
||||
};
|
||||
} catch (error) {
|
||||
|
||||
@@ -21,6 +21,9 @@ import (
|
||||
|
||||
const cacheKeySchemaVersion = "wb-derived-v1"
|
||||
|
||||
var cacheOperationDurationBucketsSeconds = []float64{0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1}
|
||||
var cachePayloadBucketsBytes = []float64{512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304}
|
||||
|
||||
var errCacheDisabled = errors.New("workbench redis cache disabled")
|
||||
|
||||
type cacheConfig struct {
|
||||
@@ -60,15 +63,38 @@ type derivedCache struct {
|
||||
}
|
||||
|
||||
type cacheMetrics struct {
|
||||
mu sync.Mutex
|
||||
Hits uint64
|
||||
Misses uint64
|
||||
Sets uint64
|
||||
Deletes uint64
|
||||
Errors uint64
|
||||
LastErrorKind string
|
||||
LastErrorOp string
|
||||
LastErrorAt time.Time
|
||||
mu sync.Mutex
|
||||
Hits uint64
|
||||
Misses uint64
|
||||
Sets uint64
|
||||
Deletes uint64
|
||||
Errors uint64
|
||||
LastErrorKind string
|
||||
LastErrorOp string
|
||||
LastErrorAt time.Time
|
||||
Requests map[string]cacheCounterSeries
|
||||
Stale map[string]cacheCounterSeries
|
||||
DBQueryAvoided map[string]cacheCounterSeries
|
||||
OperationDurations map[string]*cacheHistogramSeries
|
||||
PayloadBytes map[string]*cacheHistogramSeries
|
||||
}
|
||||
|
||||
type cacheMetricLabels struct {
|
||||
Class string
|
||||
Operation string
|
||||
Status string
|
||||
}
|
||||
|
||||
type cacheCounterSeries struct {
|
||||
Labels cacheMetricLabels
|
||||
Value uint64
|
||||
}
|
||||
|
||||
type cacheHistogramSeries struct {
|
||||
Labels cacheMetricLabels
|
||||
Buckets []uint64
|
||||
Sum float64
|
||||
Count uint64
|
||||
}
|
||||
|
||||
type cacheError struct {
|
||||
@@ -144,12 +170,15 @@ func (c *derivedCache) BuildKey(parts cacheKeyParts) (string, error) {
|
||||
}
|
||||
|
||||
func (c *derivedCache) Get(ctx context.Context, key string, target any) (bool, error) {
|
||||
startedAt := time.Now()
|
||||
class := cacheKeyClass(key)
|
||||
if !c.usable() {
|
||||
return false, nil
|
||||
}
|
||||
if err := validateCacheKey(c.config.MaxKeyBytes, key); err != nil {
|
||||
wrapped := cacheError{Op: "get", Kind: classifyCacheError(err), Err: err}
|
||||
c.metrics.recordError(wrapped)
|
||||
c.metrics.recordCacheObservation(class, "get", "error", time.Since(startedAt), 0)
|
||||
return false, wrapped
|
||||
}
|
||||
var payload []byte
|
||||
@@ -160,42 +189,51 @@ func (c *derivedCache) Get(ctx context.Context, key string, target any) (bool, e
|
||||
})
|
||||
if errors.Is(err, redis.Nil) {
|
||||
c.metrics.recordMiss()
|
||||
c.metrics.recordCacheObservation(class, "get", "miss", time.Since(startedAt), 0)
|
||||
return false, nil
|
||||
}
|
||||
if err != nil {
|
||||
wrapped := cacheError{Op: "get", Kind: classifyCacheError(err), Err: err}
|
||||
c.metrics.recordError(wrapped)
|
||||
c.metrics.recordCacheObservation(class, "get", "unavailable", time.Since(startedAt), 0)
|
||||
return false, wrapped
|
||||
}
|
||||
if len(payload) > c.config.MaxPayloadBytes {
|
||||
wrapped := cacheError{Op: "get", Kind: "cache_payload_too_large", Err: fmt.Errorf("payload bytes %d exceeds %d", len(payload), c.config.MaxPayloadBytes)}
|
||||
c.metrics.recordError(wrapped)
|
||||
c.metrics.recordCacheObservation(class, "get", "error", time.Since(startedAt), len(payload))
|
||||
return false, wrapped
|
||||
}
|
||||
if target != nil {
|
||||
if err := json.Unmarshal(payload, target); err != nil {
|
||||
wrapped := cacheError{Op: "get", Kind: "cache_payload_decode_failed", Err: err}
|
||||
c.metrics.recordError(wrapped)
|
||||
c.metrics.recordCacheObservation(class, "get", "error", time.Since(startedAt), len(payload))
|
||||
return false, wrapped
|
||||
}
|
||||
}
|
||||
c.metrics.recordHit()
|
||||
c.metrics.recordCacheObservation(class, "get", "hit", time.Since(startedAt), len(payload))
|
||||
return true, nil
|
||||
}
|
||||
|
||||
func (c *derivedCache) Set(ctx context.Context, key string, value any, ttl time.Duration) error {
|
||||
startedAt := time.Now()
|
||||
class := cacheKeyClass(key)
|
||||
if !c.usable() {
|
||||
return nil
|
||||
}
|
||||
if err := validateCacheKey(c.config.MaxKeyBytes, key); err != nil {
|
||||
wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err}
|
||||
c.metrics.recordError(wrapped)
|
||||
c.metrics.recordCacheObservation(class, "set", "error", time.Since(startedAt), 0)
|
||||
return wrapped
|
||||
}
|
||||
payload, err := cachePayloadBytes(value, c.config.MaxPayloadBytes)
|
||||
if err != nil {
|
||||
wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err}
|
||||
c.metrics.recordError(wrapped)
|
||||
c.metrics.recordCacheObservation(class, "set", "error", time.Since(startedAt), 0)
|
||||
return wrapped
|
||||
}
|
||||
err = c.withTimeout(ctx, func(opCtx context.Context) error {
|
||||
@@ -204,12 +242,36 @@ func (c *derivedCache) Set(ctx context.Context, key string, value any, ttl time.
|
||||
if err != nil {
|
||||
wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err}
|
||||
c.metrics.recordError(wrapped)
|
||||
c.metrics.recordCacheObservation(class, "set", "unavailable", time.Since(startedAt), len(payload))
|
||||
return wrapped
|
||||
}
|
||||
c.metrics.recordSet()
|
||||
c.metrics.recordCacheObservation(class, "set", "stored", time.Since(startedAt), len(payload))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *derivedCache) RecordStale(class string) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.metrics.recordStale(class)
|
||||
}
|
||||
|
||||
func (c *derivedCache) RecordDBQueryAvoided(class string) {
|
||||
if c == nil {
|
||||
return
|
||||
}
|
||||
c.metrics.recordDBQueryAvoided(class)
|
||||
}
|
||||
|
||||
func (c *derivedCache) PrometheusText(service string) string {
|
||||
if c == nil {
|
||||
var metrics cacheMetrics
|
||||
return metrics.prometheusText(service)
|
||||
}
|
||||
return c.metrics.prometheusText(service)
|
||||
}
|
||||
|
||||
func (c *derivedCache) Delete(ctx context.Context, keys ...string) error {
|
||||
if !c.usable() || len(keys) == 0 {
|
||||
return nil
|
||||
@@ -520,6 +582,15 @@ func cacheHash(value string) string {
|
||||
return hex.EncodeToString(digest[:])[:24]
|
||||
}
|
||||
|
||||
func cacheKeyClass(key string) string {
|
||||
for _, part := range strings.Split(key, ":") {
|
||||
if strings.HasPrefix(part, "class=") {
|
||||
return safeCacheToken(strings.TrimPrefix(part, "class="))
|
||||
}
|
||||
}
|
||||
return "unknown"
|
||||
}
|
||||
|
||||
func (m *cacheMetrics) recordHit() {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
@@ -556,16 +627,79 @@ func (m *cacheMetrics) recordError(err error) {
|
||||
m.LastErrorAt = time.Now().UTC()
|
||||
}
|
||||
|
||||
func (m *cacheMetrics) recordCacheObservation(class string, operation string, status string, duration time.Duration, payloadBytes int) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
labels := cacheLabels(class, operation, status)
|
||||
m.incrementCounterLocked(&m.Requests, labels)
|
||||
m.recordHistogramLocked(&m.OperationDurations, labels, duration.Seconds(), cacheOperationDurationBucketsSeconds)
|
||||
if payloadBytes > 0 {
|
||||
m.recordHistogramLocked(&m.PayloadBytes, labels, float64(payloadBytes), cachePayloadBucketsBytes)
|
||||
}
|
||||
}
|
||||
|
||||
func (m *cacheMetrics) recordStale(class string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.incrementCounterLocked(&m.Stale, cacheLabels(class, "read", "stale"))
|
||||
}
|
||||
|
||||
func (m *cacheMetrics) recordDBQueryAvoided(class string) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
m.incrementCounterLocked(&m.DBQueryAvoided, cacheLabels(class, "read", "hit"))
|
||||
}
|
||||
|
||||
func (m *cacheMetrics) incrementCounterLocked(target *map[string]cacheCounterSeries, labels cacheMetricLabels) {
|
||||
if *target == nil {
|
||||
*target = map[string]cacheCounterSeries{}
|
||||
}
|
||||
key := cacheMetricKey(labels)
|
||||
entry := (*target)[key]
|
||||
if entry.Labels.Class == "" {
|
||||
entry.Labels = labels
|
||||
}
|
||||
entry.Value++
|
||||
(*target)[key] = entry
|
||||
}
|
||||
|
||||
func (m *cacheMetrics) recordHistogramLocked(target *map[string]*cacheHistogramSeries, labels cacheMetricLabels, value float64, buckets []float64) {
|
||||
if value < 0 {
|
||||
value = 0
|
||||
}
|
||||
if *target == nil {
|
||||
*target = map[string]*cacheHistogramSeries{}
|
||||
}
|
||||
key := cacheMetricKey(labels)
|
||||
entry := (*target)[key]
|
||||
if entry == nil {
|
||||
entry = &cacheHistogramSeries{Labels: labels, Buckets: make([]uint64, len(buckets))}
|
||||
(*target)[key] = entry
|
||||
}
|
||||
for index, bucket := range buckets {
|
||||
if value <= bucket {
|
||||
entry.Buckets[index]++
|
||||
}
|
||||
}
|
||||
entry.Sum += value
|
||||
entry.Count++
|
||||
}
|
||||
|
||||
func (m *cacheMetrics) snapshot() map[string]any {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
result := map[string]any{
|
||||
"hits": int64(m.Hits),
|
||||
"misses": int64(m.Misses),
|
||||
"sets": int64(m.Sets),
|
||||
"deletes": int64(m.Deletes),
|
||||
"errors": int64(m.Errors),
|
||||
"valuesRedacted": true,
|
||||
"hits": int64(m.Hits),
|
||||
"misses": int64(m.Misses),
|
||||
"sets": int64(m.Sets),
|
||||
"deletes": int64(m.Deletes),
|
||||
"errors": int64(m.Errors),
|
||||
"requestSeries": len(m.Requests),
|
||||
"operationDurationSeries": len(m.OperationDurations),
|
||||
"payloadBytesSeries": len(m.PayloadBytes),
|
||||
"staleSeries": len(m.Stale),
|
||||
"dbQueryAvoidedSeries": len(m.DBQueryAvoided),
|
||||
"valuesRedacted": true,
|
||||
}
|
||||
if m.LastErrorKind != "" {
|
||||
result["lastErrorKind"] = m.LastErrorKind
|
||||
@@ -575,6 +709,132 @@ func (m *cacheMetrics) snapshot() map[string]any {
|
||||
return result
|
||||
}
|
||||
|
||||
func (m *cacheMetrics) prometheusText(service string) string {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
if service == "" {
|
||||
service = "hwlab-workbench-runtime"
|
||||
}
|
||||
lines := []string{
|
||||
"# HELP workbench_cache_requests_total Workbench derived Redis cache requests by low-cardinality class/status.",
|
||||
"# TYPE workbench_cache_requests_total counter",
|
||||
}
|
||||
for _, entry := range m.Requests {
|
||||
lines = append(lines, prometheusCounterLine("workbench_cache_requests_total", service, entry.Labels, entry.Value))
|
||||
}
|
||||
lines = append(lines,
|
||||
"# HELP workbench_cache_hit_ratio Workbench derived Redis cache hit ratio by class and operation.",
|
||||
"# TYPE workbench_cache_hit_ratio gauge",
|
||||
)
|
||||
for _, line := range m.prometheusHitRatioLinesLocked(service) {
|
||||
lines = append(lines, line)
|
||||
}
|
||||
lines = append(lines,
|
||||
"# HELP workbench_cache_operation_duration_seconds Workbench derived Redis cache operation duration in seconds.",
|
||||
"# TYPE workbench_cache_operation_duration_seconds histogram",
|
||||
)
|
||||
lines = append(lines, prometheusHistogramLines("workbench_cache_operation_duration_seconds", service, m.OperationDurations, cacheOperationDurationBucketsSeconds)...)
|
||||
lines = append(lines,
|
||||
"# HELP workbench_cache_payload_bytes Workbench derived Redis cache payload size in bytes.",
|
||||
"# TYPE workbench_cache_payload_bytes histogram",
|
||||
)
|
||||
lines = append(lines, prometheusHistogramLines("workbench_cache_payload_bytes", service, m.PayloadBytes, cachePayloadBucketsBytes)...)
|
||||
lines = append(lines,
|
||||
"# HELP workbench_cache_stale_total Workbench derived Redis cache stale reads detected by payload freshness metadata.",
|
||||
"# TYPE workbench_cache_stale_total counter",
|
||||
)
|
||||
for _, entry := range m.Stale {
|
||||
lines = append(lines, prometheusCounterLine("workbench_cache_stale_total", service, entry.Labels, entry.Value))
|
||||
}
|
||||
lines = append(lines,
|
||||
"# HELP workbench_cache_unavailable_total Workbench derived Redis cache unavailable/error requests.",
|
||||
"# TYPE workbench_cache_unavailable_total counter",
|
||||
)
|
||||
for _, entry := range m.Requests {
|
||||
if entry.Labels.Status == "unavailable" || entry.Labels.Status == "error" {
|
||||
lines = append(lines, prometheusCounterLine("workbench_cache_unavailable_total", service, entry.Labels, entry.Value))
|
||||
}
|
||||
}
|
||||
lines = append(lines,
|
||||
"# HELP workbench_db_query_avoided_total Workbench database reads avoided by derived Redis cache hits.",
|
||||
"# TYPE workbench_db_query_avoided_total counter",
|
||||
)
|
||||
for _, entry := range m.DBQueryAvoided {
|
||||
lines = append(lines, prometheusCounterLine("workbench_db_query_avoided_total", service, entry.Labels, entry.Value))
|
||||
}
|
||||
lines = append(lines, "")
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func (m *cacheMetrics) prometheusHitRatioLinesLocked(service string) []string {
|
||||
type totals struct{ hit, total uint64 }
|
||||
groups := map[string]totals{}
|
||||
labels := map[string]cacheMetricLabels{}
|
||||
for _, entry := range m.Requests {
|
||||
if entry.Labels.Operation != "get" {
|
||||
continue
|
||||
}
|
||||
key := entry.Labels.Class + "\x00" + entry.Labels.Operation
|
||||
group := groups[key]
|
||||
if entry.Labels.Status == "hit" {
|
||||
group.hit += entry.Value
|
||||
}
|
||||
group.total += entry.Value
|
||||
groups[key] = group
|
||||
labels[key] = cacheMetricLabels{Class: entry.Labels.Class, Operation: entry.Labels.Operation, Status: "all"}
|
||||
}
|
||||
lines := []string{}
|
||||
for key, group := range groups {
|
||||
ratio := 0.0
|
||||
if group.total > 0 {
|
||||
ratio = float64(group.hit) / float64(group.total)
|
||||
}
|
||||
lines = append(lines, prometheusGaugeLine("workbench_cache_hit_ratio", service, labels[key], ratio))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func prometheusHistogramLines(name string, service string, series map[string]*cacheHistogramSeries, buckets []float64) []string {
|
||||
lines := []string{}
|
||||
for _, entry := range series {
|
||||
for index, bucket := range buckets {
|
||||
lines = append(lines, fmt.Sprintf("%s_bucket{%s,le=\"%s\"} %d", name, prometheusLabels(service, entry.Labels), prometheusFloat(bucket), entry.Buckets[index]))
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("%s_bucket{%s,le=\"+Inf\"} %d", name, prometheusLabels(service, entry.Labels), entry.Count))
|
||||
lines = append(lines, fmt.Sprintf("%s_sum{%s} %.6f", name, prometheusLabels(service, entry.Labels), entry.Sum))
|
||||
lines = append(lines, fmt.Sprintf("%s_count{%s} %d", name, prometheusLabels(service, entry.Labels), entry.Count))
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
func prometheusCounterLine(name string, service string, labels cacheMetricLabels, value uint64) string {
|
||||
return fmt.Sprintf("%s{%s} %d", name, prometheusLabels(service, labels), value)
|
||||
}
|
||||
|
||||
func prometheusGaugeLine(name string, service string, labels cacheMetricLabels, value float64) string {
|
||||
return fmt.Sprintf("%s{%s} %.6f", name, prometheusLabels(service, labels), value)
|
||||
}
|
||||
|
||||
func prometheusLabels(service string, labels cacheMetricLabels) string {
|
||||
return fmt.Sprintf("service=\"%s\",cache_key_class=\"%s\",operation=\"%s\",cache_status=\"%s\"", prometheusEscape(service), prometheusEscape(labels.Class), prometheusEscape(labels.Operation), prometheusEscape(labels.Status))
|
||||
}
|
||||
|
||||
func prometheusEscape(value string) string {
|
||||
return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(value, "\\", "\\\\"), "\n", "\\n"), "\"", "\\\"")
|
||||
}
|
||||
|
||||
func prometheusFloat(value float64) string {
|
||||
return strconv.FormatFloat(value, 'f', -1, 64)
|
||||
}
|
||||
|
||||
func cacheLabels(class string, operation string, status string) cacheMetricLabels {
|
||||
return cacheMetricLabels{Class: safeCacheToken(first(class, "unknown")), Operation: safeCacheToken(first(operation, "unknown")), Status: safeCacheToken(first(status, "unknown"))}
|
||||
}
|
||||
|
||||
func cacheMetricKey(labels cacheMetricLabels) string {
|
||||
return labels.Class + "\x00" + labels.Operation + "\x00" + labels.Status
|
||||
}
|
||||
|
||||
func envBool(name string, fallback bool) bool {
|
||||
value := strings.ToLower(strings.TrimSpace(os.Getenv(name)))
|
||||
if value == "" {
|
||||
|
||||
@@ -94,6 +94,27 @@ func TestDisabledCacheDoesNotFailReadPath(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestCachePrometheusMetricsUseLowCardinalityLabels(t *testing.T) {
|
||||
var metrics cacheMetrics
|
||||
metrics.recordCacheObservation("turn.terminal.snapshot", "get", "hit", 12*time.Millisecond, 4096)
|
||||
metrics.recordCacheObservation("sessions.summary", "get", "miss", 25*time.Millisecond, 0)
|
||||
metrics.recordStale("sessions.summary")
|
||||
metrics.recordDBQueryAvoided("turn.terminal.snapshot")
|
||||
|
||||
text := metrics.prometheusText(serviceID)
|
||||
for _, name := range []string{"workbench_cache_requests_total", "workbench_cache_hit_ratio", "workbench_cache_operation_duration_seconds", "workbench_cache_payload_bytes", "workbench_cache_stale_total", "workbench_db_query_avoided_total"} {
|
||||
if !strings.Contains(text, name) {
|
||||
t.Fatalf("metrics missing %s in:\n%s", name, text)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(text, `cache_key_class="turn.terminal.snapshot"`) || !strings.Contains(text, `cache_status="hit"`) {
|
||||
t.Fatalf("metrics missing cache labels:\n%s", text)
|
||||
}
|
||||
if strings.Contains(text, "trc_") || strings.Contains(text, "ses_") || strings.Contains(text, "actor=") || strings.Contains(text, "cacheKey") {
|
||||
t.Fatalf("metrics leaked high-cardinality or key material:\n%s", text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSessionsSummaryCacheTTLUsesShortTTLForUnstablePages(t *testing.T) {
|
||||
config := cacheConfig{SessionsSummaryTTL: 30 * time.Second, RunningPageTTL: 5 * time.Second}
|
||||
stable := map[string]any{"sessions": []any{map[string]any{"terminal": true, "running": false, "projectionStatus": "caught-up"}}}
|
||||
|
||||
@@ -233,12 +233,23 @@ func (s *Server) Close() error {
|
||||
func (s *Server) routes() {
|
||||
s.mux.HandleFunc("/health/live", s.handleLive)
|
||||
s.mux.HandleFunc("/health/ready", s.handleReady)
|
||||
s.mux.HandleFunc("/metrics", s.handleMetrics)
|
||||
s.mux.HandleFunc("/v1/workbench-runtime/facts", s.handleFacts)
|
||||
s.mux.HandleFunc("/v1/workbench-runtime/trace-events", s.handleTraceEvents)
|
||||
s.mux.HandleFunc("/v1/workbench-runtime/sessions", s.handleSessions)
|
||||
s.mux.HandleFunc("/v1/workbench-runtime/projection-outbox", s.handleProjectionOutbox)
|
||||
}
|
||||
|
||||
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
|
||||
if r.Method != http.MethodGet {
|
||||
methodNotAllowed(w, "GET")
|
||||
return
|
||||
}
|
||||
w.Header().Set("content-type", "text/plain; version=0.0.4; charset=utf-8")
|
||||
w.WriteHeader(http.StatusOK)
|
||||
_, _ = w.Write([]byte(s.cache.PrometheusText(serviceID)))
|
||||
}
|
||||
|
||||
func (s *Server) handleLive(w http.ResponseWriter, r *http.Request) {
|
||||
cache := s.cache.Diagnostic(r.Context())
|
||||
if s.config.Cache.UnavailableLivenessFailure && cache["enabled"] == true && cache["available"] != true {
|
||||
@@ -297,6 +308,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
|
||||
plan.Meta["ttlMs"] = entry.TTLMillis
|
||||
plan.Meta["freshnessSloMs"] = entry.TTLMillis
|
||||
attachFactsCacheMetadata(payload, plan.Meta)
|
||||
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
|
||||
return payload, nil
|
||||
}
|
||||
setFactsCacheStatus(plan.Meta, "stale")
|
||||
@@ -316,6 +328,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
|
||||
plan.Meta["dbQueryDurationMs"] = dbDuration.Milliseconds()
|
||||
if plan.Key == "" {
|
||||
attachFactsCacheMetadata(payload, plan.Meta)
|
||||
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
@@ -330,6 +343,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
|
||||
setFactsCacheStatus(plan.Meta, "skipped")
|
||||
plan.Meta["skipReason"] = classifyCacheError(payloadErr)
|
||||
attachFactsCacheMetadata(payload, plan.Meta)
|
||||
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
|
||||
return payload, nil
|
||||
}
|
||||
plan.Meta["payloadBytes"] = len(payloadBytes)
|
||||
@@ -337,6 +351,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
|
||||
setFactsCacheStatus(plan.Meta, "skipped")
|
||||
plan.Meta["skipReason"] = ttlReason
|
||||
attachFactsCacheMetadata(payload, plan.Meta)
|
||||
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
|
||||
return payload, nil
|
||||
}
|
||||
entry := factsCacheEntry{CachedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: storedPayload, PayloadBytes: len(payloadBytes), TTLMillis: ttl.Milliseconds(), ValuesRedacted: true}
|
||||
@@ -344,6 +359,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
|
||||
setFactsCacheStatus(plan.Meta, "set_failed")
|
||||
plan.Meta["errorKind"] = classifyCacheError(err)
|
||||
attachFactsCacheMetadata(payload, plan.Meta)
|
||||
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
|
||||
return payload, nil
|
||||
}
|
||||
if plan.Meta["status"] != "stale" {
|
||||
@@ -351,9 +367,61 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
|
||||
}
|
||||
plan.Meta["stored"] = true
|
||||
attachFactsCacheMetadata(payload, plan.Meta)
|
||||
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
func (s *Server) observeCacheDiagnostic(ctx context.Context, class string, meta map[string]any) {
|
||||
if s == nil || meta == nil {
|
||||
return
|
||||
}
|
||||
cacheClass := first(text(meta["class"]), class, "unknown")
|
||||
status := first(text(meta["status"]), "unknown")
|
||||
if s.cache != nil {
|
||||
if status == "stale" {
|
||||
s.cache.RecordStale(cacheClass)
|
||||
}
|
||||
if boolValue(meta["dbQueryAvoided"]) {
|
||||
s.cache.RecordDBQueryAvoided(cacheClass)
|
||||
}
|
||||
}
|
||||
attrs := map[string]any{
|
||||
"cacheStatus": status,
|
||||
"cacheKeyClass": safeCacheToken(cacheClass),
|
||||
"workbench.cache.status": status,
|
||||
"workbench.cache.key_class": safeCacheToken(cacheClass),
|
||||
"workbench.cache.db_avoided": boolValue(meta["dbQueryAvoided"]),
|
||||
"workbench.cache.values_source": "diagnostic",
|
||||
"valuesRedacted": true,
|
||||
}
|
||||
if meta["cacheAgeMs"] != nil {
|
||||
age := int64FromAny(meta["cacheAgeMs"])
|
||||
attrs["cacheAgeMs"] = age
|
||||
attrs["workbench.cache.age_ms"] = age
|
||||
}
|
||||
if meta["projectionSeq"] != nil {
|
||||
seq := int64FromAny(meta["projectionSeq"])
|
||||
attrs["projectionSeq"] = seq
|
||||
attrs["workbench.projection.seq"] = seq
|
||||
}
|
||||
if meta["payloadBytes"] != nil {
|
||||
attrs["workbench.cache.payload_bytes"] = int64FromAny(meta["payloadBytes"])
|
||||
}
|
||||
if meta["ttlMs"] != nil {
|
||||
attrs["workbench.cache.ttl_ms"] = int64FromAny(meta["ttlMs"])
|
||||
}
|
||||
if meta["freshnessSloMs"] != nil {
|
||||
attrs["workbench.cache.freshness_slo_ms"] = int64FromAny(meta["freshnessSloMs"])
|
||||
}
|
||||
if revisionSource := text(meta["revisionSource"]); revisionSource != "" {
|
||||
attrs["workbench.cache.revision_source"] = revisionSource
|
||||
}
|
||||
if invalidationMode := text(meta["invalidationMode"]); invalidationMode != "" {
|
||||
attrs["workbench.cache.invalidation_mode"] = invalidationMode
|
||||
}
|
||||
_ = s.withOtelInternalSpan(ctx, "workbench-runtime.cache", attrs, func(context.Context) error { return nil })
|
||||
}
|
||||
|
||||
func factsResponsePayload(facts map[string][]any, persistence map[string]any) map[string]any {
|
||||
count := 0
|
||||
for _, rows := range facts {
|
||||
@@ -732,6 +800,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
|
||||
cacheMeta["freshnessSloMs"] = entry.TTLMillis
|
||||
applySessionsCacheRevisionMetadata(cacheMeta, sessionsSummaryRevision{ProjectionRevision: entry.ProjectionRevision, ProjectionSeqMax: entry.ProjectionSeqMax, SessionUpdatedAtMax: entry.SessionUpdatedAtMax, SessionCount: entry.SessionCount, RevisionSource: "cached_payload"})
|
||||
attachSessionsCacheMetadata(payload, cacheMeta)
|
||||
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
|
||||
return payload, nil
|
||||
}
|
||||
setSessionsCacheStatus(cacheMeta, "stale")
|
||||
@@ -751,6 +820,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
|
||||
cacheMeta["dbQueryDurationMs"] = dbDuration.Milliseconds()
|
||||
if cacheKey == "" {
|
||||
attachSessionsCacheMetadata(payload, cacheMeta)
|
||||
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
@@ -767,6 +837,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
|
||||
setSessionsCacheStatus(cacheMeta, "skipped")
|
||||
cacheMeta["skipReason"] = classifyCacheError(payloadErr)
|
||||
attachSessionsCacheMetadata(payload, cacheMeta)
|
||||
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
|
||||
return payload, nil
|
||||
}
|
||||
cacheMeta["payloadBytes"] = len(payloadBytes)
|
||||
@@ -774,6 +845,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
|
||||
setSessionsCacheStatus(cacheMeta, "skipped")
|
||||
cacheMeta["skipReason"] = ttlReason
|
||||
attachSessionsCacheMetadata(payload, cacheMeta)
|
||||
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
|
||||
return payload, nil
|
||||
}
|
||||
entry := sessionsCacheEntry{CachedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: storedPayload, PayloadBytes: len(payloadBytes), TTLMillis: ttl.Milliseconds(), ProjectionRevision: revision.ProjectionRevision, ProjectionSeqMax: revision.ProjectionSeqMax, SessionUpdatedAtMax: revision.SessionUpdatedAtMax, SessionCount: revision.SessionCount, ValuesRedacted: true}
|
||||
@@ -781,6 +853,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
|
||||
setSessionsCacheStatus(cacheMeta, "set_failed")
|
||||
cacheMeta["errorKind"] = classifyCacheError(err)
|
||||
attachSessionsCacheMetadata(payload, cacheMeta)
|
||||
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
|
||||
return payload, nil
|
||||
}
|
||||
if cacheMeta["status"] != "stale" {
|
||||
@@ -788,6 +861,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
|
||||
}
|
||||
cacheMeta["stored"] = true
|
||||
attachSessionsCacheMetadata(payload, cacheMeta)
|
||||
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
|
||||
return payload, nil
|
||||
}
|
||||
|
||||
|
||||
@@ -576,6 +576,7 @@ export interface WebPerformanceSummaryResponse {
|
||||
workbenchJourneys?: WebPerformanceRow[];
|
||||
workbenchEventPhases?: WebPerformanceRow[];
|
||||
workbenchBackendEvents?: WebPerformanceRow[];
|
||||
workbenchCacheRows?: WebPerformanceRow[];
|
||||
workbenchRows?: WebPerformanceRow[];
|
||||
rows?: WebPerformanceRow[];
|
||||
[key: string]: unknown;
|
||||
|
||||
@@ -189,6 +189,10 @@ function metricLabel(row: WebPerformanceRow): string {
|
||||
navigation_load: "页面加载完成",
|
||||
trace_event_projected: "轨迹事件投影",
|
||||
backend_event_visible: "后端事件到可见",
|
||||
cache_hit_ratio: "缓存命中率",
|
||||
db_query_avoided: "DB 查询避免率",
|
||||
cache_unavailable: "缓存不可用率",
|
||||
sessions_api_p95: "会话 API P95",
|
||||
longtask: "主线程长任务"
|
||||
};
|
||||
return labels[metric] ?? metric.replace(/[_-]/gu, " ");
|
||||
@@ -199,6 +203,7 @@ function kindLabel(kind?: string): string {
|
||||
workbench_journey: "工作台体感",
|
||||
workbench_event_phase: "事件阶段",
|
||||
workbench_backend_event_visible: "后端可见",
|
||||
workbench_cache: "缓存观测",
|
||||
web_vital: "浏览器体感",
|
||||
navigation: "页面加载",
|
||||
api: "同源 API",
|
||||
@@ -403,7 +408,7 @@ function targetStateLabel(value?: string): string {
|
||||
}
|
||||
|
||||
function cacheLabel(value?: string): string {
|
||||
const labels: Record<string, string> = { warm: "热缓存", cold: "冷加载", hit: "命中", miss: "未命中", unknown: "未知" };
|
||||
const labels: Record<string, string> = { warm: "热缓存", cold: "冷加载", hit: "命中", miss: "未命中", sessions_summary: "会话摘要", terminal_turn_snapshot: "终态轮次快照", trace_terminal_events: "终态轨迹事件", unknown: "未知" };
|
||||
return labels[String(value ?? "unknown")] ?? String(value);
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user