From 52182aa0a593b51ef4f766705c198ea6406e9138 Mon Sep 17 00:00:00 2001 From: lyon Date: Mon, 22 Jun 2026 15:10:26 +0800 Subject: [PATCH] fix: expose workbench cache observability --- internal/cloud/backend-performance.test.ts | 18 ++ internal/cloud/backend-performance.ts | 228 +++++++++++++- internal/cloud/server-workbench-http.ts | 31 +- internal/cloud/web-performance.test.ts | 18 ++ internal/cloud/web-performance.ts | 147 ++++++++- internal/cloud/workbench-facts-store.ts | 1 + internal/workbenchruntime/cache.go | 290 +++++++++++++++++- internal/workbenchruntime/cache_test.go | 21 ++ internal/workbenchruntime/service.go | 74 +++++ web/hwlab-cloud-web/src/types/index.ts | 1 + .../src/views/PerformanceView.vue | 7 +- 11 files changed, 813 insertions(+), 23 deletions(-) diff --git a/internal/cloud/backend-performance.test.ts b/internal/cloud/backend-performance.test.ts index 74bcdae1..0ee1bf49 100644 --- a/internal/cloud/backend-performance.test.ts +++ b/internal/cloud/backend-performance.test.ts @@ -56,6 +56,24 @@ test("backend performance store exposes low-cardinality request evidence for RUM assert.doesNotMatch(JSON.stringify(evidence), /ses_secret|traceId|sessionId|runId|prompt|api key/iu); }); +test("backend performance store records Workbench cache metrics without cache keys or ids", () => { + const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } }); + store.recordWorkbenchCache({ route: "/v1/workbench/turns/trc_secret_0123456789abcdef", cacheKeyClass: "turn.terminal.snapshot", cacheStatus: "hit", durationMs: 12, payloadBytes: 4096, cacheAgeMs: 5, dbQueryAvoided: true, freshnessSloMs: 300000, projectionSeq: 42 }); + store.recordWorkbenchCache({ route: "/v1/workbench/sessions", cacheKeyClass: "sessions.summary", cacheStatus: "unavailable", durationMs: 120, dbQueryAvoided: false }); + + const text = store.metricsText(); + assert.match(text, /workbench_cache_requests_total\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*cache_status="hit"[^}]*\} 1/u); + assert.match(text, /workbench_cache_hit_ratio\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*\} 1(?:\.0+)?/u); + assert.match(text, /workbench_cache_unavailable_total\{[^}]*route="\/v1\/workbench\/sessions"[^}]*cache_key_class="sessions_summary"[^}]*\} 1/u); + assert.match(text, /workbench_db_query_avoided_total\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*\} 1/u); + + const evidence = store.evidence({ window: "15m" }); + assert.equal(evidence.cacheSampleCount, 2); + assert.ok(evidence.cache.some((row) => row.route === "/v1/workbench/turns/:traceId" && row.cacheKeyClass === "turn_terminal_snapshot" && row.hitRatio === 1)); + assert.ok(evidence.cache.some((row) => row.route === "/v1/workbench/sessions" && row.unavailableCount === 1)); + assert.doesNotMatch(`${text}\n${JSON.stringify(evidence)}`, /trc_secret|0123456789abcdef|ses_secret|prompt|api key|authorization|cookie/iu); +}); + test("backend performance records Code Agent trace handler phases without high-cardinality labels", async () => { const traceId = "trc_backend_phase_trace"; const runId = "run_backend_phase"; diff --git a/internal/cloud/backend-performance.ts b/internal/cloud/backend-performance.ts index ddf7c11d..d4ec2d61 100644 --- a/internal/cloud/backend-performance.ts +++ b/internal/cloud/backend-performance.ts @@ -10,6 +10,8 @@ const PROJECTION_LAG_EVENT_BUCKETS = [0, 1, 5, 10, 50, 100, 500, 1000, 2000, 500 const PROJECTION_LAG_SECONDS_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 900, 1800, 3600]; const WORKBENCH_TURN_DURATION_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]; const WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304]; +const WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]; +const WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304]; const AGENTRUN_RESULT_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120]; const AGENTRUN_RESULT_EVENTS_BUCKETS = [0, 50, 100, 500, 1000, 2000, 5000, 10000]; const AGENTRUN_RESULT_PAGES_BUCKETS = [0, 1, 2, 5, 10, 20, 50, 100]; @@ -93,6 +95,19 @@ interface WorkbenchTurnReadInput { timedOut?: boolean; } +interface WorkbenchCacheInput { + route: string; + cacheKeyClass?: unknown; + cacheStatus?: unknown; + operation?: unknown; + durationMs?: unknown; + payloadBytes?: unknown; + cacheAgeMs?: unknown; + dbQueryAvoided?: unknown; + freshnessSloMs?: unknown; + projectionSeq?: unknown; +} + interface AgentRunResultInput { durationMs?: unknown; eventsScanned?: unknown; @@ -127,6 +142,20 @@ interface BackendRequestSample { phases: Array<{ phase: string; outcome: string; durationSeconds: number }>; } +interface WorkbenchCacheEvidenceSample { + atMs: number; + route: string; + cacheKeyClass: string; + cacheStatus: string; + operation: string; + durationSeconds: number | null; + payloadBytes: number | null; + cacheAgeMs: number | null; + dbQueryAvoided: boolean; + freshnessSloMs: number | null; + projectionSeq: number | null; +} + export function createBackendPerformanceStore(options: BackendPerformanceStoreOptions = {}) { const env = options.env ?? process.env; const maxSeries = parsePositiveInteger(env.HWLAB_BACKEND_PERFORMANCE_MAX_SERIES, options.maxSeries ?? 320); @@ -144,6 +173,12 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp const turnGetDurationSeries = new Map(); const turnGetTimeoutSeries = new Map(); const turnGetResponseBytesSeries = new Map(); + const cacheRequestSeries = new Map(); + const cacheOperationDurationSeries = new Map(); + const cachePayloadBytesSeries = new Map(); + const cacheStaleSeries = new Map(); + const cacheUnavailableSeries = new Map(); + const dbQueryAvoidedSeries = new Map(); const agentRunResultDurationSeries = new Map(); const agentRunResultPagesSeries = new Map(); const agentRunResultEventsSeries = new Map(); @@ -154,6 +189,7 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp const projectorErrorsSeries = new Map(); const projectorLastSuccessSeries = new Map(); const requestSamples: BackendRequestSample[] = []; + const cacheSamples: WorkbenchCacheEvidenceSample[] = []; const baseLabels = { service: "hwlab-cloud-api", namespace: sanitizeLabelValue(env.POD_NAMESPACE ?? env.HWLAB_METRICS_NAMESPACE ?? "hwlab-v02"), @@ -340,6 +376,55 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp } } + function recordWorkbenchCache(input: WorkbenchCacheInput) { + const route = workbenchCacheRouteTemplate(input.route); + const cacheKeyClass = normalizeCacheKeyClass(input.cacheKeyClass); + const cacheStatus = normalizeCacheStatus(input.cacheStatus); + const operation = normalizeCacheOperation(input.operation ?? "read"); + const labels = { + ...workbenchBaseLabels, + route, + cache_key_class: cacheKeyClass, + operation, + cache_status: cacheStatus + }; + incrementCounter(cacheRequestSeries, labels, maxSeries) || incrementDropped("workbench_cache_request_series_limit"); + const durationMs = finiteNonNegativeNumber(input.durationMs); + const durationSeconds = durationMs === null ? null : boundedDurationMs(durationMs, 120_000) / 1000; + if (durationSeconds !== null) { + if (!recordHistogram(cacheOperationDurationSeries, labels, durationSeconds, WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS, maxSeries)) incrementDropped("workbench_cache_operation_series_limit"); + } + const payloadBytes = finiteNonNegativeNumber(input.payloadBytes); + if (payloadBytes !== null) { + if (!recordHistogram(cachePayloadBytesSeries, labels, payloadBytes, WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS, maxSeries)) incrementDropped("workbench_cache_payload_series_limit"); + } + if (cacheStatus === "stale") { + incrementCounter(cacheStaleSeries, labels, maxSeries) || incrementDropped("workbench_cache_stale_series_limit"); + } + if (cacheStatus === "unavailable" || cacheStatus === "set_failed" || cacheStatus === "error") { + incrementCounter(cacheUnavailableSeries, labels, maxSeries) || incrementDropped("workbench_cache_unavailable_series_limit"); + } + const dbQueryAvoided = booleanValue(input.dbQueryAvoided); + if (dbQueryAvoided) { + const avoidedLabels = { ...workbenchBaseLabels, route, cache_key_class: cacheKeyClass }; + incrementCounter(dbQueryAvoidedSeries, avoidedLabels, maxSeries) || incrementDropped("workbench_db_query_avoided_series_limit"); + } + cacheSamples.push({ + atMs: Date.now(), + route, + cacheKeyClass, + cacheStatus, + operation, + durationSeconds, + payloadBytes, + cacheAgeMs: finiteNonNegativeNumber(input.cacheAgeMs), + dbQueryAvoided, + freshnessSloMs: finiteNonNegativeNumber(input.freshnessSloMs), + projectionSeq: finiteNonNegativeNumber(input.projectionSeq) + }); + while (cacheSamples.length > maxEvidenceSamples) cacheSamples.shift(); + } + function recordAgentRunResult(input: AgentRunResultInput = {}) { const eventCount = finiteNonNegativeNumber(input.eventCount ?? input.eventsScanned) ?? 0; const labels = { @@ -450,6 +535,27 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp "# HELP hwlab_workbench_turn_get_response_bytes Workbench turn/read API response bytes.", "# TYPE hwlab_workbench_turn_get_response_bytes histogram", ...renderHistogram("hwlab_workbench_turn_get_response_bytes", turnGetResponseBytesSeries, WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS), + "# HELP workbench_cache_requests_total Workbench derived cache requests observed by Cloud API read path.", + "# TYPE workbench_cache_requests_total counter", + ...renderCounters("workbench_cache_requests_total", cacheRequestSeries), + "# HELP workbench_cache_hit_ratio Workbench derived cache hit ratio by route and cache class.", + "# TYPE workbench_cache_hit_ratio gauge", + ...renderWorkbenchCacheHitRatio(cacheRequestSeries), + "# HELP workbench_cache_operation_duration_seconds Workbench derived cache operation observation duration in seconds.", + "# TYPE workbench_cache_operation_duration_seconds histogram", + ...renderHistogram("workbench_cache_operation_duration_seconds", cacheOperationDurationSeries, WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS), + "# HELP workbench_cache_payload_bytes Workbench derived cache payload size in bytes.", + "# TYPE workbench_cache_payload_bytes histogram", + ...renderHistogram("workbench_cache_payload_bytes", cachePayloadBytesSeries, WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS), + "# HELP workbench_cache_stale_total Workbench derived cache stale observations.", + "# TYPE workbench_cache_stale_total counter", + ...renderCounters("workbench_cache_stale_total", cacheStaleSeries), + "# HELP workbench_cache_unavailable_total Workbench derived cache unavailable observations.", + "# TYPE workbench_cache_unavailable_total counter", + ...renderCounters("workbench_cache_unavailable_total", cacheUnavailableSeries), + "# HELP workbench_db_query_avoided_total Workbench DB reads avoided by derived cache hits.", + "# TYPE workbench_db_query_avoided_total counter", + ...renderCounters("workbench_db_query_avoided_total", dbQueryAvoidedSeries), "# HELP hwlab_agentrun_result_duration_seconds AgentRun command result aggregation duration in seconds.", "# TYPE hwlab_agentrun_result_duration_seconds histogram", ...renderHistogram("hwlab_agentrun_result_duration_seconds", agentRunResultDurationSeries, AGENTRUN_RESULT_BUCKETS_SECONDS), @@ -491,10 +597,12 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp upstreamCount: histogramSampleCount(upstreamSeries), projectionLagSeries: projectionLagEventsSeries.size + projectionLagSecondsSeries.size, turnReadSeries: turnGetDurationSeries.size, + cacheSeries: cacheRequestSeries.size + cacheOperationDurationSeries.size + cachePayloadBytesSeries.size + cacheStaleSeries.size + cacheUnavailableSeries.size + dbQueryAvoidedSeries.size, agentRunResultSeries: agentRunResultDurationSeries.size, projectorSeries: projectorBatchDurationSeries.size + projectorCandidatesSeries.size + projectorEventsProcessedSeries.size + projectorErrorsSeries.size, droppedSeries: droppedSeries.size, - evidenceSamples: requestSamples.length + evidenceSamples: requestSamples.length, + cacheEvidenceSamples: cacheSamples.length }; } @@ -504,7 +612,9 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp const observedAt = new Date(generatedAtMs).toISOString(); const cutoff = sampleWindow.milliseconds === null ? null : generatedAtMs - sampleWindow.milliseconds; const windowSamples = requestSamples.filter((sample) => cutoff === null || (sample.atMs >= cutoff && sample.atMs <= generatedAtMs)); + const windowCacheSamples = cacheSamples.filter((sample) => cutoff === null || (sample.atMs >= cutoff && sample.atMs <= generatedAtMs)); const rows = backendEvidenceRows(windowSamples); + const cache = backendCacheEvidenceRows(windowCacheSamples); return { schemaVersion: "hwlab-backend-performance-evidence-v1", source: "cloud-api-backend-performance-store", @@ -518,8 +628,11 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp }, sampleCount: windowSamples.length, retainedSampleCount: requestSamples.length, + cacheSampleCount: windowCacheSamples.length, + retainedCacheSampleCount: cacheSamples.length, routeCount: new Set(rows.map((row) => row.route)).size, rows, + cache, valuesRedacted: true }; } @@ -530,6 +643,7 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp recordWorkbenchProjection, recordWorkbenchProjectionCursorAdvance, recordWorkbenchTurnRead, + recordWorkbenchCache, recordAgentRunResult, recordWorkbenchProjectorBatch, recordWorkbenchProjectorCandidate, @@ -570,6 +684,12 @@ function workbenchTurnRouteTemplate(input: unknown): string { return route; } +function workbenchCacheRouteTemplate(input: unknown): string { + const route = workbenchTurnRouteTemplate(input); + if (route === "/v1/workbench/traces/:id/events") return "/v1/workbench/traces/:traceId/events"; + return route; +} + function resolveBackendEvidenceWindow(input: unknown) { const key = String(input ?? DEFAULT_BACKEND_EVIDENCE_WINDOW).trim().toLowerCase(); if (Object.prototype.hasOwnProperty.call(BACKEND_EVIDENCE_WINDOWS, key)) return BACKEND_EVIDENCE_WINDOWS[key as keyof typeof BACKEND_EVIDENCE_WINDOWS]; @@ -627,6 +747,74 @@ function backendEvidenceRows(samples: BackendRequestSample[]) { }).sort((left, right) => (right.p95 - left.p95) || (right.count - left.count)); } +function backendCacheEvidenceRows(samples: WorkbenchCacheEvidenceSample[]) { + const groups = new Map(); + for (const sample of samples) { + const key = stableLabelKey({ route: sample.route, cache_key_class: sample.cacheKeyClass }); + let group = groups.get(key); + if (!group) { + group = { route: sample.route, cacheKeyClass: sample.cacheKeyClass, count: 0, hitCount: 0, missCount: 0, staleCount: 0, unavailableCount: 0, dbQueryAvoidedCount: 0, durations: [], payloads: [], ages: [], freshness: [], projectionSeqMax: null, firstAt: sample.atMs, lastAt: sample.atMs }; + groups.set(key, group); + } + group.count += 1; + if (sample.cacheStatus === "hit") group.hitCount += 1; + if (sample.cacheStatus === "miss") group.missCount += 1; + if (sample.cacheStatus === "stale") group.staleCount += 1; + if (sample.cacheStatus === "unavailable" || sample.cacheStatus === "set_failed" || sample.cacheStatus === "error") group.unavailableCount += 1; + if (sample.dbQueryAvoided) group.dbQueryAvoidedCount += 1; + if (sample.durationSeconds !== null) group.durations.push(sample.durationSeconds); + if (sample.payloadBytes !== null) group.payloads.push(sample.payloadBytes); + if (sample.cacheAgeMs !== null) group.ages.push(sample.cacheAgeMs); + if (sample.freshnessSloMs !== null) group.freshness.push(sample.freshnessSloMs); + if (sample.projectionSeq !== null) group.projectionSeqMax = Math.max(group.projectionSeqMax ?? 0, sample.projectionSeq); + group.firstAt = Math.min(group.firstAt, sample.atMs); + group.lastAt = Math.max(group.lastAt, sample.atMs); + } + return [...groups.values()].map((group) => { + const duration = exactEvidenceStats(group.durations); + const payloadBytes = group.payloads.length ? exactEvidenceStats(group.payloads) : null; + const cacheAgeMs = group.ages.length ? exactEvidenceStats(group.ages) : null; + const freshnessSloMs = group.freshness.length ? exactEvidenceStats(group.freshness) : null; + return { + route: group.route, + cacheKeyClass: group.cacheKeyClass, + count: group.count, + hitCount: group.hitCount, + missCount: group.missCount, + staleCount: group.staleCount, + unavailableCount: group.unavailableCount, + dbQueryAvoidedCount: group.dbQueryAvoidedCount, + hitRatio: group.count > 0 ? roundEvidenceValue(group.hitCount / group.count) : 0, + dbQueryAvoidedRatio: group.count > 0 ? roundEvidenceValue(group.dbQueryAvoidedCount / group.count) : 0, + unavailableRatio: group.count > 0 ? roundEvidenceValue(group.unavailableCount / group.count) : 0, + operationDuration: { ...duration, unit: "seconds" }, + payloadBytes: payloadBytes ? { count: payloadBytes.count, average: Math.round(payloadBytes.average), p50: Math.round(payloadBytes.p50), p75: Math.round(payloadBytes.p75), p95: Math.round(payloadBytes.p95), unit: "bytes" } : null, + cacheAgeMs: cacheAgeMs ? { ...cacheAgeMs, unit: "milliseconds" } : null, + freshnessSloMs: freshnessSloMs ? { ...freshnessSloMs, unit: "milliseconds" } : null, + projectionSeqMax: group.projectionSeqMax, + firstObservedAt: new Date(group.firstAt).toISOString(), + lastObservedAt: new Date(group.lastAt).toISOString(), + valuesRedacted: true + }; + }).sort((left, right) => (right.unavailableRatio - left.unavailableRatio) || (right.count - left.count) || left.route.localeCompare(right.route)); +} + function exactEvidenceStats(values: number[]) { const sorted = values.filter(Number.isFinite).sort((left, right) => left - right); const count = sorted.length; @@ -725,6 +913,21 @@ function renderGauges(metricName: string, series: Map) { return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${formatPrometheusValue(entry.value)}`); } +function renderWorkbenchCacheHitRatio(series: Map) { + const groups = new Map; hit: number; total: number }>(); + for (const entry of series.values()) { + const labels = { ...entry.labels }; + const cacheStatus = labels.cache_status; + delete labels.cache_status; + const key = stableLabelKey(labels); + const group = groups.get(key) ?? { labels, hit: 0, total: 0 }; + if (cacheStatus === "hit") group.hit += entry.value; + group.total += entry.value; + groups.set(key, group); + } + return [...groups.values()].map((group) => `${"workbench_cache_hit_ratio"}{${renderLabels(group.labels)}} ${formatPrometheusValue(group.total > 0 ? group.hit / group.total : 0)}`); +} + function renderHistogram(metricName: string, series: Map, buckets: number[]) { const lines: string[] = []; for (const entry of series.values()) { @@ -790,7 +993,22 @@ function normalizeStatusClass(value: unknown) { function normalizeStatusLabel(value: unknown) { const text = sanitizeMetricName(value, "unknown"); - return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown|running|pending|completed|failed|blocked|degraded|advanced|unchanged|scheduled|seen|skipped|caught_up|projecting|stale|missing)$/u.test(text) ? text : "unknown"; + return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown|running|pending|completed|failed|blocked|degraded|advanced|unchanged|scheduled|seen|skipped|caught_up|projecting|stale|missing|hit|miss|stored|set_failed|unavailable|disabled)$/u.test(text) ? text : "unknown"; +} + +function normalizeCacheStatus(value: unknown) { + const text = normalizeStatusLabel(value); + return /^(?:hit|miss|stale|skipped|stored|set_failed|unavailable|disabled|error|unknown)$/u.test(text) ? text : "unknown"; +} + +function normalizeCacheOperation(value: unknown) { + const text = sanitizeMetricName(value, "read"); + return /^(?:read|get|set|write|delete|ttl|ping)$/u.test(text) ? text : "read"; +} + +function normalizeCacheKeyClass(value: unknown) { + const text = sanitizeMetricName(value, "unknown").slice(0, 64); + return text || "unknown"; } function normalizeOutcome(value: unknown) { @@ -880,6 +1098,12 @@ function finiteNonNegativeNumber(value: unknown): number | null { return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; } +function booleanValue(value: unknown) { + if (value === true) return true; + const text = String(value ?? "").trim().toLowerCase(); + return text === "true" || text === "1" || text === "yes"; +} + function unixMs(value: unknown): number | null { if (value instanceof Date) { const time = value.getTime(); diff --git a/internal/cloud/server-workbench-http.ts b/internal/cloud/server-workbench-http.ts index 9724829b..944e20c7 100644 --- a/internal/cloud/server-workbench-http.ts +++ b/internal/cloud/server-workbench-http.ts @@ -300,6 +300,25 @@ function recordWorkbenchTurnReadMetric(options = {}, url, statusCode, body = {}, }); } +function recordWorkbenchCacheMetric(options = {}, url, body = {}, startedAt = nowMs()) { + const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; + if (!performanceStore?.recordWorkbenchCache) return; + const cache = body?.cache && typeof body.cache === "object" ? body.cache : null; + if (!cache) return; + performanceStore.recordWorkbenchCache({ + route: url?.pathname ?? "/v1/workbench", + cacheKeyClass: cache.class, + cacheStatus: cache.status, + operation: "read", + durationMs: nowMs() - startedAt, + payloadBytes: cache.payloadBytes, + cacheAgeMs: cache.cacheAgeMs, + dbQueryAvoided: cache.dbQueryAvoided, + freshnessSloMs: cache.freshnessSloMs, + projectionSeq: cache.projectionSeq + }); +} + function recordWorkbenchProjectionMetric(options = {}, { result = null, trace = null, projection = null, diagnostic = null } = {}) { const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; if (!performanceStore?.recordWorkbenchProjection || !projection) return; @@ -405,6 +424,7 @@ async function authenticateWorkbenchRead(request, response, options) { } async function handleWorkbenchSessionList(request, response, url, options, actor) { + const startedAt = nowMs(); if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) return sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench session list is keyed by sessionId only.")); const limit = boundedSessionListLimit(url.searchParams.get("limit")); const offset = cursorOffset(url.searchParams.get("cursor") ?? url.searchParams.get("after")); @@ -421,6 +441,7 @@ async function handleWorkbenchSessionList(request, response, url, options, actor includeSessionId: includeRouteId ?? null, actor: { id: actor.id, role: actor.role ?? "user", valuesRedacted: true } }); + recordWorkbenchCacheMetric(options, url, payload, startedAt); return sendJson(response, 200, payload); } @@ -1147,6 +1168,7 @@ async function handleWorkbenchTurnSnapshot(response, url, options, actor, rawTur contractVersion: "workbench-turn-snapshot-v1", turn: snapshot, projection, + cache: result.cache ?? null, projectionStatus: projection.projectionStatus, projectionHealth: projection.projectionHealth, lastProjectedSeq: projection.lastProjectedSeq, @@ -1162,10 +1184,12 @@ async function handleWorkbenchTurnSnapshot(response, url, options, actor, rawTur diagnostic: projection }); recordWorkbenchTurnReadMetric(options, url, 200, body, startedAt); + recordWorkbenchCacheMetric(options, url, body, startedAt); sendJson(response, 200, body); } async function handleWorkbenchTraceEventPage(response, url, options, actor, rawTraceId) { + const startedAt = nowMs(); const traceId = safeTraceId(rawTraceId); if (!traceId) return sendJson(response, 400, workbenchError("invalid_trace_id", "traceId must start with trc_.", { traceId: rawTraceId })); const readModel = createWorkbenchReadModel(options, actor); @@ -1199,7 +1223,7 @@ async function handleWorkbenchTraceEventPage(response, url, options, actor, rawT page.blocker = blocker; } const responseProjection = page.blocker ? blockedTraceEventProjection(projection, page.blocker) : projection; - sendJson(response, 200, { + const body = { ok: true, status: page.blocker ? "blocked" : "succeeded", contractVersion: "workbench-trace-events-v1", @@ -1208,6 +1232,7 @@ async function handleWorkbenchTraceEventPage(response, url, options, actor, rawT threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null), ...page, projection: responseProjection, + cache: pageResult.cache ?? metadata.cache ?? null, projectionStatus: responseProjection.projectionStatus, projectionHealth: responseProjection.projectionHealth, lastProjectedSeq: responseProjection.lastProjectedSeq, @@ -1222,7 +1247,9 @@ async function handleWorkbenchTraceEventPage(response, url, options, actor, rawT durationMs: responseProjection.durationMs, valuesRedacted: true, secretMaterialStored: false - }); + }; + recordWorkbenchCacheMetric(options, url, body, startedAt); + sendJson(response, 200, body); } function traceEventPageFromFacts(sourceEvents, options, metadata = {}) { diff --git a/internal/cloud/web-performance.test.ts b/internal/cloud/web-performance.test.ts index ec4eb792..52f711b1 100644 --- a/internal/cloud/web-performance.test.ts +++ b/internal/cloud/web-performance.test.ts @@ -62,6 +62,21 @@ test("web performance summary exposes user-perceived route latency without high- responseBytes: { count: 2, average: 2048, p50: 1024, p75: 2048, p95: 4096 }, phases: [{ phase: "agent_result_read", outcome: "ok", count: 2, average: 0.05, p50: 0.04, p75: 0.05, p95: 0.08 }] }], + cache: [{ + route: "/v1/workbench/sessions", + cacheKeyClass: "sessions_summary", + count: 4, + hitCount: 3, + missCount: 1, + staleCount: 0, + unavailableCount: 0, + dbQueryAvoidedCount: 3, + hitRatio: 0.75, + dbQueryAvoidedRatio: 0.75, + unavailableRatio: 0, + operationDuration: { count: 4, average: 0.12, p50: 0.08, p75: 0.11, p95: 0.18, unit: "seconds" }, + valuesRedacted: true + }], valuesRedacted: true } }); const json = JSON.stringify(summary); @@ -76,6 +91,9 @@ test("web performance summary exposes user-perceived route latency without high- assert.ok(summary.dashboard.cards.some((card) => card.id === "samples" && card.value === 4)); assert.ok(summary.dashboard.trends.some((trend) => trend.id === "api-timing" && trend.points.length > 0)); assert.ok(summary.dashboard.topN.some((group) => group.id === "top-api" && group.rows.length > 0)); + assert.ok(summary.workbenchCacheRows.some((row) => row.metric === "cache_hit_ratio" && row.cache === "sessions_summary" && row.p95 === 0.75)); + assert.ok(summary.workbenchCacheRows.some((row) => row.metric === "db_query_avoided" && row.p95 === 0.75)); + assert.ok(summary.workbenchCacheRows.some((row) => row.metric === "sessions_api_p95" && row.p95 === 0.18)); const matchedApi = summary.apiRoutes.find((row) => row.route === "/v1/agent/chat/result/:id" && row.statusClass === "2xx"); assert.equal(matchedApi?.backendEvidence?.status, "matched"); assert.equal(matchedApi?.backendEvidence?.diagnostic, "backend_low_sample"); diff --git a/internal/cloud/web-performance.ts b/internal/cloud/web-performance.ts index d09cd038..8cee0ac2 100644 --- a/internal/cloud/web-performance.ts +++ b/internal/cloud/web-performance.ts @@ -215,6 +215,27 @@ interface BackendPerformanceEvidenceRow { phases?: Array<{ phase?: unknown; outcome?: unknown; count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown }>; } +interface BackendPerformanceCacheEvidenceRow { + route?: unknown; + cacheKeyClass?: unknown; + count?: unknown; + hitCount?: unknown; + missCount?: unknown; + staleCount?: unknown; + unavailableCount?: unknown; + dbQueryAvoidedCount?: unknown; + hitRatio?: unknown; + dbQueryAvoidedRatio?: unknown; + unavailableRatio?: unknown; + operationDuration?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null; + payloadBytes?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null; + cacheAgeMs?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null; + freshnessSloMs?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null; + projectionSeqMax?: unknown; + firstObservedAt?: unknown; + lastObservedAt?: unknown; +} + interface BackendPerformanceEvidenceSnapshot { schemaVersion?: unknown; source?: unknown; @@ -223,6 +244,7 @@ interface BackendPerformanceEvidenceSnapshot { retainedSampleCount?: unknown; routeCount?: unknown; rows?: BackendPerformanceEvidenceRow[]; + cache?: BackendPerformanceCacheEvidenceRow[]; } interface PerformanceRowBackendEvidence { @@ -538,7 +560,8 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions = const workbenchBackendEvents = applySummaryContract(workbenchBackendEventVisibleSummaryRows(aggregate.backendEventVisibleSeries, WORKBENCH_JOURNEY_BUCKETS_SECONDS), exactStats, windowBounds) .sort(sortByProblemThenP95) .slice(0, 24); - const workbenchRows = [...workbenchJourneys, ...workbenchEventPhases, ...workbenchBackendEvents]; + const workbenchCacheRows = backendCacheEvidenceSummaryRows(backendEvidence, windowBounds); + const workbenchRows = [...workbenchJourneys, ...workbenchEventPhases, ...workbenchBackendEvents, ...workbenchCacheRows]; const apiRoutes = durationRows .filter((row) => row.metric === "api_request") .sort(sortByProblemThenP95) @@ -570,6 +593,7 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions = workbenchJourneySeries: aggregate.journeyDurationSeries.size, workbenchEventPhaseSeries: aggregate.eventPhaseSeries.size, workbenchBackendEventVisibleSeries: aggregate.backendEventVisibleSeries.size, + workbenchCacheEvidenceRows: workbenchCacheRows.length, workbenchUiEventSeries: aggregate.workbenchUiEventSeries.size, routeCount: new Set([...rows, ...workbenchRows].map((row) => row.route)).size, problemCount: problems.length, @@ -639,6 +663,7 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions = workbenchJourneys, workbenchEventPhases, workbenchBackendEvents, + workbenchCacheRows, workbenchRows: dashboard.source.workbenchRows, rows: dashboard.source.rows }; @@ -1047,6 +1072,7 @@ function applySummaryContract(rows: PerformanceSummaryRow[], exactStatsByRow: Ma function normalizeBackendEvidence(input?: BackendPerformanceEvidenceSnapshot | null) { const rows = Array.isArray(input?.rows) ? input.rows.map(normalizeBackendEvidenceRow).filter(Boolean) as BackendPerformanceEvidenceRow[] : []; + const cache = Array.isArray(input?.cache) ? input.cache.map(normalizeBackendCacheEvidenceRow).filter(Boolean) as ReturnType[] : []; return { schemaVersion: "hwlab-backend-performance-evidence-v1", source: sanitizeLabelValue(input?.source ?? "cloud-api-backend-performance-store"), @@ -1055,6 +1081,7 @@ function normalizeBackendEvidence(input?: BackendPerformanceEvidenceSnapshot | n retainedSampleCount: finiteNonNegativeInteger(input?.retainedSampleCount), routeCount: finiteNonNegativeInteger(input?.routeCount ?? new Set(rows.map((row) => row.route)).size), rows: rows.slice(0, 80), + cache: cache.slice(0, 80), available: Boolean(input && String(input.schemaVersion ?? "") === "hwlab-backend-performance-evidence-v1"), valuesRedacted: true }; @@ -1100,6 +1127,44 @@ function normalizeBackendEvidencePhase(input: { phase?: unknown; outcome?: unkno }; } +function normalizeBackendCacheEvidenceRow(input: BackendPerformanceCacheEvidenceRow) { + const route = webPerformanceRouteTemplate(input?.route ?? ""); + if (!route || route === "/") return null; + return { + route, + cacheKeyClass: sanitizeMetricName(input.cacheKeyClass, "unknown").slice(0, 64) || "unknown", + count: finiteNonNegativeInteger(input.count), + hitCount: finiteNonNegativeInteger(input.hitCount), + missCount: finiteNonNegativeInteger(input.missCount), + staleCount: finiteNonNegativeInteger(input.staleCount), + unavailableCount: finiteNonNegativeInteger(input.unavailableCount), + dbQueryAvoidedCount: finiteNonNegativeInteger(input.dbQueryAvoidedCount), + hitRatio: boundedRatio(input.hitRatio), + dbQueryAvoidedRatio: boundedRatio(input.dbQueryAvoidedRatio), + unavailableRatio: boundedRatio(input.unavailableRatio), + operationDuration: normalizeMetricStats(input.operationDuration, "seconds"), + payloadBytes: normalizeMetricStats(input.payloadBytes, "bytes"), + cacheAgeMs: normalizeMetricStats(input.cacheAgeMs, "milliseconds"), + freshnessSloMs: normalizeMetricStats(input.freshnessSloMs, "milliseconds"), + projectionSeqMax: finiteNonNegativeInteger(input.projectionSeqMax), + firstObservedAt: isoOrNull(input.firstObservedAt), + lastObservedAt: isoOrNull(input.lastObservedAt), + valuesRedacted: true + }; +} + +function normalizeMetricStats(input: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null | undefined, unit: string) { + if (!input || typeof input !== "object") return null; + return { + count: finiteNonNegativeInteger(input.count), + average: finiteNonNegativeMetric(input.average), + p50: finiteNonNegativeMetric(input.p50), + p75: finiteNonNegativeMetric(input.p75), + p95: finiteNonNegativeMetric(input.p95), + unit + }; +} + function attachBackendEvidenceToRows(rows: PerformanceSummaryRow[], evidence: ReturnType) { return rows.map((row) => { if (row.kind !== "api" || row.metric !== "api_request") return row; @@ -1188,6 +1253,71 @@ function backendEvidenceDetail(input: { diagnostic: string; count: number; backe return `${input.count} 个后端样本已匹配，后端 P95 ${formatBackendSeconds(input.backendP95)}${phaseText}。`; } +function backendCacheEvidenceSummaryRows(evidence: ReturnType, windowBounds: { windowFrom: string | null; windowTo: string; generatedAt: string }): PerformanceSummaryRow[] { + if (!evidence.available) return []; + const rows: PerformanceSummaryRow[] = []; + for (const item of evidence.cache) { + rows.push(cacheEvidenceRow({ metric: "cache_hit_ratio", route: item.route, cache: item.cacheKeyClass, value: item.hitRatio, count: item.count, unit: "score", windowBounds, outcome: "ok" })); + rows.push(cacheEvidenceRow({ metric: "db_query_avoided", route: item.route, cache: item.cacheKeyClass, value: item.dbQueryAvoidedRatio, count: item.count, unit: "score", windowBounds, outcome: "ok" })); + rows.push(cacheEvidenceRow({ metric: "cache_unavailable", route: item.route, cache: item.cacheKeyClass, value: item.unavailableRatio, count: item.count, unit: "score", windowBounds, outcome: item.unavailableCount > 0 ? "error" : "ok" })); + if (item.route === "/v1/workbench/sessions" && item.operationDuration) { + rows.push(cacheEvidenceRow({ + metric: "sessions_api_p95", + route: item.route, + cache: item.cacheKeyClass, + value: item.operationDuration.p95, + p50: item.operationDuration.p50, + p75: item.operationDuration.p75, + p95: item.operationDuration.p95, + count: item.operationDuration.count || item.count, + unit: "seconds", + windowBounds, + outcome: "ok" + })); + } + } + return rows; +} + +function cacheEvidenceRow(input: { metric: string; route: string; cache: string; value: number; p50?: number; p75?: number; p95?: number; count: number; unit: "seconds" | "score"; outcome: string; windowBounds: { windowFrom: string | null; windowTo: string; generatedAt: string } }): PerformanceSummaryRow { + const count = finiteNonNegativeInteger(input.count); + const value = finiteNonNegativeMetric(input.value); + const p50 = input.p50 === undefined ? value : finiteNonNegativeMetric(input.p50); + const p75 = input.p75 === undefined ? value : finiteNonNegativeMetric(input.p75); + const p95 = input.p95 === undefined ? value : finiteNonNegativeMetric(input.p95); + const sample = sampleState(count); + let tone: "ok" | "warn" | "blocked" | "source" = count < LOW_SAMPLE_THRESHOLD ? "source" : "ok"; + if (sample === "ok" && input.metric === "cache_unavailable" && p95 > 0) tone = "blocked"; + if (sample === "ok" && input.metric === "sessions_api_p95") tone = thresholdTone(p95, 1, 2.5); + const problem = tone === "source" ? (count <= 0 ? "no-sample" : "low-sample") : tone === "ok" ? "ok" : input.metric === "cache_unavailable" ? "cache-unavailable" : `slow-${input.metric}`; + return { + kind: "workbench_cache", + metric: input.metric, + route: input.route, + method: "-", + statusClass: "-", + outcome: input.outcome, + count, + average: value, + p50, + p75, + p95, + unit: input.unit, + tone, + problem, + lowSample: count > 0 && count < LOW_SAMPLE_THRESHOLD, + sampleState: sample, + sourceFamily: "backend_cache_evidence", + windowFrom: input.windowBounds.windowFrom, + windowTo: input.windowBounds.windowTo, + generatedAt: input.windowBounds.generatedAt, + freshness: sampleFreshness(count), + aggregationKind: "exact", + approximation: "exact", + cache: input.cache + }; +} + function formatBackendSeconds(value: number) { if (!Number.isFinite(value)) return "未知"; if (value < 1) return `${Math.round(value * 1000)}ms`; @@ -1199,6 +1329,12 @@ function finiteNonNegativeMetric(value: unknown) { return Number.isFinite(number) && number >= 0 ? roundMetric(number) : 0; } +function boundedRatio(value: unknown) { + const number = Number(value); + if (!Number.isFinite(number) || number < 0) return 0; + return Math.min(1, roundMetric(number)); +} + function finiteNonNegativeInteger(value: unknown) { const number = Number(value); return Number.isFinite(number) && number >= 0 ? Math.trunc(number) : 0; @@ -1294,12 +1430,12 @@ function summaryToneForRow(row: PerformanceSummaryRow, p95: number, count: numbe function summaryProblemForRow(row: PerformanceSummaryRow, p95: number, count: number, tone: string) { if (count <= 0) return "no-sample"; - if (row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible") return workbenchProblem(row.outcome, row.metric, count, tone); + if (row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible" || row.kind === "workbench_cache") return workbenchProblem(row.outcome, row.metric, count, tone); return performanceProblem({ metric: row.metric, outcome: row.outcome, status_class: row.statusClass }, p95, count, row.unit === "score" ? "score" : "seconds", tone); } function isPerformanceProblemRow(row: PerformanceSummaryRow) { - if ((row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible") && row.sampleState !== "ok") return false; + if ((row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible" || row.kind === "workbench_cache") && row.sampleState !== "ok") return false; return row.tone === "blocked" || row.tone === "warn" || row.outcome !== "ok" || row.statusClass === "network" || row.statusClass === "5xx" || row.statusClass === "4xx"; } @@ -1311,6 +1447,7 @@ function sourceFamilyForRow(row: PerformanceSummaryRow) { if (row.kind === "workbench_journey") return "workbench_journey"; if (row.kind === "workbench_event_phase") return "workbench_event_phase"; if (row.kind === "workbench_backend_event_visible") return "workbench_backend_event_visible"; + if (row.kind === "workbench_cache") return "backend_cache_evidence"; return "unknown"; } @@ -1812,6 +1949,10 @@ function metricLabel(row: PerformanceSummaryRow) { submit_to_first_visible: "提交到首个可见结果", submit_to_failure: "提交失败可见", backend_event_to_visible: "后端事件到可见", + cache_hit_ratio: "缓存命中率", + db_query_avoided: "DB 查询避免率", + cache_unavailable: "缓存不可用率", + sessions_api_p95: "会话 API P95", session_switch_first_visible: "切换会话首个可见", session_switch_full_load: "切换会话完整加载", workbench_open_first_visible: "打开工作台首个可见", diff --git a/internal/cloud/workbench-facts-store.ts b/internal/cloud/workbench-facts-store.ts index c218a417..a9adf9de 100644 --- a/internal/cloud/workbench-facts-store.ts +++ b/internal/cloud/workbench-facts-store.ts @@ -34,6 +34,7 @@ export function createWorkbenchFactsStore(options = {}, actor = null) { count: Number.isFinite(Number(result?.count)) ? Number(result.count) : 0, durable: true, persistence: result?.persistence ?? null, + cache: result?.cache ?? null, retry: attempt > 1 ? { attempts: attempt, maxAttempts: retryPolicy.maxAttempts, delaysMs: retryDelaysMs.slice(), valuesRedacted: true } : null }; } catch (error) { diff --git a/internal/workbenchruntime/cache.go b/internal/workbenchruntime/cache.go index 24664fba..b2e8937a 100644 --- a/internal/workbenchruntime/cache.go +++ b/internal/workbenchruntime/cache.go @@ -21,6 +21,9 @@ import ( const cacheKeySchemaVersion = "wb-derived-v1" +var cacheOperationDurationBucketsSeconds = []float64{0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1} +var cachePayloadBucketsBytes = []float64{512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304} + var errCacheDisabled = errors.New("workbench redis cache disabled") type cacheConfig struct { @@ -60,15 +63,38 @@ type derivedCache struct { } type cacheMetrics struct { - mu sync.Mutex - Hits uint64 - Misses uint64 - Sets uint64 - Deletes uint64 - Errors uint64 - LastErrorKind string - LastErrorOp string - LastErrorAt time.Time + mu sync.Mutex + Hits uint64 + Misses uint64 + Sets uint64 + Deletes uint64 + Errors uint64 + LastErrorKind string + LastErrorOp string + LastErrorAt time.Time + Requests map[string]cacheCounterSeries + Stale map[string]cacheCounterSeries + DBQueryAvoided map[string]cacheCounterSeries + OperationDurations map[string]*cacheHistogramSeries + PayloadBytes map[string]*cacheHistogramSeries +} + +type cacheMetricLabels struct { + Class string + Operation string + Status string +} + +type cacheCounterSeries struct { + Labels cacheMetricLabels + Value uint64 +} + +type cacheHistogramSeries struct { + Labels cacheMetricLabels + Buckets []uint64 + Sum float64 + Count uint64 } type cacheError struct { @@ -144,12 +170,15 @@ func (c *derivedCache) BuildKey(parts cacheKeyParts) (string, error) { } func (c *derivedCache) Get(ctx context.Context, key string, target any) (bool, error) { + startedAt := time.Now() + class := cacheKeyClass(key) if !c.usable() { return false, nil } if err := validateCacheKey(c.config.MaxKeyBytes, key); err != nil { wrapped := cacheError{Op: "get", Kind: classifyCacheError(err), Err: err} c.metrics.recordError(wrapped) + c.metrics.recordCacheObservation(class, "get", "error", time.Since(startedAt), 0) return false, wrapped } var payload []byte @@ -160,42 +189,51 @@ func (c *derivedCache) Get(ctx context.Context, key string, target any) (bool, e }) if errors.Is(err, redis.Nil) { c.metrics.recordMiss() + c.metrics.recordCacheObservation(class, "get", "miss", time.Since(startedAt), 0) return false, nil } if err != nil { wrapped := cacheError{Op: "get", Kind: classifyCacheError(err), Err: err} c.metrics.recordError(wrapped) + c.metrics.recordCacheObservation(class, "get", "unavailable", time.Since(startedAt), 0) return false, wrapped } if len(payload) > c.config.MaxPayloadBytes { wrapped := cacheError{Op: "get", Kind: "cache_payload_too_large", Err: fmt.Errorf("payload bytes %d exceeds %d", len(payload), c.config.MaxPayloadBytes)} c.metrics.recordError(wrapped) + c.metrics.recordCacheObservation(class, "get", "error", time.Since(startedAt), len(payload)) return false, wrapped } if target != nil { if err := json.Unmarshal(payload, target); err != nil { wrapped := cacheError{Op: "get", Kind: "cache_payload_decode_failed", Err: err} c.metrics.recordError(wrapped) + c.metrics.recordCacheObservation(class, "get", "error", time.Since(startedAt), len(payload)) return false, wrapped } } c.metrics.recordHit() + c.metrics.recordCacheObservation(class, "get", "hit", time.Since(startedAt), len(payload)) return true, nil } func (c *derivedCache) Set(ctx context.Context, key string, value any, ttl time.Duration) error { + startedAt := time.Now() + class := cacheKeyClass(key) if !c.usable() { return nil } if err := validateCacheKey(c.config.MaxKeyBytes, key); err != nil { wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err} c.metrics.recordError(wrapped) + c.metrics.recordCacheObservation(class, "set", "error", time.Since(startedAt), 0) return wrapped } payload, err := cachePayloadBytes(value, c.config.MaxPayloadBytes) if err != nil { wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err} c.metrics.recordError(wrapped) + c.metrics.recordCacheObservation(class, "set", "error", time.Since(startedAt), 0) return wrapped } err = c.withTimeout(ctx, func(opCtx context.Context) error { @@ -204,12 +242,36 @@ func (c *derivedCache) Set(ctx context.Context, key string, value any, ttl time. if err != nil { wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err} c.metrics.recordError(wrapped) + c.metrics.recordCacheObservation(class, "set", "unavailable", time.Since(startedAt), len(payload)) return wrapped } c.metrics.recordSet() + c.metrics.recordCacheObservation(class, "set", "stored", time.Since(startedAt), len(payload)) return nil } +func (c *derivedCache) RecordStale(class string) { + if c == nil { + return + } + c.metrics.recordStale(class) +} + +func (c *derivedCache) RecordDBQueryAvoided(class string) { + if c == nil { + return + } + c.metrics.recordDBQueryAvoided(class) +} + +func (c *derivedCache) PrometheusText(service string) string { + if c == nil { + var metrics cacheMetrics + return metrics.prometheusText(service) + } + return c.metrics.prometheusText(service) +} + func (c *derivedCache) Delete(ctx context.Context, keys ...string) error { if !c.usable() || len(keys) == 0 { return nil @@ -520,6 +582,15 @@ func cacheHash(value string) string { return hex.EncodeToString(digest[:])[:24] } +func cacheKeyClass(key string) string { + for _, part := range strings.Split(key, ":") { + if strings.HasPrefix(part, "class=") { + return safeCacheToken(strings.TrimPrefix(part, "class=")) + } + } + return "unknown" +} + func (m *cacheMetrics) recordHit() { m.mu.Lock() defer m.mu.Unlock() @@ -556,16 +627,79 @@ func (m *cacheMetrics) recordError(err error) { m.LastErrorAt = time.Now().UTC() } +func (m *cacheMetrics) recordCacheObservation(class string, operation string, status string, duration time.Duration, payloadBytes int) { + m.mu.Lock() + defer m.mu.Unlock() + labels := cacheLabels(class, operation, status) + m.incrementCounterLocked(&m.Requests, labels) + m.recordHistogramLocked(&m.OperationDurations, labels, duration.Seconds(), cacheOperationDurationBucketsSeconds) + if payloadBytes > 0 { + m.recordHistogramLocked(&m.PayloadBytes, labels, float64(payloadBytes), cachePayloadBucketsBytes) + } +} + +func (m *cacheMetrics) recordStale(class string) { + m.mu.Lock() + defer m.mu.Unlock() + m.incrementCounterLocked(&m.Stale, cacheLabels(class, "read", "stale")) +} + +func (m *cacheMetrics) recordDBQueryAvoided(class string) { + m.mu.Lock() + defer m.mu.Unlock() + m.incrementCounterLocked(&m.DBQueryAvoided, cacheLabels(class, "read", "hit")) +} + +func (m *cacheMetrics) incrementCounterLocked(target *map[string]cacheCounterSeries, labels cacheMetricLabels) { + if *target == nil { + *target = map[string]cacheCounterSeries{} + } + key := cacheMetricKey(labels) + entry := (*target)[key] + if entry.Labels.Class == "" { + entry.Labels = labels + } + entry.Value++ + (*target)[key] = entry +} + +func (m *cacheMetrics) recordHistogramLocked(target *map[string]*cacheHistogramSeries, labels cacheMetricLabels, value float64, buckets []float64) { + if value < 0 { + value = 0 + } + if *target == nil { + *target = map[string]*cacheHistogramSeries{} + } + key := cacheMetricKey(labels) + entry := (*target)[key] + if entry == nil { + entry = &cacheHistogramSeries{Labels: labels, Buckets: make([]uint64, len(buckets))} + (*target)[key] = entry + } + for index, bucket := range buckets { + if value <= bucket { + entry.Buckets[index]++ + } + } + entry.Sum += value + entry.Count++ +} + func (m *cacheMetrics) snapshot() map[string]any { m.mu.Lock() defer m.mu.Unlock() result := map[string]any{ - "hits": int64(m.Hits), - "misses": int64(m.Misses), - "sets": int64(m.Sets), - "deletes": int64(m.Deletes), - "errors": int64(m.Errors), - "valuesRedacted": true, + "hits": int64(m.Hits), + "misses": int64(m.Misses), + "sets": int64(m.Sets), + "deletes": int64(m.Deletes), + "errors": int64(m.Errors), + "requestSeries": len(m.Requests), + "operationDurationSeries": len(m.OperationDurations), + "payloadBytesSeries": len(m.PayloadBytes), + "staleSeries": len(m.Stale), + "dbQueryAvoidedSeries": len(m.DBQueryAvoided), + "valuesRedacted": true, } if m.LastErrorKind != "" { result["lastErrorKind"] = m.LastErrorKind @@ -575,6 +709,132 @@ func (m *cacheMetrics) snapshot() map[string]any { return result } +func (m *cacheMetrics) prometheusText(service string) string { + m.mu.Lock() + defer m.mu.Unlock() + if service == "" { + service = "hwlab-workbench-runtime" + } + lines := []string{ + "# HELP workbench_cache_requests_total Workbench derived Redis cache requests by low-cardinality class/status.", + "# TYPE workbench_cache_requests_total counter", + } + for _, entry := range m.Requests { + lines = append(lines, prometheusCounterLine("workbench_cache_requests_total", service, entry.Labels, entry.Value)) + } + lines = append(lines, + "# HELP workbench_cache_hit_ratio Workbench derived Redis cache hit ratio by class and operation.", + "# TYPE workbench_cache_hit_ratio gauge", + ) + for _, line := range m.prometheusHitRatioLinesLocked(service) { + lines = append(lines, line) + } + lines = append(lines, + "# HELP workbench_cache_operation_duration_seconds Workbench derived Redis cache operation duration in seconds.", + "# TYPE workbench_cache_operation_duration_seconds histogram", + ) + lines = append(lines, prometheusHistogramLines("workbench_cache_operation_duration_seconds", service, m.OperationDurations, cacheOperationDurationBucketsSeconds)...) + lines = append(lines, + "# HELP workbench_cache_payload_bytes Workbench derived Redis cache payload size in bytes.", + "# TYPE workbench_cache_payload_bytes histogram", + ) + lines = append(lines, prometheusHistogramLines("workbench_cache_payload_bytes", service, m.PayloadBytes, cachePayloadBucketsBytes)...) + lines = append(lines, + "# HELP workbench_cache_stale_total Workbench derived Redis cache stale reads detected by payload freshness metadata.", + "# TYPE workbench_cache_stale_total counter", + ) + for _, entry := range m.Stale { + lines = append(lines, prometheusCounterLine("workbench_cache_stale_total", service, entry.Labels, entry.Value)) + } + lines = append(lines, + "# HELP workbench_cache_unavailable_total Workbench derived Redis cache unavailable/error requests.", + "# TYPE workbench_cache_unavailable_total counter", + ) + for _, entry := range m.Requests { + if entry.Labels.Status == "unavailable" || entry.Labels.Status == "error" { + lines = append(lines, prometheusCounterLine("workbench_cache_unavailable_total", service, entry.Labels, entry.Value)) + } + } + lines = append(lines, + "# HELP workbench_db_query_avoided_total Workbench database reads avoided by derived Redis cache hits.", + "# TYPE workbench_db_query_avoided_total counter", + ) + for _, entry := range m.DBQueryAvoided { + lines = append(lines, prometheusCounterLine("workbench_db_query_avoided_total", service, entry.Labels, entry.Value)) + } + lines = append(lines, "") + return strings.Join(lines, "\n") +} + +func (m *cacheMetrics) prometheusHitRatioLinesLocked(service string) []string { + type totals struct{ hit, total uint64 } + groups := map[string]totals{} + labels := map[string]cacheMetricLabels{} + for _, entry := range m.Requests { + if entry.Labels.Operation != "get" { + continue + } + key := entry.Labels.Class + "\x00" + entry.Labels.Operation + group := groups[key] + if entry.Labels.Status == "hit" { + group.hit += entry.Value + } + group.total += entry.Value + groups[key] = group + labels[key] = cacheMetricLabels{Class: entry.Labels.Class, Operation: entry.Labels.Operation, Status: "all"} + } + lines := []string{} + for key, group := range groups { + ratio := 0.0 + if group.total > 0 { + ratio = float64(group.hit) / float64(group.total) + } + lines = append(lines, prometheusGaugeLine("workbench_cache_hit_ratio", service, labels[key], ratio)) + } + return lines +} + +func prometheusHistogramLines(name string, service string, series map[string]*cacheHistogramSeries, buckets []float64) []string { + lines := []string{} + for _, entry := range series { + for index, bucket := range buckets { + lines = append(lines, fmt.Sprintf("%s_bucket{%s,le=\"%s\"} %d", name, prometheusLabels(service, entry.Labels), prometheusFloat(bucket), entry.Buckets[index])) + } + lines = append(lines, fmt.Sprintf("%s_bucket{%s,le=\"+Inf\"} %d", name, prometheusLabels(service, entry.Labels), entry.Count)) + lines = append(lines, fmt.Sprintf("%s_sum{%s} %.6f", name, prometheusLabels(service, entry.Labels), entry.Sum)) + lines = append(lines, fmt.Sprintf("%s_count{%s} %d", name, prometheusLabels(service, entry.Labels), entry.Count)) + } + return lines +} + +func prometheusCounterLine(name string, service string, labels cacheMetricLabels, value uint64) string { + return fmt.Sprintf("%s{%s} %d", name, prometheusLabels(service, labels), value) +} + +func prometheusGaugeLine(name string, service string, labels cacheMetricLabels, value float64) string { + return fmt.Sprintf("%s{%s} %.6f", name, prometheusLabels(service, labels), value) +} + +func prometheusLabels(service string, labels cacheMetricLabels) string { + return fmt.Sprintf("service=\"%s\",cache_key_class=\"%s\",operation=\"%s\",cache_status=\"%s\"", prometheusEscape(service), prometheusEscape(labels.Class), prometheusEscape(labels.Operation), prometheusEscape(labels.Status)) +} + +func prometheusEscape(value string) string { + return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(value, "\\", "\\\\"), "\n", "\\n"), "\"", "\\\"") +} + +func prometheusFloat(value float64) string { + return strconv.FormatFloat(value, 'f', -1, 64) +} + +func cacheLabels(class string, operation string, status string) cacheMetricLabels { + return cacheMetricLabels{Class: safeCacheToken(first(class, "unknown")), Operation: safeCacheToken(first(operation, "unknown")), Status: safeCacheToken(first(status, "unknown"))} +} + +func cacheMetricKey(labels cacheMetricLabels) string { + return labels.Class + "\x00" + labels.Operation + "\x00" + labels.Status +} + func envBool(name string, fallback bool) bool { value := strings.ToLower(strings.TrimSpace(os.Getenv(name))) if value == "" { diff --git a/internal/workbenchruntime/cache_test.go b/internal/workbenchruntime/cache_test.go index 3b6be802..3fbd32a9 100644 --- a/internal/workbenchruntime/cache_test.go +++ b/internal/workbenchruntime/cache_test.go @@ -94,6 +94,27 @@ func TestDisabledCacheDoesNotFailReadPath(t *testing.T) { } } +func TestCachePrometheusMetricsUseLowCardinalityLabels(t *testing.T) { + var metrics cacheMetrics + metrics.recordCacheObservation("turn.terminal.snapshot", "get", "hit", 12*time.Millisecond, 4096) + metrics.recordCacheObservation("sessions.summary", "get", "miss", 25*time.Millisecond, 0) + metrics.recordStale("sessions.summary") + metrics.recordDBQueryAvoided("turn.terminal.snapshot") + + text := metrics.prometheusText(serviceID) + for _, name := range []string{"workbench_cache_requests_total", "workbench_cache_hit_ratio", "workbench_cache_operation_duration_seconds", "workbench_cache_payload_bytes", "workbench_cache_stale_total", "workbench_db_query_avoided_total"} { + if !strings.Contains(text, name) { + t.Fatalf("metrics missing %s in:\n%s", name, text) + } + } + if !strings.Contains(text, `cache_key_class="turn.terminal.snapshot"`) || !strings.Contains(text, `cache_status="hit"`) { + t.Fatalf("metrics missing cache labels:\n%s", text) + } + if strings.Contains(text, "trc_") || strings.Contains(text, "ses_") || strings.Contains(text, "actor=") || strings.Contains(text, "cacheKey") { + t.Fatalf("metrics leaked high-cardinality or key material:\n%s", text) + } +} + func TestSessionsSummaryCacheTTLUsesShortTTLForUnstablePages(t *testing.T) { config := cacheConfig{SessionsSummaryTTL: 30 * time.Second, RunningPageTTL: 5 * time.Second} stable := map[string]any{"sessions": []any{map[string]any{"terminal": true, "running": false, "projectionStatus": "caught-up"}}} diff --git a/internal/workbenchruntime/service.go b/internal/workbenchruntime/service.go index d8880ab8..00c58981 100644 --- a/internal/workbenchruntime/service.go +++ b/internal/workbenchruntime/service.go @@ -233,12 +233,23 @@ func (s *Server) Close() error { func (s *Server) routes() { s.mux.HandleFunc("/health/live", s.handleLive) s.mux.HandleFunc("/health/ready", s.handleReady) + s.mux.HandleFunc("/metrics", s.handleMetrics) s.mux.HandleFunc("/v1/workbench-runtime/facts", s.handleFacts) s.mux.HandleFunc("/v1/workbench-runtime/trace-events", s.handleTraceEvents) s.mux.HandleFunc("/v1/workbench-runtime/sessions", s.handleSessions) s.mux.HandleFunc("/v1/workbench-runtime/projection-outbox", s.handleProjectionOutbox) } +func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) { + if r.Method != http.MethodGet { + methodNotAllowed(w, "GET") + return + } + w.Header().Set("content-type", "text/plain; version=0.0.4; charset=utf-8") + w.WriteHeader(http.StatusOK) + _, _ = w.Write([]byte(s.cache.PrometheusText(serviceID))) +} + func (s *Server) handleLive(w http.ResponseWriter, r *http.Request) { cache := s.cache.Diagnostic(r.Context()) if s.config.Cache.UnavailableLivenessFailure && cache["enabled"] == true && cache["available"] != true { @@ -297,6 +308,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s plan.Meta["ttlMs"] = entry.TTLMillis plan.Meta["freshnessSloMs"] = entry.TTLMillis attachFactsCacheMetadata(payload, plan.Meta) + s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta) return payload, nil } setFactsCacheStatus(plan.Meta, "stale") @@ -316,6 +328,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s plan.Meta["dbQueryDurationMs"] = dbDuration.Milliseconds() if plan.Key == "" { attachFactsCacheMetadata(payload, plan.Meta) + s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta) return payload, nil } @@ -330,6 +343,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s setFactsCacheStatus(plan.Meta, "skipped") plan.Meta["skipReason"] = classifyCacheError(payloadErr) attachFactsCacheMetadata(payload, plan.Meta) + s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta) return payload, nil } plan.Meta["payloadBytes"] = len(payloadBytes) @@ -337,6 +351,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s setFactsCacheStatus(plan.Meta, "skipped") plan.Meta["skipReason"] = ttlReason attachFactsCacheMetadata(payload, plan.Meta) + s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta) return payload, nil } entry := factsCacheEntry{CachedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: storedPayload, PayloadBytes: len(payloadBytes), TTLMillis: ttl.Milliseconds(), ValuesRedacted: true} @@ -344,6 +359,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s setFactsCacheStatus(plan.Meta, "set_failed") plan.Meta["errorKind"] = classifyCacheError(err) attachFactsCacheMetadata(payload, plan.Meta) + s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta) return payload, nil } if plan.Meta["status"] != "stale" { @@ -351,9 +367,61 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s } plan.Meta["stored"] = true attachFactsCacheMetadata(payload, plan.Meta) + s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta) return payload, nil } +func (s *Server) observeCacheDiagnostic(ctx context.Context, class string, meta map[string]any) { + if s == nil || meta == nil { + return + } + cacheClass := first(text(meta["class"]), class, "unknown") + status := first(text(meta["status"]), "unknown") + if s.cache != nil { + if status == "stale" { + s.cache.RecordStale(cacheClass) + } + if boolValue(meta["dbQueryAvoided"]) { + s.cache.RecordDBQueryAvoided(cacheClass) + } + } + attrs := map[string]any{ + "cacheStatus": status, + "cacheKeyClass": safeCacheToken(cacheClass), + "workbench.cache.status": status, + "workbench.cache.key_class": safeCacheToken(cacheClass), + "workbench.cache.db_avoided": boolValue(meta["dbQueryAvoided"]), + "workbench.cache.values_source": "diagnostic", + "valuesRedacted": true, + } + if meta["cacheAgeMs"] != nil { + age := int64FromAny(meta["cacheAgeMs"]) + attrs["cacheAgeMs"] = age + attrs["workbench.cache.age_ms"] = age + } + if meta["projectionSeq"] != nil { + seq := int64FromAny(meta["projectionSeq"]) + attrs["projectionSeq"] = seq + attrs["workbench.projection.seq"] = seq + } + if meta["payloadBytes"] != nil { + attrs["workbench.cache.payload_bytes"] = int64FromAny(meta["payloadBytes"]) + } + if meta["ttlMs"] != nil { + attrs["workbench.cache.ttl_ms"] = int64FromAny(meta["ttlMs"]) + } + if meta["freshnessSloMs"] != nil { + attrs["workbench.cache.freshness_slo_ms"] = int64FromAny(meta["freshnessSloMs"]) + } + if revisionSource := text(meta["revisionSource"]); revisionSource != "" { + attrs["workbench.cache.revision_source"] = revisionSource + } + if invalidationMode := text(meta["invalidationMode"]); invalidationMode != "" { + attrs["workbench.cache.invalidation_mode"] = invalidationMode + } + _ = s.withOtelInternalSpan(ctx, "workbench-runtime.cache", attrs, func(context.Context) error { return nil }) +} + func factsResponsePayload(facts map[string][]any, persistence map[string]any) map[string]any { count := 0 for _, rows := range facts { @@ -732,6 +800,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[ cacheMeta["freshnessSloMs"] = entry.TTLMillis applySessionsCacheRevisionMetadata(cacheMeta, sessionsSummaryRevision{ProjectionRevision: entry.ProjectionRevision, ProjectionSeqMax: entry.ProjectionSeqMax, SessionUpdatedAtMax: entry.SessionUpdatedAtMax, SessionCount: entry.SessionCount, RevisionSource: "cached_payload"}) attachSessionsCacheMetadata(payload, cacheMeta) + s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta) return payload, nil } setSessionsCacheStatus(cacheMeta, "stale") @@ -751,6 +820,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[ cacheMeta["dbQueryDurationMs"] = dbDuration.Milliseconds() if cacheKey == "" { attachSessionsCacheMetadata(payload, cacheMeta) + s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta) return payload, nil } @@ -767,6 +837,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[ setSessionsCacheStatus(cacheMeta, "skipped") cacheMeta["skipReason"] = classifyCacheError(payloadErr) attachSessionsCacheMetadata(payload, cacheMeta) + s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta) return payload, nil } cacheMeta["payloadBytes"] = len(payloadBytes) @@ -774,6 +845,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[ setSessionsCacheStatus(cacheMeta, "skipped") cacheMeta["skipReason"] = ttlReason attachSessionsCacheMetadata(payload, cacheMeta) + s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta) return payload, nil } entry := sessionsCacheEntry{CachedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: storedPayload, PayloadBytes: len(payloadBytes), TTLMillis: ttl.Milliseconds(), ProjectionRevision: revision.ProjectionRevision, ProjectionSeqMax: revision.ProjectionSeqMax, SessionUpdatedAtMax: revision.SessionUpdatedAtMax, SessionCount: revision.SessionCount, ValuesRedacted: true} @@ -781,6 +853,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[ setSessionsCacheStatus(cacheMeta, "set_failed") cacheMeta["errorKind"] = classifyCacheError(err) attachSessionsCacheMetadata(payload, cacheMeta) + s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta) return payload, nil } if cacheMeta["status"] != "stale" { @@ -788,6 +861,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[ } cacheMeta["stored"] = true attachSessionsCacheMetadata(payload, cacheMeta) + s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta) return payload, nil } diff --git a/web/hwlab-cloud-web/src/types/index.ts b/web/hwlab-cloud-web/src/types/index.ts index 279b44d3..41712dab 100644 --- a/web/hwlab-cloud-web/src/types/index.ts +++ b/web/hwlab-cloud-web/src/types/index.ts @@ -576,6 +576,7 @@ export interface WebPerformanceSummaryResponse { workbenchJourneys?: WebPerformanceRow[]; workbenchEventPhases?: WebPerformanceRow[]; workbenchBackendEvents?: WebPerformanceRow[]; + workbenchCacheRows?: WebPerformanceRow[]; workbenchRows?: WebPerformanceRow[]; rows?: WebPerformanceRow[]; [key: string]: unknown; diff --git a/web/hwlab-cloud-web/src/views/PerformanceView.vue b/web/hwlab-cloud-web/src/views/PerformanceView.vue index 62fb9df6..b61cb196 100644 --- a/web/hwlab-cloud-web/src/views/PerformanceView.vue +++ b/web/hwlab-cloud-web/src/views/PerformanceView.vue @@ -189,6 +189,10 @@ function metricLabel(row: WebPerformanceRow): string { navigation_load: "页面加载完成", trace_event_projected: "轨迹事件投影", backend_event_visible: "后端事件到可见", + cache_hit_ratio: "缓存命中率", + db_query_avoided: "DB 查询避免率", + cache_unavailable: "缓存不可用率", + sessions_api_p95: "会话 API P95", longtask: "主线程长任务" }; return labels[metric] ?? metric.replace(/[_-]/gu, " "); @@ -199,6 +203,7 @@ function kindLabel(kind?: string): string { workbench_journey: "工作台体感", workbench_event_phase: "事件阶段", workbench_backend_event_visible: "后端可见", + workbench_cache: "缓存观测", web_vital: "浏览器体感", navigation: "页面加载", api: "同源 API", @@ -403,7 +408,7 @@ function targetStateLabel(value?: string): string { } function cacheLabel(value?: string): string { - const labels: Record = { warm: "热缓存", cold: "冷加载", hit: "命中", miss: "未命中", unknown: "未知" }; + const labels: Record = { warm: "热缓存", cold: "冷加载", hit: "命中", miss: "未命中", sessions_summary: "会话摘要", terminal_turn_snapshot: "终态轮次快照", trace_terminal_events: "终态轨迹事件", unknown: "未知" }; return labels[String(value ?? "unknown")] ?? String(value); }