Merge pull request #1897 from pikasTech/fix/1870-p7-cache-observability

[1870-P7] 接入 Workbench cache 可观测性
This commit is contained in:
Lyon
2026-06-22 15:11:41 +08:00
committed by GitHub
11 changed files with 813 additions and 23 deletions
@@ -56,6 +56,24 @@ test("backend performance store exposes low-cardinality request evidence for RUM
assert.doesNotMatch(JSON.stringify(evidence), /ses_secret|traceId|sessionId|runId|prompt|api key/iu); assert.doesNotMatch(JSON.stringify(evidence), /ses_secret|traceId|sessionId|runId|prompt|api key/iu);
}); });
test("backend performance store records Workbench cache metrics without cache keys or ids", () => {
const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } });
store.recordWorkbenchCache({ route: "/v1/workbench/turns/trc_secret_0123456789abcdef", cacheKeyClass: "turn.terminal.snapshot", cacheStatus: "hit", durationMs: 12, payloadBytes: 4096, cacheAgeMs: 5, dbQueryAvoided: true, freshnessSloMs: 300000, projectionSeq: 42 });
store.recordWorkbenchCache({ route: "/v1/workbench/sessions", cacheKeyClass: "sessions.summary", cacheStatus: "unavailable", durationMs: 120, dbQueryAvoided: false });
const text = store.metricsText();
assert.match(text, /workbench_cache_requests_total\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*cache_status="hit"[^}]*\} 1/u);
assert.match(text, /workbench_cache_hit_ratio\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*\} 1(?:\.0+)?/u);
assert.match(text, /workbench_cache_unavailable_total\{[^}]*route="\/v1\/workbench\/sessions"[^}]*cache_key_class="sessions_summary"[^}]*\} 1/u);
assert.match(text, /workbench_db_query_avoided_total\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*\} 1/u);
const evidence = store.evidence({ window: "15m" });
assert.equal(evidence.cacheSampleCount, 2);
assert.ok(evidence.cache.some((row) => row.route === "/v1/workbench/turns/:traceId" && row.cacheKeyClass === "turn_terminal_snapshot" && row.hitRatio === 1));
assert.ok(evidence.cache.some((row) => row.route === "/v1/workbench/sessions" && row.unavailableCount === 1));
assert.doesNotMatch(`${text}\n${JSON.stringify(evidence)}`, /trc_secret|0123456789abcdef|ses_secret|prompt|api key|authorization|cookie/iu);
});
test("backend performance records Code Agent trace handler phases without high-cardinality labels", async () => { test("backend performance records Code Agent trace handler phases without high-cardinality labels", async () => {
const traceId = "trc_backend_phase_trace"; const traceId = "trc_backend_phase_trace";
const runId = "run_backend_phase"; const runId = "run_backend_phase";
+226 -2
View File
@@ -10,6 +10,8 @@ const PROJECTION_LAG_EVENT_BUCKETS = [0, 1, 5, 10, 50, 100, 500, 1000, 2000, 500
const PROJECTION_LAG_SECONDS_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 900, 1800, 3600]; const PROJECTION_LAG_SECONDS_BUCKETS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 300, 900, 1800, 3600];
const WORKBENCH_TURN_DURATION_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60]; const WORKBENCH_TURN_DURATION_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60];
const WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304]; const WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304];
const WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS = [0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10];
const WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS = [512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304];
const AGENTRUN_RESULT_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120]; const AGENTRUN_RESULT_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30, 60, 120];
const AGENTRUN_RESULT_EVENTS_BUCKETS = [0, 50, 100, 500, 1000, 2000, 5000, 10000]; const AGENTRUN_RESULT_EVENTS_BUCKETS = [0, 50, 100, 500, 1000, 2000, 5000, 10000];
const AGENTRUN_RESULT_PAGES_BUCKETS = [0, 1, 2, 5, 10, 20, 50, 100]; const AGENTRUN_RESULT_PAGES_BUCKETS = [0, 1, 2, 5, 10, 20, 50, 100];
@@ -93,6 +95,19 @@ interface WorkbenchTurnReadInput {
timedOut?: boolean; timedOut?: boolean;
} }
interface WorkbenchCacheInput {
route: string;
cacheKeyClass?: unknown;
cacheStatus?: unknown;
operation?: unknown;
durationMs?: unknown;
payloadBytes?: unknown;
cacheAgeMs?: unknown;
dbQueryAvoided?: unknown;
freshnessSloMs?: unknown;
projectionSeq?: unknown;
}
interface AgentRunResultInput { interface AgentRunResultInput {
durationMs?: unknown; durationMs?: unknown;
eventsScanned?: unknown; eventsScanned?: unknown;
@@ -127,6 +142,20 @@ interface BackendRequestSample {
phases: Array<{ phase: string; outcome: string; durationSeconds: number }>; phases: Array<{ phase: string; outcome: string; durationSeconds: number }>;
} }
interface WorkbenchCacheEvidenceSample {
atMs: number;
route: string;
cacheKeyClass: string;
cacheStatus: string;
operation: string;
durationSeconds: number | null;
payloadBytes: number | null;
cacheAgeMs: number | null;
dbQueryAvoided: boolean;
freshnessSloMs: number | null;
projectionSeq: number | null;
}
export function createBackendPerformanceStore(options: BackendPerformanceStoreOptions = {}) { export function createBackendPerformanceStore(options: BackendPerformanceStoreOptions = {}) {
const env = options.env ?? process.env; const env = options.env ?? process.env;
const maxSeries = parsePositiveInteger(env.HWLAB_BACKEND_PERFORMANCE_MAX_SERIES, options.maxSeries ?? 320); const maxSeries = parsePositiveInteger(env.HWLAB_BACKEND_PERFORMANCE_MAX_SERIES, options.maxSeries ?? 320);
@@ -144,6 +173,12 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
const turnGetDurationSeries = new Map<string, HistogramSeries>(); const turnGetDurationSeries = new Map<string, HistogramSeries>();
const turnGetTimeoutSeries = new Map<string, CounterSeries>(); const turnGetTimeoutSeries = new Map<string, CounterSeries>();
const turnGetResponseBytesSeries = new Map<string, HistogramSeries>(); const turnGetResponseBytesSeries = new Map<string, HistogramSeries>();
const cacheRequestSeries = new Map<string, CounterSeries>();
const cacheOperationDurationSeries = new Map<string, HistogramSeries>();
const cachePayloadBytesSeries = new Map<string, HistogramSeries>();
const cacheStaleSeries = new Map<string, CounterSeries>();
const cacheUnavailableSeries = new Map<string, CounterSeries>();
const dbQueryAvoidedSeries = new Map<string, CounterSeries>();
const agentRunResultDurationSeries = new Map<string, HistogramSeries>(); const agentRunResultDurationSeries = new Map<string, HistogramSeries>();
const agentRunResultPagesSeries = new Map<string, HistogramSeries>(); const agentRunResultPagesSeries = new Map<string, HistogramSeries>();
const agentRunResultEventsSeries = new Map<string, HistogramSeries>(); const agentRunResultEventsSeries = new Map<string, HistogramSeries>();
@@ -154,6 +189,7 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
const projectorErrorsSeries = new Map<string, CounterSeries>(); const projectorErrorsSeries = new Map<string, CounterSeries>();
const projectorLastSuccessSeries = new Map<string, GaugeSeries>(); const projectorLastSuccessSeries = new Map<string, GaugeSeries>();
const requestSamples: BackendRequestSample[] = []; const requestSamples: BackendRequestSample[] = [];
const cacheSamples: WorkbenchCacheEvidenceSample[] = [];
const baseLabels = { const baseLabels = {
service: "hwlab-cloud-api", service: "hwlab-cloud-api",
namespace: sanitizeLabelValue(env.POD_NAMESPACE ?? env.HWLAB_METRICS_NAMESPACE ?? "hwlab-v02"), namespace: sanitizeLabelValue(env.POD_NAMESPACE ?? env.HWLAB_METRICS_NAMESPACE ?? "hwlab-v02"),
@@ -340,6 +376,55 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
} }
} }
function recordWorkbenchCache(input: WorkbenchCacheInput) {
const route = workbenchCacheRouteTemplate(input.route);
const cacheKeyClass = normalizeCacheKeyClass(input.cacheKeyClass);
const cacheStatus = normalizeCacheStatus(input.cacheStatus);
const operation = normalizeCacheOperation(input.operation ?? "read");
const labels = {
...workbenchBaseLabels,
route,
cache_key_class: cacheKeyClass,
operation,
cache_status: cacheStatus
};
incrementCounter(cacheRequestSeries, labels, maxSeries) || incrementDropped("workbench_cache_request_series_limit");
const durationMs = finiteNonNegativeNumber(input.durationMs);
const durationSeconds = durationMs === null ? null : boundedDurationMs(durationMs, 120_000) / 1000;
if (durationSeconds !== null) {
if (!recordHistogram(cacheOperationDurationSeries, labels, durationSeconds, WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS, maxSeries)) incrementDropped("workbench_cache_operation_series_limit");
}
const payloadBytes = finiteNonNegativeNumber(input.payloadBytes);
if (payloadBytes !== null) {
if (!recordHistogram(cachePayloadBytesSeries, labels, payloadBytes, WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS, maxSeries)) incrementDropped("workbench_cache_payload_series_limit");
}
if (cacheStatus === "stale") {
incrementCounter(cacheStaleSeries, labels, maxSeries) || incrementDropped("workbench_cache_stale_series_limit");
}
if (cacheStatus === "unavailable" || cacheStatus === "set_failed" || cacheStatus === "error") {
incrementCounter(cacheUnavailableSeries, labels, maxSeries) || incrementDropped("workbench_cache_unavailable_series_limit");
}
const dbQueryAvoided = booleanValue(input.dbQueryAvoided);
if (dbQueryAvoided) {
const avoidedLabels = { ...workbenchBaseLabels, route, cache_key_class: cacheKeyClass };
incrementCounter(dbQueryAvoidedSeries, avoidedLabels, maxSeries) || incrementDropped("workbench_db_query_avoided_series_limit");
}
cacheSamples.push({
atMs: Date.now(),
route,
cacheKeyClass,
cacheStatus,
operation,
durationSeconds,
payloadBytes,
cacheAgeMs: finiteNonNegativeNumber(input.cacheAgeMs),
dbQueryAvoided,
freshnessSloMs: finiteNonNegativeNumber(input.freshnessSloMs),
projectionSeq: finiteNonNegativeNumber(input.projectionSeq)
});
while (cacheSamples.length > maxEvidenceSamples) cacheSamples.shift();
}
function recordAgentRunResult(input: AgentRunResultInput = {}) { function recordAgentRunResult(input: AgentRunResultInput = {}) {
const eventCount = finiteNonNegativeNumber(input.eventCount ?? input.eventsScanned) ?? 0; const eventCount = finiteNonNegativeNumber(input.eventCount ?? input.eventsScanned) ?? 0;
const labels = { const labels = {
@@ -450,6 +535,27 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
"# HELP hwlab_workbench_turn_get_response_bytes Workbench turn/read API response bytes.", "# HELP hwlab_workbench_turn_get_response_bytes Workbench turn/read API response bytes.",
"# TYPE hwlab_workbench_turn_get_response_bytes histogram", "# TYPE hwlab_workbench_turn_get_response_bytes histogram",
...renderHistogram("hwlab_workbench_turn_get_response_bytes", turnGetResponseBytesSeries, WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS), ...renderHistogram("hwlab_workbench_turn_get_response_bytes", turnGetResponseBytesSeries, WORKBENCH_TURN_RESPONSE_BYTES_BUCKETS),
"# HELP workbench_cache_requests_total Workbench derived cache requests observed by Cloud API read path.",
"# TYPE workbench_cache_requests_total counter",
...renderCounters("workbench_cache_requests_total", cacheRequestSeries),
"# HELP workbench_cache_hit_ratio Workbench derived cache hit ratio by route and cache class.",
"# TYPE workbench_cache_hit_ratio gauge",
...renderWorkbenchCacheHitRatio(cacheRequestSeries),
"# HELP workbench_cache_operation_duration_seconds Workbench derived cache operation observation duration in seconds.",
"# TYPE workbench_cache_operation_duration_seconds histogram",
...renderHistogram("workbench_cache_operation_duration_seconds", cacheOperationDurationSeries, WORKBENCH_CACHE_OPERATION_BUCKETS_SECONDS),
"# HELP workbench_cache_payload_bytes Workbench derived cache payload size in bytes.",
"# TYPE workbench_cache_payload_bytes histogram",
...renderHistogram("workbench_cache_payload_bytes", cachePayloadBytesSeries, WORKBENCH_CACHE_PAYLOAD_BYTES_BUCKETS),
"# HELP workbench_cache_stale_total Workbench derived cache stale observations.",
"# TYPE workbench_cache_stale_total counter",
...renderCounters("workbench_cache_stale_total", cacheStaleSeries),
"# HELP workbench_cache_unavailable_total Workbench derived cache unavailable observations.",
"# TYPE workbench_cache_unavailable_total counter",
...renderCounters("workbench_cache_unavailable_total", cacheUnavailableSeries),
"# HELP workbench_db_query_avoided_total Workbench DB reads avoided by derived cache hits.",
"# TYPE workbench_db_query_avoided_total counter",
...renderCounters("workbench_db_query_avoided_total", dbQueryAvoidedSeries),
"# HELP hwlab_agentrun_result_duration_seconds AgentRun command result aggregation duration in seconds.", "# HELP hwlab_agentrun_result_duration_seconds AgentRun command result aggregation duration in seconds.",
"# TYPE hwlab_agentrun_result_duration_seconds histogram", "# TYPE hwlab_agentrun_result_duration_seconds histogram",
...renderHistogram("hwlab_agentrun_result_duration_seconds", agentRunResultDurationSeries, AGENTRUN_RESULT_BUCKETS_SECONDS), ...renderHistogram("hwlab_agentrun_result_duration_seconds", agentRunResultDurationSeries, AGENTRUN_RESULT_BUCKETS_SECONDS),
@@ -491,10 +597,12 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
upstreamCount: histogramSampleCount(upstreamSeries), upstreamCount: histogramSampleCount(upstreamSeries),
projectionLagSeries: projectionLagEventsSeries.size + projectionLagSecondsSeries.size, projectionLagSeries: projectionLagEventsSeries.size + projectionLagSecondsSeries.size,
turnReadSeries: turnGetDurationSeries.size, turnReadSeries: turnGetDurationSeries.size,
cacheSeries: cacheRequestSeries.size + cacheOperationDurationSeries.size + cachePayloadBytesSeries.size + cacheStaleSeries.size + cacheUnavailableSeries.size + dbQueryAvoidedSeries.size,
agentRunResultSeries: agentRunResultDurationSeries.size, agentRunResultSeries: agentRunResultDurationSeries.size,
projectorSeries: projectorBatchDurationSeries.size + projectorCandidatesSeries.size + projectorEventsProcessedSeries.size + projectorErrorsSeries.size, projectorSeries: projectorBatchDurationSeries.size + projectorCandidatesSeries.size + projectorEventsProcessedSeries.size + projectorErrorsSeries.size,
droppedSeries: droppedSeries.size, droppedSeries: droppedSeries.size,
evidenceSamples: requestSamples.length evidenceSamples: requestSamples.length,
cacheEvidenceSamples: cacheSamples.length
}; };
} }
@@ -504,7 +612,9 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
const observedAt = new Date(generatedAtMs).toISOString(); const observedAt = new Date(generatedAtMs).toISOString();
const cutoff = sampleWindow.milliseconds === null ? null : generatedAtMs - sampleWindow.milliseconds; const cutoff = sampleWindow.milliseconds === null ? null : generatedAtMs - sampleWindow.milliseconds;
const windowSamples = requestSamples.filter((sample) => cutoff === null || (sample.atMs >= cutoff && sample.atMs <= generatedAtMs)); const windowSamples = requestSamples.filter((sample) => cutoff === null || (sample.atMs >= cutoff && sample.atMs <= generatedAtMs));
const windowCacheSamples = cacheSamples.filter((sample) => cutoff === null || (sample.atMs >= cutoff && sample.atMs <= generatedAtMs));
const rows = backendEvidenceRows(windowSamples); const rows = backendEvidenceRows(windowSamples);
const cache = backendCacheEvidenceRows(windowCacheSamples);
return { return {
schemaVersion: "hwlab-backend-performance-evidence-v1", schemaVersion: "hwlab-backend-performance-evidence-v1",
source: "cloud-api-backend-performance-store", source: "cloud-api-backend-performance-store",
@@ -518,8 +628,11 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
}, },
sampleCount: windowSamples.length, sampleCount: windowSamples.length,
retainedSampleCount: requestSamples.length, retainedSampleCount: requestSamples.length,
cacheSampleCount: windowCacheSamples.length,
retainedCacheSampleCount: cacheSamples.length,
routeCount: new Set(rows.map((row) => row.route)).size, routeCount: new Set(rows.map((row) => row.route)).size,
rows, rows,
cache,
valuesRedacted: true valuesRedacted: true
}; };
} }
@@ -530,6 +643,7 @@ export function createBackendPerformanceStore(options: BackendPerformanceStoreOp
recordWorkbenchProjection, recordWorkbenchProjection,
recordWorkbenchProjectionCursorAdvance, recordWorkbenchProjectionCursorAdvance,
recordWorkbenchTurnRead, recordWorkbenchTurnRead,
recordWorkbenchCache,
recordAgentRunResult, recordAgentRunResult,
recordWorkbenchProjectorBatch, recordWorkbenchProjectorBatch,
recordWorkbenchProjectorCandidate, recordWorkbenchProjectorCandidate,
@@ -570,6 +684,12 @@ function workbenchTurnRouteTemplate(input: unknown): string {
return route; return route;
} }
function workbenchCacheRouteTemplate(input: unknown): string {
const route = workbenchTurnRouteTemplate(input);
if (route === "/v1/workbench/traces/:id/events") return "/v1/workbench/traces/:traceId/events";
return route;
}
function resolveBackendEvidenceWindow(input: unknown) { function resolveBackendEvidenceWindow(input: unknown) {
const key = String(input ?? DEFAULT_BACKEND_EVIDENCE_WINDOW).trim().toLowerCase(); const key = String(input ?? DEFAULT_BACKEND_EVIDENCE_WINDOW).trim().toLowerCase();
if (Object.prototype.hasOwnProperty.call(BACKEND_EVIDENCE_WINDOWS, key)) return BACKEND_EVIDENCE_WINDOWS[key as keyof typeof BACKEND_EVIDENCE_WINDOWS]; if (Object.prototype.hasOwnProperty.call(BACKEND_EVIDENCE_WINDOWS, key)) return BACKEND_EVIDENCE_WINDOWS[key as keyof typeof BACKEND_EVIDENCE_WINDOWS];
@@ -627,6 +747,74 @@ function backendEvidenceRows(samples: BackendRequestSample[]) {
}).sort((left, right) => (right.p95 - left.p95) || (right.count - left.count)); }).sort((left, right) => (right.p95 - left.p95) || (right.count - left.count));
} }
function backendCacheEvidenceRows(samples: WorkbenchCacheEvidenceSample[]) {
const groups = new Map<string, {
route: string;
cacheKeyClass: string;
count: number;
hitCount: number;
missCount: number;
staleCount: number;
unavailableCount: number;
dbQueryAvoidedCount: number;
durations: number[];
payloads: number[];
ages: number[];
freshness: number[];
projectionSeqMax: number | null;
firstAt: number;
lastAt: number;
}>();
for (const sample of samples) {
const key = stableLabelKey({ route: sample.route, cache_key_class: sample.cacheKeyClass });
let group = groups.get(key);
if (!group) {
group = { route: sample.route, cacheKeyClass: sample.cacheKeyClass, count: 0, hitCount: 0, missCount: 0, staleCount: 0, unavailableCount: 0, dbQueryAvoidedCount: 0, durations: [], payloads: [], ages: [], freshness: [], projectionSeqMax: null, firstAt: sample.atMs, lastAt: sample.atMs };
groups.set(key, group);
}
group.count += 1;
if (sample.cacheStatus === "hit") group.hitCount += 1;
if (sample.cacheStatus === "miss") group.missCount += 1;
if (sample.cacheStatus === "stale") group.staleCount += 1;
if (sample.cacheStatus === "unavailable" || sample.cacheStatus === "set_failed" || sample.cacheStatus === "error") group.unavailableCount += 1;
if (sample.dbQueryAvoided) group.dbQueryAvoidedCount += 1;
if (sample.durationSeconds !== null) group.durations.push(sample.durationSeconds);
if (sample.payloadBytes !== null) group.payloads.push(sample.payloadBytes);
if (sample.cacheAgeMs !== null) group.ages.push(sample.cacheAgeMs);
if (sample.freshnessSloMs !== null) group.freshness.push(sample.freshnessSloMs);
if (sample.projectionSeq !== null) group.projectionSeqMax = Math.max(group.projectionSeqMax ?? 0, sample.projectionSeq);
group.firstAt = Math.min(group.firstAt, sample.atMs);
group.lastAt = Math.max(group.lastAt, sample.atMs);
}
return [...groups.values()].map((group) => {
const duration = exactEvidenceStats(group.durations);
const payloadBytes = group.payloads.length ? exactEvidenceStats(group.payloads) : null;
const cacheAgeMs = group.ages.length ? exactEvidenceStats(group.ages) : null;
const freshnessSloMs = group.freshness.length ? exactEvidenceStats(group.freshness) : null;
return {
route: group.route,
cacheKeyClass: group.cacheKeyClass,
count: group.count,
hitCount: group.hitCount,
missCount: group.missCount,
staleCount: group.staleCount,
unavailableCount: group.unavailableCount,
dbQueryAvoidedCount: group.dbQueryAvoidedCount,
hitRatio: group.count > 0 ? roundEvidenceValue(group.hitCount / group.count) : 0,
dbQueryAvoidedRatio: group.count > 0 ? roundEvidenceValue(group.dbQueryAvoidedCount / group.count) : 0,
unavailableRatio: group.count > 0 ? roundEvidenceValue(group.unavailableCount / group.count) : 0,
operationDuration: { ...duration, unit: "seconds" },
payloadBytes: payloadBytes ? { count: payloadBytes.count, average: Math.round(payloadBytes.average), p50: Math.round(payloadBytes.p50), p75: Math.round(payloadBytes.p75), p95: Math.round(payloadBytes.p95), unit: "bytes" } : null,
cacheAgeMs: cacheAgeMs ? { ...cacheAgeMs, unit: "milliseconds" } : null,
freshnessSloMs: freshnessSloMs ? { ...freshnessSloMs, unit: "milliseconds" } : null,
projectionSeqMax: group.projectionSeqMax,
firstObservedAt: new Date(group.firstAt).toISOString(),
lastObservedAt: new Date(group.lastAt).toISOString(),
valuesRedacted: true
};
}).sort((left, right) => (right.unavailableRatio - left.unavailableRatio) || (right.count - left.count) || left.route.localeCompare(right.route));
}
function exactEvidenceStats(values: number[]) { function exactEvidenceStats(values: number[]) {
const sorted = values.filter(Number.isFinite).sort((left, right) => left - right); const sorted = values.filter(Number.isFinite).sort((left, right) => left - right);
const count = sorted.length; const count = sorted.length;
@@ -725,6 +913,21 @@ function renderGauges(metricName: string, series: Map<string, GaugeSeries>) {
return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${formatPrometheusValue(entry.value)}`); return [...series.values()].map((entry) => `${metricName}{${renderLabels(entry.labels)}} ${formatPrometheusValue(entry.value)}`);
} }
function renderWorkbenchCacheHitRatio(series: Map<string, CounterSeries>) {
const groups = new Map<string, { labels: Record<string, string>; hit: number; total: number }>();
for (const entry of series.values()) {
const labels = { ...entry.labels };
const cacheStatus = labels.cache_status;
delete labels.cache_status;
const key = stableLabelKey(labels);
const group = groups.get(key) ?? { labels, hit: 0, total: 0 };
if (cacheStatus === "hit") group.hit += entry.value;
group.total += entry.value;
groups.set(key, group);
}
return [...groups.values()].map((group) => `${"workbench_cache_hit_ratio"}{${renderLabels(group.labels)}} ${formatPrometheusValue(group.total > 0 ? group.hit / group.total : 0)}`);
}
function renderHistogram(metricName: string, series: Map<string, HistogramSeries>, buckets: number[]) { function renderHistogram(metricName: string, series: Map<string, HistogramSeries>, buckets: number[]) {
const lines: string[] = []; const lines: string[] = [];
for (const entry of series.values()) { for (const entry of series.values()) {
@@ -790,7 +993,22 @@ function normalizeStatusClass(value: unknown) {
function normalizeStatusLabel(value: unknown) { function normalizeStatusLabel(value: unknown) {
const text = sanitizeMetricName(value, "unknown"); const text = sanitizeMetricName(value, "unknown");
return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown|running|pending|completed|failed|blocked|degraded|advanced|unchanged|scheduled|seen|skipped|caught_up|projecting|stale|missing)$/u.test(text) ? text : "unknown"; return /^(?:ok|error|timeout|denied|network|cancelled|canceled|unknown|running|pending|completed|failed|blocked|degraded|advanced|unchanged|scheduled|seen|skipped|caught_up|projecting|stale|missing|hit|miss|stored|set_failed|unavailable|disabled)$/u.test(text) ? text : "unknown";
}
function normalizeCacheStatus(value: unknown) {
const text = normalizeStatusLabel(value);
return /^(?:hit|miss|stale|skipped|stored|set_failed|unavailable|disabled|error|unknown)$/u.test(text) ? text : "unknown";
}
function normalizeCacheOperation(value: unknown) {
const text = sanitizeMetricName(value, "read");
return /^(?:read|get|set|write|delete|ttl|ping)$/u.test(text) ? text : "read";
}
function normalizeCacheKeyClass(value: unknown) {
const text = sanitizeMetricName(value, "unknown").slice(0, 64);
return text || "unknown";
} }
function normalizeOutcome(value: unknown) { function normalizeOutcome(value: unknown) {
@@ -880,6 +1098,12 @@ function finiteNonNegativeNumber(value: unknown): number | null {
return Number.isFinite(parsed) && parsed >= 0 ? parsed : null; return Number.isFinite(parsed) && parsed >= 0 ? parsed : null;
} }
function booleanValue(value: unknown) {
if (value === true) return true;
const text = String(value ?? "").trim().toLowerCase();
return text === "true" || text === "1" || text === "yes";
}
function unixMs(value: unknown): number | null { function unixMs(value: unknown): number | null {
if (value instanceof Date) { if (value instanceof Date) {
const time = value.getTime(); const time = value.getTime();
+29 -2
View File
@@ -300,6 +300,25 @@ function recordWorkbenchTurnReadMetric(options = {}, url, statusCode, body = {},
}); });
} }
function recordWorkbenchCacheMetric(options = {}, url, body = {}, startedAt = nowMs()) {
const performanceStore = options.backendPerformanceStore ?? options.backendPerformance;
if (!performanceStore?.recordWorkbenchCache) return;
const cache = body?.cache && typeof body.cache === "object" ? body.cache : null;
if (!cache) return;
performanceStore.recordWorkbenchCache({
route: url?.pathname ?? "/v1/workbench",
cacheKeyClass: cache.class,
cacheStatus: cache.status,
operation: "read",
durationMs: nowMs() - startedAt,
payloadBytes: cache.payloadBytes,
cacheAgeMs: cache.cacheAgeMs,
dbQueryAvoided: cache.dbQueryAvoided,
freshnessSloMs: cache.freshnessSloMs,
projectionSeq: cache.projectionSeq
});
}
function recordWorkbenchProjectionMetric(options = {}, { result = null, trace = null, projection = null, diagnostic = null } = {}) { function recordWorkbenchProjectionMetric(options = {}, { result = null, trace = null, projection = null, diagnostic = null } = {}) {
const performanceStore = options.backendPerformanceStore ?? options.backendPerformance; const performanceStore = options.backendPerformanceStore ?? options.backendPerformance;
if (!performanceStore?.recordWorkbenchProjection || !projection) return; if (!performanceStore?.recordWorkbenchProjection || !projection) return;
@@ -405,6 +424,7 @@ async function authenticateWorkbenchRead(request, response, options) {
} }
async function handleWorkbenchSessionList(request, response, url, options, actor) { async function handleWorkbenchSessionList(request, response, url, options, actor) {
const startedAt = nowMs();
if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) return sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench session list is keyed by sessionId only.")); if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) return sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench session list is keyed by sessionId only."));
const limit = boundedSessionListLimit(url.searchParams.get("limit")); const limit = boundedSessionListLimit(url.searchParams.get("limit"));
const offset = cursorOffset(url.searchParams.get("cursor") ?? url.searchParams.get("after")); const offset = cursorOffset(url.searchParams.get("cursor") ?? url.searchParams.get("after"));
@@ -421,6 +441,7 @@ async function handleWorkbenchSessionList(request, response, url, options, actor
includeSessionId: includeRouteId ?? null, includeSessionId: includeRouteId ?? null,
actor: { id: actor.id, role: actor.role ?? "user", valuesRedacted: true } actor: { id: actor.id, role: actor.role ?? "user", valuesRedacted: true }
}); });
recordWorkbenchCacheMetric(options, url, payload, startedAt);
return sendJson(response, 200, payload); return sendJson(response, 200, payload);
} }
@@ -1147,6 +1168,7 @@ async function handleWorkbenchTurnSnapshot(response, url, options, actor, rawTur
contractVersion: "workbench-turn-snapshot-v1", contractVersion: "workbench-turn-snapshot-v1",
turn: snapshot, turn: snapshot,
projection, projection,
cache: result.cache ?? null,
projectionStatus: projection.projectionStatus, projectionStatus: projection.projectionStatus,
projectionHealth: projection.projectionHealth, projectionHealth: projection.projectionHealth,
lastProjectedSeq: projection.lastProjectedSeq, lastProjectedSeq: projection.lastProjectedSeq,
@@ -1162,10 +1184,12 @@ async function handleWorkbenchTurnSnapshot(response, url, options, actor, rawTur
diagnostic: projection diagnostic: projection
}); });
recordWorkbenchTurnReadMetric(options, url, 200, body, startedAt); recordWorkbenchTurnReadMetric(options, url, 200, body, startedAt);
recordWorkbenchCacheMetric(options, url, body, startedAt);
sendJson(response, 200, body); sendJson(response, 200, body);
} }
async function handleWorkbenchTraceEventPage(response, url, options, actor, rawTraceId) { async function handleWorkbenchTraceEventPage(response, url, options, actor, rawTraceId) {
const startedAt = nowMs();
const traceId = safeTraceId(rawTraceId); const traceId = safeTraceId(rawTraceId);
if (!traceId) return sendJson(response, 400, workbenchError("invalid_trace_id", "traceId must start with trc_.", { traceId: rawTraceId })); if (!traceId) return sendJson(response, 400, workbenchError("invalid_trace_id", "traceId must start with trc_.", { traceId: rawTraceId }));
const readModel = createWorkbenchReadModel(options, actor); const readModel = createWorkbenchReadModel(options, actor);
@@ -1199,7 +1223,7 @@ async function handleWorkbenchTraceEventPage(response, url, options, actor, rawT
page.blocker = blocker; page.blocker = blocker;
} }
const responseProjection = page.blocker ? blockedTraceEventProjection(projection, page.blocker) : projection; const responseProjection = page.blocker ? blockedTraceEventProjection(projection, page.blocker) : projection;
sendJson(response, 200, { const body = {
ok: true, ok: true,
status: page.blocker ? "blocked" : "succeeded", status: page.blocker ? "blocked" : "succeeded",
contractVersion: "workbench-trace-events-v1", contractVersion: "workbench-trace-events-v1",
@@ -1208,6 +1232,7 @@ async function handleWorkbenchTraceEventPage(response, url, options, actor, rawT
threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null), threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null),
...page, ...page,
projection: responseProjection, projection: responseProjection,
cache: pageResult.cache ?? metadata.cache ?? null,
projectionStatus: responseProjection.projectionStatus, projectionStatus: responseProjection.projectionStatus,
projectionHealth: responseProjection.projectionHealth, projectionHealth: responseProjection.projectionHealth,
lastProjectedSeq: responseProjection.lastProjectedSeq, lastProjectedSeq: responseProjection.lastProjectedSeq,
@@ -1222,7 +1247,9 @@ async function handleWorkbenchTraceEventPage(response, url, options, actor, rawT
durationMs: responseProjection.durationMs, durationMs: responseProjection.durationMs,
valuesRedacted: true, valuesRedacted: true,
secretMaterialStored: false secretMaterialStored: false
}); };
recordWorkbenchCacheMetric(options, url, body, startedAt);
sendJson(response, 200, body);
} }
function traceEventPageFromFacts(sourceEvents, options, metadata = {}) { function traceEventPageFromFacts(sourceEvents, options, metadata = {}) {
+18
View File
@@ -62,6 +62,21 @@ test("web performance summary exposes user-perceived route latency without high-
responseBytes: { count: 2, average: 2048, p50: 1024, p75: 2048, p95: 4096 }, responseBytes: { count: 2, average: 2048, p50: 1024, p75: 2048, p95: 4096 },
phases: [{ phase: "agent_result_read", outcome: "ok", count: 2, average: 0.05, p50: 0.04, p75: 0.05, p95: 0.08 }] phases: [{ phase: "agent_result_read", outcome: "ok", count: 2, average: 0.05, p50: 0.04, p75: 0.05, p95: 0.08 }]
}], }],
cache: [{
route: "/v1/workbench/sessions",
cacheKeyClass: "sessions_summary",
count: 4,
hitCount: 3,
missCount: 1,
staleCount: 0,
unavailableCount: 0,
dbQueryAvoidedCount: 3,
hitRatio: 0.75,
dbQueryAvoidedRatio: 0.75,
unavailableRatio: 0,
operationDuration: { count: 4, average: 0.12, p50: 0.08, p75: 0.11, p95: 0.18, unit: "seconds" },
valuesRedacted: true
}],
valuesRedacted: true valuesRedacted: true
} }); } });
const json = JSON.stringify(summary); const json = JSON.stringify(summary);
@@ -76,6 +91,9 @@ test("web performance summary exposes user-perceived route latency without high-
assert.ok(summary.dashboard.cards.some((card) => card.id === "samples" && card.value === 4)); assert.ok(summary.dashboard.cards.some((card) => card.id === "samples" && card.value === 4));
assert.ok(summary.dashboard.trends.some((trend) => trend.id === "api-timing" && trend.points.length > 0)); assert.ok(summary.dashboard.trends.some((trend) => trend.id === "api-timing" && trend.points.length > 0));
assert.ok(summary.dashboard.topN.some((group) => group.id === "top-api" && group.rows.length > 0)); assert.ok(summary.dashboard.topN.some((group) => group.id === "top-api" && group.rows.length > 0));
assert.ok(summary.workbenchCacheRows.some((row) => row.metric === "cache_hit_ratio" && row.cache === "sessions_summary" && row.p95 === 0.75));
assert.ok(summary.workbenchCacheRows.some((row) => row.metric === "db_query_avoided" && row.p95 === 0.75));
assert.ok(summary.workbenchCacheRows.some((row) => row.metric === "sessions_api_p95" && row.p95 === 0.18));
const matchedApi = summary.apiRoutes.find((row) => row.route === "/v1/agent/chat/result/:id" && row.statusClass === "2xx"); const matchedApi = summary.apiRoutes.find((row) => row.route === "/v1/agent/chat/result/:id" && row.statusClass === "2xx");
assert.equal(matchedApi?.backendEvidence?.status, "matched"); assert.equal(matchedApi?.backendEvidence?.status, "matched");
assert.equal(matchedApi?.backendEvidence?.diagnostic, "backend_low_sample"); assert.equal(matchedApi?.backendEvidence?.diagnostic, "backend_low_sample");
+144 -3
View File
@@ -215,6 +215,27 @@ interface BackendPerformanceEvidenceRow {
phases?: Array<{ phase?: unknown; outcome?: unknown; count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown }>; phases?: Array<{ phase?: unknown; outcome?: unknown; count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown }>;
} }
interface BackendPerformanceCacheEvidenceRow {
route?: unknown;
cacheKeyClass?: unknown;
count?: unknown;
hitCount?: unknown;
missCount?: unknown;
staleCount?: unknown;
unavailableCount?: unknown;
dbQueryAvoidedCount?: unknown;
hitRatio?: unknown;
dbQueryAvoidedRatio?: unknown;
unavailableRatio?: unknown;
operationDuration?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null;
payloadBytes?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null;
cacheAgeMs?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null;
freshnessSloMs?: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null;
projectionSeqMax?: unknown;
firstObservedAt?: unknown;
lastObservedAt?: unknown;
}
interface BackendPerformanceEvidenceSnapshot { interface BackendPerformanceEvidenceSnapshot {
schemaVersion?: unknown; schemaVersion?: unknown;
source?: unknown; source?: unknown;
@@ -223,6 +244,7 @@ interface BackendPerformanceEvidenceSnapshot {
retainedSampleCount?: unknown; retainedSampleCount?: unknown;
routeCount?: unknown; routeCount?: unknown;
rows?: BackendPerformanceEvidenceRow[]; rows?: BackendPerformanceEvidenceRow[];
cache?: BackendPerformanceCacheEvidenceRow[];
} }
interface PerformanceRowBackendEvidence { interface PerformanceRowBackendEvidence {
@@ -538,7 +560,8 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions =
const workbenchBackendEvents = applySummaryContract(workbenchBackendEventVisibleSummaryRows(aggregate.backendEventVisibleSeries, WORKBENCH_JOURNEY_BUCKETS_SECONDS), exactStats, windowBounds) const workbenchBackendEvents = applySummaryContract(workbenchBackendEventVisibleSummaryRows(aggregate.backendEventVisibleSeries, WORKBENCH_JOURNEY_BUCKETS_SECONDS), exactStats, windowBounds)
.sort(sortByProblemThenP95) .sort(sortByProblemThenP95)
.slice(0, 24); .slice(0, 24);
const workbenchRows = [...workbenchJourneys, ...workbenchEventPhases, ...workbenchBackendEvents]; const workbenchCacheRows = backendCacheEvidenceSummaryRows(backendEvidence, windowBounds);
const workbenchRows = [...workbenchJourneys, ...workbenchEventPhases, ...workbenchBackendEvents, ...workbenchCacheRows];
const apiRoutes = durationRows const apiRoutes = durationRows
.filter((row) => row.metric === "api_request") .filter((row) => row.metric === "api_request")
.sort(sortByProblemThenP95) .sort(sortByProblemThenP95)
@@ -570,6 +593,7 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions =
workbenchJourneySeries: aggregate.journeyDurationSeries.size, workbenchJourneySeries: aggregate.journeyDurationSeries.size,
workbenchEventPhaseSeries: aggregate.eventPhaseSeries.size, workbenchEventPhaseSeries: aggregate.eventPhaseSeries.size,
workbenchBackendEventVisibleSeries: aggregate.backendEventVisibleSeries.size, workbenchBackendEventVisibleSeries: aggregate.backendEventVisibleSeries.size,
workbenchCacheEvidenceRows: workbenchCacheRows.length,
workbenchUiEventSeries: aggregate.workbenchUiEventSeries.size, workbenchUiEventSeries: aggregate.workbenchUiEventSeries.size,
routeCount: new Set([...rows, ...workbenchRows].map((row) => row.route)).size, routeCount: new Set([...rows, ...workbenchRows].map((row) => row.route)).size,
problemCount: problems.length, problemCount: problems.length,
@@ -639,6 +663,7 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions =
workbenchJourneys, workbenchJourneys,
workbenchEventPhases, workbenchEventPhases,
workbenchBackendEvents, workbenchBackendEvents,
workbenchCacheRows,
workbenchRows: dashboard.source.workbenchRows, workbenchRows: dashboard.source.workbenchRows,
rows: dashboard.source.rows rows: dashboard.source.rows
}; };
@@ -1047,6 +1072,7 @@ function applySummaryContract(rows: PerformanceSummaryRow[], exactStatsByRow: Ma
function normalizeBackendEvidence(input?: BackendPerformanceEvidenceSnapshot | null) { function normalizeBackendEvidence(input?: BackendPerformanceEvidenceSnapshot | null) {
const rows = Array.isArray(input?.rows) ? input.rows.map(normalizeBackendEvidenceRow).filter(Boolean) as BackendPerformanceEvidenceRow[] : []; const rows = Array.isArray(input?.rows) ? input.rows.map(normalizeBackendEvidenceRow).filter(Boolean) as BackendPerformanceEvidenceRow[] : [];
const cache = Array.isArray(input?.cache) ? input.cache.map(normalizeBackendCacheEvidenceRow).filter(Boolean) as ReturnType<typeof normalizeBackendCacheEvidenceRow>[] : [];
return { return {
schemaVersion: "hwlab-backend-performance-evidence-v1", schemaVersion: "hwlab-backend-performance-evidence-v1",
source: sanitizeLabelValue(input?.source ?? "cloud-api-backend-performance-store"), source: sanitizeLabelValue(input?.source ?? "cloud-api-backend-performance-store"),
@@ -1055,6 +1081,7 @@ function normalizeBackendEvidence(input?: BackendPerformanceEvidenceSnapshot | n
retainedSampleCount: finiteNonNegativeInteger(input?.retainedSampleCount), retainedSampleCount: finiteNonNegativeInteger(input?.retainedSampleCount),
routeCount: finiteNonNegativeInteger(input?.routeCount ?? new Set(rows.map((row) => row.route)).size), routeCount: finiteNonNegativeInteger(input?.routeCount ?? new Set(rows.map((row) => row.route)).size),
rows: rows.slice(0, 80), rows: rows.slice(0, 80),
cache: cache.slice(0, 80),
available: Boolean(input && String(input.schemaVersion ?? "") === "hwlab-backend-performance-evidence-v1"), available: Boolean(input && String(input.schemaVersion ?? "") === "hwlab-backend-performance-evidence-v1"),
valuesRedacted: true valuesRedacted: true
}; };
@@ -1100,6 +1127,44 @@ function normalizeBackendEvidencePhase(input: { phase?: unknown; outcome?: unkno
}; };
} }
function normalizeBackendCacheEvidenceRow(input: BackendPerformanceCacheEvidenceRow) {
const route = webPerformanceRouteTemplate(input?.route ?? "");
if (!route || route === "/") return null;
return {
route,
cacheKeyClass: sanitizeMetricName(input.cacheKeyClass, "unknown").slice(0, 64) || "unknown",
count: finiteNonNegativeInteger(input.count),
hitCount: finiteNonNegativeInteger(input.hitCount),
missCount: finiteNonNegativeInteger(input.missCount),
staleCount: finiteNonNegativeInteger(input.staleCount),
unavailableCount: finiteNonNegativeInteger(input.unavailableCount),
dbQueryAvoidedCount: finiteNonNegativeInteger(input.dbQueryAvoidedCount),
hitRatio: boundedRatio(input.hitRatio),
dbQueryAvoidedRatio: boundedRatio(input.dbQueryAvoidedRatio),
unavailableRatio: boundedRatio(input.unavailableRatio),
operationDuration: normalizeMetricStats(input.operationDuration, "seconds"),
payloadBytes: normalizeMetricStats(input.payloadBytes, "bytes"),
cacheAgeMs: normalizeMetricStats(input.cacheAgeMs, "milliseconds"),
freshnessSloMs: normalizeMetricStats(input.freshnessSloMs, "milliseconds"),
projectionSeqMax: finiteNonNegativeInteger(input.projectionSeqMax),
firstObservedAt: isoOrNull(input.firstObservedAt),
lastObservedAt: isoOrNull(input.lastObservedAt),
valuesRedacted: true
};
}
function normalizeMetricStats(input: { count?: unknown; average?: unknown; p50?: unknown; p75?: unknown; p95?: unknown; unit?: unknown } | null | undefined, unit: string) {
if (!input || typeof input !== "object") return null;
return {
count: finiteNonNegativeInteger(input.count),
average: finiteNonNegativeMetric(input.average),
p50: finiteNonNegativeMetric(input.p50),
p75: finiteNonNegativeMetric(input.p75),
p95: finiteNonNegativeMetric(input.p95),
unit
};
}
function attachBackendEvidenceToRows(rows: PerformanceSummaryRow[], evidence: ReturnType<typeof normalizeBackendEvidence>) { function attachBackendEvidenceToRows(rows: PerformanceSummaryRow[], evidence: ReturnType<typeof normalizeBackendEvidence>) {
return rows.map((row) => { return rows.map((row) => {
if (row.kind !== "api" || row.metric !== "api_request") return row; if (row.kind !== "api" || row.metric !== "api_request") return row;
@@ -1188,6 +1253,71 @@ function backendEvidenceDetail(input: { diagnostic: string; count: number; backe
return `${input.count} 个后端样本已匹配,后端 P95 ${formatBackendSeconds(input.backendP95)}${phaseText}`; return `${input.count} 个后端样本已匹配,后端 P95 ${formatBackendSeconds(input.backendP95)}${phaseText}`;
} }
function backendCacheEvidenceSummaryRows(evidence: ReturnType<typeof normalizeBackendEvidence>, windowBounds: { windowFrom: string | null; windowTo: string; generatedAt: string }): PerformanceSummaryRow[] {
if (!evidence.available) return [];
const rows: PerformanceSummaryRow[] = [];
for (const item of evidence.cache) {
rows.push(cacheEvidenceRow({ metric: "cache_hit_ratio", route: item.route, cache: item.cacheKeyClass, value: item.hitRatio, count: item.count, unit: "score", windowBounds, outcome: "ok" }));
rows.push(cacheEvidenceRow({ metric: "db_query_avoided", route: item.route, cache: item.cacheKeyClass, value: item.dbQueryAvoidedRatio, count: item.count, unit: "score", windowBounds, outcome: "ok" }));
rows.push(cacheEvidenceRow({ metric: "cache_unavailable", route: item.route, cache: item.cacheKeyClass, value: item.unavailableRatio, count: item.count, unit: "score", windowBounds, outcome: item.unavailableCount > 0 ? "error" : "ok" }));
if (item.route === "/v1/workbench/sessions" && item.operationDuration) {
rows.push(cacheEvidenceRow({
metric: "sessions_api_p95",
route: item.route,
cache: item.cacheKeyClass,
value: item.operationDuration.p95,
p50: item.operationDuration.p50,
p75: item.operationDuration.p75,
p95: item.operationDuration.p95,
count: item.operationDuration.count || item.count,
unit: "seconds",
windowBounds,
outcome: "ok"
}));
}
}
return rows;
}
function cacheEvidenceRow(input: { metric: string; route: string; cache: string; value: number; p50?: number; p75?: number; p95?: number; count: number; unit: "seconds" | "score"; outcome: string; windowBounds: { windowFrom: string | null; windowTo: string; generatedAt: string } }): PerformanceSummaryRow {
const count = finiteNonNegativeInteger(input.count);
const value = finiteNonNegativeMetric(input.value);
const p50 = input.p50 === undefined ? value : finiteNonNegativeMetric(input.p50);
const p75 = input.p75 === undefined ? value : finiteNonNegativeMetric(input.p75);
const p95 = input.p95 === undefined ? value : finiteNonNegativeMetric(input.p95);
const sample = sampleState(count);
let tone: "ok" | "warn" | "blocked" | "source" = count < LOW_SAMPLE_THRESHOLD ? "source" : "ok";
if (sample === "ok" && input.metric === "cache_unavailable" && p95 > 0) tone = "blocked";
if (sample === "ok" && input.metric === "sessions_api_p95") tone = thresholdTone(p95, 1, 2.5);
const problem = tone === "source" ? (count <= 0 ? "no-sample" : "low-sample") : tone === "ok" ? "ok" : input.metric === "cache_unavailable" ? "cache-unavailable" : `slow-${input.metric}`;
return {
kind: "workbench_cache",
metric: input.metric,
route: input.route,
method: "-",
statusClass: "-",
outcome: input.outcome,
count,
average: value,
p50,
p75,
p95,
unit: input.unit,
tone,
problem,
lowSample: count > 0 && count < LOW_SAMPLE_THRESHOLD,
sampleState: sample,
sourceFamily: "backend_cache_evidence",
windowFrom: input.windowBounds.windowFrom,
windowTo: input.windowBounds.windowTo,
generatedAt: input.windowBounds.generatedAt,
freshness: sampleFreshness(count),
aggregationKind: "exact",
approximation: "exact",
cache: input.cache
};
}
function formatBackendSeconds(value: number) { function formatBackendSeconds(value: number) {
if (!Number.isFinite(value)) return "未知"; if (!Number.isFinite(value)) return "未知";
if (value < 1) return `${Math.round(value * 1000)}ms`; if (value < 1) return `${Math.round(value * 1000)}ms`;
@@ -1199,6 +1329,12 @@ function finiteNonNegativeMetric(value: unknown) {
return Number.isFinite(number) && number >= 0 ? roundMetric(number) : 0; return Number.isFinite(number) && number >= 0 ? roundMetric(number) : 0;
} }
function boundedRatio(value: unknown) {
const number = Number(value);
if (!Number.isFinite(number) || number < 0) return 0;
return Math.min(1, roundMetric(number));
}
function finiteNonNegativeInteger(value: unknown) { function finiteNonNegativeInteger(value: unknown) {
const number = Number(value); const number = Number(value);
return Number.isFinite(number) && number >= 0 ? Math.trunc(number) : 0; return Number.isFinite(number) && number >= 0 ? Math.trunc(number) : 0;
@@ -1294,12 +1430,12 @@ function summaryToneForRow(row: PerformanceSummaryRow, p95: number, count: numbe
function summaryProblemForRow(row: PerformanceSummaryRow, p95: number, count: number, tone: string) { function summaryProblemForRow(row: PerformanceSummaryRow, p95: number, count: number, tone: string) {
if (count <= 0) return "no-sample"; if (count <= 0) return "no-sample";
if (row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible") return workbenchProblem(row.outcome, row.metric, count, tone); if (row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible" || row.kind === "workbench_cache") return workbenchProblem(row.outcome, row.metric, count, tone);
return performanceProblem({ metric: row.metric, outcome: row.outcome, status_class: row.statusClass }, p95, count, row.unit === "score" ? "score" : "seconds", tone); return performanceProblem({ metric: row.metric, outcome: row.outcome, status_class: row.statusClass }, p95, count, row.unit === "score" ? "score" : "seconds", tone);
} }
function isPerformanceProblemRow(row: PerformanceSummaryRow) { function isPerformanceProblemRow(row: PerformanceSummaryRow) {
if ((row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible") && row.sampleState !== "ok") return false; if ((row.kind === "workbench_journey" || row.kind === "workbench_event_phase" || row.kind === "workbench_backend_event_visible" || row.kind === "workbench_cache") && row.sampleState !== "ok") return false;
return row.tone === "blocked" || row.tone === "warn" || row.outcome !== "ok" || row.statusClass === "network" || row.statusClass === "5xx" || row.statusClass === "4xx"; return row.tone === "blocked" || row.tone === "warn" || row.outcome !== "ok" || row.statusClass === "network" || row.statusClass === "5xx" || row.statusClass === "4xx";
} }
@@ -1311,6 +1447,7 @@ function sourceFamilyForRow(row: PerformanceSummaryRow) {
if (row.kind === "workbench_journey") return "workbench_journey"; if (row.kind === "workbench_journey") return "workbench_journey";
if (row.kind === "workbench_event_phase") return "workbench_event_phase"; if (row.kind === "workbench_event_phase") return "workbench_event_phase";
if (row.kind === "workbench_backend_event_visible") return "workbench_backend_event_visible"; if (row.kind === "workbench_backend_event_visible") return "workbench_backend_event_visible";
if (row.kind === "workbench_cache") return "backend_cache_evidence";
return "unknown"; return "unknown";
} }
@@ -1812,6 +1949,10 @@ function metricLabel(row: PerformanceSummaryRow) {
submit_to_first_visible: "提交到首个可见结果", submit_to_first_visible: "提交到首个可见结果",
submit_to_failure: "提交失败可见", submit_to_failure: "提交失败可见",
backend_event_to_visible: "后端事件到可见", backend_event_to_visible: "后端事件到可见",
cache_hit_ratio: "缓存命中率",
db_query_avoided: "DB 查询避免率",
cache_unavailable: "缓存不可用率",
sessions_api_p95: "会话 API P95",
session_switch_first_visible: "切换会话首个可见", session_switch_first_visible: "切换会话首个可见",
session_switch_full_load: "切换会话完整加载", session_switch_full_load: "切换会话完整加载",
workbench_open_first_visible: "打开工作台首个可见", workbench_open_first_visible: "打开工作台首个可见",
+1
View File
@@ -34,6 +34,7 @@ export function createWorkbenchFactsStore(options = {}, actor = null) {
count: Number.isFinite(Number(result?.count)) ? Number(result.count) : 0, count: Number.isFinite(Number(result?.count)) ? Number(result.count) : 0,
durable: true, durable: true,
persistence: result?.persistence ?? null, persistence: result?.persistence ?? null,
cache: result?.cache ?? null,
retry: attempt > 1 ? { attempts: attempt, maxAttempts: retryPolicy.maxAttempts, delaysMs: retryDelaysMs.slice(), valuesRedacted: true } : null retry: attempt > 1 ? { attempts: attempt, maxAttempts: retryPolicy.maxAttempts, delaysMs: retryDelaysMs.slice(), valuesRedacted: true } : null
}; };
} catch (error) { } catch (error) {
+275 -15
View File
@@ -21,6 +21,9 @@ import (
const cacheKeySchemaVersion = "wb-derived-v1" const cacheKeySchemaVersion = "wb-derived-v1"
var cacheOperationDurationBucketsSeconds = []float64{0.001, 0.0025, 0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1}
var cachePayloadBucketsBytes = []float64{512, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304}
var errCacheDisabled = errors.New("workbench redis cache disabled") var errCacheDisabled = errors.New("workbench redis cache disabled")
type cacheConfig struct { type cacheConfig struct {
@@ -60,15 +63,38 @@ type derivedCache struct {
} }
type cacheMetrics struct { type cacheMetrics struct {
mu sync.Mutex mu sync.Mutex
Hits uint64 Hits uint64
Misses uint64 Misses uint64
Sets uint64 Sets uint64
Deletes uint64 Deletes uint64
Errors uint64 Errors uint64
LastErrorKind string LastErrorKind string
LastErrorOp string LastErrorOp string
LastErrorAt time.Time LastErrorAt time.Time
Requests map[string]cacheCounterSeries
Stale map[string]cacheCounterSeries
DBQueryAvoided map[string]cacheCounterSeries
OperationDurations map[string]*cacheHistogramSeries
PayloadBytes map[string]*cacheHistogramSeries
}
type cacheMetricLabels struct {
Class string
Operation string
Status string
}
type cacheCounterSeries struct {
Labels cacheMetricLabels
Value uint64
}
type cacheHistogramSeries struct {
Labels cacheMetricLabels
Buckets []uint64
Sum float64
Count uint64
} }
type cacheError struct { type cacheError struct {
@@ -144,12 +170,15 @@ func (c *derivedCache) BuildKey(parts cacheKeyParts) (string, error) {
} }
func (c *derivedCache) Get(ctx context.Context, key string, target any) (bool, error) { func (c *derivedCache) Get(ctx context.Context, key string, target any) (bool, error) {
startedAt := time.Now()
class := cacheKeyClass(key)
if !c.usable() { if !c.usable() {
return false, nil return false, nil
} }
if err := validateCacheKey(c.config.MaxKeyBytes, key); err != nil { if err := validateCacheKey(c.config.MaxKeyBytes, key); err != nil {
wrapped := cacheError{Op: "get", Kind: classifyCacheError(err), Err: err} wrapped := cacheError{Op: "get", Kind: classifyCacheError(err), Err: err}
c.metrics.recordError(wrapped) c.metrics.recordError(wrapped)
c.metrics.recordCacheObservation(class, "get", "error", time.Since(startedAt), 0)
return false, wrapped return false, wrapped
} }
var payload []byte var payload []byte
@@ -160,42 +189,51 @@ func (c *derivedCache) Get(ctx context.Context, key string, target any) (bool, e
}) })
if errors.Is(err, redis.Nil) { if errors.Is(err, redis.Nil) {
c.metrics.recordMiss() c.metrics.recordMiss()
c.metrics.recordCacheObservation(class, "get", "miss", time.Since(startedAt), 0)
return false, nil return false, nil
} }
if err != nil { if err != nil {
wrapped := cacheError{Op: "get", Kind: classifyCacheError(err), Err: err} wrapped := cacheError{Op: "get", Kind: classifyCacheError(err), Err: err}
c.metrics.recordError(wrapped) c.metrics.recordError(wrapped)
c.metrics.recordCacheObservation(class, "get", "unavailable", time.Since(startedAt), 0)
return false, wrapped return false, wrapped
} }
if len(payload) > c.config.MaxPayloadBytes { if len(payload) > c.config.MaxPayloadBytes {
wrapped := cacheError{Op: "get", Kind: "cache_payload_too_large", Err: fmt.Errorf("payload bytes %d exceeds %d", len(payload), c.config.MaxPayloadBytes)} wrapped := cacheError{Op: "get", Kind: "cache_payload_too_large", Err: fmt.Errorf("payload bytes %d exceeds %d", len(payload), c.config.MaxPayloadBytes)}
c.metrics.recordError(wrapped) c.metrics.recordError(wrapped)
c.metrics.recordCacheObservation(class, "get", "error", time.Since(startedAt), len(payload))
return false, wrapped return false, wrapped
} }
if target != nil { if target != nil {
if err := json.Unmarshal(payload, target); err != nil { if err := json.Unmarshal(payload, target); err != nil {
wrapped := cacheError{Op: "get", Kind: "cache_payload_decode_failed", Err: err} wrapped := cacheError{Op: "get", Kind: "cache_payload_decode_failed", Err: err}
c.metrics.recordError(wrapped) c.metrics.recordError(wrapped)
c.metrics.recordCacheObservation(class, "get", "error", time.Since(startedAt), len(payload))
return false, wrapped return false, wrapped
} }
} }
c.metrics.recordHit() c.metrics.recordHit()
c.metrics.recordCacheObservation(class, "get", "hit", time.Since(startedAt), len(payload))
return true, nil return true, nil
} }
func (c *derivedCache) Set(ctx context.Context, key string, value any, ttl time.Duration) error { func (c *derivedCache) Set(ctx context.Context, key string, value any, ttl time.Duration) error {
startedAt := time.Now()
class := cacheKeyClass(key)
if !c.usable() { if !c.usable() {
return nil return nil
} }
if err := validateCacheKey(c.config.MaxKeyBytes, key); err != nil { if err := validateCacheKey(c.config.MaxKeyBytes, key); err != nil {
wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err} wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err}
c.metrics.recordError(wrapped) c.metrics.recordError(wrapped)
c.metrics.recordCacheObservation(class, "set", "error", time.Since(startedAt), 0)
return wrapped return wrapped
} }
payload, err := cachePayloadBytes(value, c.config.MaxPayloadBytes) payload, err := cachePayloadBytes(value, c.config.MaxPayloadBytes)
if err != nil { if err != nil {
wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err} wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err}
c.metrics.recordError(wrapped) c.metrics.recordError(wrapped)
c.metrics.recordCacheObservation(class, "set", "error", time.Since(startedAt), 0)
return wrapped return wrapped
} }
err = c.withTimeout(ctx, func(opCtx context.Context) error { err = c.withTimeout(ctx, func(opCtx context.Context) error {
@@ -204,12 +242,36 @@ func (c *derivedCache) Set(ctx context.Context, key string, value any, ttl time.
if err != nil { if err != nil {
wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err} wrapped := cacheError{Op: "set", Kind: classifyCacheError(err), Err: err}
c.metrics.recordError(wrapped) c.metrics.recordError(wrapped)
c.metrics.recordCacheObservation(class, "set", "unavailable", time.Since(startedAt), len(payload))
return wrapped return wrapped
} }
c.metrics.recordSet() c.metrics.recordSet()
c.metrics.recordCacheObservation(class, "set", "stored", time.Since(startedAt), len(payload))
return nil return nil
} }
func (c *derivedCache) RecordStale(class string) {
if c == nil {
return
}
c.metrics.recordStale(class)
}
func (c *derivedCache) RecordDBQueryAvoided(class string) {
if c == nil {
return
}
c.metrics.recordDBQueryAvoided(class)
}
func (c *derivedCache) PrometheusText(service string) string {
if c == nil {
var metrics cacheMetrics
return metrics.prometheusText(service)
}
return c.metrics.prometheusText(service)
}
func (c *derivedCache) Delete(ctx context.Context, keys ...string) error { func (c *derivedCache) Delete(ctx context.Context, keys ...string) error {
if !c.usable() || len(keys) == 0 { if !c.usable() || len(keys) == 0 {
return nil return nil
@@ -520,6 +582,15 @@ func cacheHash(value string) string {
return hex.EncodeToString(digest[:])[:24] return hex.EncodeToString(digest[:])[:24]
} }
func cacheKeyClass(key string) string {
for _, part := range strings.Split(key, ":") {
if strings.HasPrefix(part, "class=") {
return safeCacheToken(strings.TrimPrefix(part, "class="))
}
}
return "unknown"
}
func (m *cacheMetrics) recordHit() { func (m *cacheMetrics) recordHit() {
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
@@ -556,16 +627,79 @@ func (m *cacheMetrics) recordError(err error) {
m.LastErrorAt = time.Now().UTC() m.LastErrorAt = time.Now().UTC()
} }
func (m *cacheMetrics) recordCacheObservation(class string, operation string, status string, duration time.Duration, payloadBytes int) {
m.mu.Lock()
defer m.mu.Unlock()
labels := cacheLabels(class, operation, status)
m.incrementCounterLocked(&m.Requests, labels)
m.recordHistogramLocked(&m.OperationDurations, labels, duration.Seconds(), cacheOperationDurationBucketsSeconds)
if payloadBytes > 0 {
m.recordHistogramLocked(&m.PayloadBytes, labels, float64(payloadBytes), cachePayloadBucketsBytes)
}
}
func (m *cacheMetrics) recordStale(class string) {
m.mu.Lock()
defer m.mu.Unlock()
m.incrementCounterLocked(&m.Stale, cacheLabels(class, "read", "stale"))
}
func (m *cacheMetrics) recordDBQueryAvoided(class string) {
m.mu.Lock()
defer m.mu.Unlock()
m.incrementCounterLocked(&m.DBQueryAvoided, cacheLabels(class, "read", "hit"))
}
func (m *cacheMetrics) incrementCounterLocked(target *map[string]cacheCounterSeries, labels cacheMetricLabels) {
if *target == nil {
*target = map[string]cacheCounterSeries{}
}
key := cacheMetricKey(labels)
entry := (*target)[key]
if entry.Labels.Class == "" {
entry.Labels = labels
}
entry.Value++
(*target)[key] = entry
}
func (m *cacheMetrics) recordHistogramLocked(target *map[string]*cacheHistogramSeries, labels cacheMetricLabels, value float64, buckets []float64) {
if value < 0 {
value = 0
}
if *target == nil {
*target = map[string]*cacheHistogramSeries{}
}
key := cacheMetricKey(labels)
entry := (*target)[key]
if entry == nil {
entry = &cacheHistogramSeries{Labels: labels, Buckets: make([]uint64, len(buckets))}
(*target)[key] = entry
}
for index, bucket := range buckets {
if value <= bucket {
entry.Buckets[index]++
}
}
entry.Sum += value
entry.Count++
}
func (m *cacheMetrics) snapshot() map[string]any { func (m *cacheMetrics) snapshot() map[string]any {
m.mu.Lock() m.mu.Lock()
defer m.mu.Unlock() defer m.mu.Unlock()
result := map[string]any{ result := map[string]any{
"hits": int64(m.Hits), "hits": int64(m.Hits),
"misses": int64(m.Misses), "misses": int64(m.Misses),
"sets": int64(m.Sets), "sets": int64(m.Sets),
"deletes": int64(m.Deletes), "deletes": int64(m.Deletes),
"errors": int64(m.Errors), "errors": int64(m.Errors),
"valuesRedacted": true, "requestSeries": len(m.Requests),
"operationDurationSeries": len(m.OperationDurations),
"payloadBytesSeries": len(m.PayloadBytes),
"staleSeries": len(m.Stale),
"dbQueryAvoidedSeries": len(m.DBQueryAvoided),
"valuesRedacted": true,
} }
if m.LastErrorKind != "" { if m.LastErrorKind != "" {
result["lastErrorKind"] = m.LastErrorKind result["lastErrorKind"] = m.LastErrorKind
@@ -575,6 +709,132 @@ func (m *cacheMetrics) snapshot() map[string]any {
return result return result
} }
func (m *cacheMetrics) prometheusText(service string) string {
m.mu.Lock()
defer m.mu.Unlock()
if service == "" {
service = "hwlab-workbench-runtime"
}
lines := []string{
"# HELP workbench_cache_requests_total Workbench derived Redis cache requests by low-cardinality class/status.",
"# TYPE workbench_cache_requests_total counter",
}
for _, entry := range m.Requests {
lines = append(lines, prometheusCounterLine("workbench_cache_requests_total", service, entry.Labels, entry.Value))
}
lines = append(lines,
"# HELP workbench_cache_hit_ratio Workbench derived Redis cache hit ratio by class and operation.",
"# TYPE workbench_cache_hit_ratio gauge",
)
for _, line := range m.prometheusHitRatioLinesLocked(service) {
lines = append(lines, line)
}
lines = append(lines,
"# HELP workbench_cache_operation_duration_seconds Workbench derived Redis cache operation duration in seconds.",
"# TYPE workbench_cache_operation_duration_seconds histogram",
)
lines = append(lines, prometheusHistogramLines("workbench_cache_operation_duration_seconds", service, m.OperationDurations, cacheOperationDurationBucketsSeconds)...)
lines = append(lines,
"# HELP workbench_cache_payload_bytes Workbench derived Redis cache payload size in bytes.",
"# TYPE workbench_cache_payload_bytes histogram",
)
lines = append(lines, prometheusHistogramLines("workbench_cache_payload_bytes", service, m.PayloadBytes, cachePayloadBucketsBytes)...)
lines = append(lines,
"# HELP workbench_cache_stale_total Workbench derived Redis cache stale reads detected by payload freshness metadata.",
"# TYPE workbench_cache_stale_total counter",
)
for _, entry := range m.Stale {
lines = append(lines, prometheusCounterLine("workbench_cache_stale_total", service, entry.Labels, entry.Value))
}
lines = append(lines,
"# HELP workbench_cache_unavailable_total Workbench derived Redis cache unavailable/error requests.",
"# TYPE workbench_cache_unavailable_total counter",
)
for _, entry := range m.Requests {
if entry.Labels.Status == "unavailable" || entry.Labels.Status == "error" {
lines = append(lines, prometheusCounterLine("workbench_cache_unavailable_total", service, entry.Labels, entry.Value))
}
}
lines = append(lines,
"# HELP workbench_db_query_avoided_total Workbench database reads avoided by derived Redis cache hits.",
"# TYPE workbench_db_query_avoided_total counter",
)
for _, entry := range m.DBQueryAvoided {
lines = append(lines, prometheusCounterLine("workbench_db_query_avoided_total", service, entry.Labels, entry.Value))
}
lines = append(lines, "")
return strings.Join(lines, "\n")
}
func (m *cacheMetrics) prometheusHitRatioLinesLocked(service string) []string {
type totals struct{ hit, total uint64 }
groups := map[string]totals{}
labels := map[string]cacheMetricLabels{}
for _, entry := range m.Requests {
if entry.Labels.Operation != "get" {
continue
}
key := entry.Labels.Class + "\x00" + entry.Labels.Operation
group := groups[key]
if entry.Labels.Status == "hit" {
group.hit += entry.Value
}
group.total += entry.Value
groups[key] = group
labels[key] = cacheMetricLabels{Class: entry.Labels.Class, Operation: entry.Labels.Operation, Status: "all"}
}
lines := []string{}
for key, group := range groups {
ratio := 0.0
if group.total > 0 {
ratio = float64(group.hit) / float64(group.total)
}
lines = append(lines, prometheusGaugeLine("workbench_cache_hit_ratio", service, labels[key], ratio))
}
return lines
}
func prometheusHistogramLines(name string, service string, series map[string]*cacheHistogramSeries, buckets []float64) []string {
lines := []string{}
for _, entry := range series {
for index, bucket := range buckets {
lines = append(lines, fmt.Sprintf("%s_bucket{%s,le=\"%s\"} %d", name, prometheusLabels(service, entry.Labels), prometheusFloat(bucket), entry.Buckets[index]))
}
lines = append(lines, fmt.Sprintf("%s_bucket{%s,le=\"+Inf\"} %d", name, prometheusLabels(service, entry.Labels), entry.Count))
lines = append(lines, fmt.Sprintf("%s_sum{%s} %.6f", name, prometheusLabels(service, entry.Labels), entry.Sum))
lines = append(lines, fmt.Sprintf("%s_count{%s} %d", name, prometheusLabels(service, entry.Labels), entry.Count))
}
return lines
}
func prometheusCounterLine(name string, service string, labels cacheMetricLabels, value uint64) string {
return fmt.Sprintf("%s{%s} %d", name, prometheusLabels(service, labels), value)
}
func prometheusGaugeLine(name string, service string, labels cacheMetricLabels, value float64) string {
return fmt.Sprintf("%s{%s} %.6f", name, prometheusLabels(service, labels), value)
}
func prometheusLabels(service string, labels cacheMetricLabels) string {
return fmt.Sprintf("service=\"%s\",cache_key_class=\"%s\",operation=\"%s\",cache_status=\"%s\"", prometheusEscape(service), prometheusEscape(labels.Class), prometheusEscape(labels.Operation), prometheusEscape(labels.Status))
}
func prometheusEscape(value string) string {
return strings.ReplaceAll(strings.ReplaceAll(strings.ReplaceAll(value, "\\", "\\\\"), "\n", "\\n"), "\"", "\\\"")
}
func prometheusFloat(value float64) string {
return strconv.FormatFloat(value, 'f', -1, 64)
}
func cacheLabels(class string, operation string, status string) cacheMetricLabels {
return cacheMetricLabels{Class: safeCacheToken(first(class, "unknown")), Operation: safeCacheToken(first(operation, "unknown")), Status: safeCacheToken(first(status, "unknown"))}
}
func cacheMetricKey(labels cacheMetricLabels) string {
return labels.Class + "\x00" + labels.Operation + "\x00" + labels.Status
}
func envBool(name string, fallback bool) bool { func envBool(name string, fallback bool) bool {
value := strings.ToLower(strings.TrimSpace(os.Getenv(name))) value := strings.ToLower(strings.TrimSpace(os.Getenv(name)))
if value == "" { if value == "" {
+21
View File
@@ -94,6 +94,27 @@ func TestDisabledCacheDoesNotFailReadPath(t *testing.T) {
} }
} }
func TestCachePrometheusMetricsUseLowCardinalityLabels(t *testing.T) {
var metrics cacheMetrics
metrics.recordCacheObservation("turn.terminal.snapshot", "get", "hit", 12*time.Millisecond, 4096)
metrics.recordCacheObservation("sessions.summary", "get", "miss", 25*time.Millisecond, 0)
metrics.recordStale("sessions.summary")
metrics.recordDBQueryAvoided("turn.terminal.snapshot")
text := metrics.prometheusText(serviceID)
for _, name := range []string{"workbench_cache_requests_total", "workbench_cache_hit_ratio", "workbench_cache_operation_duration_seconds", "workbench_cache_payload_bytes", "workbench_cache_stale_total", "workbench_db_query_avoided_total"} {
if !strings.Contains(text, name) {
t.Fatalf("metrics missing %s in:\n%s", name, text)
}
}
if !strings.Contains(text, `cache_key_class="turn.terminal.snapshot"`) || !strings.Contains(text, `cache_status="hit"`) {
t.Fatalf("metrics missing cache labels:\n%s", text)
}
if strings.Contains(text, "trc_") || strings.Contains(text, "ses_") || strings.Contains(text, "actor=") || strings.Contains(text, "cacheKey") {
t.Fatalf("metrics leaked high-cardinality or key material:\n%s", text)
}
}
func TestSessionsSummaryCacheTTLUsesShortTTLForUnstablePages(t *testing.T) { func TestSessionsSummaryCacheTTLUsesShortTTLForUnstablePages(t *testing.T) {
config := cacheConfig{SessionsSummaryTTL: 30 * time.Second, RunningPageTTL: 5 * time.Second} config := cacheConfig{SessionsSummaryTTL: 30 * time.Second, RunningPageTTL: 5 * time.Second}
stable := map[string]any{"sessions": []any{map[string]any{"terminal": true, "running": false, "projectionStatus": "caught-up"}}} stable := map[string]any{"sessions": []any{map[string]any{"terminal": true, "running": false, "projectionStatus": "caught-up"}}}
+74
View File
@@ -233,12 +233,23 @@ func (s *Server) Close() error {
func (s *Server) routes() { func (s *Server) routes() {
s.mux.HandleFunc("/health/live", s.handleLive) s.mux.HandleFunc("/health/live", s.handleLive)
s.mux.HandleFunc("/health/ready", s.handleReady) s.mux.HandleFunc("/health/ready", s.handleReady)
s.mux.HandleFunc("/metrics", s.handleMetrics)
s.mux.HandleFunc("/v1/workbench-runtime/facts", s.handleFacts) s.mux.HandleFunc("/v1/workbench-runtime/facts", s.handleFacts)
s.mux.HandleFunc("/v1/workbench-runtime/trace-events", s.handleTraceEvents) s.mux.HandleFunc("/v1/workbench-runtime/trace-events", s.handleTraceEvents)
s.mux.HandleFunc("/v1/workbench-runtime/sessions", s.handleSessions) s.mux.HandleFunc("/v1/workbench-runtime/sessions", s.handleSessions)
s.mux.HandleFunc("/v1/workbench-runtime/projection-outbox", s.handleProjectionOutbox) s.mux.HandleFunc("/v1/workbench-runtime/projection-outbox", s.handleProjectionOutbox)
} }
func (s *Server) handleMetrics(w http.ResponseWriter, r *http.Request) {
if r.Method != http.MethodGet {
methodNotAllowed(w, "GET")
return
}
w.Header().Set("content-type", "text/plain; version=0.0.4; charset=utf-8")
w.WriteHeader(http.StatusOK)
_, _ = w.Write([]byte(s.cache.PrometheusText(serviceID)))
}
func (s *Server) handleLive(w http.ResponseWriter, r *http.Request) { func (s *Server) handleLive(w http.ResponseWriter, r *http.Request) {
cache := s.cache.Diagnostic(r.Context()) cache := s.cache.Diagnostic(r.Context())
if s.config.Cache.UnavailableLivenessFailure && cache["enabled"] == true && cache["available"] != true { if s.config.Cache.UnavailableLivenessFailure && cache["enabled"] == true && cache["available"] != true {
@@ -297,6 +308,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
plan.Meta["ttlMs"] = entry.TTLMillis plan.Meta["ttlMs"] = entry.TTLMillis
plan.Meta["freshnessSloMs"] = entry.TTLMillis plan.Meta["freshnessSloMs"] = entry.TTLMillis
attachFactsCacheMetadata(payload, plan.Meta) attachFactsCacheMetadata(payload, plan.Meta)
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
return payload, nil return payload, nil
} }
setFactsCacheStatus(plan.Meta, "stale") setFactsCacheStatus(plan.Meta, "stale")
@@ -316,6 +328,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
plan.Meta["dbQueryDurationMs"] = dbDuration.Milliseconds() plan.Meta["dbQueryDurationMs"] = dbDuration.Milliseconds()
if plan.Key == "" { if plan.Key == "" {
attachFactsCacheMetadata(payload, plan.Meta) attachFactsCacheMetadata(payload, plan.Meta)
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
return payload, nil return payload, nil
} }
@@ -330,6 +343,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
setFactsCacheStatus(plan.Meta, "skipped") setFactsCacheStatus(plan.Meta, "skipped")
plan.Meta["skipReason"] = classifyCacheError(payloadErr) plan.Meta["skipReason"] = classifyCacheError(payloadErr)
attachFactsCacheMetadata(payload, plan.Meta) attachFactsCacheMetadata(payload, plan.Meta)
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
return payload, nil return payload, nil
} }
plan.Meta["payloadBytes"] = len(payloadBytes) plan.Meta["payloadBytes"] = len(payloadBytes)
@@ -337,6 +351,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
setFactsCacheStatus(plan.Meta, "skipped") setFactsCacheStatus(plan.Meta, "skipped")
plan.Meta["skipReason"] = ttlReason plan.Meta["skipReason"] = ttlReason
attachFactsCacheMetadata(payload, plan.Meta) attachFactsCacheMetadata(payload, plan.Meta)
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
return payload, nil return payload, nil
} }
entry := factsCacheEntry{CachedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: storedPayload, PayloadBytes: len(payloadBytes), TTLMillis: ttl.Milliseconds(), ValuesRedacted: true} entry := factsCacheEntry{CachedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: storedPayload, PayloadBytes: len(payloadBytes), TTLMillis: ttl.Milliseconds(), ValuesRedacted: true}
@@ -344,6 +359,7 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
setFactsCacheStatus(plan.Meta, "set_failed") setFactsCacheStatus(plan.Meta, "set_failed")
plan.Meta["errorKind"] = classifyCacheError(err) plan.Meta["errorKind"] = classifyCacheError(err)
attachFactsCacheMetadata(payload, plan.Meta) attachFactsCacheMetadata(payload, plan.Meta)
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
return payload, nil return payload, nil
} }
if plan.Meta["status"] != "stale" { if plan.Meta["status"] != "stale" {
@@ -351,9 +367,61 @@ func (s *Server) queryFactsResponse(ctx context.Context, query factQuery) (map[s
} }
plan.Meta["stored"] = true plan.Meta["stored"] = true
attachFactsCacheMetadata(payload, plan.Meta) attachFactsCacheMetadata(payload, plan.Meta)
s.observeCacheDiagnostic(ctx, plan.Class, plan.Meta)
return payload, nil return payload, nil
} }
func (s *Server) observeCacheDiagnostic(ctx context.Context, class string, meta map[string]any) {
if s == nil || meta == nil {
return
}
cacheClass := first(text(meta["class"]), class, "unknown")
status := first(text(meta["status"]), "unknown")
if s.cache != nil {
if status == "stale" {
s.cache.RecordStale(cacheClass)
}
if boolValue(meta["dbQueryAvoided"]) {
s.cache.RecordDBQueryAvoided(cacheClass)
}
}
attrs := map[string]any{
"cacheStatus": status,
"cacheKeyClass": safeCacheToken(cacheClass),
"workbench.cache.status": status,
"workbench.cache.key_class": safeCacheToken(cacheClass),
"workbench.cache.db_avoided": boolValue(meta["dbQueryAvoided"]),
"workbench.cache.values_source": "diagnostic",
"valuesRedacted": true,
}
if meta["cacheAgeMs"] != nil {
age := int64FromAny(meta["cacheAgeMs"])
attrs["cacheAgeMs"] = age
attrs["workbench.cache.age_ms"] = age
}
if meta["projectionSeq"] != nil {
seq := int64FromAny(meta["projectionSeq"])
attrs["projectionSeq"] = seq
attrs["workbench.projection.seq"] = seq
}
if meta["payloadBytes"] != nil {
attrs["workbench.cache.payload_bytes"] = int64FromAny(meta["payloadBytes"])
}
if meta["ttlMs"] != nil {
attrs["workbench.cache.ttl_ms"] = int64FromAny(meta["ttlMs"])
}
if meta["freshnessSloMs"] != nil {
attrs["workbench.cache.freshness_slo_ms"] = int64FromAny(meta["freshnessSloMs"])
}
if revisionSource := text(meta["revisionSource"]); revisionSource != "" {
attrs["workbench.cache.revision_source"] = revisionSource
}
if invalidationMode := text(meta["invalidationMode"]); invalidationMode != "" {
attrs["workbench.cache.invalidation_mode"] = invalidationMode
}
_ = s.withOtelInternalSpan(ctx, "workbench-runtime.cache", attrs, func(context.Context) error { return nil })
}
func factsResponsePayload(facts map[string][]any, persistence map[string]any) map[string]any { func factsResponsePayload(facts map[string][]any, persistence map[string]any) map[string]any {
count := 0 count := 0
for _, rows := range facts { for _, rows := range facts {
@@ -732,6 +800,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
cacheMeta["freshnessSloMs"] = entry.TTLMillis cacheMeta["freshnessSloMs"] = entry.TTLMillis
applySessionsCacheRevisionMetadata(cacheMeta, sessionsSummaryRevision{ProjectionRevision: entry.ProjectionRevision, ProjectionSeqMax: entry.ProjectionSeqMax, SessionUpdatedAtMax: entry.SessionUpdatedAtMax, SessionCount: entry.SessionCount, RevisionSource: "cached_payload"}) applySessionsCacheRevisionMetadata(cacheMeta, sessionsSummaryRevision{ProjectionRevision: entry.ProjectionRevision, ProjectionSeqMax: entry.ProjectionSeqMax, SessionUpdatedAtMax: entry.SessionUpdatedAtMax, SessionCount: entry.SessionCount, RevisionSource: "cached_payload"})
attachSessionsCacheMetadata(payload, cacheMeta) attachSessionsCacheMetadata(payload, cacheMeta)
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
return payload, nil return payload, nil
} }
setSessionsCacheStatus(cacheMeta, "stale") setSessionsCacheStatus(cacheMeta, "stale")
@@ -751,6 +820,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
cacheMeta["dbQueryDurationMs"] = dbDuration.Milliseconds() cacheMeta["dbQueryDurationMs"] = dbDuration.Milliseconds()
if cacheKey == "" { if cacheKey == "" {
attachSessionsCacheMetadata(payload, cacheMeta) attachSessionsCacheMetadata(payload, cacheMeta)
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
return payload, nil return payload, nil
} }
@@ -767,6 +837,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
setSessionsCacheStatus(cacheMeta, "skipped") setSessionsCacheStatus(cacheMeta, "skipped")
cacheMeta["skipReason"] = classifyCacheError(payloadErr) cacheMeta["skipReason"] = classifyCacheError(payloadErr)
attachSessionsCacheMetadata(payload, cacheMeta) attachSessionsCacheMetadata(payload, cacheMeta)
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
return payload, nil return payload, nil
} }
cacheMeta["payloadBytes"] = len(payloadBytes) cacheMeta["payloadBytes"] = len(payloadBytes)
@@ -774,6 +845,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
setSessionsCacheStatus(cacheMeta, "skipped") setSessionsCacheStatus(cacheMeta, "skipped")
cacheMeta["skipReason"] = ttlReason cacheMeta["skipReason"] = ttlReason
attachSessionsCacheMetadata(payload, cacheMeta) attachSessionsCacheMetadata(payload, cacheMeta)
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
return payload, nil return payload, nil
} }
entry := sessionsCacheEntry{CachedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: storedPayload, PayloadBytes: len(payloadBytes), TTLMillis: ttl.Milliseconds(), ProjectionRevision: revision.ProjectionRevision, ProjectionSeqMax: revision.ProjectionSeqMax, SessionUpdatedAtMax: revision.SessionUpdatedAtMax, SessionCount: revision.SessionCount, ValuesRedacted: true} entry := sessionsCacheEntry{CachedAt: time.Now().UTC().Format(time.RFC3339Nano), Payload: storedPayload, PayloadBytes: len(payloadBytes), TTLMillis: ttl.Milliseconds(), ProjectionRevision: revision.ProjectionRevision, ProjectionSeqMax: revision.ProjectionSeqMax, SessionUpdatedAtMax: revision.SessionUpdatedAtMax, SessionCount: revision.SessionCount, ValuesRedacted: true}
@@ -781,6 +853,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
setSessionsCacheStatus(cacheMeta, "set_failed") setSessionsCacheStatus(cacheMeta, "set_failed")
cacheMeta["errorKind"] = classifyCacheError(err) cacheMeta["errorKind"] = classifyCacheError(err)
attachSessionsCacheMetadata(payload, cacheMeta) attachSessionsCacheMetadata(payload, cacheMeta)
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
return payload, nil return payload, nil
} }
if cacheMeta["status"] != "stale" { if cacheMeta["status"] != "stale" {
@@ -788,6 +861,7 @@ func (s *Server) listSessions(ctx context.Context, query sessionListQuery) (map[
} }
cacheMeta["stored"] = true cacheMeta["stored"] = true
attachSessionsCacheMetadata(payload, cacheMeta) attachSessionsCacheMetadata(payload, cacheMeta)
s.observeCacheDiagnostic(ctx, sessionsSummaryCacheClass, cacheMeta)
return payload, nil return payload, nil
} }
+1
View File
@@ -576,6 +576,7 @@ export interface WebPerformanceSummaryResponse {
workbenchJourneys?: WebPerformanceRow[]; workbenchJourneys?: WebPerformanceRow[];
workbenchEventPhases?: WebPerformanceRow[]; workbenchEventPhases?: WebPerformanceRow[];
workbenchBackendEvents?: WebPerformanceRow[]; workbenchBackendEvents?: WebPerformanceRow[];
workbenchCacheRows?: WebPerformanceRow[];
workbenchRows?: WebPerformanceRow[]; workbenchRows?: WebPerformanceRow[];
rows?: WebPerformanceRow[]; rows?: WebPerformanceRow[];
[key: string]: unknown; [key: string]: unknown;
@@ -189,6 +189,10 @@ function metricLabel(row: WebPerformanceRow): string {
navigation_load: "页面加载完成", navigation_load: "页面加载完成",
trace_event_projected: "轨迹事件投影", trace_event_projected: "轨迹事件投影",
backend_event_visible: "后端事件到可见", backend_event_visible: "后端事件到可见",
cache_hit_ratio: "缓存命中率",
db_query_avoided: "DB 查询避免率",
cache_unavailable: "缓存不可用率",
sessions_api_p95: "会话 API P95",
longtask: "主线程长任务" longtask: "主线程长任务"
}; };
return labels[metric] ?? metric.replace(/[_-]/gu, " "); return labels[metric] ?? metric.replace(/[_-]/gu, " ");
@@ -199,6 +203,7 @@ function kindLabel(kind?: string): string {
workbench_journey: "工作台体感", workbench_journey: "工作台体感",
workbench_event_phase: "事件阶段", workbench_event_phase: "事件阶段",
workbench_backend_event_visible: "后端可见", workbench_backend_event_visible: "后端可见",
workbench_cache: "缓存观测",
web_vital: "浏览器体感", web_vital: "浏览器体感",
navigation: "页面加载", navigation: "页面加载",
api: "同源 API", api: "同源 API",
@@ -403,7 +408,7 @@ function targetStateLabel(value?: string): string {
} }
function cacheLabel(value?: string): string { function cacheLabel(value?: string): string {
const labels: Record<string, string> = { warm: "热缓存", cold: "冷加载", hit: "命中", miss: "未命中", unknown: "未知" }; const labels: Record<string, string> = { warm: "热缓存", cold: "冷加载", hit: "命中", miss: "未命中", sessions_summary: "会话摘要", terminal_turn_snapshot: "终态轮次快照", trace_terminal_events: "终态轨迹事件", unknown: "未知" };
return labels[String(value ?? "unknown")] ?? String(value); return labels[String(value ?? "unknown")] ?? String(value);
} }