diff --git a/internal/cloud/web-performance.test.ts b/internal/cloud/web-performance.test.ts index 16ce8caf..58edc90d 100644 --- a/internal/cloud/web-performance.test.ts +++ b/internal/cloud/web-performance.test.ts @@ -1,3 +1,6 @@ +// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0 +// Verifies the Workbench RUM metrics contract and Prometheus label privacy boundary. + import assert from "node:assert/strict"; import { test } from "bun:test"; @@ -47,6 +50,80 @@ test("web performance summary exposes user-perceived route latency without high- assert.doesNotMatch(json, /trc_secret|sessionId|conversationId|threadId|prompt|api key/iu); }); +test("web performance store accepts v2 Workbench journey and phase metrics without high-cardinality labels", () => { + const store = createWebPerformanceStore({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); + const result = store.record({ + schemaVersion: "hwlab-web-performance-v2", + page: "/workbench/sessions/ses_secret?traceId=trc_secret", + events: [ + { + kind: "workbench_journey", + journey: "submit_to_first_visible", + route: "/workbench/sessions/ses_secret", + entry: "new", + backend: "agentrun-v01/codex", + transport: "sse", + visibility: "foreground", + outcome: "ok", + valueMs: 1234, + traceId: "trc_secret", + sessionId: "ses_secret", + runId: "run_secret" + }, + { + kind: "workbench_event_phase", + phase: "created_to_append", + eventType: "assistant_message", + backend: "agentrun-v01/codex", + transport: "sse", + outcome: "ok", + valueMs: 42, + commandId: "cmd_secret" + }, + { + kind: "workbench_backend_event_visible", + eventType: "tool", + backend: "agentrun-v01/codex", + transport: "sse", + outcome: "ok", + valueMs: 640, + conversationId: "cnv_secret" + } + ] + } as Record); + const text = store.metricsText(); + + assert.deepEqual(result, { accepted: 3, dropped: 0, received: 3 }); + assert.match(text, /hwlab_workbench_journey_total\{[^}]*journey="submit_to_first_visible"[^}]*route="\/workbench\/sessions\/:id"[^}]*backend="agentrun-v01\/codex"/u); + assert.match(text, /hwlab_workbench_event_phase_duration_seconds_bucket\{[^}]*phase="created_to_append"[^}]*event_type="assistant"[^}]*le="0\.05"\} 1/u); + assert.match(text, /hwlab_workbench_backend_event_visible_latency_seconds_bucket\{[^}]*event_type="tool_call"[^}]*backend="agentrun-v01\/codex"[^}]*le="1"\} 1/u); + assert.doesNotMatch(text, /trc_secret|ses_secret|run_secret|cmd_secret|cnv_secret|traceId|sessionId|runId|commandId|conversationId|prompt|api key/iu); +}); + +test("web performance store drops unsupported Workbench labels and counts series limit drops", () => { + const store = createWebPerformanceStore({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" }, maxSeries: 1 }); + const first = store.record({ + schemaVersion: "hwlab-web-performance-v2", + page: "/workbench", + events: [{ kind: "workbench_journey", journey: "session_switch_first_visible", route: "/workbench/sessions/ses_first", valueMs: 250, backend: "agentrun-v01/codex", transport: "sse" }] + }); + const second = store.record({ + schemaVersion: "hwlab-web-performance-v2", + page: "/workbench", + events: [ + { kind: "workbench_journey", journey: "session_switch_full_load", route: "/workbench/sessions/ses_second", valueMs: 900, backend: "run_abcdefghijklmnopqrstuvwxyz", transport: "sse" }, + { kind: "workbench_journey", journey: "trace_trc_secret", route: "/workbench/sessions/ses_third", valueMs: 100, backend: "agentrun-v01/codex", transport: "sse" } + ] + }); + const text = store.metricsText(); + + assert.deepEqual(first, { accepted: 1, dropped: 0, received: 1 }); + assert.deepEqual(second, { accepted: 0, dropped: 2, received: 2 }); + assert.match(text, /hwlab_webui_performance_dropped_total\{[^}]*reason="series_limit"\} 1/u); + assert.match(text, /hwlab_webui_performance_dropped_total\{[^}]*reason="invalid_event"\} 1/u); + assert.doesNotMatch(text, /ses_second|ses_third|trace_trc_secret|run_abcdefghijklmnopqrstuvwxyz/u); +}); + test("web performance store filters observability self-noise", () => { const store = createWebPerformanceStore({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v02", HWLAB_GITOPS_TARGET: "v02" } }); const result = store.record({ diff --git a/internal/cloud/web-performance.ts b/internal/cloud/web-performance.ts index 6fc918d3..0193101c 100644 --- a/internal/cloud/web-performance.ts +++ b/internal/cloud/web-performance.ts @@ -1,7 +1,12 @@ +// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0 +// Keeps browser-reported performance metrics low-cardinality before Prometheus export. + const DEFAULT_WEB_PERFORMANCE_BODY_LIMIT_BYTES = 64 * 1024; const DURATION_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10, 30]; const CLS_BUCKETS = [0.01, 0.05, 0.1, 0.25, 0.5, 1, 2.5]; -const ALLOWED_KINDS = new Set(["navigation", "web_vital", "api", "long_task"]); +const WORKBENCH_JOURNEY_BUCKETS_SECONDS = [0.05, 0.1, 0.25, 0.5, 1, 2, 3, 5, 8, 13, 21, 30, 60, 120]; +const WORKBENCH_EVENT_PHASE_BUCKETS_SECONDS = [0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1, 2.5, 5, 10]; +const ALLOWED_KINDS = new Set(["navigation", "web_vital", "api", "long_task", "workbench_journey", "workbench_event_phase", "workbench_backend_event_visible"]); const DURATION_METRICS = new Set([ "navigation_ttfb", "navigation_dom_content_loaded", @@ -12,6 +17,33 @@ const DURATION_METRICS = new Set([ "api_request", "long_task" ]); +const WORKBENCH_SCHEMA_VERSION = "hwlab-web-performance-v2"; +const WORKBENCH_JOURNEYS = new Set([ + "submit_to_first_visible", + "backend_event_to_visible", + "session_switch_first_visible", + "session_switch_full_load", + "workbench_open_first_visible", + "workbench_open_full_load" +]); +const WORKBENCH_EVENT_PHASES = new Set([ + "created_to_append", + "append_to_sse", + "sse_to_receive", + "receive_to_project", + "project_to_paint", + "user_submit_to_api_accepted", + "api_accepted_to_backend_event" +]); +const WORKBENCH_EVENT_TYPES = new Set(["assistant", "tool_call", "backend", "terminal", "error", "status", "request", "result", "unknown"]); +const WORKBENCH_TRANSPORTS = new Set(["sse", "rest_gap", "poll", "none", "unknown"]); +const WORKBENCH_OUTCOMES = new Set(["ok", "timeout", "error", "dropped", "stale", "partial", "empty", "network", "denied", "unknown"]); +const WORKBENCH_ENTRIES = new Set(["new", "existing", "steer", "retry", "unknown"]); +const WORKBENCH_VISIBILITY = new Set(["foreground", "background", "hidden", "unknown"]); +const WORKBENCH_CACHE_STATES = new Set(["warm", "cold", "unknown"]); +const WORKBENCH_TARGET_STATES = new Set(["running", "terminal", "empty", "unknown"]); +const WORKBENCH_SOURCES = new Set(["rail", "deeplink", "history", "direct", "hydrate", "unknown"]); +const WORKBENCH_AUTH_STATES = new Set(["warm", "login_redirect", "unknown"]); interface WebPerformanceStoreOptions { env?: Record; @@ -28,6 +60,17 @@ interface WebPerformancePayload { interface WebPerformanceEvent { kind?: unknown; metric?: unknown; + journey?: unknown; + phase?: unknown; + eventType?: unknown; + backend?: unknown; + transport?: unknown; + entry?: unknown; + visibility?: unknown; + cache?: unknown; + targetState?: unknown; + source?: unknown; + authState?: unknown; route?: unknown; valueMs?: unknown; value?: unknown; @@ -37,6 +80,14 @@ interface WebPerformanceEvent { outcome?: unknown; } +interface NormalizedPerformanceEvent { + series: "web_duration" | "layout_shift" | "workbench_journey" | "workbench_event_phase" | "workbench_backend_event_visible"; + metric: string; + value: number; + labels: Record; + counterLabels?: Record; +} + interface HistogramSeries { labels: Record; buckets: number[]; @@ -71,6 +122,11 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions = const durationSeries = new Map(); const clsSeries = new Map(); const sampleSeries = new Map(); + const droppedSeries = new Map(); + const journeyDurationSeries = new Map(); + const journeyTotalSeries = new Map(); + const eventPhaseSeries = new Map(); + const backendEventVisibleSeries = new Map(); const baseLabels = { service: "hwlab-cloud-web", namespace: sanitizeLabelValue(env.HWLAB_METRICS_NAMESPACE ?? env.POD_NAMESPACE ?? "hwlab-v02"), @@ -84,26 +140,28 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions = for (const rawEvent of events) { const event = normalizeEvent(rawEvent, payload); if (!event) { + incrementDropped("invalid_event"); dropped += 1; continue; } - const counterLabels = sampleLabels(event.labels); - incrementCounter(sampleSeries, counterLabels, maxSeries); - if (event.metric === "cls") { - recordHistogram(clsSeries, event.labels, event.value, CLS_BUCKETS, maxSeries); - } else { - recordHistogram(durationSeries, event.labels, event.value, DURATION_BUCKETS_SECONDS, maxSeries); + if (!recordNormalizedEvent(event)) { + incrementDropped("series_limit"); + dropped += 1; + continue; } accepted += 1; } return { accepted, dropped, received: events.length }; } - function normalizeEvent(rawEvent: unknown, payload: WebPerformancePayload) { + function normalizeEvent(rawEvent: unknown, payload: WebPerformancePayload): NormalizedPerformanceEvent | null { if (!rawEvent || typeof rawEvent !== "object" || Array.isArray(rawEvent)) return null; const input = rawEvent as WebPerformanceEvent; const kind = sanitizeMetricName(input.kind, "unknown"); if (!ALLOWED_KINDS.has(kind)) return null; + if (kind === "workbench_journey") return normalizeWorkbenchJourneyEvent(input, payload); + if (kind === "workbench_event_phase") return normalizeWorkbenchEventPhase(input, payload); + if (kind === "workbench_backend_event_visible") return normalizeWorkbenchBackendEventVisible(input, payload); const metric = sanitizeMetricName(input.metric, "unknown"); const isCls = metric === "cls"; if (!isCls && !DURATION_METRICS.has(metric)) return null; @@ -122,7 +180,83 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions = status_class: normalizeStatusClass(input.statusClass ?? (input.status === undefined || input.status === null ? "unknown" : statusClass(input.status))), outcome: sanitizeLabelValue(input.outcome ?? "ok", "ok") }; - return { metric, value, labels }; + return { series: isCls ? "layout_shift" : "web_duration", metric, value, labels, counterLabels: sampleLabels(labels) }; + } + + function normalizeWorkbenchJourneyEvent(input: WebPerformanceEvent, payload: WebPerformancePayload): NormalizedPerformanceEvent | null { + if (String(payload.schemaVersion ?? "") !== WORKBENCH_SCHEMA_VERSION) return null; + const journey = requiredEnum(input.journey ?? input.metric, WORKBENCH_JOURNEYS); + if (!journey) return null; + const value = durationValueSeconds(input.valueMs, 120); + if (value === null) return null; + const route = webPerformanceRouteTemplate(input.route ?? payload.page ?? "/workbench"); + const pageRoute = webPerformanceRouteTemplate(payload.page ?? "/workbench"); + if (isObservabilityNoise({ kind: "workbench_journey", metric: journey, route, pageRoute })) return null; + const labels = { + ...baseLabels, + journey, + route, + entry: optionalEnum(input.entry, WORKBENCH_ENTRIES, "unknown"), + source: optionalEnum(input.source, WORKBENCH_SOURCES, "unknown"), + target_state: optionalEnum(input.targetState ?? field(input, "target_state"), WORKBENCH_TARGET_STATES, "unknown"), + cache: optionalEnum(input.cache, WORKBENCH_CACHE_STATES, "unknown"), + auth_state: optionalEnum(input.authState ?? field(input, "auth_state"), WORKBENCH_AUTH_STATES, "unknown"), + backend: normalizeBackendLabel(input.backend), + transport: optionalEnum(input.transport, WORKBENCH_TRANSPORTS, "unknown"), + visibility: optionalEnum(input.visibility, WORKBENCH_VISIBILITY, "unknown"), + outcome: optionalEnum(input.outcome, WORKBENCH_OUTCOMES, "ok") + }; + return { series: "workbench_journey", metric: journey, value, labels, counterLabels: labels }; + } + + function normalizeWorkbenchEventPhase(input: WebPerformanceEvent, payload: WebPerformancePayload): NormalizedPerformanceEvent | null { + if (String(payload.schemaVersion ?? "") !== WORKBENCH_SCHEMA_VERSION) return null; + const phase = requiredEnum(input.phase ?? input.metric, WORKBENCH_EVENT_PHASES); + if (!phase) return null; + const value = durationValueSeconds(input.valueMs, 30); + if (value === null) return null; + const labels = { + ...baseLabels, + phase, + event_type: normalizeEventType(input.eventType ?? field(input, "event_type")), + backend: normalizeBackendLabel(input.backend), + transport: optionalEnum(input.transport, WORKBENCH_TRANSPORTS, "unknown"), + outcome: optionalEnum(input.outcome, WORKBENCH_OUTCOMES, "ok") + }; + return { series: "workbench_event_phase", metric: phase, value, labels }; + } + + function normalizeWorkbenchBackendEventVisible(input: WebPerformanceEvent, payload: WebPerformancePayload): NormalizedPerformanceEvent | null { + if (String(payload.schemaVersion ?? "") !== WORKBENCH_SCHEMA_VERSION) return null; + const value = durationValueSeconds(input.valueMs, 120); + if (value === null) return null; + const labels = { + ...baseLabels, + event_type: normalizeEventType(input.eventType ?? field(input, "event_type")), + backend: normalizeBackendLabel(input.backend), + transport: optionalEnum(input.transport, WORKBENCH_TRANSPORTS, "unknown"), + outcome: optionalEnum(input.outcome, WORKBENCH_OUTCOMES, "ok") + }; + return { series: "workbench_backend_event_visible", metric: "backend_event_to_visible", value, labels }; + } + + function recordNormalizedEvent(event: NormalizedPerformanceEvent) { + if (event.series === "layout_shift") return recordHistogramWithCounter(clsSeries, sampleSeries, event.labels, event.counterLabels ?? sampleLabels(event.labels), event.value, CLS_BUCKETS); + if (event.series === "web_duration") return recordHistogramWithCounter(durationSeries, sampleSeries, event.labels, event.counterLabels ?? sampleLabels(event.labels), event.value, DURATION_BUCKETS_SECONDS); + if (event.series === "workbench_journey") return recordHistogramWithCounter(journeyDurationSeries, journeyTotalSeries, event.labels, event.counterLabels ?? event.labels, event.value, WORKBENCH_JOURNEY_BUCKETS_SECONDS); + if (event.series === "workbench_event_phase") return recordHistogram(eventPhaseSeries, event.labels, event.value, WORKBENCH_EVENT_PHASE_BUCKETS_SECONDS, maxSeries); + return recordHistogram(backendEventVisibleSeries, event.labels, event.value, WORKBENCH_JOURNEY_BUCKETS_SECONDS, maxSeries); + } + + function recordHistogramWithCounter(histograms: Map, counters: Map, histogramLabels: Record, counterLabels: Record, value: number, buckets: number[]) { + if (!canRecordSeries(histograms, histogramLabels, maxSeries) || !canRecordSeries(counters, counterLabels, maxSeries)) return false; + recordHistogram(histograms, histogramLabels, value, buckets, maxSeries); + incrementCounter(counters, counterLabels, maxSeries); + return true; + } + + function incrementDropped(reason: string) { + incrementCounter(droppedSeries, { ...baseLabels, reason }, Math.max(maxSeries, 16)); } function metricsText() { @@ -136,6 +270,21 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions = "# HELP hwlab_webui_layout_shift_score Browser-observed cumulative layout shift score.", "# TYPE hwlab_webui_layout_shift_score histogram", ...renderHistogram("hwlab_webui_layout_shift_score", clsSeries, CLS_BUCKETS), + "# HELP hwlab_webui_performance_dropped_total Browser performance samples dropped before Prometheus export.", + "# TYPE hwlab_webui_performance_dropped_total counter", + ...renderCounters("hwlab_webui_performance_dropped_total", droppedSeries), + "# HELP hwlab_workbench_journey_total Browser-observed HWLAB Workbench journey samples accepted by cloud-api.", + "# TYPE hwlab_workbench_journey_total counter", + ...renderCounters("hwlab_workbench_journey_total", journeyTotalSeries), + "# HELP hwlab_workbench_journey_duration_seconds Browser-observed HWLAB Workbench user-visible journey durations in seconds.", + "# TYPE hwlab_workbench_journey_duration_seconds histogram", + ...renderHistogram("hwlab_workbench_journey_duration_seconds", journeyDurationSeries, WORKBENCH_JOURNEY_BUCKETS_SECONDS), + "# HELP hwlab_workbench_event_phase_duration_seconds Browser/backend phase durations for HWLAB Workbench events in seconds.", + "# TYPE hwlab_workbench_event_phase_duration_seconds histogram", + ...renderHistogram("hwlab_workbench_event_phase_duration_seconds", eventPhaseSeries, WORKBENCH_EVENT_PHASE_BUCKETS_SECONDS), + "# HELP hwlab_workbench_backend_event_visible_latency_seconds Agent backend event generated to browser-visible latency in seconds.", + "# TYPE hwlab_workbench_backend_event_visible_latency_seconds histogram", + ...renderHistogram("hwlab_workbench_backend_event_visible_latency_seconds", backendEventVisibleSeries, WORKBENCH_JOURNEY_BUCKETS_SECONDS), "" ]; return lines.join("\n"); @@ -162,6 +311,9 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions = .sort(sortByProblemThenP95) .slice(0, 16); const sampleCount = [...sampleSeries.values()].reduce((sum, entry) => sum + entry.value, 0); + const workbenchSampleCount = [...journeyTotalSeries.values()].reduce((sum, entry) => sum + entry.value, 0) + + histogramSampleCount(eventPhaseSeries) + + histogramSampleCount(backendEventVisibleSeries); return { serviceId: "hwlab-cloud-api", route: "/v1/web-performance/summary", @@ -170,10 +322,14 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions = namespace: baseLabels.namespace, gitopsTarget: baseLabels.gitops_target, summary: { - sampleCount, + sampleCount: sampleCount + workbenchSampleCount, sampleSeries: sampleSeries.size, durationSeries: durationSeries.size, clsSeries: clsSeries.size, + droppedSeries: droppedSeries.size, + workbenchJourneySeries: journeyDurationSeries.size, + workbenchEventPhaseSeries: eventPhaseSeries.size, + workbenchBackendEventVisibleSeries: backendEventVisibleSeries.size, routeCount: new Set(rows.map((row) => row.route)).size, problemCount: problems.length, status: sampleCount === 0 ? "waiting" : sampleCount < 10 ? "warming" : problems.length > 0 ? "watch" : "ok" @@ -191,7 +347,14 @@ export function createWebPerformanceStore(options: WebPerformanceStoreOptions = sampleSeries: sampleSeries.size, durationSeries: durationSeries.size, clsSeries: clsSeries.size, + droppedSeries: droppedSeries.size, + workbenchJourneySeries: journeyDurationSeries.size, + workbenchEventPhaseSeries: eventPhaseSeries.size, + workbenchBackendEventVisibleSeries: backendEventVisibleSeries.size, sampleCount: [...sampleSeries.values()].reduce((sum, entry) => sum + entry.value, 0) + + [...journeyTotalSeries.values()].reduce((sum, entry) => sum + entry.value, 0) + + histogramSampleCount(eventPhaseSeries) + + histogramSampleCount(backendEventVisibleSeries) }; } @@ -267,11 +430,16 @@ function isObservabilityNoise(event: { kind: string; metric: string; route: stri return event.route === "/v1/web-performance" || event.route.startsWith("/v1/web-performance/"); } +function canRecordSeries(series: Map, labels: Record, maxSeries: number) { + const key = stableLabelKey(labels); + return series.has(key) || series.size < maxSeries; +} + function recordHistogram(series: Map, labels: Record, value: number, buckets: number[], maxSeries: number) { const key = stableLabelKey(labels); let entry = series.get(key); if (!entry) { - if (series.size >= maxSeries) return; + if (series.size >= maxSeries) return false; entry = { labels, buckets: buckets.map(() => 0), sum: 0, count: 0 }; series.set(key, entry); } @@ -280,17 +448,23 @@ function recordHistogram(series: Map, labels: Record, labels: Record, maxSeries: number) { const key = stableLabelKey(labels); let entry = series.get(key); if (!entry) { - if (series.size >= maxSeries) return; + if (series.size >= maxSeries) return false; entry = { labels, value: 0 }; series.set(key, entry); } entry.value += 1; + return true; +} + +function histogramSampleCount(series: Map) { + return [...series.values()].reduce((sum, entry) => sum + entry.count, 0); } function renderCounters(metricName: string, series: Map) { @@ -430,10 +604,49 @@ function sanitizeMetricName(value: unknown, fallback: string) { function sanitizeLabelValue(value: unknown, fallback = "unknown") { const text = String(value ?? "").trim(); - const sanitized = text.replace(/[^A-Za-z0-9_.:-]/gu, "_").slice(0, 120); + const sanitized = text.replace(/[^A-Za-z0-9_.:/-]/gu, "_").slice(0, 120); return sanitized || fallback; } +function requiredEnum(value: unknown, allowed: Set) { + const text = sanitizeMetricName(value, ""); + return allowed.has(text) ? text : null; +} + +function optionalEnum(value: unknown, allowed: Set, fallback: string) { + const text = sanitizeMetricName(value, fallback); + return allowed.has(text) ? text : fallback; +} + +function normalizeEventType(value: unknown) { + const text = sanitizeMetricName(value, "unknown"); + const aliased = text === "assistant_message" ? "assistant" : text === "tool" ? "tool_call" : text; + return WORKBENCH_EVENT_TYPES.has(aliased) ? aliased : "unknown"; +} + +function normalizeBackendLabel(value: unknown) { + const sanitized = sanitizeLabelValue(String(value ?? "unknown").trim().toLowerCase(), "unknown").slice(0, 64); + if (!sanitized || looksHighCardinalityLabel(sanitized)) return "unknown"; + return sanitized; +} + +function looksHighCardinalityLabel(value: string) { + return /(?:^|[^a-z0-9])(?:trc|trace|thread|run|cmd|command|job|ses|session|cnv|conv|conversation|gws|box|res|pod|dp|usr|user)[_-][a-z0-9_-]{6,}/iu.test(value) + || /^[0-9a-f]{8,}(?:-[0-9a-f]{4,}){2,}$/iu.test(value) + || /^[A-Za-z0-9_-]{24,}$/u.test(value); +} + +function durationValueSeconds(valueMs: unknown, maxSeconds: number) { + const seconds = finiteNumber(valueMs, NaN) / 1000; + if (!Number.isFinite(seconds) || seconds < 0) return null; + return Math.min(seconds, maxSeconds); +} + +function field(input: unknown, key: string) { + if (!input || typeof input !== "object" || Array.isArray(input)) return undefined; + return (input as Record)[key]; +} + function escapeLabelValue(value: unknown) { return String(value ?? "").replace(/\\/gu, "\\\\").replace(/\n/gu, "\\n").replace(/"/gu, "\\\""); }