From c1d158b5bb1802f071c7b58610ee5299883cfeb8 Mon Sep 17 00:00:00 2001 From: lyon Date: Wed, 17 Jun 2026 20:41:00 +0800 Subject: [PATCH] fix: correct workbench performance visible metrics --- internal/cloud/web-performance.test.ts | 15 +++++- internal/cloud/web-performance.ts | 1 + .../scripts/workbench-performance.test.ts | 46 +++++++++++++++++++ .../src/utils/workbench-performance.ts | 39 ++++++++++++---- 4 files changed, 90 insertions(+), 11 deletions(-) diff --git a/internal/cloud/web-performance.test.ts b/internal/cloud/web-performance.test.ts index fe5704ae..b7c90b9d 100644 --- a/internal/cloud/web-performance.test.ts +++ b/internal/cloud/web-performance.test.ts @@ -80,6 +80,18 @@ test("web performance store accepts v2 Workbench journey and phase metrics witho valueMs: 42, commandId: "cmd_secret" }, + { + kind: "workbench_journey", + journey: "submit_to_failure", + route: "/workbench/sessions/ses_secret", + entry: "new", + backend: "agentrun-v01/codex", + transport: "sse", + visibility: "foreground", + outcome: "network", + valueMs: 530, + traceId: "trc_secret" + }, { kind: "workbench_backend_event_visible", eventType: "tool", @@ -93,8 +105,9 @@ test("web performance store accepts v2 Workbench journey and phase metrics witho } as Record); const text = store.metricsText(); - assert.deepEqual(result, { accepted: 3, dropped: 0, received: 3 }); + assert.deepEqual(result, { accepted: 4, dropped: 0, received: 4 }); assert.match(text, /hwlab_workbench_journey_total\{[^}]*journey="submit_to_first_visible"[^}]*route="\/workbench\/sessions\/:id"[^}]*backend="agentrun-v01\/codex"/u); + assert.match(text, /hwlab_workbench_journey_total\{[^}]*journey="submit_to_failure"[^}]*outcome="network"/u); assert.match(text, /hwlab_workbench_event_phase_duration_seconds_bucket\{[^}]*phase="created_to_append"[^}]*event_type="assistant"[^}]*le="0\.05"\} 1/u); assert.match(text, /hwlab_workbench_backend_event_visible_latency_seconds_bucket\{[^}]*event_type="tool_call"[^}]*backend="agentrun-v01\/codex"[^}]*le="1"\} 1/u); assert.doesNotMatch(text, /trc_secret|ses_secret|run_secret|cmd_secret|cnv_secret|traceId|sessionId|runId|commandId|conversationId|prompt|api key/iu); diff --git a/internal/cloud/web-performance.ts b/internal/cloud/web-performance.ts index 66e2689f..be719338 100644 --- a/internal/cloud/web-performance.ts +++ b/internal/cloud/web-performance.ts @@ -20,6 +20,7 @@ const DURATION_METRICS = new Set([ const WORKBENCH_SCHEMA_VERSION = "hwlab-web-performance-v2"; const WORKBENCH_JOURNEYS = new Set([ "submit_to_first_visible", + "submit_to_failure", "backend_event_to_visible", "session_switch_first_visible", "session_switch_full_load", diff --git a/web/hwlab-cloud-web/scripts/workbench-performance.test.ts b/web/hwlab-cloud-web/scripts/workbench-performance.test.ts index 1a08a5ba..fff0b41c 100644 --- a/web/hwlab-cloud-web/scripts/workbench-performance.test.ts +++ b/web/hwlab-cloud-web/scripts/workbench-performance.test.ts @@ -8,6 +8,7 @@ import type { ChatMessage, TraceEvent } from "../src/types/index.ts"; import { acknowledgeWorkbenchVisible, drainWorkbenchPerformanceEventsForTest, + failWorkbenchSubmitJourney, finishWorkbenchOpenFullLoad, finishWorkbenchSessionSwitchFullLoad, markWorkbenchSubmitApiAccepted, @@ -53,6 +54,37 @@ test("Workbench performance probe records journeys and strips high-cardinality i assert.doesNotMatch(payload, /trc_secret|cnv_secret|sessionId|conversationId|runId|commandId/u); }); +test("Workbench submit failures do not pollute first visible output SLI", () => { + resetWorkbenchPerformanceForTest(); + startWorkbenchSubmitJourney({ traceId: "trc_network", conversationId: "cnv_secret", entry: "new", backend: "codex", transport: "sse" }); + failWorkbenchSubmitJourney("trc_network", "network"); + + const events = drainWorkbenchPerformanceEventsForTest(); + assert.equal(events.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_first_visible"), false); + assert.ok(events.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_failure" && event.outcome === "network")); +}); + +test("Workbench submit first visible waits for assistant or tool output", () => { + resetWorkbenchPerformanceForTest(); + const wallBase = Date.now(); + const createdAt = new Date(wallBase - 12_000).toISOString(); + const appendedAt = new Date(wallBase - 11_500).toISOString(); + const backendEvent = { label: "agentrun:run:createdAgentRun", backend: "agentrun-v01/codex", createdAt, appendedAt, message: "AgentRun created." } as TraceEvent; + startWorkbenchSubmitJourney({ traceId: "trc_secret", conversationId: "cnv_secret", entry: "existing", backend: "codex", transport: "sse" }); + markWorkbenchTraceEventsReceived({ traceId: "trc_secret", transport: "rest_gap", events: [backendEvent] }); + markWorkbenchTraceProjected("trc_secret"); + acknowledgeWorkbenchVisible({ messages: [traceOnlyAgentMessage("cnv_secret", "trc_secret", [backendEvent])], activeConversationId: "cnv_secret", detailLoading: false }); + + const backendOnlyEvents = drainWorkbenchPerformanceEventsForTest(); + assert.equal(backendOnlyEvents.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_first_visible"), false); + assert.equal(backendOnlyEvents.some((event) => event.kind === "workbench_event_phase" && event.phase === "sse_to_receive"), false); + assert.ok(backendOnlyEvents.some((event) => event.kind === "workbench_backend_event_visible" && event.eventType === "backend" && event.outcome === "stale")); + + acknowledgeWorkbenchVisible({ messages: [traceOnlyAgentMessage("cnv_secret", "trc_secret", [{ type: "tool_call", command: "pwd" } as TraceEvent])], activeConversationId: "cnv_secret", detailLoading: false }); + const toolEvents = drainWorkbenchPerformanceEventsForTest(); + assert.ok(toolEvents.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_first_visible" && event.outcome === "ok")); +}); + function userMessage(conversationId: string): ChatMessage { return { id: "msg_user", role: "user", title: "User", text: "hello", status: "sent", createdAt: new Date().toISOString(), conversationId }; } @@ -70,3 +102,17 @@ function agentMessage(conversationId: string, traceId: string): ChatMessage { runnerTrace: { traceId, eventCount: 1, events: [{ type: "assistant_message" }] } }; } + +function traceOnlyAgentMessage(conversationId: string, traceId: string, events: TraceEvent[]): ChatMessage { + return { + id: "msg_agent_trace", + role: "agent", + title: "Code Agent", + text: "", + status: "running", + createdAt: new Date().toISOString(), + conversationId, + traceId, + runnerTrace: { traceId, eventCount: events.length, events } + }; +} diff --git a/web/hwlab-cloud-web/src/utils/workbench-performance.ts b/web/hwlab-cloud-web/src/utils/workbench-performance.ts index 548e31b0..c55fd8bd 100644 --- a/web/hwlab-cloud-web/src/utils/workbench-performance.ts +++ b/web/hwlab-cloud-web/src/utils/workbench-performance.ts @@ -76,6 +76,7 @@ interface TraceEventState { eventType: string; backend: string; transport: string; + visibleOutcome: WorkbenchOutcome; visibleReported: boolean; } @@ -191,7 +192,7 @@ export function failWorkbenchSubmitJourney(traceId: string | null | undefined, o state.firstVisibleReported = true; enqueue({ kind: "workbench_journey", - journey: "submit_to_first_visible", + journey: "submit_to_failure", route: state.route, entry: state.entry, backend: state.backend, @@ -216,11 +217,11 @@ export function markWorkbenchTraceEventsReceived(input: TraceEventTimingInput): const receivedAt = monotonicNow(); const receivedWallAt = wallNow(); const traceSeq = Number(input.traceSeq ?? event.seq ?? event.sourceSeq ?? NaN); - traceEvents.set(traceId, { traceId, receivedAt, projectedAt: null, eventCreatedAt, appendedAt, serverSentAt, traceSeq: Number.isFinite(traceSeq) ? Math.trunc(traceSeq) : null, eventType, backend, transport: input.transport, visibleReported: false }); + const visibleOutcome: WorkbenchOutcome = input.transport === "sse" && serverSentAt !== null ? "ok" : "stale"; + traceEvents.set(traceId, { traceId, receivedAt, projectedAt: null, eventCreatedAt, appendedAt, serverSentAt, traceSeq: Number.isFinite(traceSeq) ? Math.trunc(traceSeq) : null, eventType, backend, transport: input.transport, visibleOutcome, visibleReported: false }); if (eventCreatedAt !== null && appendedAt !== null) enqueue({ kind: "workbench_event_phase", phase: "created_to_append", eventType, backend, transport: input.transport, outcome: "ok", valueMs: Math.max(0, appendedAt - eventCreatedAt) }); if (appendedAt !== null && serverSentAt !== null) enqueue({ kind: "workbench_event_phase", phase: "append_to_sse", eventType, backend, transport: input.transport, outcome: "ok", valueMs: Math.max(0, serverSentAt - appendedAt) }); if (serverSentAt !== null) enqueue({ kind: "workbench_event_phase", phase: "sse_to_receive", eventType, backend, transport: input.transport, outcome: "ok", valueMs: Math.max(0, receivedWallAt - serverSentAt) }); - else if (eventCreatedAt !== null) enqueue({ kind: "workbench_event_phase", phase: "sse_to_receive", eventType, backend, transport: input.transport, outcome: "ok", valueMs: Math.max(0, receivedWallAt - eventCreatedAt) }); } export function markWorkbenchTraceProjected(traceId: string | null | undefined): void { @@ -250,18 +251,19 @@ export function acknowledgeWorkbenchVisible(input: { messages: ChatMessage[]; ac } for (const message of input.messages) { const traceId = safeText(message.traceId ?? message.runnerTrace?.traceId); - if (!traceId || message.role !== "agent" || !isVisibleAgentMessage(message)) continue; + if (!traceId || message.role !== "agent") continue; const submit = submitJourneys.get(traceId); - if (submit && !submit.firstVisibleReported) { + const outputVisible = isVisibleAgentOutputMessage(message); + if (submit && !submit.firstVisibleReported && outputVisible) { submit.firstVisibleReported = true; enqueue({ kind: "workbench_journey", journey: "submit_to_first_visible", route: submit.route, entry: submit.entry, backend: submit.backend, transport: submit.transport, visibility: visibilityState(), outcome: "ok", valueMs: Math.max(0, now - submit.startAt) }); submitJourneys.delete(traceId); } const event = traceEvents.get(traceId); - if (event && !event.visibleReported) { + if (event && !event.visibleReported && isVisibleTraceEventMessage(message)) { event.visibleReported = true; if (event.projectedAt !== null) enqueue({ kind: "workbench_event_phase", phase: "project_to_paint", eventType: event.eventType, backend: event.backend, transport: event.transport, outcome: "ok", valueMs: Math.max(0, now - event.projectedAt) }); - if (event.eventCreatedAt !== null) enqueue({ kind: "workbench_backend_event_visible", eventType: event.eventType, backend: event.backend, transport: event.transport, outcome: "ok", valueMs: Math.max(0, wallNow() - event.eventCreatedAt) }); + if (event.eventCreatedAt !== null) enqueue({ kind: "workbench_backend_event_visible", eventType: event.eventType, backend: event.backend, transport: event.transport, outcome: event.visibleOutcome, valueMs: Math.max(0, wallNow() - event.eventCreatedAt) }); traceEvents.delete(traceId); } } @@ -327,10 +329,17 @@ function afterNextPaint(callback: () => void): void { requestAnimationFrame(() => requestAnimationFrame(callback)); } -function isVisibleAgentMessage(message: ChatMessage): boolean { - if (message.runnerTrace && ((message.runnerTrace.events?.length ?? 0) > 0 || message.runnerTrace.eventCount)) return true; +function isVisibleAgentOutputMessage(message: ChatMessage): boolean { if (String(message.text ?? "").trim()) return true; - return !["pending", "running"].includes(String(message.status ?? "").toLowerCase()); + return traceEventsFromMessage(message).some(isAssistantOrToolOutputTraceEvent); +} + +function isVisibleTraceEventMessage(message: ChatMessage): boolean { + return traceEventsFromMessage(message).some(isUserVisibleTraceEvent); +} + +function traceEventsFromMessage(message: ChatMessage): TraceEvent[] { + return Array.isArray(message.runnerTrace?.events) ? message.runnerTrace.events : []; } function isUserVisibleTraceEvent(event: TraceEvent): boolean { @@ -338,6 +347,16 @@ function isUserVisibleTraceEvent(event: TraceEvent): boolean { return type === "assistant" || type === "tool_call" || type === "backend" || type === "terminal" || type === "error"; } +function isAssistantOrToolOutputTraceEvent(event: TraceEvent): boolean { + const type = eventTypeFromTraceEvent(event); + if (type === "tool_call") return true; + return type === "assistant" && traceEventHasDisplayText(event); +} + +function traceEventHasDisplayText(event: TraceEvent): boolean { + return [event.message, event.text, event.outputSummary, event.stdoutSummary, event.stderrSummary, event.command].some((value) => Boolean(safeText(value))); +} + function eventTypeFromTraceEvent(event: TraceEvent | null | undefined): string { const raw = safeText(event?.eventType ?? event?.type ?? event?.kind ?? event?.label ?? event?.status).toLowerCase(); if (/tool/u.test(raw)) return "tool_call";