mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1424 from pikasTech/fix/1422-workbench-performance

Browse Source 
修正 Workbench 首可见性能指标口径
This commit is contained in:
Lyon
2026-06-17 20:42:43 +08:00
committed by GitHub
parent e4f0fb1933 c1d158b5bb
commit 88d0842f9b
4 changed files with 90 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/cloud/web-performance.test.ts
+14 -1
View File
@@ -80,6 +80,18 @@ test("web performance store accepts v2 Workbench journey and phase metrics witho
valueMs: 42,
commandId: "cmd_secret"
},
{
kind: "workbench_journey",
journey: "submit_to_failure",
route: "/workbench/sessions/ses_secret",
entry: "new",
backend: "agentrun-v01/codex",
transport: "sse",
visibility: "foreground",
outcome: "network",
valueMs: 530,
traceId: "trc_secret"
},
{
kind: "workbench_backend_event_visible",
eventType: "tool",
@@ -93,8 +105,9 @@ test("web performance store accepts v2 Workbench journey and phase metrics witho
} as Record<string, unknown>);
const text = store.metricsText();
assert.deepEqual(result, { accepted: 3, dropped: 0, received: 3 });
assert.deepEqual(result, { accepted: 4, dropped: 0, received: 4 });
assert.match(text, /hwlab_workbench_journey_total\{[^}]*journey="submit_to_first_visible"[^}]*route="\/workbench\/sessions\/:id"[^}]*backend="agentrun-v01\/codex"/u);
assert.match(text, /hwlab_workbench_journey_total\{[^}]*journey="submit_to_failure"[^}]*outcome="network"/u);
assert.match(text, /hwlab_workbench_event_phase_duration_seconds_bucket\{[^}]*phase="created_to_append"[^}]*event_type="assistant"[^}]*le="0\.05"\} 1/u);
assert.match(text, /hwlab_workbench_backend_event_visible_latency_seconds_bucket\{[^}]*event_type="tool_call"[^}]*backend="agentrun-v01\/codex"[^}]*le="1"\} 1/u);
assert.doesNotMatch(text, /trc_secret|ses_secret|run_secret|cmd_secret|cnv_secret|traceId|sessionId|runId|commandId|conversationId|prompt|api key/iu);
internal/cloud/web-performance.ts
+1
View File
@@ -20,6 +20,7 @@ const DURATION_METRICS = new Set([
const WORKBENCH_SCHEMA_VERSION = "hwlab-web-performance-v2";
const WORKBENCH_JOURNEYS = new Set([
"submit_to_first_visible",
"submit_to_failure",
"backend_event_to_visible",
"session_switch_first_visible",
"session_switch_full_load",
web/hwlab-cloud-web/scripts/workbench-performance.test.ts
+46
View File
@@ -8,6 +8,7 @@ import type { ChatMessage, TraceEvent } from "../src/types/index.ts";
import {
acknowledgeWorkbenchVisible,
drainWorkbenchPerformanceEventsForTest,
failWorkbenchSubmitJourney,
finishWorkbenchOpenFullLoad,
finishWorkbenchSessionSwitchFullLoad,
markWorkbenchSubmitApiAccepted,
@@ -53,6 +54,37 @@ test("Workbench performance probe records journeys and strips high-cardinality i
assert.doesNotMatch(payload, /trc_secret|cnv_secret|sessionId|conversationId|runId|commandId/u);
});
test("Workbench submit failures do not pollute first visible output SLI", () => {
resetWorkbenchPerformanceForTest();
startWorkbenchSubmitJourney({ traceId: "trc_network", conversationId: "cnv_secret", entry: "new", backend: "codex", transport: "sse" });
failWorkbenchSubmitJourney("trc_network", "network");
const events = drainWorkbenchPerformanceEventsForTest();
assert.equal(events.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_first_visible"), false);
assert.ok(events.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_failure" && event.outcome === "network"));
});
test("Workbench submit first visible waits for assistant or tool output", () => {
resetWorkbenchPerformanceForTest();
const wallBase = Date.now();
const createdAt = new Date(wallBase - 12_000).toISOString();
const appendedAt = new Date(wallBase - 11_500).toISOString();
const backendEvent = { label: "agentrun:run:createdAgentRun", backend: "agentrun-v01/codex", createdAt, appendedAt, message: "AgentRun created." } as TraceEvent;
startWorkbenchSubmitJourney({ traceId: "trc_secret", conversationId: "cnv_secret", entry: "existing", backend: "codex", transport: "sse" });
markWorkbenchTraceEventsReceived({ traceId: "trc_secret", transport: "rest_gap", events: [backendEvent] });
markWorkbenchTraceProjected("trc_secret");
acknowledgeWorkbenchVisible({ messages: [traceOnlyAgentMessage("cnv_secret", "trc_secret", [backendEvent])], activeConversationId: "cnv_secret", detailLoading: false });
const backendOnlyEvents = drainWorkbenchPerformanceEventsForTest();
assert.equal(backendOnlyEvents.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_first_visible"), false);
assert.equal(backendOnlyEvents.some((event) => event.kind === "workbench_event_phase" && event.phase === "sse_to_receive"), false);
assert.ok(backendOnlyEvents.some((event) => event.kind === "workbench_backend_event_visible" && event.eventType === "backend" && event.outcome === "stale"));
acknowledgeWorkbenchVisible({ messages: [traceOnlyAgentMessage("cnv_secret", "trc_secret", [{ type: "tool_call", command: "pwd" } as TraceEvent])], activeConversationId: "cnv_secret", detailLoading: false });
const toolEvents = drainWorkbenchPerformanceEventsForTest();
assert.ok(toolEvents.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_first_visible" && event.outcome === "ok"));
});
function userMessage(conversationId: string): ChatMessage {
return { id: "msg_user", role: "user", title: "User", text: "hello", status: "sent", createdAt: new Date().toISOString(), conversationId };
}
@@ -70,3 +102,17 @@ function agentMessage(conversationId: string, traceId: string): ChatMessage {
runnerTrace: { traceId, eventCount: 1, events: [{ type: "assistant_message" }] }
};
}
function traceOnlyAgentMessage(conversationId: string, traceId: string, events: TraceEvent[]): ChatMessage {
return {
id: "msg_agent_trace",
role: "agent",
title: "Code Agent",
text: "",
status: "running",
createdAt: new Date().toISOString(),
conversationId,
traceId,
runnerTrace: { traceId, eventCount: events.length, events }
};
}
web/hwlab-cloud-web/src/utils/workbench-performance.ts
+29 -10
View File
@@ -76,6 +76,7 @@ interface TraceEventState {
eventType: string;
backend: string;
transport: string;
visibleOutcome: WorkbenchOutcome;
visibleReported: boolean;
}
@@ -191,7 +192,7 @@ export function failWorkbenchSubmitJourney(traceId: string | null | undefined, o
state.firstVisibleReported = true;
enqueue({
kind: "workbench_journey",
journey: "submit_to_first_visible",
journey: "submit_to_failure",
route: state.route,
entry: state.entry,
backend: state.backend,
@@ -216,11 +217,11 @@ export function markWorkbenchTraceEventsReceived(input: TraceEventTimingInput):
const receivedAt = monotonicNow();
const receivedWallAt = wallNow();
const traceSeq = Number(input.traceSeq ?? event.seq ?? event.sourceSeq ?? NaN);
traceEvents.set(traceId, { traceId, receivedAt, projectedAt: null, eventCreatedAt, appendedAt, serverSentAt, traceSeq: Number.isFinite(traceSeq) ? Math.trunc(traceSeq) : null, eventType, backend, transport: input.transport, visibleReported: false });
const visibleOutcome: WorkbenchOutcome = input.transport === "sse" && serverSentAt !== null ? "ok" : "stale";
traceEvents.set(traceId, { traceId, receivedAt, projectedAt: null, eventCreatedAt, appendedAt, serverSentAt, traceSeq: Number.isFinite(traceSeq) ? Math.trunc(traceSeq) : null, eventType, backend, transport: input.transport, visibleOutcome, visibleReported: false });
if (eventCreatedAt !== null && appendedAt !== null) enqueue({ kind: "workbench_event_phase", phase: "created_to_append", eventType, backend, transport: input.transport, outcome: "ok", valueMs: Math.max(0, appendedAt - eventCreatedAt) });
if (appendedAt !== null && serverSentAt !== null) enqueue({ kind: "workbench_event_phase", phase: "append_to_sse", eventType, backend, transport: input.transport, outcome: "ok", valueMs: Math.max(0, serverSentAt - appendedAt) });
if (serverSentAt !== null) enqueue({ kind: "workbench_event_phase", phase: "sse_to_receive", eventType, backend, transport: input.transport, outcome: "ok", valueMs: Math.max(0, receivedWallAt - serverSentAt) });
else if (eventCreatedAt !== null) enqueue({ kind: "workbench_event_phase", phase: "sse_to_receive", eventType, backend, transport: input.transport, outcome: "ok", valueMs: Math.max(0, receivedWallAt - eventCreatedAt) });
}
export function markWorkbenchTraceProjected(traceId: string | null | undefined): void {
@@ -250,18 +251,19 @@ export function acknowledgeWorkbenchVisible(input: { messages: ChatMessage[]; ac
}
for (const message of input.messages) {
const traceId = safeText(message.traceId ?? message.runnerTrace?.traceId);
if (!traceId || message.role !== "agent" || !isVisibleAgentMessage(message)) continue;
if (!traceId || message.role !== "agent") continue;
const submit = submitJourneys.get(traceId);
if (submit && !submit.firstVisibleReported) {
const outputVisible = isVisibleAgentOutputMessage(message);
if (submit && !submit.firstVisibleReported && outputVisible) {
submit.firstVisibleReported = true;
enqueue({ kind: "workbench_journey", journey: "submit_to_first_visible", route: submit.route, entry: submit.entry, backend: submit.backend, transport: submit.transport, visibility: visibilityState(), outcome: "ok", valueMs: Math.max(0, now - submit.startAt) });
submitJourneys.delete(traceId);
}
const event = traceEvents.get(traceId);
if (event && !event.visibleReported) {
if (event && !event.visibleReported && isVisibleTraceEventMessage(message)) {
event.visibleReported = true;
if (event.projectedAt !== null) enqueue({ kind: "workbench_event_phase", phase: "project_to_paint", eventType: event.eventType, backend: event.backend, transport: event.transport, outcome: "ok", valueMs: Math.max(0, now - event.projectedAt) });
if (event.eventCreatedAt !== null) enqueue({ kind: "workbench_backend_event_visible", eventType: event.eventType, backend: event.backend, transport: event.transport, outcome: "ok", valueMs: Math.max(0, wallNow() - event.eventCreatedAt) });
if (event.eventCreatedAt !== null) enqueue({ kind: "workbench_backend_event_visible", eventType: event.eventType, backend: event.backend, transport: event.transport, outcome: event.visibleOutcome, valueMs: Math.max(0, wallNow() - event.eventCreatedAt) });
traceEvents.delete(traceId);
}
}
@@ -327,10 +329,17 @@ function afterNextPaint(callback: () => void): void {
requestAnimationFrame(() => requestAnimationFrame(callback));
}
function isVisibleAgentMessage(message: ChatMessage): boolean {
if (message.runnerTrace && ((message.runnerTrace.events?.length ?? 0) > 0 || message.runnerTrace.eventCount)) return true;
function isVisibleAgentOutputMessage(message: ChatMessage): boolean {
if (String(message.text ?? "").trim()) return true;
return !["pending", "running"].includes(String(message.status ?? "").toLowerCase());
return traceEventsFromMessage(message).some(isAssistantOrToolOutputTraceEvent);
}
function isVisibleTraceEventMessage(message: ChatMessage): boolean {
return traceEventsFromMessage(message).some(isUserVisibleTraceEvent);
}
function traceEventsFromMessage(message: ChatMessage): TraceEvent[] {
return Array.isArray(message.runnerTrace?.events) ? message.runnerTrace.events : [];
}
function isUserVisibleTraceEvent(event: TraceEvent): boolean {
@@ -338,6 +347,16 @@ function isUserVisibleTraceEvent(event: TraceEvent): boolean {
return type === "assistant" || type === "tool_call" || type === "backend" || type === "terminal" || type === "error";
}
function isAssistantOrToolOutputTraceEvent(event: TraceEvent): boolean {
const type = eventTypeFromTraceEvent(event);
if (type === "tool_call") return true;
return type === "assistant" && traceEventHasDisplayText(event);
}
function traceEventHasDisplayText(event: TraceEvent): boolean {
return [event.message, event.text, event.outputSummary, event.stdoutSummary, event.stderrSummary, event.command].some((value) => Boolean(safeText(value)));
}
function eventTypeFromTraceEvent(event: TraceEvent | null | undefined): string {
const raw = safeText(event?.eventType ?? event?.type ?? event?.kind ?? event?.label ?? event?.status).toLowerCase();
if (/tool/u.test(raw)) return "tool_call";