root
406 lines
19 KiB
TypeScript
406 lines
19 KiB
TypeScript
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0
|
|
// Exercises the browser-side Workbench performance probe without sending network traffic.
|
|
|
|
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
|
|
import type { ChatMessage, TraceEvent } from "../src/types/index.ts";
|
|
import {
|
|
acknowledgeWorkbenchVisible,
|
|
drainWorkbenchPerformanceEventsForTest,
|
|
failWorkbenchSubmitJourney,
|
|
finishWorkbenchOpenFullLoad,
|
|
finishWorkbenchSessionSwitchFullLoad,
|
|
markWorkbenchSubmitApiAccepted,
|
|
markWorkbenchTraceEventsReceived,
|
|
markWorkbenchTraceProjected,
|
|
recordWorkbenchApiRequest,
|
|
recordWorkbenchRuntimeDiagnostic,
|
|
resetWorkbenchPerformanceForTest,
|
|
startWorkbenchOpenJourney,
|
|
startWorkbenchSessionSwitch,
|
|
startWorkbenchSubmitJourney
|
|
} from "../src/utils/workbench-performance.ts";
|
|
|
|
test("Workbench performance probe records journeys and strips high-cardinality identifiers", () => {
|
|
resetWorkbenchPerformanceForTest();
|
|
startWorkbenchOpenJourney({ route: "/workbench/sessions/ses_secret", cache: "cold", authState: "warm" });
|
|
acknowledgeWorkbenchVisible({ messages: [], activeSessionId: "ses_secret", detailLoading: false });
|
|
finishWorkbenchOpenFullLoad("ok");
|
|
|
|
startWorkbenchSessionSwitch({ sessionId: "ses_secret", source: "rail", targetState: "running", cache: "warm" });
|
|
acknowledgeWorkbenchVisible({ messages: [userMessage("ses_secret")], activeSessionId: "ses_secret", detailLoading: false });
|
|
finishWorkbenchSessionSwitchFullLoad("ses_secret", "ok");
|
|
|
|
const wallBase = Date.now();
|
|
const createdAt = new Date(wallBase - 300).toISOString();
|
|
const appendedAt = new Date(wallBase - 220).toISOString();
|
|
const serverSentAt = new Date(wallBase - 120).toISOString();
|
|
startWorkbenchSubmitJourney({ traceId: "trc_secret", sessionId: "ses_secret", entry: "existing", backend: "codex", transport: "sse" });
|
|
markWorkbenchSubmitApiAccepted("trc_secret");
|
|
markWorkbenchTraceEventsReceived({ traceId: "trc_secret", transport: "sse", serverSentAt, traceSeq: 42, events: [{ type: "assistant_message", eventType: "assistant", backend: "agentrun-v01/codex", createdAt, appendedAt, ts: createdAt, sourceSeq: 42 } as TraceEvent] });
|
|
markWorkbenchTraceProjected("trc_secret");
|
|
acknowledgeWorkbenchVisible({ messages: [agentMessage("ses_secret", "trc_secret")], activeSessionId: "ses_secret", detailLoading: false });
|
|
|
|
const events = drainWorkbenchPerformanceEventsForTest();
|
|
const payload = JSON.stringify(events);
|
|
assert.ok(events.some((event) => event.kind === "workbench_journey" && event.journey === "workbench_open_first_visible"));
|
|
assert.ok(events.some((event) => event.kind === "workbench_journey" && event.journey === "session_switch_first_visible"));
|
|
assert.ok(events.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_first_visible" && event.backend === "agentrun-v01/codex"));
|
|
assert.ok(events.some((event) => event.kind === "workbench_event_phase" && event.phase === "created_to_append" && event.eventType === "assistant"));
|
|
assert.ok(events.some((event) => event.kind === "workbench_event_phase" && event.phase === "append_to_sse" && event.eventType === "assistant"));
|
|
assert.ok(events.some((event) => event.kind === "workbench_event_phase" && event.phase === "sse_to_receive" && event.eventType === "assistant"));
|
|
assert.ok(events.some((event) => event.kind === "workbench_event_phase" && event.phase === "receive_to_project" && event.eventType === "assistant"));
|
|
assert.ok(events.some((event) => event.kind === "workbench_event_phase" && event.phase === "project_to_paint" && event.eventType === "assistant"));
|
|
assert.ok(events.some((event) => event.kind === "workbench_backend_event_visible" && event.eventType === "assistant"));
|
|
assert.doesNotMatch(payload, /trc_secret|ses_secret|sessionId|runId|commandId/u);
|
|
});
|
|
|
|
test("Workbench submit failures do not pollute first visible output SLI", () => {
|
|
resetWorkbenchPerformanceForTest();
|
|
startWorkbenchSubmitJourney({ traceId: "trc_network", sessionId: "ses_secret", entry: "new", backend: "codex", transport: "sse" });
|
|
failWorkbenchSubmitJourney("trc_network", "network");
|
|
|
|
const events = drainWorkbenchPerformanceEventsForTest();
|
|
assert.equal(events.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_first_visible"), false);
|
|
assert.ok(events.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_failure" && event.outcome === "network"));
|
|
});
|
|
|
|
test("Workbench runtime diagnostics expose root cause labels without raw identifiers", () => {
|
|
resetWorkbenchPerformanceForTest();
|
|
const restoreBrowserRuntime = installBrowserProbeRuntime();
|
|
try {
|
|
recordWorkbenchRuntimeDiagnostic({
|
|
module: "workbench-stream-transport",
|
|
sessionId: "ses_secret",
|
|
traceId: "trc_secret",
|
|
outcome: "network",
|
|
diagnostic: {
|
|
code: "workbench_sse_recovery",
|
|
rootCause: "eventsource-error",
|
|
recoveryAction: "refresh-session-messages,schedule-session-list",
|
|
transportState: "degraded",
|
|
scopedKey: "workbench.realtime|ses_secret|trc_secret"
|
|
}
|
|
});
|
|
|
|
const events = drainWorkbenchPerformanceEventsForTest();
|
|
const event = events.find((item) => item.kind === "workbench_ui_event" && item.eventType === "runtime_diagnostic");
|
|
assert.equal(event?.module, "workbench-stream-transport");
|
|
assert.equal(event?.diagnosticCode, "workbench_sse_recovery");
|
|
assert.equal(event?.rootCause, "eventsource-error");
|
|
assert.equal(event?.recoveryAction, "refresh-session-messages,schedule-session-list");
|
|
assert.equal(event?.transportState, "degraded");
|
|
assert.match(event?.scopedKey ?? "", /^scope_[0-9a-f]{8}$/u);
|
|
assert.doesNotMatch(JSON.stringify(events), /ses_secret|trc_secret|workbench\.realtime\|/u);
|
|
} finally {
|
|
resetWorkbenchPerformanceForTest();
|
|
restoreBrowserRuntime();
|
|
}
|
|
});
|
|
|
|
test("Workbench SSE flush diagnostics expose bounded queue cost fields", () => {
|
|
resetWorkbenchPerformanceForTest();
|
|
const restoreBrowserRuntime = installBrowserProbeRuntime();
|
|
try {
|
|
recordWorkbenchRuntimeDiagnostic({
|
|
module: "workbench-events",
|
|
sessionId: "ses_secret",
|
|
traceId: "trc_secret",
|
|
outcome: "ok",
|
|
diagnostic: {
|
|
code: "workbench_sse_flush",
|
|
reason: "yield",
|
|
eventCount: 5,
|
|
deliveredCount: 4,
|
|
dropped: 1,
|
|
chunkCount: 2,
|
|
replacedByKey: 1,
|
|
flushDurationMs: 17,
|
|
maxItemsPerChunk: 4,
|
|
maxChunkMs: 8
|
|
}
|
|
});
|
|
|
|
const events = drainWorkbenchPerformanceEventsForTest();
|
|
const event = events.find((item) => item.kind === "workbench_ui_event" && item.eventType === "runtime_diagnostic");
|
|
assert.equal(event?.state, "sample");
|
|
assert.equal(event?.reason, "sse_flush");
|
|
assert.equal(event?.valueMs, 17);
|
|
assert.equal(event?.eventCount, 5);
|
|
assert.equal(event?.deliveredCount, 4);
|
|
assert.equal(event?.droppedCount, 1);
|
|
assert.equal(event?.chunkCount, 2);
|
|
assert.equal(event?.replacedByKey, 1);
|
|
assert.equal(event?.flushDurationMs, 17);
|
|
assert.equal(event?.maxItemsPerChunk, 4);
|
|
assert.equal(event?.maxChunkMs, 8);
|
|
assert.doesNotMatch(JSON.stringify(events), /ses_secret|trc_secret/u);
|
|
} finally {
|
|
resetWorkbenchPerformanceForTest();
|
|
restoreBrowserRuntime();
|
|
}
|
|
});
|
|
|
|
test("Workbench SSE flush diagnostics request prompt web-performance delivery", async () => {
|
|
resetWorkbenchPerformanceForTest();
|
|
const timers: { delay: number; handler: () => void }[] = [];
|
|
const posts: { url: string; body: string }[] = [];
|
|
const restoreBrowserRuntime = installBrowserProbeRuntime({
|
|
setTimeout: (handler, timeout) => {
|
|
timers.push({ delay: Number(timeout ?? 0), handler: () => invokeTimerHandler(handler) });
|
|
return timers.length;
|
|
},
|
|
clearTimeout: () => undefined,
|
|
fetch: (input, init) => {
|
|
posts.push({ url: String(input), body: String(init?.body ?? "") });
|
|
return Promise.resolve(new Response(null, { status: 204 }));
|
|
}
|
|
});
|
|
try {
|
|
recordWorkbenchRuntimeDiagnostic({
|
|
module: "workbench-events",
|
|
sessionId: "ses_secret",
|
|
traceId: "trc_secret",
|
|
outcome: "ok",
|
|
diagnostic: {
|
|
code: "workbench_sse_flush",
|
|
eventCount: 3,
|
|
deliveredCount: 2,
|
|
dropped: 1,
|
|
chunkCount: 2,
|
|
flushDurationMs: 9,
|
|
maxItemsPerChunk: 4,
|
|
maxChunkMs: 8
|
|
}
|
|
});
|
|
|
|
const priorityTimer = timers.find((timer) => timer.delay === 250);
|
|
assert.ok(priorityTimer);
|
|
priorityTimer.handler();
|
|
await Promise.resolve();
|
|
|
|
assert.equal(posts.length, 1);
|
|
assert.equal(posts[0]?.url, "/v1/web-performance");
|
|
assert.match(posts[0]?.body ?? "", /workbench_sse_flush/u);
|
|
assert.match(posts[0]?.body ?? "", /"eventCount":3/u);
|
|
assert.match(posts[0]?.body ?? "", /"maxChunkMs":8/u);
|
|
assert.equal(drainWorkbenchPerformanceEventsForTest().length, 0);
|
|
} finally {
|
|
resetWorkbenchPerformanceForTest();
|
|
restoreBrowserRuntime();
|
|
}
|
|
});
|
|
|
|
test("Workbench API timing enriches from ResourceTiming and clears the browser buffer", () => {
|
|
resetWorkbenchPerformanceForTest();
|
|
let clearCount = 0;
|
|
let configuredBufferSize = 0;
|
|
const startedAtEpochMs = 1_000_000;
|
|
const entries = [resourceTimingEntry({ name: "https://hwlab.example.test/v1/workbench/sessions/ses_secret/messages?limit=100", startTime: 10, duration: 42, responseStart: 37, responseEnd: 52 })];
|
|
const restoreBrowserRuntime = installBrowserProbeRuntime({
|
|
performance: {
|
|
timeOrigin: startedAtEpochMs - 10,
|
|
now: () => 123,
|
|
getEntriesByType: (type: string) => type === "resource" ? entries : [],
|
|
setResourceTimingBufferSize: (value: number) => {
|
|
configuredBufferSize = value;
|
|
},
|
|
clearResourceTimings: () => {
|
|
clearCount += 1;
|
|
entries.splice(0, entries.length);
|
|
}
|
|
}
|
|
});
|
|
try {
|
|
recordWorkbenchApiRequest({ route: "/v1/workbench/sessions/ses_secret/messages?limit=100", method: "GET", status: 200, outcome: "ok", startedAtEpochMs, endedAtEpochMs: startedAtEpochMs + 60 });
|
|
|
|
const events = drainWorkbenchPerformanceEventsForTest();
|
|
const event = events.find((item) => item.kind === "workbench_ui_event" && item.eventType === "api_request");
|
|
assert.equal(configuredBufferSize, 2000);
|
|
assert.equal(clearCount, 1);
|
|
assert.equal(entries.length, 0);
|
|
assert.equal(event?.route, "/v1/workbench/sessions/:id/messages");
|
|
assert.equal(event?.resourceDurationMs, 42);
|
|
assert.equal(event?.resourceRequestWaitMs, 15);
|
|
assert.equal(event?.resourceResponseTransferMs, 15);
|
|
} finally {
|
|
resetWorkbenchPerformanceForTest();
|
|
restoreBrowserRuntime();
|
|
}
|
|
});
|
|
|
|
test("Workbench API timing labels metadata-only session detail distinctly", () => {
|
|
resetWorkbenchPerformanceForTest();
|
|
const restoreBrowserRuntime = installBrowserProbeRuntime();
|
|
try {
|
|
const startedAtEpochMs = 1_000_000;
|
|
recordWorkbenchApiRequest({ route: "/v1/workbench/sessions/ses_secret?includeMessages=false", method: "GET", status: 200, outcome: "ok", startedAtEpochMs, endedAtEpochMs: startedAtEpochMs + 12 });
|
|
|
|
const events = drainWorkbenchPerformanceEventsForTest();
|
|
const event = events.find((item) => item.kind === "workbench_ui_event" && item.eventType === "api_request");
|
|
assert.equal(event?.route, "/v1/workbench/sessions/:id?includeMessages=false");
|
|
} finally {
|
|
resetWorkbenchPerformanceForTest();
|
|
restoreBrowserRuntime();
|
|
}
|
|
});
|
|
|
|
test("Workbench session switch treats visible empty sessions as successful first paint", () => {
|
|
resetWorkbenchPerformanceForTest();
|
|
startWorkbenchSessionSwitch({ sessionId: "ses_empty_visible", source: "rail", targetState: "empty", cache: "cold" });
|
|
acknowledgeWorkbenchVisible({ messages: [], activeSessionId: "ses_empty_visible", detailLoading: false });
|
|
|
|
const events = drainWorkbenchPerformanceEventsForTest();
|
|
const switchEvent = events.find((event) => event.kind === "workbench_journey" && event.journey === "session_switch_first_visible");
|
|
assert.equal(switchEvent?.outcome, "ok");
|
|
assert.equal(switchEvent?.targetState, "empty");
|
|
});
|
|
|
|
test("Workbench submit first visible waits for terminal final or tool output", () => {
|
|
resetWorkbenchPerformanceForTest();
|
|
const wallBase = Date.now();
|
|
const createdAt = new Date(wallBase - 12_000).toISOString();
|
|
const appendedAt = new Date(wallBase - 11_500).toISOString();
|
|
const backendEvent = { label: "agentrun:run:createdAgentRun", backend: "agentrun-v01/codex", createdAt, appendedAt, message: "AgentRun created." } as TraceEvent;
|
|
startWorkbenchSubmitJourney({ traceId: "trc_secret", sessionId: "ses_secret", entry: "existing", backend: "codex", transport: "sse" });
|
|
markWorkbenchTraceEventsReceived({ traceId: "trc_secret", transport: "detail_history", events: [backendEvent] });
|
|
markWorkbenchTraceProjected("trc_secret");
|
|
acknowledgeWorkbenchVisible({ messages: [traceOnlyAgentMessage("ses_secret", "trc_secret", [backendEvent])], activeSessionId: "ses_secret", detailLoading: false });
|
|
|
|
const backendOnlyEvents = drainWorkbenchPerformanceEventsForTest();
|
|
assert.equal(backendOnlyEvents.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_first_visible"), false);
|
|
assert.equal(backendOnlyEvents.some((event) => event.kind === "workbench_event_phase" && event.phase === "sse_to_receive"), false);
|
|
assert.ok(backendOnlyEvents.some((event) => event.kind === "workbench_event_phase" && event.phase === "created_to_append" && event.eventType === "backend" && event.transport === "detail_history"));
|
|
assert.ok(backendOnlyEvents.some((event) => event.kind === "workbench_event_phase" && event.phase === "receive_to_project" && event.eventType === "backend" && event.transport === "detail_history"));
|
|
assert.equal(backendOnlyEvents.some((event) => event.kind === "workbench_backend_event_visible" && event.eventType === "backend" && event.outcome === "stale"), false);
|
|
|
|
acknowledgeWorkbenchVisible({ messages: [traceOnlyAgentMessage("ses_secret", "trc_secret", [{ type: "assistant_message", status: "running", message: "progress only" } as TraceEvent])], activeSessionId: "ses_secret", detailLoading: false });
|
|
const progressEvents = drainWorkbenchPerformanceEventsForTest();
|
|
assert.equal(progressEvents.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_first_visible"), false);
|
|
|
|
acknowledgeWorkbenchVisible({ messages: [traceOnlyAgentMessage("ses_secret", "trc_secret", [{ type: "tool_call", command: "pwd" } as TraceEvent])], activeSessionId: "ses_secret", detailLoading: false });
|
|
const toolEvents = drainWorkbenchPerformanceEventsForTest();
|
|
assert.ok(toolEvents.some((event) => event.kind === "workbench_journey" && event.journey === "submit_to_first_visible" && event.outcome === "ok"));
|
|
});
|
|
|
|
function userMessage(sessionId: string): ChatMessage {
|
|
return { id: "msg_user", role: "user", title: "User", text: "hello", status: "sent", createdAt: new Date().toISOString(), sessionId };
|
|
}
|
|
|
|
function agentMessage(sessionId: string, traceId: string): ChatMessage {
|
|
return {
|
|
id: "msg_agent",
|
|
role: "agent",
|
|
title: "Code Agent",
|
|
text: "visible",
|
|
status: "completed",
|
|
createdAt: new Date().toISOString(),
|
|
sessionId,
|
|
traceId,
|
|
runnerTrace: { traceId, sessionId, eventCount: 1, events: [{ type: "assistant_message" }] }
|
|
};
|
|
}
|
|
|
|
function traceOnlyAgentMessage(sessionId: string, traceId: string, events: TraceEvent[]): ChatMessage {
|
|
return {
|
|
id: "msg_agent_trace",
|
|
role: "agent",
|
|
title: "Code Agent",
|
|
text: "",
|
|
status: "running",
|
|
createdAt: new Date().toISOString(),
|
|
sessionId,
|
|
traceId,
|
|
runnerTrace: { traceId, sessionId, eventCount: events.length, events }
|
|
};
|
|
}
|
|
|
|
function installBrowserProbeRuntime(options: { performance?: FakePerformance; setTimeout?: BrowserSetTimeout; clearTimeout?: BrowserClearTimeout; fetch?: BrowserFetch } = {}): () => void {
|
|
const previousWindow = Object.getOwnPropertyDescriptor(globalThis, "window");
|
|
const previousDocument = Object.getOwnPropertyDescriptor(globalThis, "document");
|
|
const previousPerformance = Object.getOwnPropertyDescriptor(globalThis, "performance");
|
|
const previousFetch = Object.getOwnPropertyDescriptor(globalThis, "fetch");
|
|
Object.defineProperty(globalThis, "window", {
|
|
configurable: true,
|
|
value: {
|
|
location: { href: "https://hwlab.example.test/workbench", pathname: "/workbench", hash: "" },
|
|
crypto: globalThis.crypto,
|
|
addEventListener: () => undefined,
|
|
setTimeout: options.setTimeout ?? globalThis.setTimeout,
|
|
clearTimeout: options.clearTimeout ?? globalThis.clearTimeout
|
|
}
|
|
});
|
|
Object.defineProperty(globalThis, "document", {
|
|
configurable: true,
|
|
value: {
|
|
visibilityState: "visible",
|
|
addEventListener: () => undefined
|
|
}
|
|
});
|
|
if (options.performance) {
|
|
Object.defineProperty(globalThis, "performance", {
|
|
configurable: true,
|
|
value: options.performance
|
|
});
|
|
}
|
|
if (options.fetch) {
|
|
Object.defineProperty(globalThis, "fetch", {
|
|
configurable: true,
|
|
value: options.fetch
|
|
});
|
|
}
|
|
return () => {
|
|
restoreGlobalProperty("window", previousWindow);
|
|
restoreGlobalProperty("document", previousDocument);
|
|
restoreGlobalProperty("performance", previousPerformance);
|
|
restoreGlobalProperty("fetch", previousFetch);
|
|
};
|
|
}
|
|
|
|
type BrowserSetTimeout = (handler: TimerHandler, timeout?: number, ...arguments_: unknown[]) => number;
|
|
type BrowserClearTimeout = (handle?: number) => void;
|
|
type BrowserFetch = (input: RequestInfo | URL, init?: RequestInit) => Promise<Response>;
|
|
|
|
interface FakePerformance {
|
|
timeOrigin: number;
|
|
now: () => number;
|
|
getEntriesByType: (type: string) => unknown[];
|
|
setResourceTimingBufferSize?: (value: number) => void;
|
|
clearResourceTimings?: () => void;
|
|
}
|
|
|
|
function invokeTimerHandler(handler: TimerHandler): void {
|
|
if (typeof handler === "function") {
|
|
handler();
|
|
return;
|
|
}
|
|
throw new TypeError("string timer handlers are not supported by this test runtime");
|
|
}
|
|
|
|
function resourceTimingEntry(input: { name: string; startTime: number; duration: number; responseStart: number; responseEnd: number }): PerformanceResourceTiming {
|
|
return {
|
|
name: input.name,
|
|
startTime: input.startTime,
|
|
duration: input.duration,
|
|
fetchStart: input.startTime,
|
|
requestStart: input.startTime + 12,
|
|
responseStart: input.responseStart,
|
|
responseEnd: input.responseEnd,
|
|
transferSize: 1000,
|
|
encodedBodySize: 800,
|
|
decodedBodySize: 1600,
|
|
nextHopProtocol: "h2",
|
|
initiatorType: "fetch",
|
|
serverTiming: []
|
|
} as unknown as PerformanceResourceTiming;
|
|
}
|
|
|
|
function restoreGlobalProperty(key: "window" | "document" | "performance" | "fetch", descriptor: PropertyDescriptor | undefined): void {
|
|
if (descriptor) {
|
|
Object.defineProperty(globalThis, key, descriptor);
|
|
return;
|
|
}
|
|
Reflect.deleteProperty(globalThis, key);
|
|
}
|