// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0 // Verifies Cloud API backend RED metrics, Server-Timing, and AgentRun upstream timing privacy. import assert from "node:assert/strict"; import { createServer as createHttpServer } from "node:http"; import { test } from "bun:test"; import { createBackendPerformanceStore, backendPerformanceRouteTemplate, withBackendPerformanceContext } from "./backend-performance.ts"; import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { createCloudApiServer } from "./server.ts"; import { createCodeAgentChatResultStore } from "./server-code-agent-http.ts"; test("backend performance store templates routes and appends Cloud API metrics to loopback metrics", async () => { assert.equal(backendPerformanceRouteTemplate("/v1/agent/traces/trc_secret_0123456789abcdef?full=1"), "/v1/agent/traces/:id"); assert.equal(backendPerformanceRouteTemplate("/api/v1/runs/run_0123456789abcdef/commands/cmd_0123456789abcdef/result"), "/api/v1/runs/:id/commands/:id/result"); assert.equal(backendPerformanceRouteTemplate("/v1/admin/billing/users/12345/credits/adjust"), "/v1/admin/billing/users/:id/credits/adjust"); const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); const server = createCloudApiServer({ backendPerformanceStore, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { const address = server.address(); assert.ok(address && typeof address === "object"); const baseUrl = `http://127.0.0.1:${address.port}`; const summary = await fetch(`${baseUrl}/v1/web-performance/summary`); assert.equal(summary.status, 200); assert.match(summary.headers.get("server-timing") ?? "", /total;dur=/u); const metrics = await fetch(`${baseUrl}/v1/web-performance/metrics`); const text = await metrics.text(); assert.equal(metrics.status, 200); assert.match(text, /hwlab_cloud_api_requests_total\{[^}]*route="\/v1\/web-performance\/summary"[^}]*status_class="2xx"[^}]*\} 1/u); assert.match(text, /hwlab_cloud_api_request_duration_seconds_count\{[^}]*route="\/v1\/web-performance\/summary"/u); assert.doesNotMatch(text, /trc_secret|0123456789abcdef/u); } finally { await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } }); test("backend performance records Code Agent trace handler phases without high-cardinality labels", async () => { const traceId = "trc_backend_phase_trace"; const runId = "run_backend_phase"; const commandId = "cmd_backend_phase"; const traceStore = createCodeAgentTraceStore(); const codeAgentChatResults = createCodeAgentChatResultStore(); traceStore.append(traceId, { type: "message", role: "assistant", label: "assistant:start", sourceSeq: 1, commandId }); traceStore.append(traceId, { type: "result", status: "completed", terminal: true, label: "result:completed", sourceSeq: 2, commandId }); codeAgentChatResults.set(traceId, { status: "completed", conversationId: "cnv_backend_phase", sessionId: "ses_backend_phase", threadId: "thr_backend_phase", ownerUserId: "usr_backend_phase", updatedAt: "2026-06-18T00:00:00.000Z", agentRun: { adapter: "agentrun-v01", runId, commandId, terminalStatus: "completed", lastSeq: 2 }, finalResponse: { text: "done", traceId, status: "completed", updatedAt: "2026-06-18T00:00:00.000Z" }, traceSummary: { source: "agentrun-command-result", terminalStatus: "completed", eventCount: 2, updatedAt: "2026-06-18T00:00:00.000Z" } }); const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); const accessController = { required: true, async authenticate() { return { ok: true, actor: { id: "usr_backend_phase", role: "user" }, session: { id: "web_backend_phase" } }; }, async getAgentSessionByTraceId(requestedTraceId: string) { assert.equal(requestedTraceId, traceId); return { id: "ses_backend_phase", sessionId: "ses_backend_phase", projectId: "prj_hwpod_workbench", ownerUserId: "usr_backend_phase", ownerRole: "user", session: {} }; } }; const server = createCloudApiServer({ backendPerformanceStore, traceStore, codeAgentChatResults, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { const address = server.address(); assert.ok(address && typeof address === "object"); const baseUrl = `http://127.0.0.1:${address.port}`; const trace = await fetch(`${baseUrl}/v1/agent/traces/${traceId}?projectId=prj_hwpod_workbench&limit=1`); assert.equal(trace.status, 200); const serverTiming = trace.headers.get("server-timing") ?? ""; assert.match(serverTiming, /trace-project-check;dur=/u); assert.match(serverTiming, /trace-paginate;dur=/u); assert.match(serverTiming, /total;dur=/u); const payload = await trace.json(); assert.equal(payload.range.returned, 1); const metrics = await fetch(`${baseUrl}/v1/web-performance/metrics`); const text = await metrics.text(); assert.equal(metrics.status, 200); assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:id"[^}]*phase="trace_project_check"[^}]*\} 1/u); assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:id"[^}]*phase="trace_paginate"[^}]*\} 1/u); assert.doesNotMatch(text, /trc_backend_phase_trace|run_backend_phase|cmd_backend_phase|ses_backend_phase|cnv_backend_phase/u); } finally { await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); } }); test("AgentRun adapter records upstream timing through backend performance context", async () => { const calls: Array<{ method?: string; path: string; body?: any }> = []; const agentRunServer = createHttpServer(async (request, response) => { const url = new URL(request.url || "/", "http://127.0.0.1"); const chunks: Buffer[] = []; for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : null; calls.push({ method: request.method, path: url.pathname, body }); const send = (data: Record) => { response.writeHead(200, { "content-type": "application/json" }); response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_backend_perf" })}\n`); }; if (request.method === "POST" && url.pathname === "/api/v1/sessions") return send({ ok: true }); if (request.method === "POST" && url.pathname === "/api/v1/runs") return send({ id: "run_backend_perf", status: "pending", backendProfile: "codex", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/commands") return send({ id: "cmd_backend_perf", runId: "run_backend_perf", state: "pending", type: "turn", seq: 1 }); if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/runner-jobs") return send({ action: "create-kubernetes-job", runId: "run_backend_perf", commandId: body.commandId, attemptId: "attempt_backend_perf", runnerId: "runner_backend_perf", namespace: "agentrun-v01", jobName: "agentrun-v01-runner-backend-perf" }); response.writeHead(404, { "content-type": "application/json" }); response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); }); await new Promise((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve)); try { const address = agentRunServer.address(); assert.ok(address && typeof address === "object"); const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } }); const context = store.createManualContext({ route: "/v1/agent/chat", method: "POST" }); const traceStore = createCodeAgentTraceStore(); const result = await withBackendPerformanceContext(context, () => submitAgentRunChatTurn({ traceId: "trc_backend_perf", traceStore, params: { message: "backend perf smoke", ownerUserId: "usr_backend_perf", projectId: "prj_hwpod_workbench", conversationId: "cnv_backend_perf", sessionId: "ses_backend_perf", providerProfile: "codex-api" }, options: { env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${address.port}`, HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601", HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: "0123456789abcdef0123456789abcdef01234567", HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE: "codex-api", HWLAB_RUNTIME_API_URL: "http://hwlab-cloud-api.hwlab-v03.svc.cluster.local:6667", HWLAB_RUNTIME_WEB_URL: "http://hwlab-cloud-web.hwlab-v03.svc.cluster.local:8080", HWLAB_RUNTIME_NAMESPACE: "hwlab-v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_RUNTIME_ENDPOINT_LOCKED: "1", HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1", UNIDESK_MAIN_SERVER_IP: "http://74.48.78.17:18081" }, accessController: { async codeAgentToolCapabilitiesForOwner() { return { contractVersion: "admin-access-v1", tools: {} }; }, store: { async findActiveDefaultApiKeyForUser() { return { displaySecret: "hwl_live_owner_secret" }; } } } } })); assert.equal(result.status, "running"); assert.ok(calls.some((call) => call.path === "/api/v1/runs/run_backend_perf/runner-jobs")); const text = store.metricsText(); assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs"[^}]*\} 1/u); assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs\/:id\/commands"[^}]*\} 1/u); assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs\/:id\/runner-jobs"[^}]*\} 1/u); assert.doesNotMatch(text, /run_backend_perf|cmd_backend_perf|ses_backend_perf|cnv_backend_perf/u); } finally { await new Promise((resolve, reject) => agentRunServer.close((error) => error ? reject(error) : resolve())); } });