lyon
266 lines
15 KiB
TypeScript
266 lines
15 KiB
TypeScript
// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0
|
|
// Verifies the Workbench RUM metrics contract and Prometheus label privacy boundary.
|
|
|
|
import assert from "node:assert/strict";
|
|
import { test } from "bun:test";
|
|
|
|
import { createCloudApiServer } from "./server.ts";
|
|
import { createWebPerformanceStore, webPerformanceRouteTemplate } from "./web-performance.ts";
|
|
|
|
test("web performance store aggregates browser-perceived durations as Prometheus histograms", () => {
|
|
const store = createWebPerformanceStore({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v02", HWLAB_GITOPS_TARGET: "v02" } });
|
|
const result = store.record({
|
|
schemaVersion: "hwlab-web-performance-v1",
|
|
page: "/workspace",
|
|
events: [
|
|
{ kind: "web_vital", metric: "lcp", route: "/workspace", valueMs: 1234 },
|
|
{ kind: "web_vital", metric: "cls", route: "/workspace", value: 0.08 },
|
|
{ kind: "api", metric: "api_request", route: "/v1/agent/chat/result/trc_secret", method: "GET", status: 200, statusClass: "2xx", outcome: "ok", valueMs: 420 }
|
|
]
|
|
});
|
|
const text = store.metricsText();
|
|
|
|
assert.deepEqual(result, { accepted: 3, dropped: 0, received: 3 });
|
|
assert.match(text, /hwlab_webui_performance_sample_total\{service="hwlab-cloud-web",namespace="hwlab-v02",gitops_target="v02",kind="web_vital",metric="lcp",route="\/workspace"/u);
|
|
assert.match(text, /hwlab_webui_performance_duration_seconds_bucket\{[^}]*metric="api_request"[^}]*route="\/v1\/agent\/chat\/result\/:id"[^}]*le="0\.5"\} 1/u);
|
|
assert.match(text, /hwlab_webui_layout_shift_score_bucket\{[^}]*metric="cls"[^}]*le="0\.1"\} 1/u);
|
|
assert.doesNotMatch(text, /trc_secret|sessionId|conversationId|threadId|prompt|api key/iu);
|
|
});
|
|
|
|
test("web performance summary exposes user-perceived route latency without high-cardinality data", () => {
|
|
const store = createWebPerformanceStore({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v02", HWLAB_GITOPS_TARGET: "v02" } });
|
|
store.record({
|
|
schemaVersion: "hwlab-web-performance-v1",
|
|
page: "/workspace",
|
|
events: [
|
|
{ kind: "api", metric: "api_request", route: "/v1/agent/chat/result/trc_secret", method: "GET", status: 200, statusClass: "2xx", outcome: "ok", valueMs: 420 },
|
|
{ kind: "api", metric: "api_request", route: "/v1/agent/chat/result/trc_secret", method: "GET", status: 0, statusClass: "network", outcome: "timeout", valueMs: 2600 },
|
|
{ kind: "long_task", metric: "long_task", route: "/workspace", valueMs: 260 },
|
|
{ kind: "web_vital", metric: "lcp", route: "/workspace", valueMs: 3100 }
|
|
]
|
|
});
|
|
|
|
const summary = store.summary();
|
|
const json = JSON.stringify(summary);
|
|
assert.equal(summary.summary.sampleCount, 4);
|
|
assert.equal(summary.namespace, "hwlab-v02");
|
|
assert.equal(summary.dashboard.schemaVersion, "hwlab-web-performance-dashboard-v1");
|
|
assert.ok(summary.dashboard.cards.some((card) => card.id === "samples" && card.value === 4));
|
|
assert.ok(summary.dashboard.trends.some((trend) => trend.id === "api-timing" && trend.points.length > 0));
|
|
assert.ok(summary.dashboard.topN.some((group) => group.id === "top-api" && group.rows.length > 0));
|
|
assert.ok(summary.apiRoutes.some((row) => row.route === "/v1/agent/chat/result/:id" && row.metric === "api_request" && row.p95 >= 2.5));
|
|
assert.ok(summary.problems.some((row) => row.problem === "timeout" && row.tone === "blocked"));
|
|
assert.ok(summary.longTasks.some((row) => row.metric === "long_task" && row.problem === "slow-long_task" && row.tone !== "ok"));
|
|
assert.doesNotMatch(json, /trc_secret|sessionId|conversationId|threadId|prompt|api key/iu);
|
|
});
|
|
|
|
test("web performance summary filters rows by requested recent window", () => {
|
|
let nowMs = 0;
|
|
const store = createWebPerformanceStore({
|
|
env: { HWLAB_METRICS_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" },
|
|
now: () => nowMs
|
|
});
|
|
|
|
nowMs = 1_000;
|
|
store.record({
|
|
schemaVersion: "hwlab-web-performance-v1",
|
|
page: "/workbench",
|
|
events: [{ kind: "api", metric: "api_request", route: "/v1/workbench/sessions", method: "GET", status: 200, statusClass: "2xx", outcome: "ok", valueMs: 2600, traceId: "trc_old_secret" }]
|
|
} as Record<string, unknown>);
|
|
|
|
nowMs = 20 * 60 * 1000;
|
|
store.record({
|
|
schemaVersion: "hwlab-web-performance-v1",
|
|
page: "/workbench",
|
|
events: [{ kind: "api", metric: "api_request", route: "/v1/workbench/sessions", method: "GET", status: 200, statusClass: "2xx", outcome: "ok", valueMs: 320, traceId: "trc_new_secret" }]
|
|
} as Record<string, unknown>);
|
|
|
|
const recent = store.summary({ window: "5m" });
|
|
const all = store.summary({ window: "all" });
|
|
const recentJson = JSON.stringify(recent);
|
|
const allJson = JSON.stringify(all);
|
|
|
|
assert.equal(recent.sampleWindow.id, "5m");
|
|
assert.equal(recent.summary.sampleCount, 1);
|
|
assert.equal(recent.apiRoutes[0].p95, 0.5);
|
|
assert.equal(all.sampleWindow.id, "all");
|
|
assert.equal(all.summary.sampleCount, 2);
|
|
assert.equal(all.apiRoutes[0].p95, 5);
|
|
assert.doesNotMatch(recentJson, /trc_old_secret|trc_new_secret|traceId|prompt|api key/iu);
|
|
assert.doesNotMatch(allJson, /trc_old_secret|trc_new_secret|traceId|prompt|api key/iu);
|
|
});
|
|
|
|
test("web performance store accepts v2 Workbench journey and phase metrics without high-cardinality labels", () => {
|
|
const store = createWebPerformanceStore({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
|
|
const result = store.record({
|
|
schemaVersion: "hwlab-web-performance-v2",
|
|
page: "/workbench/sessions/ses_secret?traceId=trc_secret",
|
|
events: [
|
|
{
|
|
kind: "workbench_journey",
|
|
journey: "submit_to_first_visible",
|
|
route: "/workbench/sessions/ses_secret",
|
|
entry: "new",
|
|
backend: "agentrun-v01/codex",
|
|
transport: "sse",
|
|
visibility: "foreground",
|
|
outcome: "ok",
|
|
valueMs: 1234,
|
|
traceId: "trc_secret",
|
|
sessionId: "ses_secret",
|
|
runId: "run_secret"
|
|
},
|
|
{
|
|
kind: "workbench_event_phase",
|
|
phase: "created_to_append",
|
|
eventType: "assistant_message",
|
|
backend: "agentrun-v01/codex",
|
|
transport: "sse",
|
|
outcome: "ok",
|
|
valueMs: 42,
|
|
commandId: "cmd_secret"
|
|
},
|
|
{
|
|
kind: "workbench_journey",
|
|
journey: "submit_to_failure",
|
|
route: "/workbench/sessions/ses_secret",
|
|
entry: "new",
|
|
backend: "agentrun-v01/codex",
|
|
transport: "sse",
|
|
visibility: "foreground",
|
|
outcome: "network",
|
|
valueMs: 530,
|
|
traceId: "trc_secret"
|
|
},
|
|
{
|
|
kind: "workbench_backend_event_visible",
|
|
eventType: "tool",
|
|
backend: "agentrun-v01/codex",
|
|
transport: "sse",
|
|
outcome: "ok",
|
|
valueMs: 640,
|
|
conversationId: "cnv_secret"
|
|
}
|
|
]
|
|
} as Record<string, unknown>);
|
|
const text = store.metricsText();
|
|
|
|
assert.deepEqual(result, { accepted: 4, dropped: 0, received: 4 });
|
|
assert.match(text, /hwlab_workbench_journey_total\{[^}]*journey="submit_to_first_visible"[^}]*route="\/workbench\/sessions\/:id"[^}]*backend="agentrun-v01\/codex"/u);
|
|
assert.match(text, /hwlab_workbench_journey_total\{[^}]*journey="submit_to_failure"[^}]*outcome="network"/u);
|
|
assert.match(text, /hwlab_workbench_event_phase_duration_seconds_bucket\{[^}]*phase="created_to_append"[^}]*event_type="assistant"[^}]*le="0\.05"\} 1/u);
|
|
assert.match(text, /hwlab_workbench_backend_event_visible_latency_seconds_bucket\{[^}]*event_type="tool_call"[^}]*backend="agentrun-v01\/codex"[^}]*le="1"\} 1/u);
|
|
assert.doesNotMatch(text, /trc_secret|ses_secret|run_secret|cmd_secret|cnv_secret|traceId|sessionId|runId|commandId|conversationId|prompt|api key/iu);
|
|
});
|
|
|
|
test("web performance summary exposes Workbench p75 and low-sample diagnostics without high-cardinality labels", () => {
|
|
const store = createWebPerformanceStore({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
|
|
const result = store.record({
|
|
schemaVersion: "hwlab-web-performance-v2",
|
|
page: "/workbench/sessions/ses_secret?traceId=trc_secret",
|
|
events: [
|
|
{ kind: "workbench_journey", journey: "submit_to_first_visible", route: "/workbench/sessions/ses_secret", entry: "new", backend: "agentrun-v01/codex", transport: "sse", visibility: "foreground", outcome: "ok", valueMs: 1200, traceId: "trc_secret" },
|
|
{ kind: "workbench_journey", journey: "session_switch_first_visible", route: "/workbench/sessions/ses_secret", source: "rail", targetState: "running", cache: "warm", backend: "agentrun-v01/codex", transport: "rest_gap", visibility: "foreground", outcome: "ok", valueMs: 420, sessionId: "ses_secret" },
|
|
{ kind: "workbench_event_phase", phase: "created_to_append", eventType: "backend", backend: "agentrun-v01/codex", transport: "sse", outcome: "ok", valueMs: 28, commandId: "cmd_secret" },
|
|
{ kind: "workbench_backend_event_visible", eventType: "terminal", backend: "agentrun-v01/codex", transport: "sse", outcome: "ok", valueMs: 760, runId: "run_secret" }
|
|
]
|
|
} as Record<string, unknown>);
|
|
|
|
const summary = store.summary();
|
|
const json = JSON.stringify(summary);
|
|
assert.deepEqual(result, { accepted: 4, dropped: 0, received: 4 });
|
|
assert.equal(summary.summary.sampleCount, 4);
|
|
assert.equal(summary.summary.lowSampleThreshold, 5);
|
|
assert.equal(summary.dashboard.freshness.namespace, "hwlab-v03");
|
|
assert.ok(summary.dashboard.trends.some((trend) => trend.id === "workbench-experience" && trend.points.some((point) => point.label === "提交到首个可见结果")));
|
|
assert.ok(summary.dashboard.distributions.some((distribution) => distribution.id === "samples" && distribution.buckets.some((bucket) => bucket.label === "低样本")));
|
|
assert.ok(summary.workbenchJourneys.some((row) => row.metric === "submit_to_first_visible" && row.backend === "agentrun-v01/codex" && row.p75 >= row.p50 && row.lowSample === true && row.sampleState === "low-sample"));
|
|
assert.ok(summary.workbenchEventPhases.some((row) => row.phase === "created_to_append" && row.eventType === "backend" && row.transport === "sse"));
|
|
assert.ok(summary.workbenchBackendEvents.some((row) => row.metric === "backend_event_to_visible" && row.eventType === "terminal" && row.backend === "agentrun-v01/codex"));
|
|
assert.ok(summary.problems.some((row) => row.kind === "workbench_journey" && row.problem === "low-sample"));
|
|
assert.doesNotMatch(json, /trc_secret|ses_secret|run_secret|cmd_secret|traceId|sessionId|runId|commandId|conversationId|prompt|api key/iu);
|
|
});
|
|
|
|
test("web performance store drops unsupported Workbench labels and counts series limit drops", () => {
|
|
const store = createWebPerformanceStore({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" }, maxSeries: 1 });
|
|
const first = store.record({
|
|
schemaVersion: "hwlab-web-performance-v2",
|
|
page: "/workbench",
|
|
events: [{ kind: "workbench_journey", journey: "session_switch_first_visible", route: "/workbench/sessions/ses_first", valueMs: 250, backend: "agentrun-v01/codex", transport: "sse" }]
|
|
});
|
|
const second = store.record({
|
|
schemaVersion: "hwlab-web-performance-v2",
|
|
page: "/workbench",
|
|
events: [
|
|
{ kind: "workbench_journey", journey: "session_switch_full_load", route: "/workbench/sessions/ses_second", valueMs: 900, backend: "run_abcdefghijklmnopqrstuvwxyz", transport: "sse" },
|
|
{ kind: "workbench_journey", journey: "trace_trc_secret", route: "/workbench/sessions/ses_third", valueMs: 100, backend: "agentrun-v01/codex", transport: "sse" }
|
|
]
|
|
});
|
|
const text = store.metricsText();
|
|
|
|
assert.deepEqual(first, { accepted: 1, dropped: 0, received: 1 });
|
|
assert.deepEqual(second, { accepted: 0, dropped: 2, received: 2 });
|
|
assert.match(text, /hwlab_webui_performance_dropped_total\{[^}]*reason="series_limit"\} 1/u);
|
|
assert.match(text, /hwlab_webui_performance_dropped_total\{[^}]*reason="invalid_event"\} 1/u);
|
|
assert.doesNotMatch(text, /ses_second|ses_third|trace_trc_secret|run_abcdefghijklmnopqrstuvwxyz/u);
|
|
});
|
|
|
|
test("web performance store filters observability self-noise", () => {
|
|
const store = createWebPerformanceStore({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v02", HWLAB_GITOPS_TARGET: "v02" } });
|
|
const result = store.record({
|
|
schemaVersion: "hwlab-web-performance-v1",
|
|
page: "/performance",
|
|
events: [
|
|
{ kind: "web_vital", metric: "lcp", route: "/performance", valueMs: 5900 },
|
|
{ kind: "long_task", metric: "long_task", route: "/performance", valueMs: 120 },
|
|
{ kind: "api", metric: "api_request", route: "/v1/web-performance/summary", method: "GET", status: 200, statusClass: "2xx", outcome: "ok", valueMs: 620 },
|
|
{ kind: "api", metric: "api_request", route: "/v1/skills", method: "GET", status: 200, statusClass: "2xx", outcome: "ok", valueMs: 780 }
|
|
]
|
|
});
|
|
const summary = store.summary();
|
|
const rowsJson = JSON.stringify({ apiRoutes: summary.apiRoutes, webVitals: summary.webVitals, longTasks: summary.longTasks, problems: summary.problems, rows: summary.rows });
|
|
assert.deepEqual(result, { accepted: 0, dropped: 4, received: 4 });
|
|
assert.equal(summary.summary.sampleCount, 0);
|
|
assert.doesNotMatch(rowsJson, /\/performance|\/v1\/web-performance/u);
|
|
});
|
|
|
|
test("web performance route templates keep metrics low-cardinality", () => {
|
|
assert.equal(webPerformanceRouteTemplate("/v1/workbench/sessions/ses_abcdef/messages?traceId=trc_secret"), "/v1/workbench/sessions/:id/messages");
|
|
assert.equal(webPerformanceRouteTemplate("#/skills"), "/skills");
|
|
});
|
|
|
|
test("cloud api ingests WebUI performance samples and exposes metrics only on loopback host", async () => {
|
|
const server = createCloudApiServer({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v02", HWLAB_GITOPS_TARGET: "v02" } });
|
|
await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
|
|
try {
|
|
const address = server.address();
|
|
assert.ok(address && typeof address === "object");
|
|
const baseUrl = `http://127.0.0.1:${address.port}`;
|
|
const ingest = await fetch(`${baseUrl}/v1/web-performance`, {
|
|
method: "POST",
|
|
headers: { "content-type": "application/json" },
|
|
body: JSON.stringify({
|
|
schemaVersion: "hwlab-web-performance-v1",
|
|
page: "/workspace",
|
|
events: [{ kind: "web_vital", metric: "lcp", route: "/workspace", valueMs: 900 }]
|
|
})
|
|
});
|
|
assert.equal(ingest.status, 202);
|
|
const metrics = await fetch(`${baseUrl}/v1/web-performance/metrics`);
|
|
const text = await metrics.text();
|
|
assert.equal(metrics.status, 200);
|
|
assert.match(text, /hwlab_webui_performance_duration_seconds_bucket\{[^}]*metric="lcp"/u);
|
|
|
|
const publicHost = await fetch(`${baseUrl}/v1/web-performance/metrics`, { headers: { host: "74.48.78.17:19667" } });
|
|
assert.equal(publicHost.status, 404);
|
|
|
|
const summary = await fetch(`${baseUrl}/v1/web-performance/summary`);
|
|
const body = await summary.json();
|
|
assert.equal(summary.status, 200);
|
|
assert.equal(body.route, "/v1/web-performance/summary");
|
|
assert.equal(body.summary.sampleCount, 1);
|
|
} finally {
|
|
await new Promise<void>((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
|
}
|
|
});
|