pikasTech-HWLAB/internal/cloud/backend-performance.test.ts

// SPEC: PJ2026-01060505 Workbench Performance draft-2026-06-17-p0
// Verifies Cloud API backend RED metrics, Server-Timing, and AgentRun upstream timing privacy.

import assert from "node:assert/strict";
import { createServer as createHttpServer } from "node:http";
import { test } from "bun:test";

import { createBackendPerformanceStore, backendPerformanceRouteTemplate, withBackendPerformanceContext } from "./backend-performance.ts";
import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts";
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
import { createCloudApiServer } from "./server.ts";

test("backend performance store templates routes and appends Cloud API metrics to loopback metrics", async () => {
  assert.equal(backendPerformanceRouteTemplate("/v1/agent/traces/trc_secret_0123456789abcdef?full=1"), "/v1/agent/traces/:traceId");
  assert.equal(backendPerformanceRouteTemplate("/api/v1/runs/run_0123456789abcdef/commands/cmd_0123456789abcdef/result"), "/api/v1/runs/:id/commands/:id/result");
  assert.equal(backendPerformanceRouteTemplate("/v1/admin/billing/users/12345/credits/adjust"), "/v1/admin/billing/users/:id/credits/adjust");

  const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
  const accessController = { required: true, async authenticate() { return { ok: true, actor: { id: "usr_backend_metrics", role: "admin" }, session: { id: "web_backend_metrics" } }; } };
  const server = createCloudApiServer({ backendPerformanceStore, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
  try {
    const address = server.address();
    assert.ok(address && typeof address === "object");
    const baseUrl = `http://127.0.0.1:${address.port}`;
    const summary = await fetch(`${baseUrl}/v1/web-performance/summary`);
    assert.equal(summary.status, 200);
    assert.match(summary.headers.get("server-timing") ?? "", /total;dur=/u);

    const metrics = await fetch(`${baseUrl}/v1/web-performance/metrics`);
    const text = await metrics.text();
    assert.equal(metrics.status, 200);
    assert.match(text, /hwlab_cloud_api_requests_total\{[^}]*route="\/v1\/web-performance\/summary"[^}]*status_class="2xx"[^}]*\} 1/u);
    assert.match(text, /hwlab_cloud_api_request_duration_seconds_count\{[^}]*route="\/v1\/web-performance\/summary"/u);
    assert.doesNotMatch(text, /trc_secret|0123456789abcdef/u);
  } finally {
    await new Promise<void>((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
  }
});
test("backend performance store exposes low-cardinality request evidence for RUM alignment", () => {
  const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
  const context = store.createManualContext({ route: "/v1/workbench/sessions/ses_secret/messages", method: "GET" });
  context.recordPhase({ phase: "workbench_auth", durationMs: 12, outcome: "ok" });
  context.recordPhase({ phase: "workbench_message_page", durationMs: 42, outcome: "ok" });
  context.setResponseBytes(4096);
  context.finish(200);

  const evidence = store.evidence({ window: "15m" });
  assert.equal(evidence.schemaVersion, "hwlab-backend-performance-evidence-v1");
  assert.equal(evidence.sampleCount, 1);
  assert.equal(evidence.rows[0]?.route, "/v1/workbench/sessions/:id/messages");
  assert.equal(evidence.rows[0]?.method, "GET");
  assert.equal(evidence.rows[0]?.responseBytes?.p95, 4096);
  assert.ok(evidence.rows[0]?.phases.some((phase) => phase.phase === "workbench_message_page"));
  assert.doesNotMatch(JSON.stringify(evidence), /ses_secret|traceId|sessionId|runId|prompt|api key/iu);
});

test("backend performance store records Workbench cache metrics without cache keys or ids", () => {
  const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } });
  store.recordWorkbenchCache({ route: "/v1/workbench/turns/trc_secret_0123456789abcdef", cacheKeyClass: "turn.terminal.snapshot", cacheStatus: "hit", durationMs: 12, payloadBytes: 4096, cacheAgeMs: 5, dbQueryAvoided: true, freshnessSloMs: 300000, projectionSeq: 42 });
  store.recordWorkbenchCache({ route: "/v1/workbench/sessions", cacheKeyClass: "sessions.summary", cacheStatus: "unavailable", durationMs: 120, dbQueryAvoided: false });

  const text = store.metricsText();
  assert.match(text, /workbench_cache_requests_total\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*cache_status="hit"[^}]*\} 1/u);
  assert.match(text, /workbench_cache_hit_ratio\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*\} 1(?:\.0+)?/u);
  assert.match(text, /workbench_cache_unavailable_total\{[^}]*route="\/v1\/workbench\/sessions"[^}]*cache_key_class="sessions_summary"[^}]*\} 1/u);
  assert.match(text, /workbench_db_query_avoided_total\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*cache_key_class="turn_terminal_snapshot"[^}]*\} 1/u);

  const evidence = store.evidence({ window: "15m" });
  assert.equal(evidence.cacheSampleCount, 2);
  assert.ok(evidence.cache.some((row) => row.route === "/v1/workbench/turns/:traceId" && row.cacheKeyClass === "turn_terminal_snapshot" && row.hitRatio === 1));
  assert.ok(evidence.cache.some((row) => row.route === "/v1/workbench/sessions" && row.unavailableCount === 1));
  assert.doesNotMatch(`${text}\n${JSON.stringify(evidence)}`, /trc_secret|0123456789abcdef|ses_secret|prompt|api key|authorization|cookie/iu);
});

test("backend performance records Code Agent trace handler phases without high-cardinality labels", async () => {
  const traceId = "trc_backend_phase_trace";
  const runId = "run_backend_phase";
  const commandId = "cmd_backend_phase";
  const traceStore = createCodeAgentTraceStore();
  traceStore.append(traceId, { type: "message", role: "assistant", label: "assistant:start", sourceSeq: 1, commandId });
  traceStore.append(traceId, { type: "result", status: "completed", terminal: true, label: "result:completed", sourceSeq: 2, commandId });
  const projectedResult = {
    status: "completed",
    conversationId: "cnv_backend_phase",
    sessionId: "ses_backend_phase",
    threadId: "thr_backend_phase",
    ownerUserId: "usr_backend_phase",
    updatedAt: "2026-06-18T00:00:00.000Z",
    agentRun: {
      adapter: "agentrun-v01",
      runId,
      commandId,
      terminalStatus: "completed",
      lastSeq: 2
    },
    finalResponse: {
      text: "done",
      traceId,
      status: "completed",
      updatedAt: "2026-06-18T00:00:00.000Z"
    },
    traceSummary: {
      source: "agentrun-command-result",
      terminalStatus: "completed",
      eventCount: 2,
      updatedAt: "2026-06-18T00:00:00.000Z"
    },
    runnerTrace: traceStore.snapshot(traceId)
  };
  const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
  const workbenchRuntime = {
    async queryAgentTraceEvents({ traceId: requestedTraceId }: { traceId: string }) {
      assert.equal(requestedTraceId, traceId);
      return { events: traceStore.snapshot(traceId).events };
    }
  };
  const accessController = {
    required: true,
    async authenticate() {
      return { ok: true, actor: { id: "usr_backend_phase", role: "user" }, session: { id: "web_backend_phase" } };
    },
    async getAgentSessionByTraceId(requestedTraceId: string) {
      assert.equal(requestedTraceId, traceId);
      return {
        id: "ses_backend_phase",
        sessionId: "ses_backend_phase",
        projectId: "prj_hwpod_workbench",
        ownerUserId: "usr_backend_phase",
        ownerRole: "user",
        session: { traceResults: { [traceId]: projectedResult } }
      };
    }
  };
  const server = createCloudApiServer({ backendPerformanceStore, traceStore, workbenchRuntime, accessController, env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
  await new Promise<void>((resolve) => server.listen(0, "127.0.0.1", resolve));
  try {
    const address = server.address();
    assert.ok(address && typeof address === "object");
    const baseUrl = `http://127.0.0.1:${address.port}`;
    const trace = await fetch(`${baseUrl}/v1/agent/traces/${traceId}?projectId=prj_hwpod_workbench&limit=1`);
    assert.equal(trace.status, 200);
    const serverTiming = trace.headers.get("server-timing") ?? "";
    assert.match(serverTiming, /trace-project-check;dur=/u);
    assert.match(serverTiming, /trace-paginate;dur=/u);
    assert.match(serverTiming, /total;dur=/u);
    const payload = await trace.json();
    assert.equal(payload.range.returned, 1);

    const metrics = await fetch(`${baseUrl}/v1/web-performance/metrics`);
    const text = await metrics.text();
    assert.equal(metrics.status, 200);
    assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:traceId"[^}]*phase="trace_project_check"[^}]*\} 1/u);
    assert.match(text, /hwlab_cloud_api_request_phase_duration_seconds_count\{[^}]*route="\/v1\/agent\/traces\/:traceId"[^}]*phase="trace_paginate"[^}]*\} 1/u);
    assert.doesNotMatch(text, /trc_backend_phase_trace|run_backend_phase|cmd_backend_phase|ses_backend_phase|cnv_backend_phase/u);
  } finally {
    await new Promise<void>((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
  }
});

test("AgentRun adapter records upstream timing through backend performance context", async () => {
  const calls: Array<{ method?: string; path: string; body?: any }> = [];
  const agentRunServer = createHttpServer(async (request, response) => {
    const url = new URL(request.url || "/", "http://127.0.0.1");
    const chunks: Buffer[] = [];
    for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk));
    const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : null;
    calls.push({ method: request.method, path: url.pathname, body });
    const send = (data: Record<string, unknown>) => {
      response.writeHead(200, { "content-type": "application/json" });
      response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_backend_perf" })}\n`);
    };
    if (request.method === "POST" && url.pathname === "/api/v1/sessions") return send({ ok: true });
    if (request.method === "POST" && url.pathname === "/api/v1/runs") return send({ id: "run_backend_perf", status: "pending", backendProfile: "codex", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef });
    if (request.method === "POST" && url.pathname === "/api/v1/runs/run_backend_perf/commands") return send({ id: "cmd_backend_perf", runId: "run_backend_perf", state: "pending", type: "turn", seq: 1, dispatchIntent: { id: "dispatch_backend_perf", state: "pending", runnerJobId: "rjob_backend_perf", attemptCount: 0, durable: true } });
    response.writeHead(404, { "content-type": "application/json" });
    response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`);
  });
  await new Promise<void>((resolve) => agentRunServer.listen(0, "127.0.0.1", resolve));
  try {
    const address = agentRunServer.address();
    assert.ok(address && typeof address === "object");
    const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03" } });
    const context = store.createManualContext({ route: "/v1/agent/chat", method: "POST" });
    const traceStore = createCodeAgentTraceStore();

    const result = await withBackendPerformanceContext(context, () => submitAgentRunChatTurn({
      traceId: "trc_backend_perf",
      traceStore,
      params: {
        message: "backend perf smoke",
        ownerUserId: "usr_backend_perf",
        projectId: "prj_hwpod_workbench",
        conversationId: "cnv_backend_perf",
        sessionId: "ses_backend_perf",
        providerProfile: "codex-api"
      },
      options: {
        env: {
          HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01",
          AGENTRUN_MGR_URL: `http://127.0.0.1:${address.port}`,
          HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1",
          HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601",
          HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: "0123456789abcdef0123456789abcdef01234567",
          HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE: "codex-api",
          HWLAB_RUNTIME_API_URL: "http://hwlab-cloud-api.hwlab-v03.svc.cluster.local:6667",
          HWLAB_RUNTIME_WEB_URL: "http://hwlab-cloud-web.hwlab-v03.svc.cluster.local:8080",
          HWLAB_RUNTIME_NAMESPACE: "hwlab-v03",
          HWLAB_RUNTIME_LANE: "v03",
          HWLAB_RUNTIME_ENDPOINT_LOCKED: "1",
          HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1",
          UNIDESK_MAIN_SERVER_IP: "http://74.48.78.17:18081"
        },
        accessController: {
          async codeAgentToolCapabilitiesForOwner() { return { contractVersion: "admin-access-v1", tools: {} }; },
          store: { async findActiveDefaultApiKeyForUser() { return { displaySecret: "hwl_live_owner_secret" }; } }
        }
      }
    }));

    assert.equal(result.status, "running");
    assert.ok(!calls.some((call) => call.path === "/api/v1/runs/run_backend_perf/runner-jobs"));
    const text = store.metricsText();
    assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs"[^}]*\} 1/u);
    assert.match(text, /hwlab_cloud_api_upstream_duration_seconds_count\{[^}]*component="agentrun"[^}]*route="\/api\/v1\/runs\/:id\/commands"[^}]*\} 1/u);
    assert.doesNotMatch(text, /runner-jobs/u);
    assert.doesNotMatch(text, /run_backend_perf|cmd_backend_perf|ses_backend_perf|cnv_backend_perf/u);
  } finally {
    await new Promise<void>((resolve, reject) => agentRunServer.close((error) => error ? reject(error) : resolve()));
  }
});

test("backend performance store records Workbench projection and AgentRun result metrics without identifiers", () => {
  const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } });

  store.recordWorkbenchProjection({
    sourceLatestSeq: 2626,
    lastProjectedSeq: 137,
    sourceLatestEventAt: "2026-06-19T02:00:30.000Z",
    projectedLatestEventAt: "2026-06-19T02:00:00.000Z",
    status: "running",
    reason: "projecting",
    projectionStatus: "projecting",
    source: "agentrun-v01",
  });
  store.recordWorkbenchTurnRead({ route: "/v1/workbench/turns/trc_secret_0123456789abcdef", status: "running", degradedReason: "projecting", durationMs: 3364, responseBytes: 4096 });
  store.recordAgentRunResult({ durationMs: 9800, eventCount: 2626, eventsScanned: 2626, pagesScanned: 6, status: "timeout", budget: 2500, timedOut: true });
  store.recordWorkbenchProjectorBatch({ phase: "agentrun_events_fetch", status: "ok", durationMs: 42 });
  store.recordWorkbenchProjectorCandidate({ status: "seen", reason: "read_side_refresh" });
  store.recordWorkbenchProjectorEvents({ eventKind: "assistant_message", status: "ok", count: 2 });
  store.recordWorkbenchProjectorError({ reason: "agentrun_timeout" });

  const text = store.metricsText();
  assert.match(text, /hwlab_workbench_projection_lag_events_bucket\{[^}]*projection_status="projecting"[^}]*\}/u);
  assert.match(text, /hwlab_workbench_projection_stuck_traces\{[^}]*reason="projecting"[^}]*\} 1/u);
  assert.match(text, /hwlab_workbench_turn_get_duration_seconds_count\{[^}]*route="\/v1\/workbench\/turns\/:traceId"[^}]*degraded_reason="projecting"[^}]*\} 1/u);
  assert.match(text, /hwlab_agentrun_result_duration_seconds_count\{[^}]*event_count_bucket="2001_5000"[^}]*status="timeout"[^}]*\} 1/u);
  assert.match(text, /hwlab_agentrun_result_timeout_total\{[^}]*budget="turn_2_5s"[^}]*status="timeout"[^}]*\} 1/u);
  assert.match(text, /hwlab_workbench_projector_batch_duration_seconds_count\{[^}]*phase="agentrun_events_fetch"[^}]*status="ok"[^}]*\} 1/u);
  assert.match(text, /hwlab_workbench_projector_last_success_unixtime\{[^}]*node="D601"[^}]*\} /u);
  assert.doesNotMatch(text, /trc_secret|0123456789abcdef|run_secret|cmd_secret|ses_secret|cnv_secret/u);
});

test("backend performance reports zero projection lag for caught-up terminal traces", () => {
  const store = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } });

  store.recordWorkbenchProjection({
    sourceLatestSeq: 601,
    lastProjectedSeq: 501,
    sourceLatestEventAt: "2026-06-19T03:00:00.000Z",
    projectedLatestEventAt: "2026-06-19T02:00:00.000Z",
    terminalSourceAt: "2026-06-19T02:00:00.000Z",
    terminalProjectedAt: "2026-06-19T02:00:00.020Z",
    status: "completed",
    reason: "none",
    projectionStatus: "caught-up",
    source: "agentrun-v01",
  });

  const text = store.metricsText();
  const lagEventsSumLine = text.split("\n").find((line) => line.startsWith("hwlab_workbench_projection_lag_events_sum") && line.includes('projection_status="caught_up"'));
  const lagEventsBucketLine = text.split("\n").find((line) => line.startsWith("hwlab_workbench_projection_lag_events_bucket") && line.includes('projection_status="caught_up"') && line.includes('le="0"'));
  const lagSumLine = text.split("\n").find((line) => line.startsWith("hwlab_workbench_projection_lag_seconds_sum") && line.includes('projection_status="caught_up"'));
  const lagBucketLine = text.split("\n").find((line) => line.startsWith("hwlab_workbench_projection_lag_seconds_bucket") && line.includes('projection_status="caught_up"') && line.includes('le="0.05"'));
  const stuckLine = text.split("\n").find((line) => line.startsWith("hwlab_workbench_projection_stuck_traces") && line.includes('projection_status="caught_up"'));
  const terminalDelayBucketLine = text.split("\n").find((line) => line.startsWith("hwlab_workbench_terminal_projection_delay_seconds_bucket") && line.includes('projection_status="caught_up"') && line.includes('le="0.05"'));
  assert.ok(lagEventsSumLine);
  assert.match(lagEventsSumLine, / 0\.000000$/u);
  assert.ok(lagEventsBucketLine?.endsWith(" 1"));
  assert.ok(lagSumLine);
  assert.match(lagSumLine, / 0\.000000$/u);
  assert.ok(lagBucketLine?.endsWith(" 1"));
  assert.ok(stuckLine?.endsWith(" 0"));
  assert.ok(terminalDelayBucketLine?.endsWith(" 1"));
});