pikasTech-HWLAB/internal/cloud/server-workbench-realtime-http.test.ts

// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery.
// Responsibility: Workbench realtime and projection blocker regression tests.

import assert from "node:assert/strict";
import { test } from "bun:test";

import { createCloudApiServer } from "./server.ts";
import { createBackendPerformanceStore } from "./backend-performance.ts";
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
import { classifyWorkbenchReadModelFailure } from "./server-workbench-http.ts";
import { codeAgentTurnStatusPayload, createCodeAgentChatResultStore } from "./server-code-agent-http.ts";
import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts";

import {
  ACTOR,
  getJson,
  waitForCondition,
  createDurableFactsRuntimeStore,
  createRuntimeStoreFromFacts,
  buildDurableFactsForSession,
  normalizeTestMessages,
  testTimingProjection,
  timestampIso,
  elapsedMs,
  normalizeTestEvents,
  filterFacts,
  workbenchTestFactOrder,
  matchesFact,
  matchesTraceFact,
  testFactFamilySet,
  mergeFacts,
  emptyFacts,
  normalizeTestStatus,
  getSseEvents,
  parseSseBlock,
  createFakeKafkaFactory,
  waitFor
} from "./server-workbench-http-test-helpers.ts";

test("workbench realtime stream surfaces facts blocker instead of legacy trace fallback", async () => {
  const traceStore = createCodeAgentTraceStore();
  const results = createCodeAgentChatResultStore();
  const traceId = "trc_workbench_realtime";
  const session = {
    id: "ses_workbench_realtime",
    projectId: "prj_hwpod_workbench",
    agentId: "hwlab-code-agent",
    status: "running",
    ownerUserId: ACTOR.id,
    conversationId: "cnv_workbench_realtime",
    threadId: "thread-workbench-realtime",
    lastTraceId: traceId,
    updatedAt: "2026-06-17T02:00:00.000Z",
    session: { sessionStatus: "running", lastTraceId: traceId, messages: [{ role: "user", text: "stream", traceId }] }
  };
  traceStore.append(traceId, { type: "assistant", status: "completed", label: "assistant:completed", terminal: true, message: "legacy trace answer" });
  results.set(traceId, { status: "completed", traceId, ownerUserId: ACTOR.id, sessionId: session.id, threadId: session.threadId, finalResponse: { text: "legacy result answer" } });
  const accessController = {
    store: {
      async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
      async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
    },
    async ensureBootstrap() {},
    async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
  };
  const server = createCloudApiServer({ accessController, traceStore, codeAgentChatResults: results, env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000" } });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const eventsPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}`, 4);
    const events = await eventsPromise;
    assert.deepEqual(events.map((event) => event.event), [
      "workbench.connected",
      "workbench.error",
      "workbench.trace.snapshot",
      "workbench.turn.snapshot"
    ]);
    assert.equal(events[0].data.filters.sessionId, session.id);
    assert.equal(events[1].data.error.code, "workbench_facts_trace_missing");
    assert.equal(events[2].data.traceId, traceId);
    assert.equal(events[2].data.snapshot.status, "unknown");
    assert.equal(events[2].data.snapshot.eventCount, 0);
    assert.equal(events[3].data.turn.traceId, traceId);
    assert.equal(events[3].data.turn.status, "unknown");
    assert.equal(events[3].data.turn.terminal, false);
    assert.equal(events[3].data.turn.finalResponse, null);
    assert.equal(JSON.stringify(events).includes("legacy result answer"), false);
    assert.equal(JSON.stringify(events).includes("legacy trace answer"), false);
  } finally {
    await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
  }
});
test("workbench realtime stream forwards HWLAB Kafka events after initial connection", async () => {
  const fakeKafka = createFakeKafkaFactory();
  const traceId = "trc_workbench_realtime_after_seq";
  const session = {
    id: "ses_workbench_realtime_after_seq",
    projectId: "prj_hwpod_workbench",
    agentId: "hwlab-code-agent",
    status: "running",
    ownerUserId: ACTOR.id,
    conversationId: "cnv_workbench_realtime_after_seq",
    threadId: "thread-workbench-realtime-after-seq",
    lastTraceId: traceId,
    updatedAt: "2026-06-24T14:00:00.000Z",
    session: { sessionStatus: "running", lastTraceId: traceId }
  };
  const outboxQueries = [];
  const workbenchRuntime = {
    async readWorkbenchProjectionOutbox(params = {}) {
      outboxQueries.push({ ...params });
      return [
        { outboxSeq: 11, projectedSeq: 7, traceId, sessionId: session.id, turnId: traceId, commitType: "event", terminal: false, sealed: false, createdAt: "2026-06-24T14:00:01.000Z" }
      ];
    }
  };
  const accessController = {
    store: {
      async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
      async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
    },
    async ensureBootstrap() {},
    async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
  };
  const serverWithKafka = createCloudApiServer({
    accessController,
    workbenchRuntime,
    kafkaFactory: fakeKafka.factory,
    env: {
      HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000",
      HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092",
      HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api",
      HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1"
    }
  });
  await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = serverWithKafka.address();
    setTimeout(() => { void fakeKafka.emit({
      eventType: "hwlab.trace.event.projected",
      sessionId: "ses_agentrun_workbench_realtime_after_seq",
      traceId,
      context: { sourceSeq: 7, runId: "run_workbench_realtime_after_seq", commandId: "cmd_workbench_realtime_after_seq" },
      event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:test", message: "Kafka realtime event", sourceSeq: 7 }
    }); }, 25);
    const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}&afterSeq=10`, 2);
    assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]);
    assert.equal(events[0].id, "10");
    assert.equal(events[1].id, "hwlab.event.v1:0:0");
    assert.equal(events[1].data.realtimeSource, "kafka");
    assert.equal(events[1].data.realtimeAuthority, "workbench-realtime-authority-v2");
    assert.equal(events[1].data.entity.family, "traceEvents");
    assert.equal(events[1].data.entity.version, 7);
    assert.equal(events[1].data.entity.projectionRevision, "kafka:hwlab.event.v1:0:0");
    assert.equal(events[1].data.event.message, "Kafka realtime event");
    assert.equal(events[1].data.kafka.topic, "hwlab.event.v1");
    assert.equal(events[1].data.cursor.traceSeq, 7);
    assert.deepEqual(outboxQueries, []);
  } finally {
    await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve()));
  }
});

test("workbench session realtime Kafka stream does not pin subscription to stale lastTraceId", async () => {
  const fakeKafka = createFakeKafkaFactory();
  const staleTraceId = "trc_workbench_realtime_stale_trace";
  const liveTraceId = "trc_workbench_realtime_live_trace";
  const session = {
    id: "ses_workbench_realtime_session_wide",
    projectId: "prj_hwpod_workbench",
    agentId: "hwlab-code-agent",
    status: "running",
    ownerUserId: ACTOR.id,
    conversationId: "cnv_workbench_realtime_session_wide",
    threadId: "thread-workbench-realtime-session-wide",
    lastTraceId: staleTraceId,
    updatedAt: "2026-06-24T14:00:00.000Z",
    session: { sessionStatus: "running", lastTraceId: staleTraceId }
  };
  const accessController = {
    store: {
      async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
      async getAgentSessionByTraceId() { return null; }
    },
    async ensureBootstrap() {},
    async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
  };
  const serverWithKafka = createCloudApiServer({
    accessController,
    workbenchRuntime: {
      async queryWorkbenchFacts(params = {}) {
        return {
          facts: {
            sessions: [{ sessionId: session.id, ownerUserId: ACTOR.id, threadId: session.threadId, lastTraceId: staleTraceId, status: "running", valuesRedacted: true }],
            messages: [],
            parts: [],
            turns: [],
            checkpoints: []
          },
          count: 1,
          persistence: { adapter: "test-session-wide-kafka", durable: true },
          params
        };
      }
    },
    kafkaFactory: fakeKafka.factory,
    env: {
      HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000",
      HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092",
      HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api",
      HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1"
    }
  });
  await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = serverWithKafka.address();
    setTimeout(() => { void fakeKafka.emit({
      eventType: "hwlab.trace.event.projected",
      sessionId: "ses_agentrun_workbench_realtime_session_wide",
      traceId: liveTraceId,
      context: { runId: "run_workbench_realtime_session_wide", commandId: "cmd_workbench_realtime_session_wide" },
      event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:live", message: "Kafka live trace event" }
    }); }, 100);
    const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}`, 4);
    assert.equal(events[0].event, "workbench.connected");
    const liveEvent = events.find((event) => event.event === "workbench.trace.event" && event.data?.traceId === liveTraceId);
    assert.ok(liveEvent, JSON.stringify(events.map((event) => ({ event: event.event, traceId: event.data?.traceId, message: event.data?.event?.message }))));
    assert.equal(liveEvent.data.event.message, "Kafka live trace event");
    assert.equal(liveEvent.data.realtimeAuthority, "workbench-realtime-authority-v2");
  } finally {
    await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve()));
  }
});

test("workbench read model exposes runtime trace projection query failures as projection blockers", async () => {
  const traceStore = createCodeAgentTraceStore();
  const results = createCodeAgentChatResultStore();
  const traceId = "trc_workbench_projection_store_unavailable";
  const session = {
    id: "ses_workbench_projection_store_unavailable",
    projectId: "prj_hwpod_workbench",
    agentId: "hwlab-code-agent",
    status: "running",
    ownerUserId: ACTOR.id,
    conversationId: "cnv_workbench_projection_store_unavailable",
    threadId: "thread-workbench-projection-store-unavailable",
    lastTraceId: traceId,
    updatedAt: "2026-06-18T01:10:00.000Z",
    session: {
      sessionStatus: "running",
      lastTraceId: traceId,
      messages: [
        { role: "user", text: "projection store unavailable", traceId, status: "sent", createdAt: "2026-06-18T01:09:58.000Z" },
        { role: "agent", text: "", traceId, status: "running", createdAt: "2026-06-18T01:09:59.000Z" }
      ],
      valuesRedacted: true,
      secretMaterialStored: false
    }
  };
  traceStore.append(traceId, { seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: "2026-06-18T01:10:00.000Z" });
  results.set(traceId, { status: "running", traceId, ownerUserId: ACTOR.id, sessionId: session.id, threadId: session.threadId });
  const accessController = {
    store: {
      async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
      async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
    },
    async ensureBootstrap() {},
    async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
  };
  const runtimeStore = {
    async queryWorkbenchFacts() {
      const error = new Error("runtime query failed");
      error.code = "TEST_RUNTIME_QUERY_FAILED";
      throw error;
    }
  };
  const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`);
    assert.equal(turn.status, 503);
    assert.equal(turn.body.error.code, "projection_store_unavailable");

    const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`);
    assert.equal(trace.status, 503);
    assert.equal(trace.body.error.code, "projection_store_unavailable");
  } finally {
    await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
  }
});

test("workbench trace events remain visible after session lastTraceId moves", async () => {
  const oldTraceId = "trc_workbench_history_old_trace";
  const newTraceId = "trc_workbench_history_new_trace";
  const session = {
    id: "ses_workbench_history_trace",
    projectId: "prj_hwpod_workbench",
    agentId: "hwlab-code-agent",
    status: "running",
    ownerUserId: ACTOR.id,
    conversationId: "cnv_workbench_history_trace",
    threadId: "thread-workbench-history-trace",
    lastTraceId: newTraceId,
    updatedAt: "2026-06-20T10:10:00.000Z",
    session: {
      messages: [
        { role: "user", text: "old turn", traceId: oldTraceId, status: "sent", createdAt: "2026-06-20T10:00:00.000Z" },
        { role: "agent", text: "old final", traceId: oldTraceId, status: "completed", createdAt: "2026-06-20T10:00:01.000Z" }
      ],
      valuesRedacted: true
    }
  };
  const oldFacts = buildDurableFactsForSession({
    session: { ...session, status: "completed", lastTraceId: oldTraceId, updatedAt: "2026-06-20T10:01:00.000Z" },
    status: "completed",
    finalText: "old final",
    runId: "run_workbench_history_old_trace",
    commandId: "cmd_workbench_history_old_trace",
    events: [
      { projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:00:00.000Z" },
      { projectedSeq: 2, sourceSeq: 2, type: "result", status: "completed", label: "runner:completed", terminal: true, createdAt: "2026-06-20T10:01:00.000Z" }
    ],
    lastProjectedSeq: 2
  });
  const newFacts = buildDurableFactsForSession({
    session: { ...session, session: { messages: [{ role: "user", text: "new turn", traceId: newTraceId, status: "sent" }] } },
    status: "running",
    runId: "run_workbench_history_new_trace",
    commandId: "cmd_workbench_history_new_trace",
    events: [{ projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:10:00.000Z" }],
    lastProjectedSeq: 1
  });
  const facts = emptyFacts();
  mergeFacts(facts, oldFacts);
  mergeFacts(facts, newFacts);
  facts.sessions = newFacts.sessions;
  const queries = [];
  const runtimeStore = {
    async queryWorkbenchFacts(params = {}) {
      queries.push(params);
      const filtered = filterFacts(facts, params);
      return {
        facts: filtered,
        count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0),
        persistence: { adapter: "test-durable-workbench-facts", durable: true }
      };
    }
  };
  const accessController = {
    async ensureBootstrap() {},
    async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
  };
  const server = createCloudApiServer({ accessController, runtimeStore });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(oldTraceId)}/events?limit=10`);
    assert.equal(response.status, 200);
    assert.equal(response.body.status, "succeeded");
    assert.equal(response.body.sessionId, session.id);
    assert.equal(response.body.traceId, oldTraceId);
    assert.equal(response.body.events.length, 2);
    assert.equal(response.body.events.at(-1).status, "completed");
    assert.equal(response.body.error, undefined);
    assert.equal(queries.some((query) => query.sessionId === session.id && Array.isArray(query.families) && query.families.includes("sessions")), true);
  } finally {
    await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
  }
});

test("workbench trace event page keeps per-trace terminal status after later session cancel", async () => {
  const completedTraceId = "trc_workbench_completed_before_cancel";
  const canceledTraceId = "trc_workbench_later_cancel";
  const session = {
    id: "ses_workbench_completed_before_cancel",
    projectId: "prj_hwpod_workbench",
    agentId: "hwlab-code-agent",
    status: "canceled",
    ownerUserId: ACTOR.id,
    conversationId: "cnv_workbench_completed_before_cancel",
    threadId: "thread-workbench-completed-before-cancel",
    lastTraceId: canceledTraceId,
    updatedAt: "2026-06-20T10:20:30.000Z",
    session: {
      messages: [
        { role: "user", text: "completed first", traceId: completedTraceId, status: "sent", createdAt: "2026-06-20T10:20:00.000Z" },
        { role: "agent", text: "completed final", traceId: completedTraceId, status: "completed", createdAt: "2026-06-20T10:20:10.000Z" },
        { role: "user", text: "cancel later", traceId: canceledTraceId, status: "sent", createdAt: "2026-06-20T10:20:20.000Z" },
        { role: "agent", text: "hwlab-user-cancel", traceId: canceledTraceId, status: "canceled", createdAt: "2026-06-20T10:20:30.000Z" }
      ],
      valuesRedacted: true
    }
  };
  const facts = emptyFacts();
  mergeFacts(facts, buildDurableFactsForSession({
    session: { ...session, status: "completed", lastTraceId: completedTraceId, updatedAt: "2026-06-20T10:20:10.000Z" },
    status: "completed",
    finalText: "completed final",
    events: [
      { projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:20:00.000Z" },
      { projectedSeq: 2, sourceSeq: 2, type: "result", status: "completed", label: "runner:completed", terminal: true, createdAt: "2026-06-20T10:20:10.000Z" }
    ],
    lastProjectedSeq: 2
  }));
  mergeFacts(facts, buildDurableFactsForSession({
    session: { ...session, status: "canceled", lastTraceId: canceledTraceId, session: { messages: session.session.messages.slice(2), valuesRedacted: true } },
    status: "canceled",
    finalText: "hwlab-user-cancel",
    events: [
      { projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:20:20.000Z" },
      { projectedSeq: 2, sourceSeq: 2, type: "cancel", status: "canceled", label: "runner:canceled", terminal: true, createdAt: "2026-06-20T10:20:30.000Z" }
    ],
    lastProjectedSeq: 2
  }));
  facts.sessions = facts.sessions.filter((item) => item.lastTraceId === canceledTraceId);
  const runtimeStore = {
    async queryWorkbenchFacts(params = {}) {
      const filtered = filterFacts(facts, params);
      return {
        facts: filtered,
        count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0),
        persistence: { adapter: "test-durable-workbench-facts", durable: true }
      };
    }
  };
  const accessController = {
    async ensureBootstrap() {},
    async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
  };
  const server = createCloudApiServer({ accessController, workbenchRuntime: runtimeStore });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(completedTraceId)}/events?limit=10`);
    assert.equal(response.status, 200);
    assert.equal(response.body.traceStatus, "completed");
    assert.equal(response.body.events.at(-1).status, "completed");
  } finally {
    await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
  }
});