diff --git a/internal/agent/agentrun-dispatch.mjs b/internal/agent/agentrun-dispatch.mjs index d68af0c1..f08612b1 100644 --- a/internal/agent/agentrun-dispatch.mjs +++ b/internal/agent/agentrun-dispatch.mjs @@ -214,20 +214,21 @@ export function createHwlabAgentRunDispatchAssembly(options = {}) { providerProfile: backendProfile, dispatchSource: "hwlab-code-agent" }, + dispatch: { + kind: "kubernetes-job", + input: { + idempotencyKey: options.runnerJobIdempotencyKey ?? `hwlab:${traceId}:runner-job`, + ...(options.attemptId ? { attemptId: nonEmptyString(options.attemptId, "attemptId") } : {}), + transientEnv + } + }, idempotencyKey: options.commandIdempotencyKey ?? `hwlab:${traceId}:turn` }; - const runnerJobPayload = { - idempotencyKey: options.runnerJobIdempotencyKey ?? `hwlab:${traceId}:runner-job`, - ...(options.attemptId ? { attemptId: nonEmptyString(options.attemptId, "attemptId") } : {}), - transientEnv - }; - return { managerUrl, runPayload, commandPayload, - runnerJobPayload, boundaries: { valuesPrinted: false, githubCredentialSource: includeGithubToolCredential ? "toolCredentials" : null, @@ -249,18 +250,19 @@ export async function dispatchHwlabAgentRun(options = {}) { const run = await postJson(fetchImpl, managerUrl, "/api/v1/runs", assembly.runPayload); const runId = nonEmptyString(run.id, "run.id"); const command = await postJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/commands`, assembly.commandPayload); - const commandId = nonEmptyString(command.id, "command.id"); - const runnerJobPayload = { ...assembly.runnerJobPayload, commandId }; - const runnerJob = await postJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(runId)}/runner-jobs`, runnerJobPayload); + nonEmptyString(command.id, "command.id"); + const dispatchIntent = command.dispatchIntent; + if (!dispatchIntent || dispatchIntent.durable !== true || !optionalString(dispatchIntent.id)) { + throw new Error("AgentRun command response requires a durable dispatchIntent"); + } return { managerUrl, run, command, - runnerJob, + dispatchIntent, requests: { runPayload: assembly.runPayload, - commandPayload: assembly.commandPayload, - runnerJobPayload + commandPayload: assembly.commandPayload }, boundaries: assembly.boundaries }; diff --git a/internal/agent/agentrun-dispatch.test.mjs b/internal/agent/agentrun-dispatch.test.mjs index 85ed6721..deac7702 100644 --- a/internal/agent/agentrun-dispatch.test.mjs +++ b/internal/agent/agentrun-dispatch.test.mjs @@ -61,7 +61,7 @@ test("HWLAB AgentRun assembly grants GitHub and UniDesk SSH through toolCredenti assert.equal(tools.some((item) => item.tool === "github" && item.projection.envName === "GH_TOKEN"), true); assert.equal(tools.some((item) => item.tool === "unidesk-ssh" && item.projection.envName === "UNIDESK_SSH_CLIENT_TOKEN"), true); - const transientEnv = assembly.runnerJobPayload.transientEnv; + const transientEnv = assembly.commandPayload.dispatch.input.transientEnv; assert.equal(transientEnv.find((item) => item.name === DEFAULT_UNIDESK_MAIN_SERVER_ENV)?.value, "https://unidesk.example.test"); assert.equal(transientEnv.some((item) => item.name === "UNIDESK_SSH_CLIENT_TOKEN"), false); assert.equal(assembly.boundaries.unideskSshCredentialSource, "toolCredentials"); @@ -112,7 +112,7 @@ test("HWLAB AgentRun assembly does not duplicate explicit UniDesk main-server en unideskMainServerIp: "https://unidesk.example.test", transientEnv: [{ name: DEFAULT_UNIDESK_MAIN_SERVER_ENV, value: "https://explicit.example.test", sensitive: false }] }); - const entries = assembly.runnerJobPayload.transientEnv.filter((item) => item.name === DEFAULT_UNIDESK_MAIN_SERVER_ENV); + const entries = assembly.commandPayload.dispatch.input.transientEnv.filter((item) => item.name === DEFAULT_UNIDESK_MAIN_SERVER_ENV); assert.equal(entries.length, 1); assert.equal(entries[0].value, "https://explicit.example.test"); }); @@ -128,14 +128,13 @@ test("HWLAB AgentRun assembly requires full commit and main server when UniDesk ); }); -test("dispatchHwlabAgentRun posts run command and runner job with commandId", async () => { +test("dispatchHwlabAgentRun atomically requests durable dispatch with the command", async () => { const requests = []; const fetchImpl = async (url, init) => { const request = { url, method: init.method, body: JSON.parse(init.body) }; requests.push(request); if (url.endsWith("/api/v1/runs")) return jsonResponse({ id: "run_hwlab_001" }); - if (url.endsWith("/api/v1/runs/run_hwlab_001/commands")) return jsonResponse({ id: "cmd_hwlab_001" }); - if (url.endsWith("/api/v1/runs/run_hwlab_001/runner-jobs")) return jsonResponse({ id: "job_hwlab_001", jobName: "agentrun-v01-runner-selftest" }); + if (url.endsWith("/api/v1/runs/run_hwlab_001/commands")) return jsonResponse({ id: "cmd_hwlab_001", dispatchIntent: { id: "dispatch_hwlab_001", state: "pending", runnerJobId: "rjob_hwlab_001", durable: true } }); return jsonResponse({ error: "unexpected path" }, { status: 404 }); }; const result = await dispatchHwlabAgentRun({ @@ -148,21 +147,19 @@ test("dispatchHwlabAgentRun posts run command and runner job with commandId", as assert.equal(result.run.id, "run_hwlab_001"); assert.equal(result.command.id, "cmd_hwlab_001"); - assert.equal(result.runnerJob.jobName, "agentrun-v01-runner-selftest"); + assert.equal(result.dispatchIntent.id, "dispatch_hwlab_001"); assert.deepEqual(requests.map((item) => new URL(item.url).pathname), [ "/api/v1/runs", - "/api/v1/runs/run_hwlab_001/commands", - "/api/v1/runs/run_hwlab_001/runner-jobs" + "/api/v1/runs/run_hwlab_001/commands" ]); - assert.equal(requests[2].body.commandId, "cmd_hwlab_001"); - assert.equal(requests[2].body.transientEnv.some((item) => item.name === "UNIDESK_SSH_CLIENT_TOKEN"), false); + assert.equal(requests[1].body.dispatch.kind, "kubernetes-job"); + assert.equal(requests[1].body.dispatch.input.transientEnv.some((item) => item.name === "UNIDESK_SSH_CLIENT_TOKEN"), false); }); test("dispatchHwlabAgentRun unwraps AgentRun manager response envelopes", async () => { const fetchImpl = async (url, init) => { if (url.endsWith("/api/v1/runs")) return jsonResponse({ ok: true, data: { id: "run_hwlab_envelope" } }); - if (url.endsWith("/api/v1/runs/run_hwlab_envelope/commands")) return jsonResponse({ ok: true, data: { id: "cmd_hwlab_envelope" } }); - if (url.endsWith("/api/v1/runs/run_hwlab_envelope/runner-jobs")) return jsonResponse({ ok: true, data: { id: "job_hwlab_envelope", jobName: "agentrun-v01-runner-envelope" } }); + if (url.endsWith("/api/v1/runs/run_hwlab_envelope/commands")) return jsonResponse({ ok: true, data: { id: "cmd_hwlab_envelope", dispatchIntent: { id: "dispatch_hwlab_envelope", state: "pending", runnerJobId: "rjob_hwlab_envelope", durable: true } } }); return jsonResponse({ ok: false, failureKind: "unexpected-path" }, { status: 404 }); }; @@ -176,7 +173,7 @@ test("dispatchHwlabAgentRun unwraps AgentRun manager response envelopes", async assert.equal(result.run.id, "run_hwlab_envelope"); assert.equal(result.command.id, "cmd_hwlab_envelope"); - assert.equal(result.runnerJob.jobName, "agentrun-v01-runner-envelope"); + assert.equal(result.dispatchIntent.id, "dispatch_hwlab_envelope"); }); function jsonResponse(body, { status = 200 } = {}) { diff --git a/internal/cloud/kafka-event-bridge.test.ts b/internal/cloud/kafka-event-bridge.test.ts index 5dcfc727..8b4aa3e3 100644 --- a/internal/cloud/kafka-event-bridge.test.ts +++ b/internal/cloud/kafka-event-bridge.test.ts @@ -6,9 +6,14 @@ import { test } from "bun:test"; import { createCloudApiBunServer } from "./bun-server.ts"; import { buildCloudApiReadiness } from "./health-contract.ts"; -import { decodeCanonicalAgentRunKafkaMessage, projectAgentRunKafkaEventToHwlabEvent, relayHwlabKafkaOutboxOnce, startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts"; +import { decodeCanonicalAgentRunKafkaMessage, kafkaEventBridgeConfig, liveKafkaOtelSpanAttributes, projectAgentRunKafkaEventToHwlabEvent, publishAgentRunKafkaMessageLive, relayHwlabKafkaOutboxOnce, shouldEmitLiveKafkaOtelSpan, startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts"; const PROJECTOR_ENV = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "true", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true", HWLAB_KAFKA_ENABLED: "true", HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true", HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092", @@ -16,6 +21,7 @@ const PROJECTOR_ENV = Object.freeze({ HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1", HWLAB_KAFKA_CLIENT_ID: "hwlab-projector-test", HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-projector-test-v1", + HWLAB_KAFKA_PROJECTOR_GROUP_ID: "hwlab-projector-test-v1", HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "10", HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS: "60000", HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE: "1", @@ -24,6 +30,22 @@ const PROJECTOR_ENV = Object.freeze({ HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS: "1000" }); +const LIVE_ENV = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false", + HWLAB_KAFKA_ENABLED: "true", + HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true", + HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092", + HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC: "agentrun.event.v1", + HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1", + HWLAB_KAFKA_CLIENT_ID: "hwlab-live-test", + HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-live-bridge-fixed", + HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID: "hwlab-live-fanout-fixed" +}); + test("projects AgentRun assistant_message Kafka event into HWLAB trace event", () => { const projected = projectAgentRunKafkaEventToHwlabEvent({ schema: "agentrun.event.v1", @@ -57,7 +79,10 @@ test("projects AgentRun assistant_message Kafka event into HWLAB trace event", ( assert.equal(projected.eventType, "hwlab.trace.event.projected"); assert.equal(projected.source, "hwlab-test"); assert.equal(projected.traceId, "trc_kafka_projection"); + assert.equal(projected.hwlabSessionId, "ses_kafka_projection"); assert.equal(projected.sessionId, "ses_kafka_projection"); + assert.equal(projected.runId, "run_kafka_projection"); + assert.equal(projected.commandId, "cmd_kafka_projection"); assert.equal(projected.context.runId, "run_kafka_projection"); assert.equal(projected.context.commandId, "cmd_kafka_projection"); assert.equal(projected.context.sourceSeq, 7); @@ -74,7 +99,7 @@ test("projects AgentRun assistant_message Kafka event into HWLAB trace event", ( assert.equal(projected.valuesPrinted, false); }); -test("projects final AgentRun assistant text as structured final response", () => { +test("projects final AgentRun assistant text without sealing before terminal_status", () => { const projected = projectAgentRunKafkaEventToHwlabEvent({ schema: "agentrun.event.v1", eventType: "agentrun.run.event", @@ -84,7 +109,8 @@ test("projects final AgentRun assistant text as structured final response", () = }, { source: "hwlab-test" }); assert.equal(projected.event.assistantText, "durable final answer"); - assert.equal(projected.event.terminal, true); + assert.equal(projected.event.status, "running"); + assert.equal(projected.event.terminal, false); assert.equal(projected.event.finalResponse.text, "durable final answer"); assert.equal(projected.event.finalResponse.traceId, "trc_final"); }); @@ -115,6 +141,202 @@ test("projects AgentRun terminal_status Kafka event into terminal HWLAB event", assert.equal(projected.event.message, "AgentRun completed"); }); +test("live Kafka mode publishes canonical AgentRun input directly and fans one broker event to all SSE subscribers", async () => { + const consumerConfigs = []; + const subscriptions = []; + const runOptions = []; + const runOrder = []; + const produced = []; + const otelSpans = []; + const consumers = [0, 1].map((index) => ({ + async connect() {}, + async subscribe(input) { subscriptions[index] = input; }, + async run(input) { runOptions[index] = input; runOrder.push(index); }, + async stop() {}, + async disconnect() {} + })); + const producer = { + async connect() {}, + async send(input) { produced.push(input); return [{ topicName: input.topic, partition: 4, baseOffset: "91" }]; }, + async disconnect() {} + }; + let consumerIndex = 0; + const kafkaFactory = () => ({ + consumer(config) { + consumerConfigs.push(config); + return consumers[consumerIndex++]; + }, + producer() { return producer; } + }); + const forbiddenRuntimeStore = new Proxy({}, { + get() { throw new Error("live mode must not touch the runtime store"); } + }); + const bridge = startHwlabKafkaEventBridge({ + env: LIVE_ENV, + runtimeStore: forbiddenRuntimeStore, + kafkaFactory, + logger: null, + otelSpanEmitter(name, traceId, env, options) { otelSpans.push({ name, traceId, env, options }); } + }); + await bridge.ready; + + assert.deepEqual(consumerConfigs.map((item) => item.groupId), ["hwlab-live-bridge-fixed", "hwlab-live-fanout-fixed"]); + assert.deepEqual(runOrder, [1, 0], "fanout group must be ready before direct publish can create a new HWLAB event"); + assert.deepEqual(subscriptions, [ + { topic: "agentrun.event.v1", fromBeginning: false }, + { topic: "hwlab.event.v1", fromBeginning: false } + ]); + const message = canonicalMessage({ offset: "70" }); + await runOptions[0].eachMessage({ topic: "agentrun.event.v1", partition: 0, message }); + assert.equal(produced.length, 1); + assert.equal(produced[0].topic, "hwlab.event.v1"); + const envelope = JSON.parse(produced[0].messages[0].value); + assert.equal(envelope.schema, "hwlab.event.v1"); + assert.equal(envelope.traceId, "trc_projector_test"); + assert.equal(envelope.hwlabSessionId, "ses_projector_test"); + assert.equal(envelope.runId, "run_projector_test"); + assert.equal(envelope.commandId, "cmd_projector_test"); + assert.equal(envelope.event.assistantText, "projected answer"); + assert.equal(otelSpans[0].name, "hwlab.kafka.live.direct_publish"); + assert.equal(otelSpans[0].traceId, "trc_projector_test"); + assert.deepEqual(otelSpans[0].options.attributes, { + businessTraceId: "trc_projector_test", + hwlabSessionId: "ses_projector_test", + runId: "run_projector_test", + commandId: "cmd_projector_test", + topic: "hwlab.event.v1", + partition: 4, + offset: "91", + sourceTopic: "agentrun.event.v1", + sourcePartition: 0, + sourceOffset: "70", + eventType: "assistant_message", + terminal: false, + valuesRedacted: true + }); + + const first = []; + const second = []; + const firstTransports = []; + bridge.subscribeLiveHwlabEvents((value, transport) => { first.push(value); firstTransports.push(transport); }); + bridge.subscribeLiveHwlabEvents((value) => second.push(value)); + await runOptions[1].eachBatch({ + batch: { topic: "hwlab.event.v1", partition: 0, highWatermark: "1", messages: [{ offset: "0", value: Buffer.from(JSON.stringify(envelope)) }] }, + resolveOffset() {}, + async heartbeat() {}, + isRunning: () => true, + isStale: () => false + }); + assert.equal(bridge.liveSubscriberCount(), 2); + assert.deepEqual(first, [envelope]); + assert.deepEqual(second, [envelope]); + assert.deepEqual(firstTransports, [{ topic: "hwlab.event.v1", partition: 0, offset: "0" }]); + assert.equal(otelSpans[1].name, "hwlab.kafka.live.fanout_receive"); + assert.equal(otelSpans[1].options.attributes.topic, "hwlab.event.v1"); + assert.equal(otelSpans[1].options.attributes.partition, 0); + assert.equal(otelSpans[1].options.attributes.offset, "0"); + assert.equal(consumerConfigs.length, 2, "browser subscribers must not allocate Kafka consumer groups"); + assert.equal((await bridge.status()).consumerLag.total, 0); + await bridge.stop(); +}); + +test("live Kafka publisher has no PG inbox, facts, checkpoint, or outbox dependency", async () => { + const sent = []; + const runtimeStore = new Proxy({}, { get() { throw new Error("unexpected DB access"); } }); + void runtimeStore; + const result = await publishAgentRunKafkaMessageLive({ topic: "agentrun.event.v1", partition: 3, message: canonicalMessage({ offset: "71" }) }, { + producer: { async send(input) { sent.push(input); } }, + config: { clientId: "hwlab-live-test", hwlabTopic: "hwlab.event.v1" }, + logger: null + }); + assert.equal(result.published, true); + assert.equal(sent.length, 1); + assert.equal(JSON.parse(sent[0].messages[0].value).sourceEvent.offset, "71"); +}); + +test("live Kafka OTel sampling is deterministic and bounded without payload attributes", () => { + const envelope = (agentRunEventType, sourceSeq, event = {}) => ({ + schema: "hwlab.event.v1", + eventId: `hwlab:evt_otel_${sourceSeq}`, + sourceEventId: `evt_otel_${sourceSeq}`, + traceId: "trc_live_otel", + hwlabSessionId: "ses_live_otel", + runId: "run_live_otel", + commandId: "cmd_live_otel", + context: { agentRunEventType, sourceSeq, valuesRedacted: true }, + event: { type: "backend", eventType: "backend", terminal: false, message: "must-not-leak", payload: { password: "must-not-leak" }, ...event } + }); + + assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("assistant_message", 1, { type: "assistant" })), true); + assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("tool_call", 2, { type: "tool" })), true); + assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("terminal_status", 3, { type: "result", terminal: true })), true); + const decisions = Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" }))); + assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => (index + 1) % 32 === 0)); + assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" })))); + + const attributes = liveKafkaOtelSpanAttributes(envelope("assistant_message", 1, { type: "assistant" }), { + topic: "hwlab.event.v1", + partition: 2, + offset: "14" + }); + assert.deepEqual(attributes, { + businessTraceId: "trc_live_otel", + hwlabSessionId: "ses_live_otel", + runId: "run_live_otel", + commandId: "cmd_live_otel", + topic: "hwlab.event.v1", + partition: 2, + offset: "14", + sourceTopic: null, + sourcePartition: null, + sourceOffset: null, + eventType: "assistant_message", + terminal: false, + valuesRedacted: true + }); + assert.equal(JSON.stringify(attributes).includes("must-not-leak"), false); +}); + +test("composable Kafka capabilities reject consumer group collisions", () => { + assert.throws( + () => kafkaEventBridgeConfig({ + ...LIVE_ENV, + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true", + HWLAB_KAFKA_PROJECTOR_GROUP_ID: LIVE_ENV.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID, + HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "1000" + }), + (error) => error?.code === "hwlab_kafka_consumer_groups_not_distinct" + ); +}); + +test("projection realtime subscribes to projection commits without enabling Kafka ingest or relay", async () => { + let subscribeCount = 0; + let unsubscribeCount = 0; + const bridge = startHwlabKafkaEventBridge({ + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true" + }, + runtimeStore: { + async assertWorkbenchTransactionalRealtimeReady() { return { ready: true }; }, + async subscribeWorkbenchProjectionCommits() { + subscribeCount += 1; + return () => { unsubscribeCount += 1; }; + } + }, + kafkaFactory() { throw new Error("projection realtime must not create a Kafka client"); }, + logger: null + }); + + await bridge.ready; + assert.equal(subscribeCount, 1); + await bridge.stop(); + assert.equal(unsubscribeCount, 1); +}); + test("projects AgentRun run-created envelope even when no source event is present", () => { const projected = projectAgentRunKafkaEventToHwlabEvent({ schema: "agentrun.event.v1", @@ -226,6 +448,26 @@ test("Kafka projector signals committed projection but does not commit a stale g await bridge.stop(); }); +test("Kafka projector commits a suppressed post-seal offset without publishing a projection signal", async () => { + const calls = []; + let signal = null; + const harness = kafkaProjectorHarness({ + async commitAgentRunKafkaProjection() { + calls.push("db"); + return { projectedSeq: 9, projectionWritten: false, suppressedAfterSeal: true }; + } + }, calls); + const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null }); + bridge.subscribeProjectionCommits((value) => { signal = value; }); + await bridge.ready; + + await harness.runBatch(canonicalMessage({ offset: "44" })); + + assert.equal(signal, null); + assert.deepEqual(calls.slice(-3), ["resolve:44", "commit:45", "heartbeat"]); + await bridge.stop(); +}); + test("Kafka projector persists malformed and non-HWLAB events before committing offsets", async () => { const calls = []; const harness = kafkaProjectorHarness({}, calls); @@ -307,6 +549,52 @@ test("HWLAB Kafka relay claims just in time and forwards fencing token", async ( assert.deepEqual(calls, ["claim:1", "send:1", "complete:1:2:relay-owner", "claim:1", "send:2", "complete:2:1:relay-owner"]); }); +test("HWLAB Kafka relay retries a failed send and clears backlog on the next cycle", async () => { + const calls = []; + let pending = true; + let attemptCount = 0; + const runtimeStore = { + async claimHwlabKafkaOutbox() { + if (!pending) return []; + attemptCount += 1; + return [{ outboxSeq: 7, eventId: "evt-relay-retry", topic: "hwlab.event.v1", partitionKey: "trc-relay-retry", payload: { eventId: "evt-relay-retry" }, headers: {}, attemptCount, leaseOwner: "relay-retry-owner", leaseExpiresAt: `2026-07-10T10:0${attemptCount}:30.000Z` }]; + }, + async completeHwlabKafkaOutbox(item) { + calls.push(`complete:${item.attemptCount}`); + pending = false; + }, + async retryHwlabKafkaOutbox(item, error, retryAt) { + calls.push(`retry:${item.attemptCount}:${error.message}:${retryAt}`); + }, + async hwlabKafkaProjectorStatus() { + return { backlogCount: pending ? 1 : 0, retryCount: pending && attemptCount > 0 ? 1 : 0, failedInboxCount: 0, dlqCount: 0 }; + } + }; + let sendCount = 0; + const producer = { + async send() { + sendCount += 1; + calls.push(`send:${sendCount}`); + if (sendCount === 1) throw new Error("broker unavailable"); + } + }; + const config = { relay: { batchSize: 1, leaseMs: 30000, sendTimeoutMs: 10000, retryBackoffMs: 1000 } }; + + const failedCycle = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-retry-owner", now: () => "2026-07-10T10:00:00.000Z", logger: null }); + assert.deepEqual(failedCycle, { claimedCount: 1, deliveredCount: 0, retryCount: 1, valuesPrinted: false }); + assert.deepEqual(await runtimeStore.hwlabKafkaProjectorStatus(), { backlogCount: 1, retryCount: 1, failedInboxCount: 0, dlqCount: 0 }); + + const recoveredCycle = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-retry-owner", now: () => "2026-07-10T10:00:02.000Z", logger: null }); + assert.deepEqual(recoveredCycle, { claimedCount: 1, deliveredCount: 1, retryCount: 0, valuesPrinted: false }); + assert.deepEqual(await runtimeStore.hwlabKafkaProjectorStatus(), { backlogCount: 0, retryCount: 0, failedInboxCount: 0, dlqCount: 0 }); + assert.deepEqual(calls, [ + "send:1", + "retry:1:broker unavailable:2026-07-10T10:00:01.000Z", + "send:2", + "complete:2" + ]); +}); + function kafkaProjectorHarness(overrides = {}, calls = []) { let runOptions = null; let stale = false; @@ -320,6 +608,8 @@ function kafkaProjectorHarness(overrides = {}, calls = []) { }; const producer = { async connect() {}, async send() {}, async disconnect() {} }; const runtimeStore = { + async assertWorkbenchTransactionalRealtimeReady() { return { ready: true }; }, + async subscribeWorkbenchProjectionCommits() { return () => {}; }, async commitAgentRunKafkaProjection() { return { projectedSeq: 1 }; }, async recordFailedAgentRunKafkaMessage() { calls.push("dlq"); }, async recordIgnoredAgentRunKafkaMessage() { calls.push("ignored"); return { ignored: true }; }, diff --git a/internal/cloud/kafka-event-bridge.ts b/internal/cloud/kafka-event-bridge.ts index c31f411e..f61412ca 100644 --- a/internal/cloud/kafka-event-bridge.ts +++ b/internal/cloud/kafka-event-bridge.ts @@ -4,36 +4,52 @@ import { createHash, randomUUID } from "node:crypto"; import { Kafka, logLevel } from "kafkajs"; +import { emitCodeAgentOtelSpan } from "./otel-trace.ts"; import { buildWorkbenchProjectionEventFacts } from "./workbench-projection-writer.ts"; +import { + WORKBENCH_REALTIME_CAPABILITY_ENVS, + workbenchRealtimeCapabilities +} from "./workbench-realtime-capabilities.ts"; const TRUE_VALUES = new Set(["1", "true", "yes", "on"]); const DEFAULT_AGENTRUN_EVENT_TOPIC = "agentrun.event.v1"; const DEFAULT_HWLAB_EVENT_TOPIC = "hwlab.event.v1"; const DEFAULT_STDIO_TOPIC = "codex-stdio.raw.v1"; const DEFAULT_CLIENT_ID = "hwlab-v03-cloud-api"; -const DEFAULT_GROUP_ID = "hwlab-v03-agentrun-event-bridge"; const DEFAULT_QUERY_TIMEOUT_MS = 5000; const DEFAULT_QUERY_LIMIT = 50; -const REQUIRED_PROJECTOR_ENV = Object.freeze([ +const LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS = 32; +const REQUIRED_KAFKA_ENV = Object.freeze([ "HWLAB_KAFKA_BOOTSTRAP_SERVERS", "HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC", "HWLAB_KAFKA_EVENT_TOPIC", - "HWLAB_KAFKA_CLIENT_ID", - "HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID", - "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS", - "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS", - "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE", - "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS", - "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS", - "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS" + "HWLAB_KAFKA_CLIENT_ID" ]); export function kafkaEventBridgeConfig(env = process.env) { - if (!truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED)) return null; - const missing = REQUIRED_PROJECTOR_ENV.filter((name) => !stringValue(env[name])); + const featureEnvPresent = Object.values(WORKBENCH_REALTIME_CAPABILITY_ENVS).some((name) => stringValue(env[name])); + const legacyKafkaEnabled = truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED); + if (!featureEnvPresent && !legacyKafkaEnabled) return null; + const capabilities = workbenchRealtimeCapabilities(env); + const kafkaEnabled = capabilities.directPublish || capabilities.liveKafkaSse || capabilities.transactionalProjector || capabilities.projectionOutboxRelay; + const transactionalEnabled = capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime; + if (!kafkaEnabled && !transactionalEnabled) return null; + const required = [...REQUIRED_KAFKA_ENV]; + if (!kafkaEnabled) required.length = 0; + if (capabilities.directPublish) required.push("HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID"); + if (capabilities.liveKafkaSse) required.push("HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID"); + if (capabilities.transactionalProjector) required.push("HWLAB_KAFKA_PROJECTOR_GROUP_ID", "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS"); + if (capabilities.projectionOutboxRelay) required.push( + "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS", + "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE", + "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS", + "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS", + "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS" + ); + const missing = required.filter((name) => !stringValue(env[name])); if (missing.length > 0) { - const error = new Error(`HWLAB Kafka projector configuration is incomplete: ${missing.join(", ")}`); - error.code = "hwlab_kafka_projector_config_incomplete"; + const error = new Error(`HWLAB Kafka capability configuration is incomplete: ${missing.join(", ")}`); + error.code = "hwlab_kafka_realtime_config_incomplete"; error.missing = missing; throw error; } @@ -41,34 +57,58 @@ export function kafkaEventBridgeConfig(env = process.env) { const agentRunTopic = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC); const hwlabTopic = stringValue(env.HWLAB_KAFKA_EVENT_TOPIC); const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID); - const groupId = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID); - const relay = { + const relay = capabilities.projectionOutboxRelay ? { intervalMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS, "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS"), batchSize: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE, "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE"), leaseMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS, "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS"), sendTimeoutMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS, "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS"), retryBackoffMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS, "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS") - }; - if (relay.sendTimeoutMs + 5000 >= relay.leaseMs) { + } : null; + if (relay && relay.sendTimeoutMs + 5000 >= relay.leaseMs) { const error = new Error("HWLAB Kafka relay lease must exceed producer send timeout by at least 5000ms."); error.code = "hwlab_kafka_relay_lease_invalid"; throw error; } + const enabledConsumerGroups = [ + capabilities.directPublish ? stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID) : null, + capabilities.transactionalProjector ? stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID) : null, + capabilities.liveKafkaSse ? stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID) : null + ].filter(Boolean); + if (new Set(enabledConsumerGroups).size !== enabledConsumerGroups.length) { + const error = new Error("HWLAB Kafka direct publisher, projector, and live SSE consumer groups must be distinct when enabled together."); + error.code = "hwlab_kafka_consumer_groups_not_distinct"; + error.groupIds = enabledConsumerGroups; + throw error; + } return { + capabilities, brokers, agentRunTopic, hwlabTopic, clientId, - groupId, - projectorHeartbeatIntervalMs: strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS"), + directPublishGroupId: stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID), + projectorGroupId: stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID), + hwlabEventGroupId: stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID), + projectorHeartbeatIntervalMs: capabilities.transactionalProjector ? strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS") : null, relay }; } -export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString() } = {}) { +export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString(), otelSpanEmitter = emitCodeAgentOtelSpan } = {}) { const config = kafkaEventBridgeConfig(env); if (!config) return { started: false, reason: "disabled_or_unconfigured", stop() {}, subscribeProjectionCommits() { return () => {}; }, valuesPrinted: false }; - requireKafkaProjectorStore(runtimeStore); + const components = []; + if (config.capabilities.directPublish || config.capabilities.liveKafkaSse) { + components.push(startLiveHwlabKafkaEventBridge({ config, env, logger, kafkaFactory, otelSpanEmitter })); + } + if (config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay || config.capabilities.projectionRealtime) { + components.push(startTransactionalHwlabKafkaEventBridge({ config, logger, kafkaFactory, runtimeStore, now })); + } + return components.length === 1 ? components[0] : combineKafkaEventBridgeComponents(config, components); +} + +function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString() } = {}) { + requireKafkaProjectorStore(runtimeStore, config.capabilities); let stopped = false; let consumer = null; @@ -86,16 +126,17 @@ export function startHwlabKafkaEventBridge({ env = process.env, logger = console let startupError = null; let startupReady = false; const ready = (async () => { - const kafka = kafkaFactory(config); - consumer = kafka.consumer({ groupId: config.groupId, allowAutoTopicCreation: false }); - producer = kafka.producer({ allowAutoTopicCreation: false }); - await producer.connect(); - await consumer.connect(); - if (typeof runtimeStore.subscribeWorkbenchProjectionCommits === "function") { + await runtimeStore.assertWorkbenchTransactionalRealtimeReady(); + const kafka = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay ? kafkaFactory(config) : null; + if (config.capabilities.transactionalProjector) consumer = kafka.consumer({ groupId: config.projectorGroupId, allowAutoTopicCreation: false }); + if (config.capabilities.projectionOutboxRelay) producer = kafka.producer({ allowAutoTopicCreation: false }); + if (producer) await producer.connect(); + if (consumer) await consumer.connect(); + if (config.capabilities.projectionRealtime && typeof runtimeStore.subscribeWorkbenchProjectionCommits === "function") { stopProjectionNotifications = await runtimeStore.subscribeWorkbenchProjectionCommits(publishProjectionCommit); } - await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false }); - await consumer.run({ + if (consumer) await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false }); + if (consumer) await consumer.run({ autoCommit: false, eachBatchAutoResolve: false, eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => { @@ -122,6 +163,7 @@ export function startHwlabKafkaEventBridge({ env = process.env, logger = console } }); const runRelay = async () => { + if (!config.capabilities.projectionOutboxRelay) return; if (stopped || relayRunning) return; relayRunning = true; try { @@ -130,15 +172,18 @@ export function startHwlabKafkaEventBridge({ env = process.env, logger = console relayRunning = false; } }; - relayTimer = setInterval(() => { void runRelay().catch((error) => logWarn(logger, "hwlab-kafka-outbox-relay-failed", { message: errorMessage(error), valuesPrinted: false })); }, config.relay.intervalMs); - relayTimer.unref?.(); - await runRelay(); + if (config.capabilities.projectionOutboxRelay) { + relayTimer = setInterval(() => { void runRelay().catch((error) => logWarn(logger, "hwlab-kafka-outbox-relay-failed", { message: errorMessage(error), valuesPrinted: false })); }, config.relay.intervalMs); + relayTimer.unref?.(); + await runRelay(); + } startupReady = true; logInfo(logger, "hwlab-kafka-event-bridge-started", { agentRunTopic: config.agentRunTopic, hwlabTopic: config.hwlabTopic, clientId: config.clientId, - groupId: config.groupId, + projectorGroupId: config.projectorGroupId, + capabilities: config.capabilities, valuesPrinted: false }); })(); @@ -168,7 +213,13 @@ export function startHwlabKafkaEventBridge({ env = process.env, logger = console async status() { if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true }; if (!startupReady) return { status: "initializing", valuesRedacted: true }; - return { status: "ready", ...(await runtimeStore.hwlabKafkaProjectorStatus()), valuesRedacted: true }; + const transactionalReadiness = typeof runtimeStore.workbenchTransactionalRealtimeReadiness === "function" + ? await runtimeStore.workbenchTransactionalRealtimeReadiness() + : { ready: true, valuesRedacted: true }; + const projectorStatus = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay + ? await runtimeStore.hwlabKafkaProjectorStatus() + : {}; + return { status: transactionalReadiness.ready === false ? "blocked" : "ready", capabilities: config.capabilities, transactionalReadiness, ...projectorStatus, valuesRedacted: true }; }, startupStatus() { if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true }; @@ -183,6 +234,263 @@ export function startHwlabKafkaEventBridge({ env = process.env, logger = console }; } +function startLiveHwlabKafkaEventBridge({ config, env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) { + let stopped = false; + let bridgeConsumer = null; + let fanoutConsumer = null; + let producer = null; + let startupError = null; + let startupReady = false; + const subscribers = new Set(); + const lagByPartition = new Map(); + const publishLiveEnvelope = (envelope, transport) => { + for (const listener of [...subscribers]) { + try { + listener(envelope, transport); + } catch (error) { + logWarn(logger, "hwlab-live-kafka-subscriber-failed", { message: errorMessage(error), valuesPrinted: false }); + } + } + }; + const ready = (async () => { + const kafka = kafkaFactory(config); + if (config.capabilities.directPublish) { + bridgeConsumer = kafka.consumer({ groupId: config.directPublishGroupId, allowAutoTopicCreation: false }); + producer = kafka.producer({ allowAutoTopicCreation: false }); + } + if (config.capabilities.liveKafkaSse) fanoutConsumer = kafka.consumer({ groupId: config.hwlabEventGroupId, allowAutoTopicCreation: false }); + if (producer) await producer.connect(); + if (bridgeConsumer) await bridgeConsumer.connect(); + if (fanoutConsumer) await fanoutConsumer.connect(); + if (bridgeConsumer) await bridgeConsumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false }); + if (fanoutConsumer) await fanoutConsumer.subscribe({ topic: config.hwlabTopic, fromBeginning: false }); + if (fanoutConsumer) await fanoutConsumer.run({ + eachBatchAutoResolve: false, + eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => { + let lastOffset = null; + for (const message of batch.messages) { + if (stopped || !isRunning() || isStale()) break; + const envelope = parseJson(message?.value ? Buffer.from(message.value).toString("utf8") : ""); + if (!envelope || envelope.schema !== "hwlab.event.v1") { + logWarn(logger, "hwlab-live-kafka-envelope-ignored", { reason: "schema-invalid", valuesPrinted: false }); + } else { + const transport = { topic: batch.topic, partition: batch.partition, offset: message.offset }; + emitLiveKafkaOtelSpan("hwlab.kafka.live.fanout_receive", envelope, transport, { env, otelSpanEmitter }); + publishLiveEnvelope(envelope, transport); + } + resolveOffset(message.offset); + lastOffset = message.offset; + await heartbeat(); + } + if (lastOffset !== null) lagByPartition.set(batch.partition, kafkaPartitionLag(batch.highWatermark, lastOffset)); + } + }); + if (bridgeConsumer) await bridgeConsumer.run({ + eachMessage: async ({ topic, partition, message }) => { + if (stopped) return; + await publishAgentRunKafkaMessageLive({ topic, partition, message }, { producer, config, env, logger, otelSpanEmitter }); + } + }); + startupReady = true; + logInfo(logger, "hwlab-live-kafka-event-bridge-started", { + capabilities: config.capabilities, + agentRunTopic: config.agentRunTopic, + hwlabTopic: config.hwlabTopic, + clientId: config.clientId, + directPublishGroupId: config.directPublishGroupId, + hwlabEventGroupId: config.hwlabEventGroupId, + valuesPrinted: false + }); + })(); + void ready.catch((error) => { + startupError = error; + logWarn(logger, "hwlab-live-kafka-event-bridge-start-failed", { + message: errorMessage(error), + agentRunTopic: config.agentRunTopic, + hwlabTopic: config.hwlabTopic, + valuesPrinted: false + }); + }); + return { + started: true, + ...config, + ready, + async stop() { + stopped = true; + subscribers.clear(); + await Promise.allSettled([bridgeConsumer?.stop?.(), fanoutConsumer?.stop?.()]); + await Promise.allSettled([bridgeConsumer?.disconnect?.(), fanoutConsumer?.disconnect?.(), producer?.disconnect?.()]); + }, + async status() { + if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true }; + return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, subscriberCount: subscribers.size, consumerLag: liveKafkaLagSummary(lagByPartition), lossPossible: true, valuesRedacted: true }; + }, + startupStatus() { + if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true }; + return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, valuesRedacted: true }; + }, + subscribeProjectionCommits() { return () => {}; }, + subscribeLiveHwlabEvents(listener) { + if (typeof listener !== "function") return () => {}; + subscribers.add(listener); + return () => subscribers.delete(listener); + }, + liveSubscriberCount() { return subscribers.size; }, + valuesPrinted: false + }; +} + +function kafkaPartitionLag(highWatermark, lastOffset) { + try { + const lag = BigInt(String(highWatermark ?? "0")) - (BigInt(String(lastOffset ?? "0")) + 1n); + return Number(lag > 0n ? lag : 0n); + } catch { + return null; + } +} + +function liveKafkaLagSummary(lagByPartition) { + const partitions = [...lagByPartition.entries()].map(([partition, lag]) => ({ partition, lag })).sort((left, right) => left.partition - right.partition); + const known = partitions.map((item) => item.lag).filter((lag) => Number.isFinite(lag)); + return { + known: known.length > 0, + total: known.length > 0 ? known.reduce((sum, lag) => sum + lag, 0) : null, + partitions, + valuesRedacted: true + }; +} + +function combineKafkaEventBridgeComponents(config, components) { + const ready = Promise.all(components.map((component) => component.ready)); + const componentStartup = () => components.map((component) => component.startupStatus?.() ?? { status: "initializing" }); + return { + started: true, + ...config, + ready, + async stop() { await Promise.allSettled(components.map((component) => component.stop())); }, + async status() { + const statuses = await Promise.all(components.map((component) => component.status())); + const blocked = statuses.find((status) => status.status === "blocked"); + const initializing = statuses.some((status) => status.status !== "ready"); + return { + status: blocked ? "blocked" : initializing ? "initializing" : "ready", + capabilities: config.capabilities, + components: statuses, + errorCode: blocked?.errorCode ?? null, + message: blocked?.message ?? null, + valuesRedacted: true + }; + }, + startupStatus() { + const statuses = componentStartup(); + const blocked = statuses.find((status) => status.status === "blocked"); + return { + status: blocked ? "blocked" : statuses.every((status) => status.status === "ready") ? "ready" : "initializing", + capabilities: config.capabilities, + components: statuses, + errorCode: blocked?.errorCode ?? null, + message: blocked?.message ?? null, + valuesRedacted: true + }; + }, + subscribeProjectionCommits(listener) { + const stops = components.map((component) => component.subscribeProjectionCommits?.(listener)).filter((stop) => typeof stop === "function"); + return () => stops.forEach((stop) => stop()); + }, + subscribeLiveHwlabEvents(listener) { + const owner = components.find((component) => typeof component.subscribeLiveHwlabEvents === "function"); + return owner?.subscribeLiveHwlabEvents(listener) ?? (() => {}); + }, + liveSubscriberCount() { + const owner = components.find((component) => typeof component.liveSubscriberCount === "function"); + return owner?.liveSubscriberCount() ?? 0; + }, + valuesPrinted: false + }; +} + +export async function publishAgentRunKafkaMessageLive(kafkaMessage = {}, { producer, config, env = process.env, logger = console, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) { + const transport = kafkaTransport(kafkaMessage); + const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage); + if (!decoded.ok) { + logWarn(logger, "hwlab-live-kafka-message-invalid", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false }); + return { handled: true, invalid: true, published: false, valuesPrinted: false }; + } + if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) { + return { handled: true, ignored: true, published: false, valuesPrinted: false }; + } + const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, { + source: config.clientId, + sourceTopic: transport.sourceTopic, + sourcePartition: transport.sourcePartition, + sourceOffset: transport.sourceOffset, + sourceKey: transport.sourceKey, + inputSha256: transport.inputSha256 + }); + const publishMetadata = await producer.send({ + topic: config.hwlabTopic, + messages: [{ key: kafkaMessageKey(projected), value: JSON.stringify(projected), headers: kafkaHeaders(projected) }] + }); + const publishedRecord = Array.isArray(publishMetadata) ? publishMetadata[0] : null; + emitLiveKafkaOtelSpan("hwlab.kafka.live.direct_publish", projected, { + topic: config.hwlabTopic, + partition: publishedRecord?.partition, + offset: publishedRecord?.baseOffset ?? publishedRecord?.offset, + sourceTopic: transport.sourceTopic, + sourcePartition: transport.sourcePartition, + sourceOffset: transport.sourceOffset + }, { env, otelSpanEmitter }); + return { handled: true, published: true, envelope: projected, sessionId: projected.sessionId, traceId: projected.traceId, valuesPrinted: false }; +} + +export function shouldEmitLiveKafkaOtelSpan(envelope = {}) { + const event = objectValue(envelope.event); + const context = objectValue(envelope.context); + const eventType = firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType)?.toLowerCase(); + if (event.terminal === true || eventType === "terminal" || eventType === "terminal_status") return true; + if (["assistant", "assistant_message", "tool", "tool_call"].includes(eventType)) return true; + if (eventType !== "command_output" && event.type !== "output") return true; + const sourceSeq = integerValue(context.sourceSeq ?? event.sourceSeq); + if (Number.isInteger(sourceSeq) && sourceSeq > 0) return sourceSeq % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0; + const identity = firstText(envelope.sourceEventId, envelope.eventId, envelope.traceId, envelope.hwlabSessionId, envelope.sessionId); + if (!identity) return false; + return createHash("sha256").update(identity).digest().readUInt32BE(0) % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0; +} + +export function liveKafkaOtelSpanAttributes(envelope = {}, transport = {}) { + const event = objectValue(envelope.event); + const context = objectValue(envelope.context); + const sourceEvent = objectValue(envelope.sourceEvent); + return { + businessTraceId: firstText(envelope.traceId, event.traceId), + hwlabSessionId: firstText(envelope.hwlabSessionId, envelope.sessionId, event.sessionId), + runId: firstText(envelope.runId, context.runId, event.runId), + commandId: firstText(envelope.commandId, context.commandId, event.commandId), + topic: firstText(transport.topic, sourceEvent.topic), + partition: integerValue(transport.partition ?? sourceEvent.partition), + offset: firstText(transport.offset, sourceEvent.offset), + sourceTopic: firstText(transport.sourceTopic), + sourcePartition: integerValue(transport.sourcePartition), + sourceOffset: firstText(transport.sourceOffset), + eventType: firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType), + terminal: event.terminal === true, + valuesRedacted: true + }; +} + +export function emitLiveKafkaOtelSpan(name, envelope = {}, transport = {}, { env = process.env, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) { + if (typeof otelSpanEmitter !== "function" || !shouldEmitLiveKafkaOtelSpan(envelope)) return { emitted: false, valuesRedacted: true }; + const attributes = liveKafkaOtelSpanAttributes(envelope, transport); + if (!attributes.businessTraceId) return { emitted: false, valuesRedacted: true }; + try { + const pending = otelSpanEmitter(name, attributes.businessTraceId, env, { attributes }); + void Promise.resolve(pending).catch(() => undefined); + } catch { + // OTel is best effort and must never change the live Kafka delivery path. + } + return { emitted: true, valuesRedacted: true }; +} + export async function projectAgentRunKafkaMessage(kafkaMessage = {}, { runtimeStore, config, logger = console } = {}) { const transport = kafkaTransport(kafkaMessage); const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage); @@ -215,7 +523,17 @@ export async function projectAgentRunKafkaMessage(kafkaMessage = {}, { runtimeSt headers: kafkaHeaders(projected) }); if (result.conflict) logWarn(logger, "hwlab-kafka-message-conflict", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: result.diagnostic?.code, valuesPrinted: false }); - return { handled: true, projected: !result.duplicate && !result.conflict, duplicate: result.duplicate === true, conflict: result.conflict === true, projectedSeq: result.projectedSeq, sessionId: projected.sessionId, traceId: projected.traceId, valuesPrinted: false }; + return { + handled: true, + projected: !result.duplicate && !result.conflict && result.projectionWritten !== false && result.suppressedAfterSeal !== true, + duplicate: result.duplicate === true, + conflict: result.conflict === true, + suppressedAfterSeal: result.suppressedAfterSeal === true, + projectedSeq: result.projectedSeq, + sessionId: projected.sessionId, + traceId: projected.traceId, + valuesPrinted: false + }; } export async function relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner, now = () => new Date().toISOString(), logger = console } = {}) { @@ -303,7 +621,10 @@ export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {}) source: stringValue(options.source) || DEFAULT_CLIENT_ID, producedAt, traceId, + hwlabSessionId: sessionId, sessionId, + runId, + commandId, context: { conversationId: firstText(input.conversationId, hwlab.conversationId, context.conversationId, run.conversationId, payload.conversationId), threadId: firstText(input.threadId, hwlab.threadId, context.threadId, run.threadId, payload.threadId), @@ -402,10 +723,15 @@ function kafkaTransport(kafkaMessage = {}) { }; } -function requireKafkaProjectorStore(runtimeStore) { - const required = ["commitAgentRunKafkaProjection", "recordFailedAgentRunKafkaMessage", "recordIgnoredAgentRunKafkaMessage", "claimHwlabKafkaOutbox", "completeHwlabKafkaOutbox", "retryHwlabKafkaOutbox", "hwlabKafkaProjectorStatus"]; +function requireKafkaProjectorStore(runtimeStore, capabilities = {}) { + const required = []; + if (capabilities.transactionalProjector) required.push("commitAgentRunKafkaProjection", "recordFailedAgentRunKafkaMessage", "recordIgnoredAgentRunKafkaMessage"); + if (capabilities.projectionOutboxRelay) required.push("claimHwlabKafkaOutbox", "completeHwlabKafkaOutbox", "retryHwlabKafkaOutbox"); + if (capabilities.projectionRealtime) required.push("subscribeWorkbenchProjectionCommits"); + if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay) required.push("hwlabKafkaProjectorStatus"); + if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime) required.push("assertWorkbenchTransactionalRealtimeReady"); const missing = required.filter((name) => typeof runtimeStore?.[name] !== "function"); - if (missing.length > 0) throw contractError("hwlab_kafka_projector_store_invalid", `Kafka projector requires a durable runtime store: ${missing.join(", ")}`); + if (missing.length > 0) throw contractError("hwlab_kafka_projector_store_invalid", `Kafka durable capabilities require a runtime store: ${missing.join(", ")}`); } export async function queryKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, traceId = null, sessionId = null, runId = null, commandId = null, limit = DEFAULT_QUERY_LIMIT, timeoutMs = DEFAULT_QUERY_TIMEOUT_MS, fromBeginning = true, kafkaFactory = defaultKafkaFactory } = {}) { @@ -468,6 +794,60 @@ export async function queryKafkaEventStream({ env = process.env, stream = "hwlab }; } +export async function openKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, traceId = null, sessionId = null, runId = null, commandId = null, fromBeginning = false, onEvent, onError = null, kafkaFactory = defaultKafkaFactory } = {}) { + if (typeof onEvent !== "function") throw new Error("onEvent callback is required for Kafka event streaming."); + const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS); + if (brokers.length === 0) throw new Error("HWLAB_KAFKA_BOOTSTRAP_SERVERS is required for Kafka event streams."); + const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID; + const resolvedTopic = stringValue(topic) || kafkaTopicForStream(stream, env); + const filters = compactObject({ traceId, sessionId, runId, commandId }); + const kafka = kafkaFactory({ brokers, clientId: `${clientId}-debug-sse` }); + const groupId = `${clientId}-debug-sse-${Date.now()}-${randomUUID().slice(0, 8)}`; + const consumer = kafka.consumer({ groupId, allowAutoTopicCreation: false }); + let running = false; + let stopped = false; + await consumer.connect(); + await consumer.subscribe({ topic: resolvedTopic, fromBeginning: fromBeginning === true }); + running = true; + try { + await consumer.run({ + eachMessage: async ({ topic: messageTopic, partition, message }) => { + if (stopped) return; + const valueText = message.value ? Buffer.from(message.value).toString("utf8") : ""; + const value = parseJson(valueText); + if (!value || !eventMatchesFilters(value, filters)) return; + await onEvent({ + topic: messageTopic, + partition, + offset: message.offset, + key: message.key ? Buffer.from(message.key).toString("utf8") : null, + timestamp: message.timestamp ?? null, + value + }); + } + }); + } catch (error) { + if (typeof onError === "function") onError(error); + stopped = true; + if (running) await consumer.stop().catch(() => undefined); + await boundedDisconnect(consumer); + throw error; + } + return { + ok: true, + stream, + topic: resolvedTopic, + groupId, + filters, + async stop() { + stopped = true; + if (running) await consumer.stop().catch(() => undefined); + await boundedDisconnect(consumer); + }, + valuesPrinted: false + }; +} + function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, command, traceId, sessionId, sourceSeq, runId, commandId }) { const base = { traceId, @@ -484,22 +864,22 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, }; const type = firstText(sourceEvent.type, payload.type, input.eventType) || "event"; if (type === "assistant_message") { - const terminal = payload.replyAuthority === true || payload.final === true; + const authoritative = payload.replyAuthority === true || payload.final === true; const assistantText = textPayload(payload, "assistant message"); return { ...base, type: "assistant", eventType: "assistant", - status: terminal ? "completed" : "running", + status: "running", label: "agentrun:assistant:message", message: assistantText, text: assistantText, assistantText, - finalResponse: terminal ? { text: assistantText, status: "completed", traceId, source: "agentrun.kafka", valuesPrinted: false } : null, + finalResponse: authoritative ? { text: assistantText, status: "completed", traceId, source: "agentrun.kafka", valuesPrinted: false } : null, itemId: firstText(payload.itemId), replyAuthority: payload.replyAuthority === true, final: payload.final === true, - terminal + terminal: false }; } if (type === "backend_status") { @@ -511,6 +891,24 @@ function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, const normalized = terminalStatus === "cancelled" ? "canceled" : terminalStatus; return { ...base, type: "result", eventType: "terminal", status: normalized, label: `agentrun:terminal:${terminalStatus}`, errorCode: firstText(payload.failureKind, payload.errorCode, sourceEvent.failureKind, sourceEvent.errorCode), failureKind: firstText(payload.failureKind, sourceEvent.failureKind), message: textPayload({ ...sourceEvent, ...payload }, `AgentRun terminal status ${terminalStatus}`), terminal: true }; } + if (type === "tool_call") { + const toolName = firstText(payload.toolName, payload.name, payload.method) || "tool"; + return { + ...base, + type: "tool", + eventType: "tool", + status: firstText(payload.status) || "running", + label: `agentrun:tool:${toolName}`, + message: textPayload(payload, `AgentRun tool ${toolName}`), + toolName, + method: firstText(payload.method), + itemId: firstText(payload.itemId), + exitCode: integerValue(payload.exitCode), + durationMs: integerValue(payload.durationMs), + outputSummary: firstText(payload.outputSummary, payload.summary?.text), + terminal: false + }; + } if (type === "command_output") { const stream = payload.stream === "stderr" ? "stderr" : "stdout"; return { ...base, type: "output", eventType: "status", status: "running", label: `agentrun:output:${stream}`, stream, message: textPayload(payload, ""), outputTruncated: payload.outputTruncated === true }; diff --git a/internal/cloud/server-agent-chat.test.ts b/internal/cloud/server-agent-chat.test.ts index 42bf5a97..db888cdf 100644 --- a/internal/cloud/server-agent-chat.test.ts +++ b/internal/cloud/server-agent-chat.test.ts @@ -14,6 +14,7 @@ import { validateCodeAgentChatSchema } from "./code-agent-chat.ts"; import { createCodexStdioSessionManager } from "./codex-stdio-session.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { submitAgentRunChatTurn } from "./code-agent-agentrun-adapter.ts"; +import { buildWorkbenchProjectionEventFacts } from "./workbench-projection-writer.ts"; import { codexStdioChatFixture, codexStdioReadyFixture, @@ -31,6 +32,60 @@ import { const TEST_AGENT_ACTOR = { id: "usr_agent_owner", username: "agent-owner", displayName: "Agent Owner", role: "user", status: "active" }; const TEST_AUTH_SESSION = { id: "auth_ses_agent_owner" }; +async function waitForTraceEvent(traceStore, traceId, predicate, message = "expected trace event") { + for (let index = 0; index < 100; index += 1) { + const event = traceStore.snapshot(traceId).events.find(predicate); + if (event) return event; + await delay(5); + } + assert.fail(message); +} + +async function commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId, + runId = null, + commandId = null, + status = "completed", + text, + failureKind = null, + projectedSeq = 1, + sourceSeq = 1, + createdAt = "2026-07-10T10:00:00.000Z" +}) { + const built = buildWorkbenchProjectionEventFacts({ + projectedSeq, + projectedAt: createdAt, + event: { + traceId, + sessionId, + runId, + commandId, + sourceEventId: `evt_${traceId.slice(4)}_${sourceSeq}`, + sourceSeq, + type: "assistant", + eventType: "assistant", + label: "agentrun:kafka:assistant-final", + status, + terminal: true, + final: true, + replyAuthority: true, + text, + failureKind, + errorCode: failureKind, + createdAt + } + }); + assert.equal(built.written, true); + await runtimeStore.writeWorkbenchFacts({ facts: built.facts }, { + source: "test-kafka-projector", + traceId, + sessionId, + valuesPrinted: false + }); + return built; +} + function testAgentSessionRecord(input = {}) { const sessionId = input.sessionId ?? input.id; return { @@ -111,7 +166,7 @@ test("cloud api trace resource paginates events by sinceSeq", async () => { const traceStore = createCodeAgentTraceStore(); const traceId = "trc_trace_pagination"; for (let index = 0; index < 5; index += 1) { - traceStore.append(traceId, { type: "trace", status: "observed", label: `event:${index}` }); + traceStore.append(traceId, { type: "trace", status: index === 4 ? "completed" : "observed", terminal: index === 4, label: `event:${index}` }); } const server = createCloudApiServer({ traceStore, @@ -148,6 +203,11 @@ test("cloud api trace resource paginates events by sinceSeq", async () => { assert.equal(finalPage.status, 200); const finalBody = await finalPage.json(); assert.deepEqual(finalBody.events.map((event) => event.label), ["event:4"]); + assert.equal(finalBody.events[0].terminal, false); + assert.equal(finalBody.events[0].diagnosticOnly, true); + assert.equal(finalBody.events[0].authority, "trace-store-diagnostics"); + assert.equal(finalBody.terminal, false); + assert.equal(finalBody.terminalAuthority, "workbench-projection"); assert.equal(finalBody.hasMore, false); assert.equal(finalBody.truncated, false); assert.equal(finalBody.fullTraceLoaded, true); @@ -158,6 +218,116 @@ test("cloud api trace resource paginates events by sinceSeq", async () => { } }); +test("cloud api trace pagination assigns a collision-free response cursor across diagnostic and projected seq", async () => { + const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); + const traceId = "trc_trace_pagination_collision"; + const sessionId = "ses_trace_pagination_collision"; + traceStore.append(traceId, { seq: 1, type: "diagnostic", status: "running", label: "diagnostic:seq-1", terminal: false }); + const built = buildWorkbenchProjectionEventFacts({ + projectedSeq: 1, + projectedAt: "2026-07-10T10:00:00.000Z", + event: { + traceId, + sessionId, + sourceEventId: "evt_trace_pagination_collision_1", + sourceSeq: 1, + type: "backend", + label: "projection:seq-1", + status: "running", + terminal: false, + createdAt: "2026-07-10T10:00:00.000Z" + } + }); + await runtimeStore.writeWorkbenchFacts({ facts: built.facts }, { traceId, sessionId, valuesPrinted: false }); + const server = createCloudApiServer({ + traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, + accessController: { + required: false, + async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; } + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const first = await fetch(`http://127.0.0.1:${port}/v1/agent/traces/${traceId}?limit=1`); + assert.equal(first.status, 200); + const firstBody = await first.json(); + assert.equal(firstBody.events.length, 1); + assert.equal(firstBody.events[0].seq, 1); + assert.equal(firstBody.events[0].projectedSeq, 1); + assert.equal(firstBody.nextSinceSeq, 1); + assert.equal(firstBody.hasMore, true); + + const second = await fetch(`http://127.0.0.1:${port}/v1/agent/traces/${traceId}?sinceSeq=${firstBody.nextSinceSeq}&limit=1`); + assert.equal(second.status, 200); + const secondBody = await second.json(); + assert.equal(secondBody.events.length, 1); + assert.equal(secondBody.events[0].seq, 2); + assert.equal(secondBody.events[0].sourceSeq, 1); + assert.equal(secondBody.events[0].diagnosticOnly, true); + assert.equal(secondBody.hasMore, false); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("Code Agent steer fails closed when terminal projection authority is unavailable", async () => { + const traceId = "trc_steer_projection_store_unavailable"; + const codeAgentChatResults = new Map([[traceId, { + accepted: true, + status: "running", + traceId, + ownerUserId: TEST_AGENT_ACTOR.id, + sessionId: "ses_steer_projection_store_unavailable", + agentRun: { + adapter: "agentrun-v01", + runId: "run_steer_projection_store_unavailable", + commandId: "cmd_steer_projection_store_unavailable", + dispatchIntentId: "dispatch_steer_projection_store_unavailable", + durableDispatch: true + } + }]]); + let projectionQueryCount = 0; + const workbenchRuntime = { + async queryWorkbenchFacts() { + projectionQueryCount += 1; + const error = new Error("projection database unavailable"); + error.code = "TEST_PROJECTION_DB_UNAVAILABLE"; + throw error; + } + }; + const server = createCloudApiServer({ + codeAgentChatResults, + workbenchRuntime, + accessController: { + required: false, + async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; } + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/steer`, { + method: "POST", + headers: { "content-type": "application/json", "x-trace-id": traceId }, + body: JSON.stringify({ traceId, message: "must not dispatch while terminal authority is unreadable" }) + }); + assert.equal(response.status, 503); + const body = await response.json(); + assert.equal(body.accepted, false); + assert.equal(body.status, "blocked"); + assert.equal(body.error.code, "projection_store_unavailable"); + assert.ok(projectionQueryCount > 0); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + test("cloud api turn and trace endpoints reject mismatched requested project", async () => { const traceStore = createCodeAgentTraceStore(); const traceId = "trc_issue1429_project_scope"; @@ -210,15 +380,13 @@ test("AgentRun adapter filters resource tools and credentials through access cap }; if (request.method === "POST" && url.pathname === "/api/v1/sessions") return send({ ok: true }); if (request.method === "POST" && url.pathname === "/api/v1/runs") return send({ id: "run_access_tools", status: "pending", backendProfile: "deepseek", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_access_tools/commands") return send({ id: "cmd_access_tools", runId: "run_access_tools", state: "pending", type: "turn", seq: 1 }); - if (request.method === "POST" && url.pathname === "/api/v1/runs/run_access_tools/runner-jobs") return send({ - action: "create-kubernetes-job", + if (request.method === "POST" && url.pathname === "/api/v1/runs/run_access_tools/commands") return send({ + id: "cmd_access_tools", runId: "run_access_tools", - commandId: body.commandId, - attemptId: "attempt_access_tools", - runnerId: "runner_access_tools", - namespace: "agentrun-v01", - jobName: "agentrun-v01-runner-access-tools" + state: "pending", + type: "turn", + seq: 1, + dispatchIntent: { id: "dispatch_access_tools", state: "pending", runnerJobId: "rjob_access_tools", attemptCount: 0, durable: true } }); response.writeHead(404, { "content-type": "application/json" }); response.end(`${JSON.stringify({ ok: false, message: `unexpected ${request.method} ${url.pathname}` })}\n`); @@ -288,8 +456,9 @@ test("AgentRun adapter filters resource tools and credentials through access cap assert.equal(Object.hasOwn(createRun.body.resourceBundleRef, "skillRefs"), false); assert.equal(Object.hasOwn(createRun.body.resourceBundleRef, "workspaceFiles"), false); assert.deepEqual(createRun.body.executionPolicy.secretScope.toolCredentials, []); - const runnerJob = calls.find((call) => call.method === "POST" && call.path === "/api/v1/runs/run_access_tools/runner-jobs"); - assert.equal(runnerJob.body.transientEnv.some((entry) => entry.name === "HWLAB_API_KEY"), false); + const command = calls.find((call) => call.method === "POST" && call.path === "/api/v1/runs/run_access_tools/commands"); + assert.equal(command.body.dispatch.input.transientEnv.some((entry) => entry.name === "HWLAB_API_KEY"), false); + assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); } finally { await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); } @@ -297,6 +466,10 @@ test("AgentRun adapter filters resource tools and credentials through access cap test("cloud api /v1/agent/chat parse and params errors use structured blocker envelope", async () => { const server = createCloudApiServer({ + accessController: { + required: false, + async authenticate() { return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; } + }, env: { HWLAB_CODE_AGENT_MODEL: "gpt-test" } @@ -402,7 +575,9 @@ test("cloud api /v1/agent/chat supports short submit and result polling", async assert.equal(accepted.turnId, traceId); assert.equal(accepted.userMessageId, "msg_server-test-short-submit_user"); assert.equal(accepted.assistantMessageId, "msg_server-test-short-submit_agent"); - assert.equal(accepted.resultUrl, `/v1/agent/chat/result/${traceId}`); + assert.equal(accepted.resultUrl, undefined); + assert.equal(accepted.controlSemantics, "submit-and-project-sse"); + assert.match(accepted.workbenchEventsUrl, new RegExp(`traceId=${traceId}$`, "u")); const acceptedTurn = await fetch(`http://127.0.0.1:${port}/v1/agent/turns/${encodeURIComponent(traceId)}`, { headers: { cookie: manualSession.cookie } }); assert.equal(acceptedTurn.status, 200); @@ -411,7 +586,7 @@ test("cloud api /v1/agent/chat supports short submit and result polling", async assert.equal(acceptedTurnPayload.userMessageId, accepted.userMessageId); assert.equal(acceptedTurnPayload.assistantMessageId, accepted.assistantMessageId); - const payload = await pollAgentResult(port, traceId); + const payload = await pollAgentResult(port, traceId, { headers: { cookie: manualSession.cookie } }); validateCodeAgentChatSchema(payload); assert.equal(payload.status, "completed"); assert.equal(payload.traceId, traceId); @@ -521,8 +696,36 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte assert.equal(body.payload.prompt.includes("sk-test-hwpod-secret"), false); } else { assert.equal(Object.hasOwn(body.payload, "conversationContext"), false); + assert.equal(body.dispatch?.kind, "kubernetes-job"); + const transientEnv = Object.fromEntries(body.dispatch.input.transientEnv.map((entry) => [entry.name, entry.value])); + assert.equal(transientEnv.HWLAB_RUNTIME_API_URL, "http://hwlab-cloud-api.hwlab-v03.svc.cluster.local:6667"); + assert.equal(transientEnv.HWLAB_RUNTIME_WEB_URL, "http://hwlab-cloud-web.hwlab-v03.svc.cluster.local:8080"); + assert.equal(transientEnv.HWLAB_RUNTIME_NAMESPACE, "hwlab-v03"); + assert.equal(transientEnv.HWLAB_RUNTIME_LANE, "v03"); + assert.equal(transientEnv.HWLAB_RUNTIME_ENDPOINT_LOCKED, "1"); + assert.equal(transientEnv.HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME, "1"); + assert.equal(transientEnv.UNIDESK_MAIN_SERVER_IP, "http://74.48.78.17:18081"); + assert.equal(transientEnv.HWLAB_CODE_AGENT_PROVIDER_PROFILE, "deepseek"); + assert.equal(transientEnv.HWLAB_CODE_AGENT_PARENT_TRACE_ID, "trc_server-test-agentrun-adapter"); + assert.equal(Object.hasOwn(transientEnv, "UNIDESK_SSH_CLIENT_TOKEN"), false); + assert.equal(Object.hasOwn(transientEnv, "GH_TOKEN"), false); } - return send({ id: secondTurn ? "cmd_hwlab_adapter_second" : "cmd_hwlab_adapter", runId: "run_hwlab_adapter", state: "pending", type: "turn", seq: secondTurn ? 2 : 1 }); + return send({ + id: secondTurn ? "cmd_hwlab_adapter_second" : "cmd_hwlab_adapter", + runId: "run_hwlab_adapter", + state: "pending", + type: "turn", + seq: secondTurn ? 2 : 1, + ...(secondTurn ? {} : { + dispatchIntent: { + id: "dispatch_hwlab_adapter", + state: "pending", + runnerJobId: "rjob_hwlab_adapter", + attemptCount: 0, + durable: true + } + }) + }); } if (request.method === "GET" && url.pathname === "/api/v1/runs/run_hwlab_adapter/commands") { return send({ items: [ @@ -624,8 +827,11 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte const agentRunPort = agentRunServer.address().port; const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); const server = createCloudApiServer({ traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, @@ -698,12 +904,27 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte assert.equal(accepted.turnId, traceId); assert.equal(accepted.userMessageId, "msg_server-test-agentrun-adapter_user"); assert.equal(accepted.assistantMessageId, "msg_server-test-agentrun-adapter_agent"); + assert.equal(accepted.admission.state, "promoted"); + assert.equal(accepted.admission.commandId, "cmd_hwlab_adapter"); + assert.equal(accepted.admission.dispatchIntentId, "dispatch_hwlab_adapter"); + assert.equal(accepted.admission.durableDispatch, true); - for (let index = 0; index < 50 && !traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:result:completed"); index += 1) await delay(10); - const projectedTrace = traceStore.snapshot(traceId); - assert.equal(projectedTrace.status, "completed"); - assert.ok(projectedTrace.events.some((event) => event.label === "agentrun:assistant:message")); - assert.ok(projectedTrace.events.some((event) => event.label === "agentrun:result:completed")); + await waitForTraceEvent( + traceStore, + traceId, + (event) => event.label === "agentrun:dispatch-intent:admitted", + "AgentRun command must be durably admitted before Kafka projection" + ); + await commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId: "ses_server-test-agentrun", + runId: "run_hwlab_adapter", + commandId: "cmd_hwlab_adapter", + text: "AgentRun adapter 已接管 HWLAB Code Agent。", + projectedSeq: 1, + sourceSeq: 5, + createdAt: "2026-06-01T00:00:02.000Z" + }); const acceptedTurn = await fetch(`http://127.0.0.1:${port}/v1/agent/turns/${encodeURIComponent(traceId)}`, { headers: { cookie: "hwlab_session=test-stub-session" } }); assert.equal(acceptedTurn.status, 200); @@ -713,51 +934,27 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte assert.equal(acceptedTurnPayload.assistantMessageId, accepted.assistantMessageId); const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); assert.equal(payload.status, "completed"); assert.equal(payload.turnId, traceId); assert.equal(payload.userMessageId, accepted.userMessageId); assert.equal(payload.assistantMessageId, accepted.assistantMessageId); - assert.equal(payload.provider, "deepseek"); - assert.equal(payload.backend, "agentrun-v01/deepseek"); - assert.equal(payload.infrastructureBackend, "agentrun-v01/deepseek"); - assert.equal(payload.capabilityLevel, "agentrun-v01-shared-code-agent-session"); - assert.equal(payload.sessionMode, "agentrun-v01-durable-session"); - assert.equal(payload.implementationType, "agentrun-v01-shared-execution-infra"); - assert.equal(payload.runner.kind, "agentrun-v01-shared-runner"); - assert.equal(payload.runner.provider, "deepseek"); - assert.equal(payload.runner.codexStdio, false); - assert.equal(payload.runner.delegatedToAgentRun, true); - assert.equal(payload.providerTrace.protocol, "agentrun-v01-jsonrpc"); - assert.equal(payload.providerTrace.command, "agentrun.v01.command.turn"); - assert.equal(payload.providerTrace.runnerKind, "agentrun-v01-shared-runner"); - assert.equal(payload.providerTrace.terminalStatus, "completed"); - assert.equal(payload.longLivedSessionGate.status, "pass"); - assert.equal(payload.longLivedSessionGate.provider, "deepseek"); - assert.equal(payload.longLivedSessionGate.codexStdio, false); - assert.equal(payload.longLivedSessionGate.delegatedToAgentRun, true); assert.equal(payload.agentRun.runId, "run_hwlab_adapter"); assert.equal(payload.agentRun.commandId, "cmd_hwlab_adapter"); - assert.equal(payload.agentRun.jobName, "agentrun-v01-runner-hwlab-adapter"); + assert.equal(payload.agentRun.dispatchIntentId, "dispatch_hwlab_adapter"); + assert.equal(payload.agentRun.runnerJobId, "rjob_hwlab_adapter"); + assert.equal(payload.agentRun.durableDispatch, true); + assert.equal(payload.projection.sourceRunId, "run_hwlab_adapter"); + assert.equal(payload.projection.sourceCommandId, "cmd_hwlab_adapter"); + assert.equal(payload.projection.projectionStatus, "caught-up"); const serializedPayload = JSON.stringify(payload); assert.equal(serializedPayload.includes("repo-owned-codex"), false); assert.equal(serializedPayload.includes("codex-app-server-stdio"), false); assert.equal(serializedPayload.includes("\"provider\":\"codex-stdio\""), false); assert.equal(serializedPayload.includes("\"codexStdio\":true"), false); - assert.match(payload.reply.content, /AgentRun adapter/u); - assert.ok(payload.runnerTrace.events.some((event) => event.label === "agentrun:backend:runner-job-created")); - const bundleEvent = payload.runnerTrace.events.find((event) => event.label === "agentrun:backend:resource-bundle-materialized"); - assert.equal(bundleEvent?.details?.kind, "gitbundle"); - assert.deepEqual(bundleEvent?.details?.bundles?.names, ["hwlab-tools", "hwlab-agent-skills"]); - assert.deepEqual(bundleEvent?.details?.tools?.names, ["hwpod", "hwpod-ctl", "hwpod-compiler", "unidesk-ssh", "hwlab-code-agent"]); - assert.deepEqual(bundleEvent?.details?.promptRefs?.names, ["hwlab-v02-runtime"]); - assert.deepEqual(bundleEvent?.details?.skillDirs?.names, ["hwpod-cli", "hwpod-ctl", "hwlab-agent-runtime", "hwlab-code-agent"]); - const promptEvent = payload.runnerTrace.events.find((event) => event.label === "agentrun:backend:initial-prompt-assembly"); - assert.equal(promptEvent?.details?.initialPromptInjected, true); - assert.equal(promptEvent?.details?.reason, "thread-start"); - assert.equal(promptEvent?.details?.initialPrompt?.promptRefCount, 1); - assert.equal(promptEvent?.details?.initialPrompt?.skillCount, 4); - assert.ok(calls.some((call) => call.path === "/api/v1/runs/run_hwlab_adapter/runner-jobs")); + assert.equal(payload.finalResponse.text, "AgentRun adapter 已接管 HWLAB Code Agent。"); + assert.equal(calls.some((call) => call.path === "/api/v1/runs/run_hwlab_adapter/runner-jobs"), false); + assert.equal(calls.some((call) => call.path === "/api/v1/runs/run_hwlab_adapter/events"), false); + assert.equal(calls.some((call) => call.path.endsWith("/result")), false); const inspectByTrace = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/inspect?traceId=${traceId}`); assert.equal(inspectByTrace.status, 200); @@ -766,9 +963,9 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte assert.equal(inspectByTraceBody.latestTraceId, traceId); assert.equal(inspectByTraceBody.session.sessionId, "ses_server-test-agentrun"); assert.equal(inspectByTraceBody.session.conversationId, "cnv_server-test-agentrun"); - assert.equal(inspectByTraceBody.session.threadId, "019e8078-db67-7750-a5d9-1a99f3abd445"); + assert.equal(inspectByTraceBody.session.threadId, null); assert.equal(inspectByTraceBody.conversationFacts.conversationId, "cnv_server-test-agentrun"); - assert.equal(inspectByTraceBody.conversationFacts.threadId, "019e8078-db67-7750-a5d9-1a99f3abd445"); + assert.equal(inspectByTraceBody.conversationFacts.threadId, null); assert.equal(inspectByTraceBody.runnerTrace.traceId, traceId); assert.equal(inspectByTraceBody.valuesRedacted, true); assert.equal(JSON.stringify(inspectByTraceBody).includes("hwl_live_test_secret"), false); @@ -779,20 +976,18 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte const requestTraceEvent = traceBody.events.find((event) => event.label === "agentrun:request:accepted"); assert.equal(requestTraceEvent.eventType, "request"); assert.equal(requestTraceEvent.backend, "agentrun-v01/deepseek"); - const runnerJobTraceEvent = traceBody.events.find((event) => event.label === "agentrun:runner-job:created"); - assert.equal(runnerJobTraceEvent.eventType, "backend"); - assert.equal(runnerJobTraceEvent.backend, "agentrun-v01/deepseek"); - assert.ok(traceBody.events.some((event) => event.runId === "run_hwlab_adapter")); - assert.ok(traceBody.events.some((event) => event.label === "agentrun:assistant:message")); - assert.equal(traceBody.events.some((event) => event.details?.initialPromptInjected === true), true); - const commandTraceEvent = traceBody.events.find((event) => event.label === "item/commandExecution:completed"); - assert.equal(commandTraceEvent.eventType, "tool_call"); - assert.equal(commandTraceEvent.backend, "agentrun-v01/deepseek"); - assert.ok(Date.parse(commandTraceEvent.appendedAt) >= Date.parse(commandTraceEvent.createdAt)); - assert.equal(commandTraceEvent.toolName, "commandExecution"); - assert.equal(commandTraceEvent.createdAt, "2026-06-01T00:00:00.500Z"); - assert.match(commandTraceEvent.command, /hwpod inspect --dry-run/u); - assert.match(commandTraceEvent.stdoutSummary, /hwpod-cli\.plan/u); + assert.equal(requestTraceEvent.terminal, false); + assert.equal(requestTraceEvent.diagnosticOnly, true); + assert.equal(requestTraceEvent.authority, "trace-store-diagnostics"); + const dispatchTraceEvent = traceBody.events.find((event) => event.label === "agentrun:dispatch-intent:admitted"); + assert.equal(dispatchTraceEvent.eventType, "backend"); + assert.equal(dispatchTraceEvent.terminal, false); + assert.equal(dispatchTraceEvent.diagnosticOnly, true); + const projectionTraceEvent = traceBody.events.find((event) => event.label === "agentrun:kafka:assistant-final"); + assert.equal(projectionTraceEvent.authority, "workbench-projection"); + assert.equal(projectionTraceEvent.terminal, true); + assert.equal(traceBody.terminal, true); + assert.equal(traceBody.terminalAuthority, "workbench-projection"); conversationMessages.push( { id: "msg_693_user", @@ -860,28 +1055,31 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte }) }); assert.equal(second.status, 202); + await waitForTraceEvent( + traceStore, + secondTraceId, + (event) => event.label === "agentrun:runner-job:reused", + "second turn must reuse the admitted AgentRun runner" + ); + await commitTestKafkaTerminal(runtimeStore, { + traceId: secondTraceId, + sessionId: "ses_server-test-agentrun", + runId: "run_hwlab_adapter", + commandId: "cmd_hwlab_adapter_second", + text: "AgentRun adapter 复用已有 runner 完成第二轮。", + projectedSeq: 2, + sourceSeq: 9, + createdAt: "2026-06-01T00:00:05.000Z" + }); const secondPayload = await pollAgentResult(port, secondTraceId); - validateCodeAgentChatSchema(secondPayload); assert.equal(secondPayload.status, "completed"); - assert.equal(secondPayload.provider, "deepseek"); - assert.equal(secondPayload.backend, "agentrun-v01/deepseek"); - assert.equal(secondPayload.runner.kind, "agentrun-v01-shared-runner"); - assert.equal(secondPayload.runner.codexStdio, false); assert.equal(secondPayload.agentRun.runId, "run_hwlab_adapter"); assert.equal(secondPayload.agentRun.commandId, "cmd_hwlab_adapter_second"); - assert.equal(secondPayload.providerTrace.commandId, "cmd_hwlab_adapter_second"); - assert.equal(secondPayload.providerTrace.traceId, secondTraceId); - assert.equal(secondPayload.agentRun.providerTrace.commandId, "cmd_hwlab_adapter_second"); - assert.equal(secondPayload.agentRun.providerTrace.traceId, secondTraceId); - assert.equal(secondPayload.agentRun.jobName, "agentrun-v01-runner-hwlab-adapter-second"); - assert.equal(secondPayload.sessionReuse.reused, true); - assert.equal(secondPayload.agentRun.runnerJobCount, 1); - assert.match(secondPayload.reply.content, /复用已有 runner/u); - assert.ok(secondPayload.runnerTrace.events.some((event) => event.label === "agentrun:run:reused")); - assert.ok(secondPayload.runnerTrace.events.some((event) => event.label === "agentrun:runner-job:ensured")); - assert.equal(secondPayload.runnerTrace.events.some((event) => String(event.text ?? event.message ?? "").includes("旧 command 尾部")), false); + assert.equal(secondPayload.agentRun.runnerJobCount, 0); + assert.match(secondPayload.finalResponse.text, /复用已有 runner/u); + assert.equal(secondPayload.projection.sourceCommandId, "cmd_hwlab_adapter_second"); assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs").length, 1); - assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs/run_hwlab_adapter/runner-jobs").length, 2); + assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs/run_hwlab_adapter/runner-jobs").length, 0); assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs/run_hwlab_adapter/commands").length, 2); } finally { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); @@ -889,7 +1087,7 @@ test("cloud api /v1/agent/chat delegates v0.3 turns to AgentRun v0.1 over adapte } }); -test("cloud api keeps admitted user message when billing preflight fails before AgentRun dispatch (#1619)", async () => { +test("cloud api rejects billing failure before durable AgentRun admission without false running state (#1619)", async () => { const traceId = "trc_issue1619_billing_after_admission"; const sessionId = "ses_issue1619_billing_after_admission"; const conversationId = "cnv_issue1619_billing_after_admission"; @@ -923,7 +1121,7 @@ test("cloud api keeps admitted user message when billing preflight fails before configured: true, async billingPreflight(body) { billingCalls.push(body); - assert.ok(ownerWrites.some((write) => write.traceId === traceId && write.status === "running"), "durable admission must be recorded before billing preflight"); + assert.equal(ownerWrites.some((write) => write.traceId === traceId && write.status === "running"), false, "billing must run before UI-authoritative admission"); return { ok: false, status: 503, error: { code: "billing_preflight_unavailable", message: "billing preflight unavailable" } }; } }, @@ -962,29 +1160,21 @@ test("cloud api keeps admitted user message when billing preflight fails before message: "测试一下和 cpython 对比的性能" }) }); - assert.equal(submit.status, 202); - const accepted = await submit.json(); - assert.equal(accepted.accepted, true); - assert.equal(accepted.traceId, traceId); - - const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); - assert.equal(payload.status, "failed"); - assert.equal(payload.error.code, "billing_preflight_unavailable"); - assert.match(payload.finalResponse.text, /billing preflight unavailable/u); + assert.equal(submit.status, 503); + const rejected = await submit.json(); + assert.equal(rejected.accepted, false); + assert.equal(rejected.status, "failed"); + assert.equal(rejected.traceId, traceId); + assert.equal(rejected.error.code, "billing_preflight_unavailable"); assert.equal(billingCalls.length, 1); assert.equal(agentRunCalls.length, 0); - assert.ok(ownerWrites.some((write) => write.traceId === traceId && write.status === "running")); - assert.ok(ownerWrites.some((write) => write.traceId === traceId && write.status === "failed")); + assert.equal(ownerWrites.some((write) => write.traceId === traceId && ["running", "failed"].includes(write.status)), false); const stored = ownerSessions.get(sessionId); const userMessage = stored?.session?.messages?.find((message) => message.role === "user"); - assert.match(userMessage?.text ?? "", /cpython/u); - - const turn = await fetch(`http://127.0.0.1:${port}/v1/agent/turns/${traceId}`, { headers: { cookie: "hwlab_session=test-stub-session" } }); - assert.equal(turn.status, 200); - const turnBody = await turn.json(); - assert.equal(turnBody.status, "failed"); - assert.equal(turnBody.error.code, "billing_preflight_unavailable"); + assert.equal(userMessage, undefined); + const diagnosticTrace = traceStore.snapshot(traceId); + assert.equal(diagnosticTrace.events.some((event) => event.label === "turn:admitting"), true); + assert.equal(diagnosticTrace.events.some((event) => event.terminal === true), false); } finally { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); @@ -994,7 +1184,7 @@ test("cloud api keeps admitted user message when billing preflight fails before test("cloud api AgentRun adapter reports persistent thread resume when a completed run needs a new runner", async () => { const calls = []; const hwlabSessionId = "ses_server-test-thread-resume"; - const agentRunSessionId = "ses_agentrun_deepseek_server_test_thread_resume"; + const agentRunSessionId = "ses_agentrun_server_test_thread_resume"; const conversationId = "cnv_server-test-thread-resume"; const threadId = "019e90e0-9535-7130-a894-47ef4e127206"; const ownerSessions = new Map([[hwlabSessionId, testAgentSessionRecord({ @@ -1045,7 +1235,7 @@ test("cloud api AgentRun adapter reports persistent thread resume when a complet if (request.method === "POST" && url.pathname === "/api/v1/sessions") { assert.equal(body.sessionId, agentRunSessionId); assert.equal(body.backendProfile, "deepseek"); - return send({ sessionId: body.sessionId, backendProfile: body.backendProfile }); + return send({ sessionId: body.sessionId, backendProfile: body.backendProfile, threadId }); } if (request.method === "POST" && url.pathname === "/api/v1/runs") { assert.equal(body.sessionRef.sessionId, agentRunSessionId); @@ -1058,9 +1248,22 @@ test("cloud api AgentRun adapter reports persistent thread resume when a complet assert.equal(body.type, "turn"); assert.equal(body.payload.sessionId, agentRunSessionId); assert.equal(body.payload.hwlabSessionId, hwlabSessionId); - assert.equal(body.payload.threadId, null); + assert.equal(body.payload.threadId, threadId); assert.match(body.payload.prompt, /ISSUE812_RESUME/u); - return send({ id: "cmd_issue812_resume", runId: "run_issue812_resume", state: "pending", type: "turn", seq: 1 }); + return send({ + id: "cmd_issue812_resume", + runId: "run_issue812_resume", + state: "pending", + type: "turn", + seq: 1, + dispatchIntent: { + id: "dispatch_issue812_resume", + state: "pending", + runnerJobId: "rjob_issue812_resume", + attemptCount: 0, + durable: true + } + }); } if (request.method === "GET" && url.pathname === "/api/v1/runs/run_issue812_resume/commands") { return send({ items: [ @@ -1118,8 +1321,11 @@ test("cloud api AgentRun adapter reports persistent thread resume when a complet record.session.agentRun.managerUrl = `http://127.0.0.1:${agentRunPort}`; } const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); const server = createCloudApiServer({ traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, @@ -1169,38 +1375,47 @@ test("cloud api AgentRun adapter reports persistent thread resume when a complet }); assert.equal(submit.status, 202); + await waitForTraceEvent(traceStore, traceId, (event) => event.label === "agentrun:dispatch-intent:admitted"); + await commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId: hwlabSessionId, + runId: "run_issue812_resume", + commandId: "cmd_issue812_resume", + text: "ISSUE812_RESUME_OK", + projectedSeq: 1, + sourceSeq: 4, + createdAt: "2026-06-04T00:00:02.000Z" + }); + const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); assert.equal(payload.status, "completed"); assert.equal(payload.sessionId, hwlabSessionId); assert.equal(payload.threadId, threadId); assert.equal(payload.agentRun.runId, "run_issue812_resume"); assert.equal(payload.agentRun.reused, false); assert.equal(payload.agentRun.runnerReused, false); - assert.equal(payload.agentRun.threadReused, false); - assert.equal(payload.agentRun.persistentResume, false); - assert.equal(payload.sessionReuse.threadId, threadId); - assert.equal(payload.sessionReuse.reused, true); - assert.equal(payload.sessionReuse.status, "thread-resumed"); - assert.equal(payload.sessionReuse.runnerReused, false); - assert.equal(payload.sessionReuse.threadReused, true); - assert.equal(payload.sessionReuse.persistentResume, true); - assert.equal(payload.reply.content, "ISSUE812_RESUME_OK"); + assert.equal(payload.agentRun.threadReused, true); + assert.equal(payload.agentRun.persistentResume, true); + assert.equal(payload.finalResponse.text, "ISSUE812_RESUME_OK"); + assert.equal(payload.agentRun.dispatchIntentId, "dispatch_issue812_resume"); assert.equal(calls.some((call) => call.method === "GET" && call.path === "/api/v1/runs/run_issue812_previous"), true); assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs").length, 1); - assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs/run_issue812_resume/runner-jobs").length, 1); + assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs/run_issue812_resume/runner-jobs").length, 0); + assert.equal(calls.some((call) => call.path.endsWith("/result")), false); } finally { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); } }); -test("cloud api AgentRun adapter exposes invalid tool-call attribution in result payload", async () => { +test("cloud api AgentRun adapter exposes invalid tool-call attribution through Kafka projection trace", async () => { + const calls = []; const agentRunServer = createHttpServer(async (request, response) => { const url = new URL(request.url || "/", "http://127.0.0.1"); const chunks = []; for await (const chunk of request) chunks.push(Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk)); const body = chunks.length ? JSON.parse(Buffer.concat(chunks).toString("utf8")) : null; + calls.push({ method: request.method, path: url.pathname, body }); const send = (data) => { response.writeHead(200, { "content-type": "application/json" }); response.end(`${JSON.stringify({ ok: true, data, traceId: "trc_fake_agentrun_invalid_tool" })}\n`); @@ -1209,7 +1424,20 @@ test("cloud api AgentRun adapter exposes invalid tool-call attribution in result return send({ id: "run_invalid_tool", status: "pending", backendProfile: "deepseek", sessionRef: body.sessionRef, resourceBundleRef: body.resourceBundleRef }); } if (request.method === "POST" && url.pathname === "/api/v1/runs/run_invalid_tool/commands") { - return send({ id: "cmd_invalid_tool", runId: "run_invalid_tool", state: "pending", type: "turn", seq: 1 }); + return send({ + id: "cmd_invalid_tool", + runId: "run_invalid_tool", + state: "pending", + type: "turn", + seq: 1, + dispatchIntent: { + id: "dispatch_invalid_tool", + state: "pending", + runnerJobId: "rjob_invalid_tool", + attemptCount: 0, + durable: true + } + }); } if (request.method === "GET" && url.pathname === "/api/v1/runs/run_invalid_tool/commands") { return send({ items: [ @@ -1264,7 +1492,12 @@ test("cloud api AgentRun adapter exposes invalid tool-call attribution in result conversationId: "cnv_invalid_tool", status: "idle" })]]); + const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); const server = createCloudApiServer({ + traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, @@ -1301,27 +1534,35 @@ test("cloud api AgentRun adapter exposes invalid tool-call attribution in result body: JSON.stringify({ conversationId: "cnv_invalid_tool", sessionId: "ses_server-test-invalid-tool", message: "invalid tool-call attribution" }) }); assert.equal(submit.status, 202); + await waitForTraceEvent(traceStore, traceId, (event) => event.label === "agentrun:dispatch-intent:admitted"); + await commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId: "ses_server-test-invalid-tool", + runId: "run_invalid_tool", + commandId: "cmd_invalid_tool", + status: "failed", + text: "invalid params, invalid function arguments json string, tool_call_id: call_function_selftest_2 (2013)", + failureKind: "provider-invalid-tool-call", + projectedSeq: 1, + sourceSeq: 2, + createdAt: "2026-06-02T00:00:01.000Z" + }); const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); assert.equal(payload.status, "failed"); - assert.equal(payload.providerTrace.failureKind, "provider-invalid-tool-call"); - assert.equal(payload.error.code, "provider-invalid-tool-call"); - assert.equal(payload.error.category, "provider_invalid_tool_call"); - assert.match(payload.error.userMessage, /无效 tool-call arguments JSON/u); - assert.match(payload.error.userMessage, /AgentRun\/provider/u); - assert.match(payload.error.userMessage, /不是 HWPOD/u); assert.equal(payload.finalResponse.status, "failed"); assert.match(payload.finalResponse.text, /invalid function arguments json string/u); - assert.equal(payload.traceSummary.terminalStatus, "failed"); - assert.match(payload.traceSummary.finalAssistantRow.textPreview, /invalid function arguments json string/u); - assert.equal(payload.blocker?.category ?? payload.error.category, "provider_invalid_tool_call"); - assert.match(payload.blocker?.summary ?? payload.error.userMessage, /invalid function arguments json string/u); - assert.equal(payload.session.status, "failed"); - assert.equal(payload.session.lifecycle.requiresNewSession, false); - assert.equal(payload.sessionReuse.threadId, "thread_invalid_tool"); - assert.equal(payload.sessionReuse.status, "failed-resumable"); assert.equal(payload.agentRun.reuseEligible, true); - assert.equal(payload.reuseEligible, true); + assert.equal(payload.projection.sourceRunId, "run_invalid_tool"); + assert.equal(payload.projection.sourceCommandId, "cmd_invalid_tool"); + const trace = await fetch(`http://127.0.0.1:${port}/v1/agent/traces/${traceId}`); + assert.equal(trace.status, 200); + const traceBody = await trace.json(); + const projectedFailure = traceBody.events.find((event) => event.label === "agentrun:kafka:assistant-final"); + assert.equal(projectedFailure.authority, "workbench-projection"); + assert.equal(projectedFailure.failureKind, "provider-invalid-tool-call"); + assert.equal(projectedFailure.terminal, true); + assert.equal(calls.some((call) => call.path.endsWith("/result")), false); + assert.equal(calls.some((call) => call.path.endsWith("/events")), false); const serializedPayload = JSON.stringify(payload); assert.equal(serializedPayload.includes("repo-owned-codex"), false); assert.equal(serializedPayload.includes("codex-app-server-stdio"), false); @@ -1332,7 +1573,7 @@ test("cloud api AgentRun adapter exposes invalid tool-call attribution in result } }); -test("cloud api AgentRun adapter retries evicted session storage with a fresh session", async () => { +test("cloud api surfaces manager-side evicted-session recovery through Kafka projection", async () => { const calls = []; const agentRunServer = createHttpServer(async (request, response) => { const url = new URL(request.url || "/", "http://127.0.0.1"); @@ -1456,7 +1697,12 @@ test("cloud api AgentRun adapter retries evicted session storage with a fresh se threadId: "thread_evicted", status: "idle" })]]); + const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); const server = createCloudApiServer({ + traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, @@ -1493,20 +1739,27 @@ test("cloud api AgentRun adapter retries evicted session storage with a fresh se body: JSON.stringify({ conversationId: "cnv_evicted", sessionId: "ses_server-test-evicted", threadId: "thread_evicted", message: "resume evicted session" }) }); assert.equal(submit.status, 202); + await waitForTraceEvent(traceStore, traceId, (event) => event.label === "agentrun:dispatch-intent:admitted"); + await commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId: "ses_server-test-evicted", + runId: "run_evicted_retry", + commandId: "cmd_evicted_retry", + text: "EVICTED_RETRY_OK", + projectedSeq: 1, + sourceSeq: 3, + createdAt: "2026-06-02T00:00:04.000Z" + }); const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); assert.equal(payload.status, "completed"); - assert.equal(payload.reply.content, "EVICTED_RETRY_OK"); - assert.equal(payload.agentRun.runId, "run_evicted_retry"); - assert.equal(payload.agentRun.commandId, "cmd_evicted_retry"); - assert.equal(payload.agentRun.freshSessionRetryOf.runId, "run_evicted"); - assert.equal(payload.agentRun.freshSessionRetryOf.commandId, "cmd_evicted"); - assert.match(payload.agentRun.sessionId, /-reset-/u); - assert.equal(payload.session.threadId, "thread_evicted_retry"); - assert.equal(payload.sessionReuse.status, "thread-resumed"); - assert.equal(payload.error, undefined); - assert.equal(ownerSessions.get("ses_server-test-evicted")?.status, "completed"); - assert.equal(calls.some((call) => call.method === "POST" && call.path === "/api/v1/runs/run_evicted_retry/commands"), true); + assert.equal(payload.finalResponse.text, "EVICTED_RETRY_OK"); + assert.equal(payload.agentRun.runId, "run_evicted"); + assert.equal(payload.agentRun.commandId, "cmd_evicted"); + assert.equal(payload.projection.sourceRunId, "run_evicted_retry"); + assert.equal(payload.projection.sourceCommandId, "cmd_evicted_retry"); + assert.equal(calls.some((call) => call.method === "POST" && call.path === "/api/v1/runs/run_evicted_retry/commands"), false); + assert.equal(calls.some((call) => call.path.endsWith("/result")), false); + assert.equal(calls.some((call) => call.path.endsWith("/events")), false); assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); } finally { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); @@ -1601,7 +1854,12 @@ test("cloud api AgentRun adapter maps minimax-m3 provider profile to AgentRun ba conversationId: "cnv_server-test-minimax-m3", status: "idle" })]]); + const traceStore = createCodeAgentTraceStore(); + const runtimeStore = createCloudRuntimeStore(); const server = createCloudApiServer({ + traceStore, + runtimeStore, + workbenchRuntime: runtimeStore, env: { HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", AGENTRUN_MGR_URL: `http://127.0.0.1:${agentRunPort}`, @@ -1646,23 +1904,31 @@ test("cloud api AgentRun adapter maps minimax-m3 provider profile to AgentRun ba const accepted = await submit.json(); assert.equal(accepted.shortConnection, true); + await waitForTraceEvent(traceStore, traceId, (event) => event.label === "agentrun:dispatch-intent:admitted"); + await commitTestKafkaTerminal(runtimeStore, { + traceId, + sessionId: "ses_server-test-minimax-m3", + runId: "run_hwlab_minimax_m3", + commandId: "cmd_hwlab_minimax_m3", + text: "AGENTRUN_MINIMAX_M3_OK", + projectedSeq: 1, + sourceSeq: 3, + createdAt: "2026-06-02T00:00:02.000Z" + }); + const payload = await pollAgentResult(port, traceId); - validateCodeAgentChatSchema(payload); assert.equal(payload.status, "completed"); - assert.equal(payload.provider, "minimax-m3"); - assert.equal(payload.backend, "agentrun-v01/minimax-m3"); - assert.equal(payload.infrastructureBackend, "agentrun-v01/minimax-m3"); - assert.equal(payload.runner.kind, "agentrun-v01-shared-runner"); - assert.equal(payload.runner.provider, "minimax-m3"); - assert.equal(payload.runner.codexStdio, false); assert.equal(payload.agentRun.backendProfile, "minimax-m3"); assert.equal(payload.sessionId, "ses_server-test-minimax-m3"); assert.equal(payload.agentRun.sessionId, "ses_agentrun_server_test_minimax_m3"); - assert.equal(payload.providerTrace.backendProfile, "minimax-m3"); - assert.equal(payload.agentRun.jobName, "agentrun-v01-runner-hwlab-minimax-m3"); - assert.equal(payload.reply.content, "AGENTRUN_MINIMAX_M3_OK"); + assert.equal(payload.agentRun.dispatchIntentId, "dispatch_hwlab_minimax_m3"); + assert.equal(payload.finalResponse.text, "AGENTRUN_MINIMAX_M3_OK"); + assert.equal(payload.projection.sourceRunId, "run_hwlab_minimax_m3"); + assert.equal(payload.projection.sourceCommandId, "cmd_hwlab_minimax_m3"); assert.equal(calls.filter((call) => call.method === "POST" && call.path === "/api/v1/runs").length, 1); assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false); + assert.equal(calls.some((call) => call.path.endsWith("/events")), false); + assert.equal(calls.some((call) => call.path.endsWith("/result")), false); } finally { await new Promise((resolve, reject) => server.close((error) => (error ? reject(error) : resolve()))); await new Promise((resolve, reject) => agentRunServer.close((error) => (error ? reject(error) : resolve()))); diff --git a/internal/cloud/server-code-agent-admission-http.ts b/internal/cloud/server-code-agent-admission-http.ts index 293c9a70..edb557f9 100644 --- a/internal/cloud/server-code-agent-admission-http.ts +++ b/internal/cloud/server-code-agent-admission-http.ts @@ -17,8 +17,11 @@ import { createCodeAgentErrorPayload, handleCodeAgentChat } from "./code-agent-c import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle.ts"; import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts"; -import { writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts"; +import { promoteWorkbenchTurnAdmission as persistWorkbenchTurnAdmissionPromotion, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts"; import { createWorkbenchReadModel } from "./workbench-read-model.ts"; +import { + workbenchRealtimeCapabilities +} from "./workbench-realtime-capabilities.ts"; import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts"; import { firstHeaderValue, @@ -38,6 +41,7 @@ import { normalizeCodeAgentProviderProfile, firstNonEmptyValue, recordCodeAgentSessionInputFact, + buildCodeAgentSessionInputFact, stableCodeAgentInputId, codeAgentInputDelivery, codeAgentInputStatus, @@ -209,7 +213,7 @@ export async function handleCodeAgentChatHttp(request, response, options) { traceId }); } catch (error) { - if (isCodeAgentAdmissionUnavailableError(error)) { + if (isCodeAgentSubmissionUnavailableError(error)) { sendJson(response, error.statusCode ?? 503, error.payload); return; } @@ -240,6 +244,16 @@ export async function handleCodeAgentChatHttp(request, response, options) { lastEventAt: submitted?.lastEventAt ?? admittedTiming.lastEventAt, finishedAt: submitted?.finishedAt ?? admittedTiming.finishedAt, durationMs: submitted?.durationMs ?? admittedTiming.durationMs, + realtimeCapabilities: workbenchRealtimeCapabilities(options.env ?? process.env), + admission: submitted?.agentRun ? { + state: submitted.admissionState ?? "promoted", + runId: submitted.agentRun.runId ?? null, + commandId: submitted.agentRun.commandId ?? null, + dispatchIntentId: submitted.agentRun.dispatchIntentId ?? null, + runnerJobId: submitted.agentRun.runnerJobId ?? null, + durableDispatch: submitted.agentRun.durableDispatch === true, + valuesRedacted: true + } : null, traceUrl, turnUrl, workbenchEventsUrl, @@ -1018,8 +1032,6 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) { projectId: params.projectId ?? null, ...codeAgentAdmissionTimingFields(timestamp) }; - await recordCodeAgentTurnAdmission({ payload: acceptedPayload, params, options, traceId }); - results.set(traceId, annotateOwner(acceptedPayload, params)); void emitCodeAgentOtelSpan("provider_decision", traceId, runtimeEnv, { attributes: { ...agentChatOtelAttributes(params), @@ -1033,68 +1045,133 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) { } }); if (adapterEnabled) { - const run = async () => { - let initial = null; - let executionOptions = options; + const realtimeCapabilities = workbenchRealtimeCapabilities(runtimeEnv); + const transactionalAdmission = realtimeCapabilities.transactionalProjector; + const pendingPromotion = results.get(traceId); + if (pendingPromotion?.admissionState === "promotion-pending" && pendingPromotion?.agentRun?.durableDispatch === true) { + const retryParams = { ...params, userBillingReservation: pendingPromotion.userBillingReservation ?? null }; try { - initial = initialAgentRunChatResult({ params, options, traceId }); - results.set(traceId, annotateOwner(initial, params)); - const billingStartedAt = Date.now(); - const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false }); - void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, { - startTimeMs: billingStartedAt, - attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) }, - status: billingPreflight?.blocked ? "error" : "ok", - error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null - }); - if (billingPreflight?.blocked) { - await settleAdmittedCodeAgentFailure({ - base: initial, - params, - options, - traceId, - results, - failure: billingPreflight, - traceLabel: "billing:preflight:failed" - }); - return; - } - const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null }; - initial = withCodeAgentBillingReservation(initial, dispatchParams); - results.set(traceId, annotateOwner(initial, dispatchParams)); - executionOptions = { ...options, ...(await codeAgentChatExecutionOptions(options, dispatchParams)) }; - const dispatchStartedAt = Date.now(); - const payload = await submitAgentRunChatTurn({ params: dispatchParams, options: executionOptions, traceId, traceStore, results }); - void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { - startTimeMs: dispatchStartedAt, - attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, runId: payload?.agentRun?.runId ?? null, commandId: payload?.agentRun?.commandId ?? null, providerProfile: dispatchParams.providerProfile ?? null } - }); - if (isCodeAgentResultCanceled(results.get(traceId))) return; - const owned = annotateOwner(withCodeAgentBillingReservation(payload, dispatchParams), dispatchParams); - await recordCodeAgentSessionOwner({ payload: owned, params: dispatchParams, options: executionOptions, status: "running" }); - results.set(traceId, owned); + await promoteCodeAgentTurnAdmission({ payload: pendingPromotion, params: retryParams, options, traceId }); } catch (error) { - if (isCodeAgentResultCanceled(results.get(traceId))) return; - const dispatchErrorAttributes = codeAgentDispatchErrorOtelAttributes(error); - void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { - status: "error", - error, - attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, ...dispatchErrorAttributes } - }); - await settleAdmittedCodeAgentFailure({ - base: initial ?? codeAgentAdmittedFailureBasePayload({ params, options, traceId }), + throw codeAgentPromotionUnavailableError(error, { payload: pendingPromotion, params: retryParams, options, traceId }); + } + const promoted = annotateOwner({ ...pendingPromotion, accepted: true, status: "running", admissionState: "promoted" }, retryParams); + results.set(traceId, promoted); + return promoted; + } + + const lifecycle = codeAgentTurnLifecycleFields(traceId, params); + if (transactionalAdmission) { + await persistCodeAgentAdmissionInputState({ + params, + options, + traceId, + status: "admitting", + messageId: lifecycle.userMessageId + }); + } + + traceStore.append(traceId, { + type: "request", + status: "admitting", + label: "turn:admitting", + message: "Code Agent turn is validating billing and durable AgentRun dispatch before UI admission.", + terminal: false, + waitingFor: "billing-and-durable-dispatch", + valuesPrinted: false + }); + const initial = initialAgentRunChatResult({ params, options, traceId }); + const billingStartedAt = Date.now(); + const billingPreflight = await preflightCodeAgentBilling({ params, options, traceId, sendResponse: false }); + void emitCodeAgentOtelSpan("billing_preflight", traceId, options.env ?? process.env, { + startTimeMs: billingStartedAt, + attributes: { sessionId: safeSessionId(params.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, params).turnId, blocked: Boolean(billingPreflight?.blocked) }, + status: billingPreflight?.blocked ? "error" : "ok", + error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null + }); + if (billingPreflight?.blocked) { + if (transactionalAdmission) { + await persistCodeAgentAdmissionInputState({ params, - options: executionOptions, + options, traceId, - results, - failure: codeAgentDispatchFailure(error), - traceLabel: "agentrun:dispatch:failed" + status: "blocked", + messageId: lifecycle.userMessageId, + error: billingPreflight.payload?.error ?? billingPreflight.blocker ?? { code: "billing_preflight_blocked" } }); } - }; - setImmediate(() => { run(); }); - return annotateOwner(acceptedPayload, params); + traceStore.append(traceId, { + type: "billing", + status: "blocked", + label: "billing:preflight:blocked-before-admission", + errorCode: billingPreflight.payload?.error?.code ?? billingPreflight.blocker?.code ?? "billing_preflight_blocked", + terminal: false, + valuesPrinted: false + }); + throw codeAgentSubmissionUnavailableError(billingPreflight, { params, options, traceId }); + } + + const dispatchParams = { ...params, userBillingReservation: billingPreflight?.reservation ?? null }; + const executionOptions = { ...options, ...(await codeAgentChatExecutionOptions(options, dispatchParams)) }; + const dispatchStartedAt = Date.now(); + let payload; + try { + payload = await submitAgentRunChatTurn({ params: dispatchParams, options: executionOptions, traceId, traceStore, results }); + } catch (error) { + const failure = codeAgentDispatchFailure(error); + const rejected = codeAgentSubmissionUnavailablePayload(failure, { params: dispatchParams, options: executionOptions, traceId }); + await finalizeCodeAgentBillingUsage({ payload: rejected, params: dispatchParams, options: executionOptions }); + if (transactionalAdmission) { + await persistCodeAgentAdmissionInputState({ + params: dispatchParams, + options: executionOptions, + traceId, + status: "failed", + messageId: lifecycle.userMessageId, + error: failure.payload?.error ?? error + }); + } + void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { + startTimeMs: dispatchStartedAt, + status: "error", + error, + attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, ...codeAgentDispatchErrorOtelAttributes(error) } + }); + throw codeAgentSubmissionUnavailableError(failure, { params: dispatchParams, options: executionOptions, traceId }); + } + void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, { + startTimeMs: dispatchStartedAt, + attributes: { sessionId: safeSessionId(dispatchParams.sessionId) || null, turnId: codeAgentTurnLifecycleFields(traceId, dispatchParams).turnId, runId: payload?.agentRun?.runId ?? null, commandId: payload?.agentRun?.commandId ?? null, providerProfile: dispatchParams.providerProfile ?? null } + }); + const pending = annotateOwner({ + ...withCodeAgentBillingReservation(payload, dispatchParams), + accepted: false, + status: "admitting", + admissionState: "promotion-pending" + }, dispatchParams); + results.set(traceId, pending); + try { + if (transactionalAdmission) { + await persistCodeAgentAdmissionInputState({ + params: dispatchParams, + options: executionOptions, + traceId, + status: "admitting", + messageId: lifecycle.userMessageId, + commandId: payload.agentRun?.commandId ?? null, + agentRun: payload.agentRun ?? null + }); + } + await promoteCodeAgentTurnAdmission({ payload: pending, params: dispatchParams, options: executionOptions, traceId }); + } catch (error) { + throw codeAgentPromotionUnavailableError(error, { payload: pending, params: dispatchParams, options: executionOptions, traceId }); + } + const promoted = annotateOwner({ ...pending, accepted: true, status: "running", admissionState: "promoted" }, dispatchParams); + results.set(traceId, promoted); + return promoted; } + await recordCodeAgentTurnAdmission({ payload: acceptedPayload, params, options, traceId }); + results.set(traceId, annotateOwner(acceptedPayload, params)); traceStore.ensure(traceId, { runnerKind: "codex-app-server-stdio-runner", workspace: options.workspace ?? options.env?.HWLAB_CODE_AGENT_CODEX_WORKSPACE ?? options.env?.HWLAB_CODE_AGENT_WORKSPACE ?? null, @@ -1233,6 +1310,122 @@ async function recordCodeAgentTurnAdmission({ payload = {}, params = {}, options return ownerRecord; } +async function promoteCodeAgentTurnAdmission({ payload = {}, params = {}, options = {}, traceId } = {}) { + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const ownerRecord = await recordCodeAgentSessionOwner({ payload, params, options, status: "admitting" }); + if (!ownerRecord) { + const error = new Error("Code Agent turn promotion did not durably persist the admitting owner record."); + error.code = "workbench_admission_owner_not_persisted"; + throw error; + } + const lifecycle = codeAgentTurnLifecycleFields(traceId, params); + const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env); + if (!realtimeCapabilities.transactionalProjector) { + const runningOwner = await recordCodeAgentSessionOwner({ payload: { ...payload, status: "running" }, params, options, status: "running" }); + if (!runningOwner) { + const error = new Error("Code Agent live turn did not durably promote the session owner to running."); + error.code = "workbench_admission_owner_not_promoted"; + error.valuesRedacted = true; + throw error; + } + traceStore.append(traceId, { + type: "request", + status: "admitted", + label: "turn:admitted-live-kafka", + message: "Code Agent turn is live after AgentRun durable dispatch; lifecycle visibility is owned only by hwlab.event.v1.", + sessionId: safeSessionId(params.sessionId) || null, + runId: payload.agentRun?.runId ?? null, + commandId: payload.agentRun?.commandId ?? null, + dispatchIntentId: payload.agentRun?.dispatchIntentId ?? null, + waitingFor: "hwlab-live-kafka-sse", + terminal: false, + valuesPrinted: false + }); + return { written: false, promoted: true, outboxCommitted: false, realtimeCapabilities, liveOnly: realtimeCapabilities.liveKafkaSse, valuesPrinted: false }; + } + const inputFact = buildCodeAgentSessionInputFact({ + params, + options, + traceId, + delivery: "queue", + status: "promoted", + messageId: lifecycle.userMessageId, + commandId: payload.agentRun?.commandId ?? null, + agentRun: payload.agentRun ?? null + }); + const promotion = await persistWorkbenchTurnAdmissionPromotion({ + runtimeStore: options.runtimeStore, + session: ownerRecord, + inputFact, + payload, + params, + ownerUserId: options.actor?.id ?? params.ownerUserId, + ownerRole: options.actor?.role ?? params.ownerRole, + projectId: params.projectId ?? ownerRecord.projectId ?? null, + conversationId: params.conversationId ?? ownerRecord.conversationId ?? null, + threadId: payload.agentRun?.threadId ?? params.threadId ?? ownerRecord.threadId ?? null + }); + const runningOwner = await recordCodeAgentSessionOwner({ payload: { ...payload, status: "running" }, params, options, status: "running" }); + if (!runningOwner) { + const error = new Error("Code Agent turn promotion did not durably promote the session owner to running."); + error.code = "workbench_admission_owner_not_promoted"; + error.valuesRedacted = true; + throw error; + } + traceStore.append(traceId, { + type: "request", + status: "admitted", + label: "turn:admitted", + message: "Code Agent turn was promoted only after billing and durable AgentRun dispatch admission committed.", + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(payload.agentRun?.threadId ?? params.threadId) || null, + runId: payload.agentRun?.runId ?? null, + commandId: payload.agentRun?.commandId ?? null, + dispatchIntentId: payload.agentRun?.dispatchIntentId ?? null, + runnerJobId: payload.agentRun?.runnerJobId ?? null, + waitingFor: "agentrun-kafka-projection", + terminal: false, + valuesPrinted: false + }); + void emitCodeAgentOtelSpan("durable_admission", traceId, options.env ?? process.env, { + attributes: { + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(payload.agentRun?.threadId ?? params.threadId) || null, + turnId: lifecycle.turnId, + runId: payload.agentRun?.runId ?? null, + commandId: payload.agentRun?.commandId ?? null, + dispatchIntentId: payload.agentRun?.dispatchIntentId ?? null, + outboxCommitted: promotion?.outboxCommitted === true + } + }); + return promotion; +} + +async function persistCodeAgentAdmissionInputState({ params = {}, options = {}, traceId, status, messageId = null, commandId = null, agentRun = null, error = null } = {}) { + try { + const written = await recordCodeAgentSessionInputFact({ + params, + options, + traceId, + delivery: "queue", + status, + messageId, + commandId, + agentRun, + error + }); + if (written?.written !== true) { + const unavailable = new Error("Code Agent admission input state was not durably persisted."); + unavailable.code = "workbench_admission_input_not_persisted"; + unavailable.valuesRedacted = true; + throw unavailable; + } + return written; + } catch (cause) { + throw codeAgentAdmissionUnavailableError(cause, { params, options, traceId }); + } +} + function codeAgentAdmissionUnavailableError(error, { params = {}, options = {}, traceId } = {}) { const payload = codeAgentAdmissionUnavailablePayload({ error, params, options, traceId }); return Object.assign(new Error(payload.error.message), { @@ -1247,6 +1440,89 @@ function isCodeAgentAdmissionUnavailableError(error) { return error?.code === "admission_unavailable" && error?.payload; } +function isCodeAgentSubmissionUnavailableError(error) { + return Boolean(error?.payload && Number.isInteger(error?.statusCode) && ( + isCodeAgentAdmissionUnavailableError(error) + || error.code === "submission_unavailable" + || error.code === "admission_promotion_unavailable" + )); +} + +function codeAgentSubmissionUnavailableError(failure = {}, context = {}) { + const payload = codeAgentSubmissionUnavailablePayload(failure, context); + return Object.assign(new Error(payload.error.message), { + code: "submission_unavailable", + statusCode: failure.statusCode ?? 503, + payload, + alreadyClassified: true + }); +} + +function codeAgentSubmissionUnavailablePayload(failure = {}, { params = {}, options = {}, traceId } = {}) { + const source = failure.payload ?? failure; + const error = source.error ?? {}; + const blocker = source.blocker ?? null; + const message = error.message ?? blocker?.summary ?? "Code Agent submission failed before durable admission."; + return { + ...source, + ok: false, + accepted: false, + status: "failed", + traceId: safeTraceId(traceId) || null, + ...codeAgentOtelTraceFields(traceId, options.env ?? process.env), + ...codeAgentTurnLifecycleFields(traceId, params), + conversationId: safeConversationId(params.conversationId) || null, + sessionId: safeSessionId(params.sessionId) || null, + threadId: safeOpaqueId(params.threadId) || null, + error: { + ...error, + code: error.code ?? blocker?.code ?? "submission_unavailable", + layer: error.layer ?? blocker?.layer ?? "admission", + retryable: error.retryable !== false, + message, + route: error.route ?? "/v1/agent/chat" + }, + blocker: blocker ? { ...blocker, summary: blocker.summary ?? message } : null, + valuesRedacted: true, + secretMaterialStored: false + }; +} + +function codeAgentPromotionUnavailableError(error, { payload = {}, params = {}, options = {}, traceId } = {}) { + const message = error?.message ?? "Local Workbench admission promotion failed after AgentRun durable dispatch."; + const responsePayload = codeAgentSubmissionUnavailablePayload({ + error: { + code: "admission_promotion_unavailable", + layer: "admission", + retryable: true, + message, + userMessage: "AgentRun 已持久接纳本次请求,但本地可见性提交尚未完成;请使用同一 traceId 重试补提交。", + route: "/v1/agent/chat" + }, + blocker: { + code: "admission_promotion_unavailable", + layer: "admission", + retryable: true, + summary: message + } + }, { params, options, traceId }); + responsePayload.admission = { + state: "promotion-pending", + runId: payload.agentRun?.runId ?? null, + commandId: payload.agentRun?.commandId ?? null, + dispatchIntentId: payload.agentRun?.dispatchIntentId ?? null, + runnerJobId: payload.agentRun?.runnerJobId ?? null, + durableDispatch: payload.agentRun?.durableDispatch === true, + valuesRedacted: true + }; + return Object.assign(new Error(message), { + code: "admission_promotion_unavailable", + statusCode: 503, + payload: responsePayload, + alreadyClassified: true + }); +} + function codeAgentAdmissionUnavailablePayload({ error, params = {}, options = {}, traceId } = {}) { const message = error?.message ?? "Code Agent turn admission is unavailable; the user message was not persisted."; return { diff --git a/internal/cloud/server-code-agent-http-support.ts b/internal/cloud/server-code-agent-http-support.ts index 07629b68..a466597d 100644 --- a/internal/cloud/server-code-agent-http-support.ts +++ b/internal/cloud/server-code-agent-http-support.ts @@ -72,27 +72,39 @@ function firstNonEmptyValue(...values) { return null; } -async function recordCodeAgentSessionInputFact({ params = {}, options = {}, traceId, delivery = "queue", status = "admitted", messageId = null, commandId = null, error = null } = {}) { +async function recordCodeAgentSessionInputFact({ params = {}, options = {}, traceId, delivery = "queue", status = "admitted", messageId = null, commandId = null, agentRun = null, error = null } = {}) { const runtimeStore = options.runtimeStore ?? null; if (typeof runtimeStore?.writeWorkbenchFacts !== "function") return null; + const fact = buildCodeAgentSessionInputFact({ params, options, traceId, delivery, status, messageId, commandId, agentRun, error }); + if (!fact) return null; + return runtimeStore.writeWorkbenchFacts({ facts: { inputs: [fact] } }, codeAgentSessionInputRequestMeta(fact, params, options)); +} + +function buildCodeAgentSessionInputFact({ params = {}, options = {}, traceId, delivery = "queue", status = "admitted", messageId = null, commandId = null, agentRun = null, error = null } = {}) { const resolvedTraceId = safeTraceId(traceId ?? params.traceId) || null; const lifecycle = codeAgentTurnLifecycleFields(resolvedTraceId, params); const sessionId = safeSessionId(params.sessionId ?? params.session?.sessionId) || null; if (!sessionId) return null; const resolvedMessageId = safeMessageId(messageId ?? params.messageId ?? lifecycle.userMessageId) || lifecycle.userMessageId; - const resolvedCommandId = textValue(commandId ?? params.commandId ?? params.agentRun?.commandId) || null; + const resolvedCommandId = textValue(commandId ?? agentRun?.commandId ?? params.commandId ?? params.agentRun?.commandId) || null; const normalizedDelivery = codeAgentInputDelivery(delivery); const normalizedStatus = codeAgentInputStatus(status); const now = new Date().toISOString(); const sourceEventId = `${sessionId}:${lifecycle.turnId}:${normalizedDelivery}:${resolvedTraceId ?? resolvedCommandId ?? resolvedMessageId}`; - const inputId = stableCodeAgentInputId({ sessionId, turnId: lifecycle.turnId, traceId: resolvedTraceId, messageId: resolvedMessageId, commandId: resolvedCommandId, delivery: normalizedDelivery, sourceEventId }); - const fact = { + // Command allocation happens after the initial admitting write. Keep one stable + // row across admitting -> blocked/failed/promoted transitions for the trace. + const inputId = stableCodeAgentInputId({ sessionId, turnId: lifecycle.turnId, traceId: resolvedTraceId, messageId: resolvedMessageId, delivery: normalizedDelivery, sourceEventId }); + return { inputId, sessionId, turnId: lifecycle.turnId, traceId: resolvedTraceId, messageId: resolvedMessageId, commandId: resolvedCommandId, + runId: textValue(agentRun?.runId ?? params.agentRun?.runId) || null, + dispatchIntentId: textValue(agentRun?.dispatchIntentId ?? params.agentRun?.dispatchIntentId) || null, + runnerJobId: textValue(agentRun?.runnerJobId ?? params.agentRun?.runnerJobId) || null, + durableDispatch: agentRun?.durableDispatch === true || params.agentRun?.durableDispatch === true, delivery: normalizedDelivery, status: normalizedStatus, errorCode: textValue(error?.code ?? error?.error?.code) || null, @@ -104,15 +116,18 @@ async function recordCodeAgentSessionInputFact({ params = {}, options = {}, trac createdAt: now, updatedAt: now }; - return runtimeStore.writeWorkbenchFacts({ facts: { inputs: [fact] } }, { - sessionId, - traceId: resolvedTraceId, - turnId: lifecycle.turnId, - messageId: resolvedMessageId, +} + +function codeAgentSessionInputRequestMeta(fact, params = {}, options = {}) { + return { + sessionId: fact.sessionId, + traceId: fact.traceId, + turnId: fact.turnId, + messageId: fact.messageId, ownerUserId: options.actor?.id ?? params.ownerUserId, projectId: params.projectId ?? DEFAULT_CODE_AGENT_PROJECT_ID, valuesPrinted: false - }); + }; } function stableCodeAgentInputId(value) { @@ -127,7 +142,7 @@ function codeAgentInputDelivery(value) { function codeAgentInputStatus(value) { const text = textValue(value).toLowerCase().replace(/-/gu, "_"); - return ["admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; + return ["admitting", "admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; } function codeAgentResultTimingFields(...sources) { @@ -1507,6 +1522,7 @@ export { normalizeCodeAgentProviderProfile, firstNonEmptyValue, recordCodeAgentSessionInputFact, + buildCodeAgentSessionInputFact, stableCodeAgentInputId, codeAgentInputDelivery, codeAgentInputStatus, diff --git a/internal/cloud/server-code-agent-trace-http.ts b/internal/cloud/server-code-agent-trace-http.ts index 84f6e35b..5ab319f9 100644 --- a/internal/cloud/server-code-agent-trace-http.ts +++ b/internal/cloud/server-code-agent-trace-http.ts @@ -210,6 +210,7 @@ export async function handleCodeAgentSteerHttp(request, response, options) { const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; const durableResult = await loadPersistedAgentRunResult(traceId, options); const currentResult = durableResult ?? options.codeAgentChatResults?.get?.(traceId); + const projectionContext = await readCodeAgentCompatProjection(traceId, options); if (!canAccessOwnedResult(currentResult, options.actor)) { sendJson(response, 403, { ok: false, @@ -219,6 +220,28 @@ export async function handleCodeAgentSteerHttp(request, response, options) { }); return; } + const projectionStoreBlocker = codeAgentProjectionStoreBlocker(projectionContext); + if (projectionStoreBlocker) { + sendJson(response, 503, { + ok: false, + accepted: false, + status: "blocked", + traceId, + route: "/v1/agent/chat/steer", + error: { + ...projectionStoreBlocker, + code: "projection_store_unavailable", + layer: projectionStoreBlocker.layer ?? "runtime-store", + category: projectionStoreBlocker.category ?? "projection-store-unavailable", + retryable: true, + message: projectionStoreBlocker.message ?? "Workbench projection store is unavailable; steer is blocked until terminal authority can be checked.", + traceId, + route: "/v1/agent/chat/steer" + }, + valuesPrinted: false + }); + return; + } if (!currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) { sendJson(response, 404, { ok: false, @@ -230,7 +253,11 @@ export async function handleCodeAgentSteerHttp(request, response, options) { }); return; } - if (isTraceCommandTerminalStatus(currentResult.status)) { + const projectedTerminalStatus = normalizeTurnStatus( + projectionContext.trace?.terminalEvidence?.status, + projectionContext.trace?.status + ); + if (isTraceCommandTerminalStatus(projectedTerminalStatus)) { sendJson(response, 409, { ok: false, accepted: false, @@ -359,10 +386,13 @@ export async function handleCodeAgentTraceHttp(request, response, url, options) return; } + const diagnosticTrace = (requestOptions.traceStore ?? defaultCodeAgentTraceStore).snapshot(traceId); + const trace = mergeCodeAgentDiagnosticTrace(context.trace, diagnosticTrace, traceId); + const diagnosticFound = trace.events.length > 0; const body = await measureCodeAgentHttpPhase(requestOptions, "trace_paginate", () => { - return paginateTraceSnapshot(context.trace, pageOptions); + return paginateTraceSnapshot(trace, pageOptions); }); - const statusCode = context.found ? 200 : 404; + const statusCode = context.found || diagnosticFound ? 200 : 404; void emitCodeAgentOtelSpan("trace_events_read", traceId, requestOptions.env ?? process.env, { startTimeMs: startedAt, attributes: { @@ -383,6 +413,63 @@ export async function handleCodeAgentTraceHttp(request, response, url, options) sendJson(response, statusCode, codeAgentCompatProjectionPayload(body, context)); } +function mergeCodeAgentDiagnosticTrace(projectionTrace, diagnosticTrace, traceId) { + const projected = projectionTrace && typeof projectionTrace === "object" ? projectionTrace : {}; + const diagnostics = diagnosticTrace && typeof diagnosticTrace === "object" ? diagnosticTrace : {}; + const merged = new Map(); + for (const [source, events] of [["diagnostic", diagnostics.events], ["projection", projected.events]]) { + for (const [index, event] of (Array.isArray(events) ? events : []).entries()) { + const originalSeq = traceEventSequence(event, index); + const key = textValue(event?.id ?? event?.sourceEventId) || `${originalSeq}:${textValue(event?.label ?? event?.type)}:${textValue(event?.createdAt ?? event?.occurredAt)}`; + const normalized = source === "projection" + ? { + ...event, + projectedSeq: Number.isFinite(Number(event?.projectedSeq)) ? Number(event.projectedSeq) : originalSeq, + authority: "workbench-projection" + } + : { + ...event, + sourceSeq: Number.isFinite(Number(event?.sourceSeq)) ? Number(event.sourceSeq) : originalSeq, + terminal: false, + sealed: false, + diagnosticOnly: true, + authority: "trace-store-diagnostics" + }; + merged.set(key, { event: normalized, source, originalSeq, sourceIndex: index, key }); + } + } + const events = [...merged.values()] + .sort((left, right) => left.originalSeq - right.originalSeq + || (left.source === right.source ? 0 : left.source === "projection" ? -1 : 1) + || left.sourceIndex - right.sourceIndex + || left.key.localeCompare(right.key)) + .map(({ event }, index) => ({ ...event, seq: index + 1, paginationSeq: index + 1 })); + const projectionFound = projected.status && projected.status !== "missing"; + return { + ...projected, + traceId, + status: projectionFound ? projected.status : events.length > 0 ? "running" : "missing", + terminal: projected.terminal === true, + sealed: projected.sealed === true, + events, + eventCount: events.length, + diagnosticEventCount: events.filter((event) => event.diagnosticOnly === true).length, + terminalAuthority: "workbench-projection", + valuesRedacted: true + }; +} + +function codeAgentProjectionStoreBlocker(context = {}) { + const candidates = [ + context?.projection?.blocker, + context?.trace?.projection?.blocker, + context?.trace?.blocker, + context?.result?.projection?.blocker, + context?.result?.blocker + ]; + return candidates.find((blocker) => blocker?.code === "projection_store_unavailable") ?? null; +} + function codeAgentTracePageOptions(url) { const sinceSeq = nonNegativeInteger(url.searchParams.get("sinceSeq") ?? url.searchParams.get("since"), 0); const requestedLimit = parsePositiveInteger(url.searchParams.get("limit"), DEFAULT_CODE_AGENT_TRACE_PAGE_LIMIT); diff --git a/internal/cloud/server-code-agent-turn-http.ts b/internal/cloud/server-code-agent-turn-http.ts index ae9c5244..399cba68 100644 --- a/internal/cloud/server-code-agent-turn-http.ts +++ b/internal/cloud/server-code-agent-turn-http.ts @@ -166,7 +166,7 @@ export async function handleCodeAgentChatResultHttp(request, response, url, opti return; } const result = context.result; - if (result && result.status !== "running") { + if (result && result.status !== "running" && !codeAgentResultUsesProjectionTerminalAuthority(result)) { sendJson(response, 200, codeAgentCompatProjectionPayload(compactCodeAgentChatResultPayload(result, options), context)); return; } @@ -412,24 +412,38 @@ async function getCodeAgentSessionByTraceId(traceId, options = {}) { export function codeAgentTurnStatusPayload({ traceId, result, snapshot, resultPollError, refreshError, options }) { const resultObject = result && typeof result === "object" ? result : null; const snapshotObject = snapshot && typeof snapshot === "object" ? snapshot : null; + const projectionTerminalAuthority = codeAgentResultUsesProjectionTerminalAuthority(resultObject); const lifecycle = codeAgentTurnLifecycleFields(traceId, resultObject ?? snapshotObject ?? {}); const events = Array.isArray(snapshotObject?.events) ? snapshotObject.events : Array.isArray(resultObject?.runnerTrace?.events) ? resultObject.runnerTrace.events : []; const lastEvent = events.at(-1) ?? null; - const finalResponse = resultObject?.finalResponse ?? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(resultObject ?? snapshotObject ?? {}, traceId); - const terminalStatus = codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId); - const terminalSealBlocked = Boolean(terminalStatus && isTurnTerminalStatus(terminalStatus) && !codeAgentTerminalFinalText(finalResponse, resultObject, snapshotObject)); - const rawStatus = normalizeTurnStatus( - terminalStatus, - resultObject?.agentRun?.commandState, - resultObject?.agentRun?.status, - resultObject?.agentRun?.runStatus, - codeAgentRunningStatus(snapshotObject?.status), - codeAgentRunningStatus(snapshotObject?.traceStatus), - codeAgentRunningStatus(snapshotObject?.runnerTrace?.status), - codeAgentRunningStatus(resultObject?.status), - resultObject?.agentRun?.terminalStatus, - snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus - ); + const finalResponse = projectionTerminalAuthority + ? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(snapshotObject ?? {}, traceId) + : resultObject?.finalResponse ?? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(resultObject ?? snapshotObject ?? {}, traceId); + const terminalStatus = projectionTerminalAuthority + ? codeAgentProjectionTerminalStatus(snapshotObject) + : codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId); + const terminalSealBlocked = Boolean(terminalStatus && isTurnTerminalStatus(terminalStatus) && !codeAgentTerminalFinalText(finalResponse, projectionTerminalAuthority ? null : resultObject, snapshotObject)); + const rawStatus = projectionTerminalAuthority + ? normalizeTurnStatus( + terminalStatus, + codeAgentRunningStatus(snapshotObject?.status), + codeAgentRunningStatus(snapshotObject?.traceStatus), + codeAgentRunningStatus(snapshotObject?.runnerTrace?.status), + codeAgentRunningStatus(resultObject?.status), + "running" + ) + : normalizeTurnStatus( + terminalStatus, + resultObject?.agentRun?.commandState, + resultObject?.agentRun?.status, + resultObject?.agentRun?.runStatus, + codeAgentRunningStatus(snapshotObject?.status), + codeAgentRunningStatus(snapshotObject?.traceStatus), + codeAgentRunningStatus(snapshotObject?.runnerTrace?.status), + codeAgentRunningStatus(resultObject?.status), + resultObject?.agentRun?.terminalStatus, + snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus + ); const status = terminalSealBlocked ? "running" : rawStatus; const found = Boolean(resultObject || (snapshotObject && snapshotObject.status !== "missing") || snapshotObject?.persisted === true); const running = isTurnRunningStatus(status); @@ -473,6 +487,22 @@ export function codeAgentTurnStatusPayload({ traceId, result, snapshot, resultPo }; } +function codeAgentResultUsesProjectionTerminalAuthority(result = null) { + const agentRun = result?.agentRun && typeof result.agentRun === "object" ? result.agentRun : null; + if (!agentRun || agentRun.adapter !== "agentrun-v01") return false; + return agentRun.durableDispatch === true + || Boolean(textValue(agentRun.dispatchIntentId ?? agentRun.commandId)); +} + +function codeAgentProjectionTerminalStatus(snapshotObject = null) { + const status = normalizeTurnStatus( + snapshotObject?.terminalEvidence?.status, + snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus, + snapshotObject?.status + ); + return status && isTurnTerminalStatus(status) && codeAgentSnapshotHasTerminalAuthority(snapshotObject) ? status : null; +} + function codeAgentTerminalFinalText(finalResponse = null, resultObject = null, snapshotObject = null) { const candidates = [ finalResponse, diff --git a/internal/cloud/server-live-builds.test.ts b/internal/cloud/server-live-builds.test.ts index 58a40a92..0aa6b330 100644 --- a/internal/cloud/server-live-builds.test.ts +++ b/internal/cloud/server-live-builds.test.ts @@ -149,6 +149,10 @@ test("cloud api aggregates live HWLAB build times from health and controlled dep }; const server = createCloudApiServer({ + accessController: { + required: false, + async authenticate() { return { ok: true, actor: { id: "usr_live_builds", role: "admin" }, session: { id: "uss_live_builds" } }; } + }, env: { PATH: "", HWLAB_COMMIT_ID: "apiabcdef123456", @@ -285,6 +289,10 @@ test("cloud api ignores old repository reports and keeps missing buildCreatedAt const observe = async () => { const server = createCloudApiServer({ + accessController: { + required: false, + async authenticate() { return { ok: true, actor: { id: "usr_live_builds", role: "admin" }, session: { id: "uss_live_builds" } }; } + }, env: { PATH: "", HWLAB_GATEWAY_URL: "http://live.test/hwlab-gateway" diff --git a/internal/cloud/server-m3-http.test.ts b/internal/cloud/server-m3-http.test.ts index 592a664d..79fa0b86 100644 --- a/internal/cloud/server-m3-http.test.ts +++ b/internal/cloud/server-m3-http.test.ts @@ -146,6 +146,10 @@ test("cloud api /v1/diagnostics/gate exposes Chinese single-table rows from live now: () => "2026-05-23T00:00:00.000Z" }); const server = createCloudApiServer({ + accessController: { + required: false, + async authenticate() { return { ok: true, actor: { id: "usr_m3_diagnostics", role: "admin" }, session: { id: "uss_m3_diagnostics" } }; } + }, runtimeStore, env: { HWLAB_M3_GATEWAY_SIMU_1_URL: "http://gateway-1", diff --git a/internal/cloud/server-test-helpers.ts b/internal/cloud/server-test-helpers.ts index 982096dd..a9bcd85e 100644 --- a/internal/cloud/server-test-helpers.ts +++ b/internal/cloud/server-test-helpers.ts @@ -683,15 +683,16 @@ function sessionCookieFromResponse(response) { return value.split(";")[0] || ""; } -export async function pollAgentResult(port, traceId) { +export async function pollAgentResult(port, traceId, init = {}) { let last = null; for (let attempt = 0; attempt < 20; attempt += 1) { - const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/result/${encodeURIComponent(traceId)}`); + const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/result/${encodeURIComponent(traceId)}`, init); last = { status: response.status, body: await response.json() }; if (response.status === 200) return last.body; + if (response.status !== 202) throw new Error(`Code Agent result polling failed: ${JSON.stringify(last)}`); await delay(10); } throw new Error(`Code Agent result did not complete: ${JSON.stringify(last)}`); diff --git a/internal/cloud/server-workbench-read-http.ts b/internal/cloud/server-workbench-read-http.ts index 01012dd7..78fb4c86 100644 --- a/internal/cloud/server-workbench-read-http.ts +++ b/internal/cloud/server-workbench-read-http.ts @@ -17,6 +17,7 @@ import { createWorkbenchReadModel } from "./workbench-read-model.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; +import { workbenchRealtimeCapabilities } from "./workbench-realtime-capabilities.ts"; import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; import * as workbenchFacts from "./server-workbench-facts.ts"; @@ -156,6 +157,19 @@ export async function handleWorkbenchReadModelHttp(request, response, url, optio if (!auth) return; if (url.pathname === "/v1/workbench/sync") { + const capabilities = workbenchRealtimeCapabilities(options.env ?? process.env); + if (!capabilities.projectionRealtime) { + sendJson(response, 503, { + ok: false, + error: { + code: "workbench_projection_realtime_disabled", + message: "Workbench projection sync/replay capability is disabled.", + capabilities, + valuesRedacted: true + } + }); + return; + } await (perf ? perf.measure("workbench_sync", () => handleWorkbenchSyncHttp(request, response, url, options, auth.actor)) : handleWorkbenchSyncHttp(request, response, url, options, auth.actor)); return; } diff --git a/internal/cloud/server-workbench-realtime-http.test.ts b/internal/cloud/server-workbench-realtime-http.test.ts index 0e422577..17811e7d 100644 --- a/internal/cloud/server-workbench-realtime-http.test.ts +++ b/internal/cloud/server-workbench-realtime-http.test.ts @@ -37,6 +37,14 @@ import { parseSseBlock } from "./server-workbench-http-test-helpers.ts"; +const TRANSACTIONAL_REALTIME_ENV = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true" +}); + test("projection outbox emits immutable assistant versions in row order", () => { const traceId = "trc_outbox_versions"; const sessionId = "ses_outbox_versions"; @@ -53,6 +61,267 @@ test("projection outbox emits immutable assistant versions in row order", () => assert.deepEqual(events.map((item) => item.payload.cursor.outboxSeq), [10, 11]); }); +test("live Kafka SSE transparently fans out one envelope without DB, snapshot, cursor, replay, or SSE id", async () => { + const sessionId = "ses_live_kafka_sse"; + const traceId = "trc_live_kafka_sse"; + const subscribers = new Set(); + const otelSpans = []; + let releaseBridgeReady; + const bridgeReady = new Promise((resolve) => { releaseBridgeReady = resolve; }); + const bridge = { + capabilities: { directPublish: true, liveKafkaSse: true, transactionalProjector: false, projectionOutboxRelay: false, projectionRealtime: false }, + ready: bridgeReady, + subscribeLiveHwlabEvents(listener) { + subscribers.add(listener); + return () => subscribers.delete(listener); + }, + async stop() {} + }; + let dbCalls = 0; + const workbenchRuntime = new Proxy({}, { + get() { + dbCalls += 1; + throw new Error("live SSE must not access Workbench runtime facts"); + } + }); + const server = createCloudApiServer({ + accessController: realtimeAccessController({ + sessions: [{ id: sessionId, ownerUserId: ACTOR.id, lastTraceId: traceId }] + }), + workbenchRuntime, + kafkaEventBridge: bridge, + otelSpanEmitter(name, emittedTraceId, env, spanOptions) { otelSpans.push({ name, emittedTraceId, env, spanOptions }); }, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const envelope = { + schema: "hwlab.event.v1", + eventType: "hwlab.trace.event.projected", + eventId: "hwlab:evt_live_kafka_sse", + traceId, + hwlabSessionId: sessionId, + sessionId, + runId: "run_live_kafka_sse", + commandId: "cmd_live_kafka_sse", + context: { runId: "run_live_kafka_sse", commandId: "cmd_live_kafka_sse", sourceSeq: 4, valuesRedacted: true }, + event: { type: "assistant", eventType: "assistant", status: "running", traceId, sessionId, text: "live increment", sourceSeq: 4, terminal: false, valuesPrinted: false }, + valuesPrinted: false + }; + + try { + const { port } = server.address(); + const firstPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${sessionId}&afterSeq=999`, 2); + const secondPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${sessionId}&traceId=${traceId}&afterSeq=888`, 2); + await waitForCondition(() => subscribers.size === 2); + for (const listener of subscribers) listener(envelope, { topic: "hwlab.event.v1", partition: 6, offset: "101" }); + releaseBridgeReady(); + const [first, second] = await Promise.all([firstPromise, secondPromise]); + for (const events of [first, second]) { + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "hwlab.event.v1"]); + assert.deepEqual(events[0].data.capabilities, bridge.capabilities); + assert.equal(events[0].data.liveOnly, true); + assert.equal(events[0].data.replay, false); + assert.equal(events[0].data.lossPossible, true); + assert.equal(events[0].data.cursor, undefined); + assert.equal(events[1].id, null); + assert.deepEqual(events[1].data, envelope); + } + assert.equal(otelSpans.length, 2); + for (const span of otelSpans) { + assert.equal(span.name, "hwlab.workbench.live_sse.business_event_write"); + assert.equal(span.emittedTraceId, traceId); + assert.deepEqual(span.spanOptions.attributes, { + businessTraceId: traceId, + hwlabSessionId: sessionId, + runId: "run_live_kafka_sse", + commandId: "cmd_live_kafka_sse", + topic: "hwlab.event.v1", + partition: 6, + offset: "101", + sourceTopic: null, + sourcePartition: null, + sourceOffset: null, + eventType: "assistant", + terminal: false, + valuesRedacted: true + }); + assert.equal(JSON.stringify(span.spanOptions.attributes).includes("live increment"), false); + } + assert.equal(dbCalls, 0); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("live Kafka SSE heartbeat keeps the transport open without DB, cursor, snapshot, or replay", async () => { + const sessionId = "ses_live_kafka_heartbeat"; + let dbCalls = 0; + const server = createCloudApiServer({ + accessController: realtimeAccessController({ sessions: [{ id: sessionId, ownerUserId: ACTOR.id }] }), + workbenchRuntime: new Proxy({}, { + get() { + dbCalls += 1; + throw new Error("live heartbeat must not access Workbench projection state"); + } + }), + kafkaEventBridge: { + capabilities: { liveKafkaSse: true }, + ready: Promise.resolve(), + subscribeLiveHwlabEvents() { return () => {}; }, + async stop() {} + }, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false", + HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "5" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}`, 2); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.heartbeat"]); + assert.equal(events[1].id, null); + assert.equal(events[1].data.liveOnly, true); + assert.equal(events[1].data.replay, false); + assert.equal(events[1].data.cursor, undefined); + assert.equal(events[1].data.snapshot, undefined); + assert.equal(dbCalls, 0); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("live Kafka SSE rejects foreign or inconsistent ownership scopes before fanout subscription", async () => { + const ownedSession = { id: "ses_live_kafka_owned", ownerUserId: ACTOR.id, lastTraceId: "trc_live_kafka_owned" }; + const foreignSession = { id: "ses_live_kafka_foreign", ownerUserId: "usr_live_kafka_foreign", lastTraceId: "trc_live_kafka_foreign" }; + let subscriptions = 0; + let projectionReads = 0; + const server = createCloudApiServer({ + accessController: realtimeAccessController({ sessions: [ownedSession, foreignSession] }), + workbenchRuntime: new Proxy({}, { + get() { + projectionReads += 1; + throw new Error("live SSE authorization must not access Workbench projection state"); + } + }), + kafkaEventBridge: { + capabilities: { liveKafkaSse: true }, + ready: Promise.resolve(), + subscribeLiveHwlabEvents() { + subscriptions += 1; + return () => {}; + }, + async stop() {} + }, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const foreignSessionResponse = await getJson(port, `/v1/workbench/events?sessionId=${foreignSession.id}`); + const foreignTraceResponse = await getJson(port, `/v1/workbench/events?traceId=${foreignSession.lastTraceId}`); + const inconsistentResponse = await getJson(port, `/v1/workbench/events?sessionId=${ownedSession.id}&traceId=${foreignSession.lastTraceId}`); + const missingResponse = await getJson(port, "/v1/workbench/events?sessionId=ses_live_kafka_missing"); + + for (const response of [foreignSessionResponse, foreignTraceResponse, inconsistentResponse, missingResponse]) { + assert.equal(response.status, 404); + assert.equal(response.body.error.code, "workbench_realtime_scope_not_found"); + } + assert.equal(subscriptions, 0); + assert.equal(projectionReads, 0); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("live Kafka SSE fails closed when ownership lookup is not configured", async () => { + let subscriptions = 0; + const server = createCloudApiServer({ + accessController: { + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_realtime_unconfigured" } }; } + }, + kafkaEventBridge: { + capabilities: { liveKafkaSse: true }, + ready: Promise.resolve(), + subscribeLiveHwlabEvents() { + subscriptions += 1; + return () => {}; + }, + async stop() {} + }, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const response = await getJson(server.address().port, "/v1/workbench/events?sessionId=ses_live_kafka_unconfigured"); + assert.equal(response.status, 503); + assert.equal(response.body.error.code, "workbench_realtime_authorization_unconfigured"); + assert.equal(subscriptions, 0); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + +test("live Kafka SSE lets admin subscribe to another owner's consistent session and trace", async () => { + const session = { id: "ses_live_kafka_admin", ownerUserId: "usr_live_kafka_owner", lastTraceId: "trc_live_kafka_admin" }; + const admin = { ...ACTOR, id: "usr_live_kafka_admin", role: "admin" }; + let subscriptions = 0; + const server = createCloudApiServer({ + accessController: realtimeAccessController({ actor: admin, sessions: [session] }), + kafkaEventBridge: { + capabilities: { liveKafkaSse: true }, + ready: Promise.resolve(), + subscribeLiveHwlabEvents() { + subscriptions += 1; + return () => {}; + }, + async stop() {} + }, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${session.id}&traceId=${session.lastTraceId}`, 1); + assert.equal(events[0].event, "workbench.connected"); + assert.deepEqual(events[0].data.filters, { sessionId: session.id, traceId: session.lastTraceId }); + assert.equal(subscriptions, 1); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + test("workbench realtime initial connection emits current snapshot without replaying historical outbox", async () => { const sessionId = "ses_realtime_snapshot_only"; const traceId = "trc_realtime_snapshot_only"; @@ -76,7 +345,7 @@ test("workbench realtime initial connection emits current snapshot without repla const server = createCloudApiServer({ accessController: realtimeAccessController(), workbenchRuntime: runtime, - env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { @@ -92,6 +361,50 @@ test("workbench realtime initial connection emits current snapshot without repla } }); +test("live and projection realtime remain independently reachable when both capabilities are enabled", async () => { + const sessionId = "ses_composable_realtime"; + const traceId = "trc_composable_realtime"; + let projectionReads = 0; + const runtime = { + async readAtomicWorkbenchProjectionSync() { + projectionReads += 1; + return { + facts: { + sessions: [{ sessionId, lastTraceId: traceId }], + messages: [{ messageId: "msg_composable_realtime", sessionId, traceId, role: "agent", text: "projection snapshot", projectedSeq: 1 }], + turns: [] + }, + events: [], + cutoffOutboxSeq: 1, + cursorOutboxSeq: 1, + hasMore: false + }; + } + }; + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: runtime, + env: { + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true", + HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", + HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + try { + const events = await getSseEvents(server.address().port, `/v1/workbench/projection-events?sessionId=${sessionId}`, 2); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot"]); + assert.equal(events[0].data.realtimeSource, "projection-outbox"); + assert.equal(projectionReads, 1); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + test("synchronous projection notification is buffered until the unique initial snapshot completes", async () => { const sessionId = "ses_realtime_initial_barrier"; const traceId = "trc_realtime_initial_barrier"; @@ -137,7 +450,7 @@ test("synchronous projection notification is buffered until the unique initial s accessController: realtimeAccessController(), workbenchRuntime: runtime, kafkaEventBridge, - env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { @@ -178,7 +491,7 @@ test("initial atomic snapshot failure emits an error and closes SSE for reconnec subscribeProjectionCommits() { return () => { unsubscribeCount += 1; }; }, async stop() {} }, - env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); const controller = new AbortController(); @@ -202,6 +515,66 @@ test("initial atomic snapshot failure emits an error and closes SSE for reconnec } }); +test("delta scan failure after a successful snapshot closes SSE for cursor reconnect", async () => { + const sessionId = "ses_realtime_delta_failure"; + const traceId = "trc_realtime_delta_failure"; + let unsubscribeCount = 0; + let calls = 0; + const server = createCloudApiServer({ + accessController: realtimeAccessController(), + workbenchRuntime: { + async readAtomicWorkbenchProjectionSync() { + calls += 1; + if (calls === 1) { + return { + facts: { + sessions: [{ sessionId, lastTraceId: traceId }], + messages: [{ messageId: "msg_delta_failure", sessionId, traceId, role: "agent", text: "durable snapshot", projectedSeq: 5 }], + turns: [] + }, + events: [], + cutoffOutboxSeq: 5, + cursorOutboxSeq: 5, + hasMore: false + }; + } + const error = new Error("delta scan failed"); + error.code = "workbench_delta_scan_failed"; + throw error; + } + }, + kafkaEventBridge: { + subscribeProjectionCommits(listener) { + listener({ sessionId, traceId }); + return () => { unsubscribeCount += 1; }; + }, + async stop() {} + }, + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const controller = new AbortController(); + let timeout = null; + try { + const response = await fetch(`http://127.0.0.1:${server.address().port}/v1/workbench/events?sessionId=${sessionId}`, { signal: controller.signal }); + assert.equal(response.status, 200); + const body = await Promise.race([ + response.text(), + new Promise((_, reject) => { timeout = setTimeout(() => reject(new Error("delta failure left SSE open")), 500); }) + ]); + const events = body.trim().split("\n\n").filter(Boolean).map(parseSseBlock); + assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.message.snapshot", "workbench.error"]); + assert.equal(events[2].data.error.code, "workbench_delta_scan_failed"); + assert.equal(body.includes("workbench.heartbeat"), false); + assert.equal(calls, 2); + await waitForCondition(() => unsubscribeCount === 1); + } finally { + if (timeout) clearTimeout(timeout); + controller.abort(); + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + test("projector commit during initial snapshot wakes one coalesced delta scan without idle polling", async () => { const sessionId = "ses_realtime_commit_wakeup"; const traceId = "trc_realtime_commit_wakeup"; @@ -229,7 +602,7 @@ test("projector commit during initial snapshot wakes one coalesced delta scan wi accessController: realtimeAccessController(), workbenchRuntime: runtime, kafkaEventBridge, - env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { @@ -263,7 +636,7 @@ test("workbench realtime early disconnect releases commit subscription during in accessController: realtimeAccessController(), workbenchRuntime: { async readAtomicWorkbenchProjectionSync() { await gate; return { facts: {}, events: [], cutoffOutboxSeq: 0, cursorOutboxSeq: 0, hasMore: false }; } }, kafkaEventBridge: { subscribeProjectionCommits() { subscribed += 1; return () => { unsubscribed += 1; }; }, async stop() {} }, - env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } + env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); let clientRequest = null; @@ -303,7 +676,7 @@ test("workbench realtime resets a cursor ahead of scoped cutoff with one fresh s return { facts: {}, events: [], cutoffOutboxSeq: 5, cursorOutboxSeq: 5, hasMore: false }; } }; - const server = createCloudApiServer({ accessController: realtimeAccessController(), workbenchRuntime: runtime, env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); + const server = createCloudApiServer({ accessController: realtimeAccessController(), workbenchRuntime: runtime, env: { ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { const events = await getSseEvents(server.address().port, `/v1/workbench/events?sessionId=${sessionId}&afterSeq=42`, 2); @@ -354,6 +727,7 @@ test("workbench realtime stream surfaces facts blocker instead of legacy trace f codeAgentChatResults: results, workbenchRuntime, env: { + ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } @@ -420,6 +794,7 @@ test("workbench session realtime follows durable outbox beyond stale lastTraceId accessController, workbenchRuntime, env: { + ...TRANSACTIONAL_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } @@ -663,9 +1038,13 @@ test("workbench trace event page keeps per-trace terminal status after later ses } }); -function realtimeAccessController() { +function realtimeAccessController({ actor = ACTOR, sessions = [] } = {}) { + const bySessionId = new Map(sessions.map((session) => [session.id, session])); + const byTraceId = new Map(sessions.filter((session) => session.lastTraceId).map((session) => [session.lastTraceId, session])); return { async ensureBootstrap() {}, - async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_realtime_test" } }; } + async authenticate() { return { ok: true, actor, session: { id: "uss_realtime_test" } }; }, + async getAgentSession(sessionId) { return bySessionId.get(sessionId) ?? null; }, + async getAgentSessionByTraceId(traceId) { return byTraceId.get(traceId) ?? null; } }; } diff --git a/internal/cloud/server-workbench-realtime-http.ts b/internal/cloud/server-workbench-realtime-http.ts index 65fd37f6..b83fdb14 100644 --- a/internal/cloud/server-workbench-realtime-http.ts +++ b/internal/cloud/server-workbench-realtime-http.ts @@ -14,11 +14,15 @@ import { } from "./server-http-utils.ts"; import { createWorkbenchReadModel } from "./workbench-read-model.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; +import { emitLiveKafkaOtelSpan } from "./kafka-event-bridge.ts"; import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts"; import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts"; import { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts"; import { durableTraceStatus, RUNNING_STATUSES, TERMINAL_STATUSES } from "./workbench-turn-projection.ts"; import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts"; +import { + workbenchRealtimeCapabilities +} from "./workbench-realtime-capabilities.ts"; import * as workbenchFacts from "./server-workbench-facts.ts"; const DEFAULT_PAGE_LIMIT = 50; @@ -178,10 +182,21 @@ export async function drainWorkbenchRealtimeConnections(options = {}) { export async function handleWorkbenchRealtimeHttp(request, response, url, options = {}) { try { if (request.method !== "GET") return methodNotAllowed(response, "GET"); + const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env); + const projectionOnly = url.pathname === "/v1/workbench/projection-events"; + if (!projectionOnly && realtimeCapabilities.liveKafkaSse) { + await handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, options); + return; + } + if (!realtimeCapabilities.projectionRealtime) { + sendJson(response, 503, workbenchError("workbench_realtime_disabled", "No Workbench realtime SSE capability is enabled.")); + return; + } const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId")); const requestedTraceId = safeTraceId(url.searchParams.get("traceId")); const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS); attachWorkbenchRealtimeOtelContext(request, { + route: projectionOnly ? "/v1/workbench/projection-events" : "/v1/workbench/events", sessionId: requestedSessionId, traceId: requestedTraceId, heartbeatMs, @@ -387,6 +402,7 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option } while (scanRequested && isActive()); } catch (error) { await emitProjectionScanError(error); + if (!response.writableEnded) response.end(); } finally { scanRunning = false; } @@ -436,10 +452,226 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option } } +async function handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, options = {}) { + const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId")); + const requestedTraceId = safeTraceId(url.searchParams.get("traceId")); + const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env); + const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS); + attachWorkbenchRealtimeOtelContext(request, { + sessionId: requestedSessionId, + traceId: requestedTraceId, + heartbeatMs, + realtimeSource: "live-kafka-sse", + realtimeCapabilities + }); + const perf = options.backendPerformance; + const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options); + if (!auth) return; + if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) { + sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench realtime is keyed by sessionId/traceId only.")); + return; + } + if (!requestedSessionId && !requestedTraceId) { + sendJson(response, 400, workbenchError("workbench_realtime_scope_required", "Workbench realtime requires sessionId or traceId.")); + return; + } + const authorization = await authorizeLiveKafkaWorkbenchRealtimeScope(options, auth.actor, { + sessionId: requestedSessionId, + traceId: requestedTraceId + }); + if (!authorization.ok) { + sendJson(response, authorization.status, authorization.body); + return; + } + const bridge = options.kafkaEventBridge; + if (bridge?.capabilities?.liveKafkaSse !== true || typeof bridge?.subscribeLiveHwlabEvents !== "function") { + sendJson(response, 503, workbenchError("workbench_live_kafka_unconfigured", "Workbench live Kafka fanout is not configured.")); + return; + } + + let closed = false; + let realtimeCloseReason = "client_close"; + let realtimeCloseSignal = null; + const realtimeStartedAtMs = Date.now(); + const cleanup = []; + const isActive = () => !closed && !response.destroyed && !response.writableEnded; + const closeConnection = () => { + if (closed) return; + closed = true; + emitWorkbenchRealtimeClosedOtelSpan(request, options.env, { + reason: realtimeCloseReason, + signal: realtimeCloseSignal, + startedAtMs: realtimeStartedAtMs, + sessionId: requestedSessionId, + threadId: null, + traceId: requestedTraceId, + activeConnectionCount: activeWorkbenchRealtimeConnections.size + }); + for (const item of cleanup.splice(0)) item(); + }; + + response.once("close", closeConnection); + request.once?.("aborted", closeConnection); + request.socket?.once?.("close", closeConnection); + cleanup.push(() => request.off?.("aborted", closeConnection)); + cleanup.push(() => request.socket?.off?.("close", closeConnection)); + + const bufferedEnvelopes = []; + let liveDeliveryStarted = false; + let enqueueEvent = null; + const unsubscribe = bridge.subscribeLiveHwlabEvents((envelope, transport) => { + if (!liveKafkaEnvelopeMatches(envelope, requestedSessionId, requestedTraceId)) return; + if (!liveDeliveryStarted || typeof enqueueEvent !== "function") { + bufferedEnvelopes.push({ envelope, transport }); + return; + } + void enqueueEvent("hwlab.event.v1", envelope, transport); + }); + if (typeof unsubscribe === "function") cleanup.push(unsubscribe); + + try { + await bridge.ready; + } catch (error) { + closeConnection(); + throw error; + } + if (!isActive()) { + closeConnection(); + return; + } + + response.writeHead(200, { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-store, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + "x-content-type-options": "nosniff" + }); + if (typeof response.flushHeaders === "function") response.flushHeaders(); + + const writeEvent = async (name, payload, transport = null) => { + if (!isActive()) return false; + const block = `event: ${name}\ndata: ${JSON.stringify(payload)}\n\n`; + const writable = response.write(block); + if (!writable) await waitForResponseDrain(response, isActive); + const active = isActive(); + if (active && name === "hwlab.event.v1") { + emitLiveKafkaOtelSpan("hwlab.workbench.live_sse.business_event_write", payload, transport, { + env: options.env, + otelSpanEmitter: options.otelSpanEmitter ?? emitCodeAgentOtelSpan + }); + } + return active; + }; + let writeChain = Promise.resolve(true); + enqueueEvent = (name, payload, transport = null) => { + writeChain = writeChain.then(() => writeEvent(name, payload, transport)); + return writeChain; + }; + + const realtimeConnection = { + close(fields = {}) { + if (!isActive()) return false; + realtimeCloseReason = textValue(fields.reason) || "server_shutdown"; + realtimeCloseSignal = textValue(fields.signal) || null; + void enqueueEvent("workbench.server_draining", { + type: "server.draining", + status: "closing", + reason: realtimeCloseReason, + signal: realtimeCloseSignal, + capabilities: realtimeCapabilities, + lossPossible: true + }).finally(() => { + if (!response.writableEnded) response.end(); + }); + return true; + }, + isActive + }; + activeWorkbenchRealtimeConnections.add(realtimeConnection); + cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection)); + + await enqueueEvent("workbench.connected", { + type: "connected", + status: "connected", + capabilities: realtimeCapabilities, + realtimeSource: "hwlab.event.v1", + deliverySemantics: "live-only", + liveOnly: true, + replay: false, + replaySupported: false, + lossPossible: true, + filters: { sessionId: requestedSessionId, traceId: requestedTraceId } + }); + liveDeliveryStarted = true; + const readyBufferedEnvelopes = bufferedEnvelopes.splice(0); + for (const { envelope, transport } of readyBufferedEnvelopes) void enqueueEvent("hwlab.event.v1", envelope, transport); + await writeChain; + if (!isActive()) { + closeConnection(); + return; + } + const heartbeatTimer = setInterval(() => { + void enqueueEvent("workbench.heartbeat", { + type: "heartbeat", + status: "connected", + realtimeSource: "hwlab.event.v1", + liveOnly: true, + replay: false, + lossPossible: true, + serverSentAt: new Date().toISOString(), + valuesPrinted: false + }); + }, heartbeatMs); + heartbeatTimer.unref?.(); + cleanup.push(() => clearInterval(heartbeatTimer)); + emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env); +} + +async function authorizeLiveKafkaWorkbenchRealtimeScope(options, actor, { sessionId, traceId }) { + const access = options.accessController; + if ((sessionId && typeof access?.getAgentSession !== "function") || (traceId && typeof access?.getAgentSessionByTraceId !== "function")) { + return { + ok: false, + status: 503, + body: workbenchError("workbench_realtime_authorization_unconfigured", "Workbench live realtime ownership authorization is not configured.") + }; + } + + const [sessionById, sessionByTrace] = await Promise.all([ + sessionId ? access.getAgentSession(sessionId) : null, + traceId ? access.getAgentSessionByTraceId(traceId) : null + ]); + const sessionByIdId = sessionById ? safeSessionId(sessionById.id ?? sessionById.sessionId) : null; + const sessionByTraceId = sessionByTrace ? safeSessionId(sessionByTrace.id ?? sessionByTrace.sessionId) : null; + const scopeMissing = (sessionId && sessionByIdId !== sessionId) + || (traceId && !sessionByTraceId) + || (sessionId && traceId && sessionByIdId !== sessionByTraceId); + const resolvedSessions = [sessionById, sessionByTrace].filter(Boolean); + const ownedByActor = actor?.role === "admin" || (Boolean(actor?.id) + && resolvedSessions.length > 0 + && resolvedSessions.every((session) => textValue(session.ownerUserId) === actor.id)); + if (scopeMissing || !ownedByActor) { + return { + ok: false, + status: 404, + body: workbenchError("workbench_realtime_scope_not_found", "Workbench realtime scope is not visible to the current actor.") + }; + } + return { ok: true, sessionId: sessionByIdId ?? sessionByTraceId }; +} + +function liveKafkaEnvelopeMatches(envelope, sessionId, traceId) { + if (!envelope || envelope.schema !== "hwlab.event.v1") return false; + if (sessionId && safeSessionId(envelope.hwlabSessionId ?? envelope.sessionId) !== sessionId) return false; + if (traceId && safeTraceId(envelope.traceId) !== traceId) return false; + return true; +} + export function attachWorkbenchRealtimeOtelContext(request, fields = {}) { const context = request?.hwlabHttpRequestContext; if (!context) return; - context.route = "/v1/workbench/events"; + context.route = textValue(fields.route) || "/v1/workbench/events"; context.otelAttributes = { ...(context.otelAttributes ?? {}), "workbench.route": "events", @@ -449,7 +681,9 @@ export function attachWorkbenchRealtimeOtelContext(request, fields = {}) { "workbench.thread_id": fields.threadId ?? null, "workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null, "workbench.sse.realtime_source": fields.realtimeSource ?? null, - "workbench.sse.outbox_mode": true + "workbench.sse.live_kafka_enabled": fields.realtimeCapabilities?.liveKafkaSse ?? null, + "workbench.sse.projection_realtime_enabled": fields.realtimeCapabilities?.projectionRealtime ?? null, + "workbench.sse.outbox_mode": fields.realtimeCapabilities?.liveKafkaSse === true ? false : true }; } @@ -500,7 +734,7 @@ export function emitWorkbenchRealtimeAcceptedOtelSpan(request, env = process.env endTimeMs: endedAtMs, statusCode: 200, method: "GET", - route: "/v1/workbench/events", + route: context.route ?? "/v1/workbench/events", attributes: { ...(context.otelAttributes ?? {}), "http.closed_by": "sse-accepted", @@ -521,7 +755,7 @@ export function emitWorkbenchRealtimeClosedOtelSpan(request, env = process.env, endTimeMs: endedAtMs, statusCode: 200, method: "GET", - route: "/v1/workbench/events", + route: context.route ?? "/v1/workbench/events", attributes: { ...(context.otelAttributes ?? {}), "http.closed_by": reason, diff --git a/internal/cloud/server.ts b/internal/cloud/server.ts index f2af88db..56f3bc29 100644 --- a/internal/cloud/server.ts +++ b/internal/cloud/server.ts @@ -69,6 +69,7 @@ import { } from "./server-code-agent-http.ts"; import { drainWorkbenchRealtimeConnections, handleWorkbenchReadModelHttp, handleWorkbenchRealtimeHttp } from "./server-workbench-http.ts"; import { handleWorkbenchDebugFakeSseHttp } from "./workbench-debug-fake-sse.ts"; +import { handleWorkbenchKafkaSseDebugHttp } from "./workbench-kafka-sse-debug.ts"; import { handleWorkbenchLaunchHttp } from "./server-workbench-launch-http.ts"; import { startWorkbenchEmptySessionGc } from "./workbench-empty-session-gc.ts"; import { startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts"; @@ -367,9 +368,17 @@ export async function buildHealthPayload(options = {}) { async function kafkaProjectorHealth(projector, { liveProbe = false } = {}) { if (!projector?.started) return { started: false, reason: projector?.reason ?? "disabled", valuesRedacted: true }; - if (liveProbe) return { started: true, groupId: projector.groupId, agentRunTopic: projector.agentRunTopic, hwlabTopic: projector.hwlabTopic, ...(projector.startupStatus?.() ?? { status: "initializing" }), statusSource: "startup-state", valuesRedacted: true }; + const identity = { + capabilities: projector.capabilities ?? null, + directPublishGroupId: projector.directPublishGroupId ?? null, + projectorGroupId: projector.projectorGroupId ?? null, + hwlabEventGroupId: projector.hwlabEventGroupId ?? null, + agentRunTopic: projector.agentRunTopic, + hwlabTopic: projector.hwlabTopic + }; + if (liveProbe) return { started: true, ...identity, ...(projector.startupStatus?.() ?? { status: "initializing" }), statusSource: "startup-state", valuesRedacted: true }; try { - return { started: true, groupId: projector.groupId, agentRunTopic: projector.agentRunTopic, hwlabTopic: projector.hwlabTopic, ...(await projector.status()), valuesRedacted: true }; + return { started: true, ...identity, ...(await projector.status()), valuesRedacted: true }; } catch (error) { return { started: true, status: "blocked", errorCode: error?.code ?? "hwlab_kafka_projector_status_failed", message: error?.message ?? "Kafka projector status query failed.", valuesRedacted: true }; } @@ -734,7 +743,7 @@ async function handleRestAdapter(request, response, url, options) { return; } - if (url.pathname === "/v1/workbench/events") { + if (url.pathname === "/v1/workbench/events" || url.pathname === "/v1/workbench/projection-events") { await handleWorkbenchRealtimeHttp(request, response, url, options); return; } @@ -749,6 +758,11 @@ async function handleRestAdapter(request, response, url, options) { return; } + if (url.pathname === "/v1/workbench/debug/kafka-sse" || url.pathname.startsWith("/v1/workbench/debug/kafka-sse/")) { + await handleWorkbenchKafkaSseDebugHttp(request, response, url, options); + return; + } + if (url.pathname === "/v1/workbench/sync" || url.pathname === "/v1/workbench/sessions" || url.pathname.startsWith("/v1/workbench/sessions/") || url.pathname.startsWith("/v1/workbench/turns/") || url.pathname.startsWith("/v1/workbench/traces/")) { await handleWorkbenchReadModelHttp(request, response, url, options); return; @@ -980,8 +994,8 @@ function navIdForRestPath(pathname, method = "GET") { if (pathname === "/v1/project-management" || pathname.startsWith("/v1/project-management/")) return "project.mdtodo"; if (pathname === "/v1/api-keys" || pathname === "/v1/api-keys/default" || pathname.startsWith("/v1/api-keys/")) return "user.apiKeys"; if (pathname === "/v1/users/me/profile" || pathname === "/v1/users/me/password") return "system.settings"; - if (pathname === "/v1/workbench/debug/fake-sse" || pathname.startsWith("/v1/workbench/debug/fake-sse/")) return "workbench.debug"; - if (pathname === "/v1/workbench/events" || pathname === "/v1/workbench/sync" || pathname === "/v1/workbench/launches" || pathname === "/v1/workbench/sessions" || pathname.startsWith("/v1/workbench/sessions/") || pathname.startsWith("/v1/workbench/turns/") || pathname.startsWith("/v1/workbench/traces/")) return "workbench.code"; + if (pathname === "/v1/workbench/debug/fake-sse" || pathname.startsWith("/v1/workbench/debug/fake-sse/") || pathname === "/v1/workbench/debug/kafka-sse" || pathname.startsWith("/v1/workbench/debug/kafka-sse/")) return "workbench.debug"; + if (pathname === "/v1/workbench/events" || pathname === "/v1/workbench/projection-events" || pathname === "/v1/workbench/sync" || pathname === "/v1/workbench/launches" || pathname === "/v1/workbench/sessions" || pathname.startsWith("/v1/workbench/sessions/") || pathname.startsWith("/v1/workbench/turns/") || pathname.startsWith("/v1/workbench/traces/")) return "workbench.code"; if (pathname === "/v1/agent/chat" || pathname === "/v1/agent/sessions" || pathname.startsWith("/v1/agent/sessions/") || pathname === "/v1/agent/chat/inspect" || pathname.startsWith("/v1/agent/chat/result/") || pathname.startsWith("/v1/agent/turns/") || pathname.startsWith("/v1/agent/traces/") || pathname === "/v1/agent/chat/cancel" || pathname === "/v1/agent/chat/steer") return "workbench.code"; if (pathname === "/v1/admin/provider-profiles" || pathname.startsWith("/v1/admin/provider-profiles/")) return "admin.providerProfiles"; if (pathname === "/v1/admin/secrets" || pathname.startsWith("/v1/admin/secrets/")) return "admin.secrets"; diff --git a/internal/cloud/web-performance.test.ts b/internal/cloud/web-performance.test.ts index 809980d1..227ece53 100644 --- a/internal/cloud/web-performance.test.ts +++ b/internal/cloud/web-performance.test.ts @@ -491,7 +491,13 @@ test("web performance route templates keep metrics low-cardinality", () => { }); test("cloud api ingests WebUI performance samples and exposes metrics only on loopback host", async () => { - const server = createCloudApiServer({ env: { HWLAB_METRICS_NAMESPACE: "hwlab-v02", HWLAB_GITOPS_TARGET: "v02" } }); + const server = createCloudApiServer({ + accessController: { + required: false, + async authenticate() { return { ok: true, actor: { id: "usr_web_performance", role: "admin" }, session: { id: "uss_web_performance" } }; } + }, + env: { HWLAB_METRICS_NAMESPACE: "hwlab-v02", HWLAB_GITOPS_TARGET: "v02" } + }); await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); try { const address = server.address(); diff --git a/internal/cloud/workbench-facts-store.ts b/internal/cloud/workbench-facts-store.ts index d5b0b99f..f636d149 100644 --- a/internal/cloud/workbench-facts-store.ts +++ b/internal/cloud/workbench-facts-store.ts @@ -206,7 +206,7 @@ function objectValue(value) { } async function durableTraceSnapshot(runtimeStore, traceId) { - if (!runtimeStore || typeof runtimeStore.queryAgentTraceEvents !== "function") return null; + if (!runtimeStore || typeof runtimeStore.queryWorkbenchFacts !== "function") return null; const safeId = safeTraceId(traceId); if (!safeId) return null; const cached = terminalDurableTraceCache.get(safeId); @@ -217,11 +217,11 @@ async function durableTraceSnapshot(runtimeStore, traceId) { } let result; try { - result = await runtimeStore.queryAgentTraceEvents({ traceId: safeId }); + result = await runtimeStore.queryWorkbenchFacts({ traceId: safeId, families: ["traceEvents"] }); } catch (error) { return projectionStoreUnavailableTrace(safeId, error); } - const events = Array.isArray(result?.events) ? result.events : []; + const events = Array.isArray(result?.facts?.traceEvents) ? result.facts.traceEvents : []; if (events.length === 0) return null; const normalizedEvents = events.map((event, index) => ({ ...event, traceId: safeId, seq: eventSeq(event, index) })); const firstEvent = normalizedEvents[0] ?? null; diff --git a/internal/cloud/workbench-kafka-sse-debug.test.ts b/internal/cloud/workbench-kafka-sse-debug.test.ts new file mode 100644 index 00000000..f9e200a5 --- /dev/null +++ b/internal/cloud/workbench-kafka-sse-debug.test.ts @@ -0,0 +1,308 @@ +import assert from "node:assert/strict"; +import { createServer } from "node:http"; +import { test } from "bun:test"; + +import { handleWorkbenchKafkaSseDebugHttp } from "./workbench-kafka-sse-debug.ts"; + +test("workbench Kafka SSE debug endpoint streams filtered raw HWLAB Kafka events", async () => { + const fakeKafka = createFakeKafkaFactory(); + const env = { + HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", + HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", + HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" + }; + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, accessController: fakeAccessController(), logger: null }); + }); + await listen(server); + const abort = new AbortController(); + try { + const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab&sessionId=ses_kafka_sse_debug`, { signal: abort.signal }); + assert.equal(response.status, 200); + assert.match(response.headers.get("content-type") ?? "", /text\/event-stream/u); + const reader = response.body?.getReader(); + assert.ok(reader, "SSE response must expose a readable stream"); + const connected = await readUntil(reader, "hwlab.kafka.connected"); + assert.match(connected, /hwlab\.event\.v1/u); + assert.match(connected, /"consumerReady":true/u); + assert.match(connected, new RegExp(String(fakeKafka.groupId).replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"), "u")); + + await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", sessionId: "ses_other", event: { label: "ignored" } }); + await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", sessionId: "ses_kafka_sse_debug", traceId: "trc_kafka_sse_debug", context: { runId: "run_kafka_sse_debug", commandId: "cmd_kafka_sse_debug" }, event: { label: "agentrun:event:debug", status: "running" } }); + const streamed = await readUntil(reader, "trc_kafka_sse_debug"); + assert.match(streamed, /hwlab\.kafka\.event/u); + assert.match(streamed, /ses_kafka_sse_debug/u); + assert.doesNotMatch(streamed, /ses_other/u); + } finally { + abort.abort(); + await close(server); + } +}); + +test("workbench Kafka SSE debug endpoint streams filtered codex stdio Kafka events", async () => { + const fakeKafka = createFakeKafkaFactory(); + const env = { + HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", + HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", + HWLAB_KAFKA_STDIO_TOPIC: "codex-stdio.raw.v1" + }; + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, accessController: fakeAccessController(), logger: null }); + }); + await listen(server); + const abort = new AbortController(); + try { + const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=stdio&traceId=trc_stdio_sse_debug&fromBeginning=true`, { signal: abort.signal }); + assert.equal(response.status, 200); + const reader = response.body?.getReader(); + assert.ok(reader, "SSE response must expose a readable stream"); + const connected = await readUntil(reader, "hwlab.kafka.connected"); + assert.match(connected, /codex-stdio\.raw\.v1/u); + + await fakeKafka.emit({ traceId: "trc_other", stream: "stdout", line: "ignored" }); + await fakeKafka.emit({ trace_id: "trc_stdio_sse_debug", metadata: { session_id: "ses_stdio_sse_debug", run_id: "run_stdio_sse_debug" }, stream: "stdout", line: "visible stdio frame" }); + const streamed = await readUntil(reader, "visible stdio frame"); + assert.match(streamed, /hwlab\.kafka\.event/u); + assert.match(streamed, /codex-stdio\.raw\.v1/u); + assert.match(streamed, /ses_stdio_sse_debug/u); + assert.doesNotMatch(streamed, /trc_other/u); + } finally { + abort.abort(); + await close(server); + } +}); + +test("workbench Kafka SSE debug endpoint resolves traceId to AgentRun Kafka ids", async () => { + const fakeKafka = createFakeKafkaFactory(); + const traceStore = { + snapshot: () => ({ + traceId: "trc_resolved_kafka_debug", + events: [ + { type: "backend", runId: "run_resolved_kafka_debug", commandId: "cmd_resolved_kafka_debug", sessionId: "ses_agentrun_resolved_kafka_debug" } + ], + lastEvent: null + }) + }; + const env = { + HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", + HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", + HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" + }; + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { env, kafkaFactory: fakeKafka.factory, traceStore, accessController: fakeAccessController(), logger: null }); + }); + await listen(server); + const abort = new AbortController(); + try { + const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab&traceId=trc_resolved_kafka_debug`, { signal: abort.signal }); + assert.equal(response.status, 200); + const reader = response.body?.getReader(); + assert.ok(reader, "SSE response must expose a readable stream"); + const connected = await readUntil(reader, "resolvedFilters"); + assert.match(connected, /trc_resolved_kafka_debug/u); + assert.match(connected, /run_resolved_kafka_debug/u); + + await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", traceId: null, sessionId: null, context: { runId: "run_other", commandId: "cmd_other" }, event: { label: "ignored" } }); + await fakeKafka.emit({ eventType: "hwlab.trace.event.projected", traceId: null, sessionId: null, context: { runId: "run_resolved_kafka_debug", commandId: "cmd_resolved_kafka_debug" }, event: { label: "resolved", status: "running" } }); + const streamed = await readUntil(reader, "run_resolved_kafka_debug"); + assert.match(streamed, /hwlab\.kafka\.event/u); + assert.match(streamed, /resolved/u); + assert.doesNotMatch(streamed, /run_other/u); + } finally { + abort.abort(); + await close(server); + } +}); + +test("workbench Kafka SSE debug endpoint is admin-only", async () => { + const fakeKafka = createFakeKafkaFactory(); + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { + env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092" }, + kafkaFactory: fakeKafka.factory, + accessController: fakeAccessController("user"), + logger: null + }); + }); + await listen(server); + try { + const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab`); + assert.equal(response.status, 403); + assert.match(await response.text(), /admin_required/u); + assert.equal(fakeKafka.consumerCreated, false); + } finally { + await close(server); + } +}); + +test("workbench Kafka SSE debug endpoint reports connected only after consumer run is ready", async () => { + const fakeKafka = createFakeKafkaFactory({ deferRun: true }); + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { + env: { + HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", + HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", + HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" + }, + kafkaFactory: fakeKafka.factory, + accessController: fakeAccessController(), + logger: null + }); + }); + await listen(server); + const abort = new AbortController(); + try { + const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab`, { signal: abort.signal }); + const reader = response.body?.getReader(); + assert.ok(reader); + const firstRead = reader.read(); + const early = await Promise.race([firstRead.then(() => "event"), delay(25).then(() => "pending")]); + assert.equal(early, "pending"); + fakeKafka.releaseRun(); + const first = await firstRead; + assert.equal(first.done, false); + const connected = new TextDecoder().decode(first.value); + assert.match(connected, /hwlab\.kafka\.connected/u); + assert.match(connected, /"consumerReady":true/u); + assert.match(connected, /"groupId":"test-hwlab-cloud-api-debug-sse-/u); + } finally { + abort.abort(); + fakeKafka.releaseRun(); + await close(server); + } +}); + +test("workbench Kafka SSE debug endpoint stops a consumer that becomes ready after client close", async () => { + const fakeKafka = createFakeKafkaFactory({ deferRun: true }); + const server = createServer((request, response) => { + const url = new URL(request.url ?? "/", "http://127.0.0.1"); + void handleWorkbenchKafkaSseDebugHttp(request, response, url, { + env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092", HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api", HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1" }, + kafkaFactory: fakeKafka.factory, + accessController: fakeAccessController(), + logger: null + }); + }); + await listen(server); + const abort = new AbortController(); + try { + const response = await fetch(`${serverUrl(server)}/v1/workbench/debug/kafka-sse/events?stream=hwlab`, { signal: abort.signal }); + const reader = response.body?.getReader(); + assert.ok(reader); + const firstRead = reader.read().catch(() => null); + await delay(20); + abort.abort(); + fakeKafka.releaseRun(); + await waitUntil(() => fakeKafka.stopCalls > 0); + const first = await firstRead; + const text = first && !first.done && first.value ? new TextDecoder().decode(first.value) : ""; + assert.doesNotMatch(text, /hwlab\.kafka\.connected/u); + assert.equal(fakeKafka.disconnectCalls > 0, true); + } finally { + abort.abort(); + fakeKafka.releaseRun(); + await close(server); + } +}); + +function createFakeKafkaFactory(options: { deferRun?: boolean } = {}) { + let eachMessage: ((input: any) => Promise) | null = null; + let subscribedTopic = "hwlab.event.v1"; + let releaseRun: (() => void) | null = null; + const runGate = options.deferRun ? new Promise((resolve) => { releaseRun = resolve; }) : null; + let groupId: string | null = null; + let stopCalls = 0; + let disconnectCalls = 0; + let consumerCreated = false; + const consumer = { + connect: async () => undefined, + subscribe: async (input: { topic?: string }) => { subscribedTopic = String(input.topic ?? subscribedTopic); }, + run: async (input: any) => { + if (runGate) await runGate; + eachMessage = input.eachMessage; + }, + stop: async () => { stopCalls += 1; }, + disconnect: async () => { disconnectCalls += 1; } + }; + return { + factory: () => ({ + consumer: (input: { groupId?: string }) => { + consumerCreated = true; + groupId = String(input.groupId ?? ""); + return consumer; + } + }), + get groupId() { return groupId; }, + get stopCalls() { return stopCalls; }, + get disconnectCalls() { return disconnectCalls; }, + get consumerCreated() { return consumerCreated; }, + releaseRun: () => releaseRun?.(), + emit: async (value: Record) => { + assert.ok(eachMessage, "consumer.run must be called before emitting fake Kafka events"); + await eachMessage({ + topic: subscribedTopic, + partition: 0, + message: { + offset: String(value.sessionId === "ses_other" ? 1 : 2), + key: Buffer.from(String(value.sessionId ?? "unknown")), + timestamp: "2026-07-09T18:30:00.000Z", + value: Buffer.from(JSON.stringify(value)) + } + }); + } + }; +} + +function fakeAccessController(role: "admin" | "user" = "admin") { + return { + async ensureBootstrap() {}, + async authenticate() { + return { ok: true, status: 200, actor: { id: role === "admin" ? "usr_debug_admin" : "usr_debug_user", role } }; + } + }; +} + +async function delay(ms: number) { + await new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function waitUntil(predicate: () => boolean, timeoutMs = 1000) { + const deadline = Date.now() + timeoutMs; + while (!predicate()) { + if (Date.now() >= deadline) throw new Error("condition did not become true before timeout"); + await delay(10); + } +} + +async function readUntil(reader: ReadableStreamDefaultReader, pattern: string): Promise { + const decoder = new TextDecoder(); + let text = ""; + const deadline = Date.now() + 3000; + while (!text.includes(pattern)) { + if (Date.now() > deadline) throw new Error(`SSE stream did not include ${pattern}: ${text}`); + const next = await reader.read(); + if (next.done) break; + text += decoder.decode(next.value, { stream: true }); + } + return text; +} + +async function listen(server: ReturnType) { + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); +} + +async function close(server: ReturnType) { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); +} + +function serverUrl(server: ReturnType): string { + const address = server.address(); + assert.equal(typeof address, "object"); + assert.ok(address && typeof address.port === "number"); + return `http://127.0.0.1:${address.port}`; +} diff --git a/internal/cloud/workbench-kafka-sse-debug.ts b/internal/cloud/workbench-kafka-sse-debug.ts new file mode 100644 index 00000000..10d4b13a --- /dev/null +++ b/internal/cloud/workbench-kafka-sse-debug.ts @@ -0,0 +1,229 @@ +/* + * SPEC: PJ2026-0104010803 Workbench事件流可见性 draft-2026-07-09-p0-kafka-authority. + * Responsibility: debug-only SSE passthrough for HWLAB Kafka event envelopes. + */ +import { sendJson } from "./server-http-utils.ts"; +import { openKafkaEventStream } from "./kafka-event-bridge.ts"; +import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts"; +import { authenticateWorkbenchRead } from "./server-workbench-read-http.ts"; + +const CONTRACT_VERSION = "workbench-debug-kafka-sse-v1"; +const DEFAULT_STREAM = "hwlab"; +const ALLOWED_STREAMS = new Set(["stdio", "agentrun", "hwlab"]); + +export async function handleWorkbenchKafkaSseDebugHttp(request, response, url, options = {}) { + const route = routeSuffix(url.pathname); + try { + const auth = await authenticateWorkbenchRead(request, response, options); + if (!auth) return; + if (auth.actor?.role !== "admin") { + return sendJson(response, 403, debugError("admin_required", "Only admin users can access raw Workbench Kafka debug streams.")); + } + if (route === "") { + if (request.method !== "GET") return methodNotAllowed(response, "GET"); + return sendJson(response, 200, describeKafkaSseDebug(url, options.env ?? process.env)); + } + if (route === "/events") { + if (request.method !== "GET") return methodNotAllowed(response, "GET"); + return openKafkaDebugSse(request, response, url, options); + } + return sendJson(response, 404, debugError("workbench_debug_kafka_sse_route_not_found", "Workbench debug Kafka SSE route is not implemented.", { route })); + } catch (error) { + options.logger?.warn?.({ + event: "workbench_debug_kafka_sse_failed", + route, + errorName: error?.name ?? "Error", + message: error instanceof Error ? error.message : String(error ?? "unknown"), + valuesRedacted: true + }); + if (response.headersSent) return writeSse(response, "hwlab.kafka.error", { ok: false, error: errorMessagePayload(error) }); + return sendJson(response, 500, debugError("workbench_debug_kafka_sse_failed", "Workbench debug Kafka SSE request failed.")); + } +} + +function describeKafkaSseDebug(url, env) { + const stream = streamFromUrl(url); + const requestedFilters = filtersFromUrl(url); + const resolvedFilters = resolveKafkaDebugFilters(requestedFilters, { traceStore: defaultCodeAgentTraceStore }); + return { + ok: true, + contractVersion: CONTRACT_VERSION, + stream, + topic: topicForStream(stream, env), + filters: requestedFilters, + resolvedFilters, + eventsRoute: `/v1/workbench/debug/kafka-sse/events${url.search || ""}`, + valuesPrinted: false + }; +} + +async function openKafkaDebugSse(request, response, url, options) { + const env = options.env ?? process.env; + const stream = streamFromUrl(url); + const filters = filtersFromUrl(url); + const resolvedFilters = resolveKafkaDebugFilters(filters, options); + response.writeHead(200, { + "content-type": "text/event-stream; charset=utf-8", + "cache-control": "no-store, no-transform", + connection: "keep-alive", + "x-accel-buffering": "no", + "x-content-type-options": "nosniff" + }); + if (typeof response.flushHeaders === "function") response.flushHeaders(); + let kafkaStream = null; + let closed = false; + const close = () => { + if (closed) return; + closed = true; + void kafkaStream?.stop?.(); + }; + response.once("close", close); + request.once?.("aborted", close); + request.socket?.once?.("close", close); + try { + kafkaStream = await openKafkaEventStream({ + env, + stream, + fromBeginning: url.searchParams.get("fromBeginning") === "1" || url.searchParams.get("fromBeginning") === "true", + ...resolvedFilters, + kafkaFactory: options.kafkaFactory, + onEvent: async (record) => { + writeSse(response, "hwlab.kafka.event", { + ok: true, + contractVersion: CONTRACT_VERSION, + stream, + topic: record.topic, + partition: record.partition, + offset: record.offset, + key: record.key, + timestamp: record.timestamp, + value: record.value, + serverSentAt: new Date().toISOString(), + valuesPrinted: false + }); + } + }); + if (closed) { + await kafkaStream.stop?.(); + return; + } + writeSse(response, "hwlab.kafka.connected", { + ok: true, + contractVersion: CONTRACT_VERSION, + consumerReady: true, + groupId: kafkaStream.groupId, + stream, + topic: kafkaStream.topic ?? topicForStream(stream, env), + filters, + resolvedFilters, + serverSentAt: new Date().toISOString(), + valuesPrinted: false + }); + } catch (error) { + writeSse(response, "hwlab.kafka.error", { ok: false, error: errorMessagePayload(error), valuesPrinted: false }); + } +} + +function resolveKafkaDebugFilters(filters = {}, options = {}) { + const traceId = textValue(filters.traceId); + if (!traceId) return filters; + const traceStore = options.traceStore ?? defaultCodeAgentTraceStore; + const snapshot = typeof traceStore?.snapshot === "function" ? traceStore.snapshot(traceId) : null; + const resolved = collectTraceLinkedIds(snapshot); + const hasResolvedKafkaKey = Boolean(resolved.runId || resolved.commandId || resolved.sessionId); + if (!hasResolvedKafkaKey) return filters; + const resolvedSessionId = resolved.runId || resolved.commandId ? null : resolved.sessionId; + return compactObject({ + sessionId: filters.sessionId || resolvedSessionId, + runId: filters.runId || resolved.runId, + commandId: filters.commandId || resolved.commandId + }); +} + +function collectTraceLinkedIds(snapshot) { + const out = { sessionId: null, runId: null, commandId: null }; + const events = Array.isArray(snapshot?.events) ? snapshot.events : []; + for (const event of events) collectIdsFromRecord(event, out); + collectIdsFromRecord(snapshot?.lastEvent, out); + return out; +} + +function collectIdsFromRecord(record, out) { + const value = record && typeof record === "object" && !Array.isArray(record) ? record : {}; + const agentRun = value.agentRun && typeof value.agentRun === "object" && !Array.isArray(value.agentRun) ? value.agentRun : {}; + const payload = value.payload && typeof value.payload === "object" && !Array.isArray(value.payload) ? value.payload : {}; + out.sessionId ||= textValue(value.sessionId ?? value.sourceSessionId ?? agentRun.sessionId ?? payload.sessionId); + out.runId ||= textValue(value.runId ?? value.sourceRunId ?? agentRun.runId ?? payload.runId); + out.commandId ||= textValue(value.commandId ?? value.sourceCommandId ?? agentRun.commandId ?? payload.commandId); +} + +function writeSse(response, eventName, payload) { + if (response.destroyed || response.writableEnded) return false; + try { + response.write(`event: ${eventName}\n`); + const id = sseEventId(payload); + if (id) response.write(`id: ${id}\n`); + response.write(`data: ${JSON.stringify(payload)}\n\n`); + return true; + } catch { + return false; + } +} + +function routeSuffix(pathname) { + return String(pathname || "").replace(/^\/v1\/workbench\/debug\/kafka-sse/u, ""); +} + +function streamFromUrl(url) { + const stream = String(url.searchParams.get("stream") || DEFAULT_STREAM).trim(); + return ALLOWED_STREAMS.has(stream) ? stream : DEFAULT_STREAM; +} + +function filtersFromUrl(url) { + return compactObject({ + traceId: safeId(url.searchParams.get("traceId") || url.searchParams.get("trace-id")), + sessionId: safeId(url.searchParams.get("sessionId") || url.searchParams.get("session-id")), + runId: safeId(url.searchParams.get("runId") || url.searchParams.get("run-id")), + commandId: safeId(url.searchParams.get("commandId") || url.searchParams.get("command-id")) + }); +} + +function topicForStream(stream, env) { + if (stream === "stdio") return textValue(env.HWLAB_KAFKA_STDIO_TOPIC ?? env.AGENTRUN_KAFKA_STDIO_TOPIC) || "codex-stdio.raw.v1"; + if (stream === "agentrun") return textValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC) || "agentrun.event.v1"; + return textValue(env.HWLAB_KAFKA_EVENT_TOPIC) || "hwlab.event.v1"; +} + +function methodNotAllowed(response, allow) { + response.setHeader("allow", allow); + return sendJson(response, 405, debugError("method_not_allowed", `Method not allowed; expected ${allow}.`)); +} + +function debugError(code, message, extra = {}) { + return { ok: false, error: { code, message, ...extra }, valuesPrinted: false }; +} + +function errorMessagePayload(error) { + return { code: "kafka_sse_error", message: error instanceof Error ? error.message : String(error ?? "unknown") }; +} + +function safeId(value) { + const text = textValue(value); + return text && /^[A-Za-z0-9_.:-]{3,220}$/u.test(text) ? text : null; +} + +function textValue(value) { + const text = typeof value === "string" ? value.trim() : value === null || value === undefined ? "" : String(value).trim(); + return text.length > 0 ? text : null; +} + +function compactObject(value) { + return Object.fromEntries(Object.entries(value).filter(([, entry]) => textValue(entry))); +} + +function sseEventId(payload) { + const topic = textValue(payload?.topic); + const partition = textValue(payload?.partition); + const offset = textValue(payload?.offset); + return topic && partition && offset ? `${topic}:${partition}:${offset}` : null; +} diff --git a/internal/cloud/workbench-projection-writer.test.ts b/internal/cloud/workbench-projection-writer.test.ts index 5d95effc..3336f1c8 100644 --- a/internal/cloud/workbench-projection-writer.test.ts +++ b/internal/cloud/workbench-projection-writer.test.ts @@ -6,6 +6,7 @@ import { recordWorkbenchSessionOwner, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts"; +import { projectionOutboxRealtimeEvents } from "./server-workbench-realtime-http.ts"; test("HTTP lifecycle records ownership without writing projection facts", async () => { const calls = []; @@ -159,11 +160,46 @@ test("completed terminal without final response remains visibly unsealed", () => } }); - assert.equal(built.facts.turns[0].terminal, false); - assert.equal(built.facts.turns[0].finalResponse, null); - assert.equal(built.facts.turns[0].diagnostic.terminalSealBlocked, true); - assert.equal(built.facts.checkpoints[0].projectionHealth, "degraded"); - assert.equal(built.facts.checkpoints[0].sealed, false); + const session = built.facts.sessions[0]; + const message = built.facts.messages[0]; + const turn = built.facts.turns[0]; + const traceEvent = built.facts.traceEvents[0]; + const checkpoint = built.facts.checkpoints[0]; + for (const fact of [session, message, turn, traceEvent, checkpoint]) { + assert.equal(fact.status ?? "running", "running"); + assert.equal(fact.terminal, false); + assert.equal(fact.sealed, false); + } + assert.equal(built.terminal, false); + assert.equal(built.sourceTerminal, true); + assert.equal(turn.finalResponse, null); + assert.equal(turn.diagnostic.terminalSealBlocked, true); + assert.equal(traceEvent.diagnostic.sourceStatus, "completed"); + assert.equal(traceEvent.finishedAt, null); + assert.equal(traceEvent.durationMs, null); + assert.equal(checkpoint.projectionStatus, "projecting"); + assert.equal(checkpoint.projectionHealth, "degraded"); + + const [sse] = projectionOutboxRealtimeEvents({ + events: [{ + outboxSeq: 1, + entityFamily: "traceEvents", + entityId: traceEvent.id, + projectedSeq: 1, + projectionRevision: 1, + traceId: traceEvent.traceId, + sessionId: traceEvent.sessionId, + commitType: "event", + terminal: traceEvent.terminal, + sealed: traceEvent.sealed, + payload: { family: "traceEvents", fact: traceEvent } + }] + }); + assert.equal(sse.name, "workbench.trace.event"); + assert.equal(sse.payload.terminal, false); + assert.equal(sse.payload.sealed, false); + assert.equal(sse.payload.snapshot.status, "running"); + assert.equal(sse.payload.event.terminal, false); }); test("late running event cannot overwrite a sealed terminal", () => { @@ -195,6 +231,38 @@ test("late running event cannot overwrite a sealed terminal", () => { assert.equal("facts" in built, false); }); +test("late terminal event cannot overwrite a different sealed terminal", () => { + const built = buildWorkbenchProjectionEventFacts({ + projectedSeq: 11, + previousCheckpoint: { + traceId: "trc_sealed_terminal", + sessionId: "ses_sealed_terminal", + projectedSeq: 10, + sourceSeq: 10, + status: "completed", + terminal: true, + sealed: true, + finalResponse: { text: "immutable completed answer", status: "completed" } + }, + event: { + traceId: "trc_sealed_terminal", + sessionId: "ses_sealed_terminal", + sourceEventId: "evt_late_failed_terminal", + sourceSeq: 11, + type: "terminal_status", + status: "failed", + terminal: true, + errorCode: "late-failure" + } + }); + + assert.equal(built.written, false); + assert.equal(built.suppressedAfterSeal, true); + assert.equal(built.projectedSeq, 10); + assert.equal(built.sourceEventId, "evt_late_failed_terminal"); + assert.equal("facts" in built, false); +}); + test("same canonical source event derives deterministic fact identities", () => { const input = { projectedSeq: 3, diff --git a/internal/cloud/workbench-projection-writer.ts b/internal/cloud/workbench-projection-writer.ts index 4af8ee7b..ab05e8d0 100644 --- a/internal/cloud/workbench-projection-writer.ts +++ b/internal/cloud/workbench-projection-writer.ts @@ -1,7 +1,7 @@ /* * SPEC: PJ2026-0104010803 Workbench唯一投影; HWLAB#2464 transactional Kafka projector. - * Responsibility: persist admission-only session ownership and build pure Kafka projection facts. - * Message/turn/checkpoint facts are committed only by commitAgentRunKafkaProjection. + * Responsibility: atomically promote durable admission into initial nonterminal Workbench facts and build pure Kafka projection facts. + * Local admission may create the first running session/message/turn snapshot; all later lifecycle and terminal authority comes only from commitAgentRunKafkaProjection. */ import { createHash } from "node:crypto"; @@ -27,6 +27,118 @@ export async function writeWorkbenchSessionAdmissionFact({ runtimeStore = null, if (typeof runtimeStore?.writeWorkbenchSessionAdmissionFact !== "function") { throw workbenchProjectionError("workbench_admission_store_unconfigured", "Workbench admission requires the durable session admission store."); } + const fact = buildWorkbenchSessionAdmissionFact({ session, ownerUserId, ownerRole, projectId, conversationId, threadId, status, now }); + return runtimeStore.writeWorkbenchSessionAdmissionFact({ fact }, { + sessionId: fact.sessionId, + ownerUserId: fact.ownerUserId, + projectId: fact.projectId, + valuesPrinted: false + }); +} + +export async function promoteWorkbenchTurnAdmission({ runtimeStore = null, session = null, inputFact = null, payload = {}, params = {}, ownerUserId = null, ownerRole = null, projectId = null, conversationId = null, threadId = null, now = new Date().toISOString() } = {}) { + if (typeof runtimeStore?.promoteWorkbenchTurnAdmission !== "function") { + throw workbenchProjectionError("workbench_admission_promotion_store_unconfigured", "Workbench admission promotion requires the durable atomic promotion store."); + } + if (!inputFact || typeof inputFact !== "object") { + throw workbenchProjectionError("workbench_admission_input_required", "Workbench admission promotion requires an input fact."); + } + const facts = buildWorkbenchAdmissionPromotionFacts({ session, inputFact, payload, params, ownerUserId, ownerRole, projectId, conversationId, threadId, now }); + const sessionFact = facts.sessions[0]; + return runtimeStore.promoteWorkbenchTurnAdmission({ facts }, { + sessionId: sessionFact.sessionId, + traceId: inputFact.traceId ?? null, + turnId: inputFact.turnId ?? null, + messageId: inputFact.messageId ?? null, + ownerUserId: sessionFact.ownerUserId, + projectId: sessionFact.projectId, + valuesPrinted: false + }); +} + +export function buildWorkbenchAdmissionPromotionFacts({ session = null, inputFact = null, payload = {}, params = {}, ownerUserId = null, ownerRole = null, projectId = null, conversationId = null, threadId = null, now = new Date().toISOString() } = {}) { + const sessionFact = buildWorkbenchSessionAdmissionFact({ session, ownerUserId, ownerRole, projectId, conversationId, threadId, status: "running", now }); + const traceId = safeTraceId(payload.traceId ?? params.traceId ?? inputFact?.traceId); + if (!traceId) throw workbenchProjectionError("workbench_admission_trace_required", "Workbench admission promotion requires traceId."); + const turnId = textValue(payload.turnId ?? params.turnId ?? inputFact?.turnId) || traceId; + const userMessageId = textValue(payload.userMessageId ?? params.userMessageId ?? inputFact?.messageId) || eventProjectionUserMessageId(traceId, {}); + const assistantMessageId = textValue(payload.assistantMessageId ?? params.assistantMessageId) || eventProjectionAssistantMessageId(traceId, {}); + const sourceEventId = `${traceId}:admission-promotion`; + const occurredAt = timestampValue(payload.createdAt ?? now); + const timing = eventTimingProjection({ startedAt: occurredAt, lastEventAt: occurredAt, terminal: false }); + const userText = textValue(params.message ?? params.prompt ?? params.text); + const common = { + sessionId: sessionFact.sessionId, + turnId, + traceId, + projectedSeq: 0, + sourceSeq: 0, + sourceEventId, + terminal: false, + sealed: false, + timing, + startedAt: timing.startedAt, + lastEventAt: timing.lastEventAt, + finishedAt: null, + durationMs: null, + createdAt: occurredAt, + updatedAt: occurredAt, + valuesPrinted: false + }; + const userMessage = { ...common, messageId: userMessageId, role: "user", status: "sent", text: userText }; + const assistantMessage = { ...common, messageId: assistantMessageId, role: "agent", status: "running", text: "" }; + const turn = { + ...common, + messageId: assistantMessageId, + status: "running", + finalResponse: null, + assistantText: "", + failureKind: null, + diagnostic: { + authority: "local-admission-promotion", + admissionState: "promoted", + sourceRunId: textValue(payload.agentRun?.runId) || null, + sourceCommandId: textValue(payload.agentRun?.commandId) || null, + dispatchIntentId: textValue(payload.agentRun?.dispatchIntentId) || null, + runnerJobId: textValue(payload.agentRun?.runnerJobId) || null, + valuesRedacted: true + } + }; + return { + inputs: [{ + ...inputFact, + status: "promoted", + commandId: textValue(payload.agentRun?.commandId ?? inputFact.commandId) || null, + runId: textValue(payload.agentRun?.runId) || null, + dispatchIntentId: textValue(payload.agentRun?.dispatchIntentId) || null, + runnerJobId: textValue(payload.agentRun?.runnerJobId) || null, + durableDispatch: payload.agentRun?.durableDispatch === true, + updatedAt: occurredAt, + valuesRedacted: true, + secretMaterialStored: false + }], + sessions: [{ + ...sessionFact, + status: "running", + lastTraceId: traceId, + sessionJson: { + ...sessionFact.sessionJson, + admissionState: "promoted", + agentRun: payload.agentRun ?? sessionFact.sessionJson?.agentRun ?? null, + valuesRedacted: true, + secretMaterialStored: false + }, + updatedAt: occurredAt + }], + messages: [userMessage, assistantMessage], + parts: userText ? messagePartFacts(userMessage) : [], + turns: [turn], + traceEvents: [], + checkpoints: [] + }; +} + +export function buildWorkbenchSessionAdmissionFact({ session = null, ownerUserId = null, ownerRole = null, projectId = null, conversationId = null, threadId = null, status = "idle", now = new Date().toISOString() } = {}) { if (!session || typeof session !== "object") { throw workbenchProjectionError("workbench_admission_session_required", "Workbench admission requires a session record."); } @@ -38,7 +150,7 @@ export async function writeWorkbenchSessionAdmissionFact({ runtimeStore = null, const updatedAt = timestampValue(session.updatedAt ?? createdAt); const normalizedStatus = normalizeWorkbenchStatus(session.status ?? session.session?.sessionStatus ?? status) || "idle"; const sessionJson = workbenchSessionJsonWithLaunchContext(plainObjectValue(session.session) ?? {}, { payload: session }); - const fact = { + return { sessionId, ownerUserId: textValue(session.ownerUserId ?? ownerUserId) || null, ownerRole: textValue(session.ownerRole ?? ownerRole) || null, @@ -65,12 +177,6 @@ export async function writeWorkbenchSessionAdmissionFact({ runtimeStore = null, updatedAt, valuesRedacted: true }; - return runtimeStore.writeWorkbenchSessionAdmissionFact({ fact }, { - sessionId, - ownerUserId: fact.ownerUserId, - projectId: fact.projectId, - valuesPrinted: false - }); } export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = {}, previousCheckpoint = null, projectedSeq, projectedAt = new Date().toISOString() } = {}) { @@ -82,12 +188,12 @@ export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = { const sessionId = textValue(event.sessionId ?? requestMeta.sessionId) || null; const turnId = textValue(event.turnId) || traceId; const eventType = textValue(event.eventType ?? event.type ?? event.label) || "event"; - const terminal = event.terminal === true || TERMINAL_STATUSES.has(normalizeWorkbenchStatus(event.status)); - const status = terminal ? normalizeWorkbenchStatus(event.status) : "running"; + const sourceTerminal = event.terminal === true || TERMINAL_STATUSES.has(normalizeWorkbenchStatus(event.status)); + const sourceStatus = sourceTerminal ? normalizeWorkbenchStatus(event.status) : "running"; const previousTiming = normalizeTimingProjection(previousCheckpoint?.timing) ?? normalizeTimingProjection(previousCheckpoint); const sourceOccurredAt = timestampValue(event.createdAt ?? event.occurredAt ?? event.updatedAt ?? projectedAt); const occurredAt = latestTimestamp(previousTiming?.lastEventAt, sourceOccurredAt) ?? sourceOccurredAt; - if (checkpointIsTerminal(previousCheckpoint) && !terminal) { + if (checkpointIsTerminal(previousCheckpoint)) { return { written: false, suppressedAfterSeal: true, @@ -103,37 +209,39 @@ export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = { const explicitDurationMs = durationValue(event.durationMs ?? explicitEventTiming?.durationMs); const explicitStartedAt = optionalTimestampValue(event.startedAt ?? event.traceStartedAt ?? event.runnerStartedAt ?? explicitEventTiming?.startedAt); const provisionalLastEventAt = latestTimestamp(previousTiming?.lastEventAt, explicitEventTiming?.lastEventAt, occurredAt); - const provisionalFinishedAt = terminal ? latestTimestamp(explicitEventTiming?.finishedAt, optionalTimestampValue(event.finishedAt), provisionalLastEventAt, occurredAt) : null; + const provisionalFinishedAt = sourceTerminal ? latestTimestamp(explicitEventTiming?.finishedAt, optionalTimestampValue(event.finishedAt), provisionalLastEventAt, occurredAt) : null; const startedAt = previousTiming?.startedAt ?? explicitStartedAt ?? startedAtFromDuration(provisionalFinishedAt, explicitDurationMs) ?? (sourceSeq <= 1 ? occurredAt : null); const lastEventAt = provisionalLastEventAt; - const finishedAt = terminal ? latestTimestamp(provisionalFinishedAt, lastEventAt) : null; - const timing = eventTimingProjection({ startedAt, lastEventAt, finishedAt, terminal, durationMs: explicitDurationMs ?? previousTiming?.durationMs }); - const timingAuthorityIssue = terminalTimingAuthorityIssue(timing, { terminal, traceId, source: "kafka", status, label: event.label, sourceSeq }); + const finishedAt = sourceTerminal ? latestTimestamp(provisionalFinishedAt, lastEventAt) : null; const sourceEventId = workbenchSourceEventId(event, { traceId, sourceSeq, occurredAt }); const eventId = textValue(event.id) || stableFactId("wte", { traceId, sourceEventId }); const previousFinalText = finalResponseTextValue(previousCheckpoint?.finalResponse, previousCheckpoint?.assistantText, previousCheckpoint?.finalText); const liveAssistantText = isAssistantMessageTraceEvent(event) ? finalResponseTextValue(event.assistantText, event.text, event.reply, event.message) : finalResponseTextValue(event.assistantText, event.reply); - const eventTerminalFinalResponse = terminal ? terminalFinalResponse(status, event, { evidence: event, finalResponse: event.finalResponse }) : null; - const messageFinalText = terminal ? finalResponseTextValue(event.finalResponse, liveAssistantText, previousFinalText, eventTerminalFinalResponse) : null; + const eventTerminalFinalResponse = sourceTerminal ? terminalFinalResponse(sourceStatus, event, { evidence: event, finalResponse: event.finalResponse }) : null; + const messageFinalText = sourceTerminal ? finalResponseTextValue(event.finalResponse, liveAssistantText, previousFinalText, eventTerminalFinalResponse) : null; const checkpointAssistantText = messageFinalText ?? liveAssistantText ?? previousFinalText ?? null; - const checkpointFinalResponse = terminal && messageFinalText - ? { ...(eventTerminalFinalResponse ?? {}), text: messageFinalText, status, traceId, source: "agentrun-kafka-projector", valuesPrinted: false } + const checkpointFinalResponse = sourceTerminal && messageFinalText + ? { ...(eventTerminalFinalResponse ?? {}), text: messageFinalText, status: sourceStatus, traceId, source: "agentrun-kafka-projector", valuesPrinted: false } : previousCheckpoint?.finalResponse ?? null; - const terminalSealBlocked = terminal && status === "completed" && !messageFinalText; - const factTerminal = terminal && !terminalSealBlocked; + const terminalSealBlocked = sourceTerminal && sourceStatus === "completed" && !messageFinalText; + const effectiveTerminal = sourceTerminal && !terminalSealBlocked; + const effectiveStatus = effectiveTerminal ? sourceStatus : "running"; + const timing = eventTimingProjection({ startedAt, lastEventAt, finishedAt, terminal: effectiveTerminal, durationMs: explicitDurationMs ?? previousTiming?.durationMs }); + const timingAuthorityIssue = terminalTimingAuthorityIssue(timing, { terminal: effectiveTerminal, traceId, source: "kafka", status: effectiveStatus, label: event.label, sourceSeq }); + const projectionDiagnostic = { timingAuthorityIssue, terminalSealBlocked, sourceTerminal, sourceStatus, authority: "agentrun-kafka-projector", valuesRedacted: true }; const userMessageFact = sessionId && previousCheckpoint?.userMessage ? normalizeMessageFact(previousCheckpoint.userMessage, 0, { traceId, sessionId, turnId, terminal: false, terminalStatus: "sent", finalText: null, timestamp: projectedAt, timing }) : null; const messageFact = sessionId ? normalizeMessageFact({ messageId: eventProjectionAssistantMessageId(traceId, event), role: "agent", - status, + status: effectiveStatus, traceId, turnId, text: messageFinalText ?? liveAssistantText ?? previousFinalText ?? "", projectedSeq: durableProjectedSeq, sourceSeq, sourceEventId, - terminal: factTerminal, - sealed: factTerminal, + terminal: effectiveTerminal, + sealed: effectiveTerminal, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, @@ -142,10 +250,10 @@ export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = { createdAt: occurredAt, updatedAt: projectedAt, source: "agentrun-kafka-projector" - }, userMessageFact ? 1 : 0, { traceId, sessionId, turnId, terminal, terminalStatus: status, finalText: messageFinalText, terminalSealBlocked, timestamp: projectedAt, timing }) : null; + }, userMessageFact ? 1 : 0, { traceId, sessionId, turnId, terminal: effectiveTerminal, terminalStatus: effectiveStatus, finalText: messageFinalText, terminalSealBlocked, timestamp: projectedAt, timing }) : null; const facts = { - sessions: sessionId ? [{ sessionId, status, lastTraceId: traceId, projectedSeq: durableProjectedSeq, sourceSeq, sourceEventId, terminal: factTerminal, sealed: factTerminal, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs, updatedAt: projectedAt }] : [], - traceEvents: [{ ...event, id: eventId, traceId, sessionId, turnId, messageId: textValue(event.messageId) || null, sourceSeq, sourceEventId, projectedSeq: durableProjectedSeq, eventType, terminal, sealed: terminal, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs, createdAt: occurredAt, occurredAt, updatedAt: projectedAt }], + sessions: sessionId ? [{ sessionId, status: effectiveStatus, lastTraceId: traceId, projectedSeq: durableProjectedSeq, sourceSeq, sourceEventId, terminal: effectiveTerminal, sealed: effectiveTerminal, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs, updatedAt: projectedAt }] : [], + traceEvents: [{ ...event, id: eventId, traceId, sessionId, turnId, messageId: textValue(event.messageId) || null, status: effectiveStatus, sourceSeq, sourceEventId, projectedSeq: durableProjectedSeq, eventType, terminal: effectiveTerminal, sealed: effectiveTerminal, diagnostic: projectionDiagnostic, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs, createdAt: occurredAt, occurredAt, updatedAt: projectedAt }], messages: [userMessageFact, messageFact].filter(Boolean), parts: [userMessageFact, messageFact].filter(Boolean).flatMap((message) => messagePartFacts(message, { finalText: message === messageFact ? messageFinalText : null })), turns: sessionId ? [{ @@ -153,16 +261,16 @@ export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = { sessionId, traceId, messageId: messageFact?.messageId ?? null, - status, + status: effectiveStatus, projectedSeq: durableProjectedSeq, sourceSeq, sourceEventId, - terminal: factTerminal, - sealed: factTerminal, + terminal: effectiveTerminal, + sealed: effectiveTerminal, finalResponse: checkpointFinalResponse, assistantText: checkpointAssistantText, failureKind: textValue(event.failureKind ?? event.errorCode) || null, - diagnostic: { timingAuthorityIssue, terminalSealBlocked, authority: "agentrun-kafka-projector", valuesRedacted: true }, + diagnostic: projectionDiagnostic, timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, @@ -180,10 +288,10 @@ export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = { projectedSeq: durableProjectedSeq, sourceSeq, sourceEventId, - projectionStatus: terminal ? "caught_up" : "projecting", + projectionStatus: effectiveTerminal ? "caught_up" : "projecting", projectionHealth: terminalSealBlocked ? "degraded" : "healthy", - terminal: factTerminal, - sealed: factTerminal, + terminal: effectiveTerminal, + sealed: effectiveTerminal, assistantText: checkpointAssistantText, finalResponse: checkpointFinalResponse, timing, @@ -191,11 +299,11 @@ export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = { lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs, - diagnostic: { lastEventLabel: textValue(event.label) || null, lastEventStatus: textValue(event.status) || null, timingAuthorityIssue, terminalSealBlocked, authority: "agentrun-kafka-projector", valuesRedacted: true }, + diagnostic: { ...projectionDiagnostic, lastEventLabel: textValue(event.label) || null, lastEventStatus: textValue(event.status) || null }, updatedAt: projectedAt }] }; - return { written: true, suppressedAfterSeal: false, traceId, sessionId, sourceSeq, sourceEventId, projectedSeq: durableProjectedSeq, facts, terminal, valuesPrinted: false }; + return { written: true, suppressedAfterSeal: false, traceId, sessionId, sourceSeq, sourceEventId, projectedSeq: durableProjectedSeq, facts, terminal: effectiveTerminal, sourceTerminal, terminalSealBlocked, valuesPrinted: false }; } function eventProjectionAssistantMessageId(traceId, event = {}) { diff --git a/internal/cloud/workbench-realtime-authority-contract.test.ts b/internal/cloud/workbench-realtime-authority-contract.test.ts index e6cd55bf..f1a6e689 100644 --- a/internal/cloud/workbench-realtime-authority-contract.test.ts +++ b/internal/cloud/workbench-realtime-authority-contract.test.ts @@ -7,6 +7,59 @@ import { test } from "bun:test"; import { createCloudApiServer } from "./server.ts"; const ACTOR = { id: "usr_workbench_realtime_authority", username: "reader", displayName: "Reader", role: "user", status: "active" }; +const PROJECTION_REALTIME_ENV = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true" +}); +const LIVE_REALTIME_ENV = Object.freeze({ + HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false", + HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" +}); + +test("workbench sync fails closed before projection storage in live Kafka capability", async () => { + let syncReads = 0; + const server = createCloudApiServer({ + accessController: createAccessController(), + workbenchRuntime: { + async readAtomicWorkbenchProjectionSync() { + syncReads += 1; + throw new Error("live Kafka sync must not reach projection storage"); + } + }, + kafkaEventBridge: { + started: true, + capabilities: { + directPublish: true, + liveKafkaSse: true, + transactionalProjector: false, + projectionOutboxRelay: false, + projectionRealtime: false + }, + ready: Promise.resolve(), + subscribeLiveHwlabEvents() { return () => {}; }, + async stop() {} + }, + env: { ...LIVE_REALTIME_ENV } + }); + await listen(server); + + try { + const { port } = server.address(); + const response = await getJson(port, "/v1/workbench/sync?sessionId=ses_live_only&since=0"); + assert.equal(response.status, 503); + assert.equal(response.body.error.code, "workbench_projection_realtime_disabled"); + assert.equal(response.body.error.capabilities.liveKafkaSse, true); + assert.equal(syncReads, 0); + } finally { + await close(server); + } +}); test("workbench sync and SSE read the same atomic projection outbox", async () => { const sessionId = "ses_realtime_authority_p1"; @@ -52,6 +105,7 @@ test("workbench sync and SSE read the same atomic projection outbox", async () = accessController: createAccessController(), workbenchRuntime: runtime, env: { + ...PROJECTION_REALTIME_ENV, HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000", HWLAB_WORKBENCH_OUTBOX_TAIL_BATCH_SIZE: "100" } @@ -98,7 +152,7 @@ test("workbench sync delta includes all authority families and marks trace rows }), outboxRows: [] }); - const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime }); + const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime, env: { ...PROJECTION_REALTIME_ENV } }); await listen(server); try { @@ -128,7 +182,7 @@ test("workbench sync preserves session and trace scope for narrow replay", async const traceId = "trc_realtime_authority_narrow"; const outboxQueries = []; const runtime = createRuntime({ facts: durableFacts({ sessionId, traceId }), outboxQueries }); - const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime }); + const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime, env: { ...PROJECTION_REALTIME_ENV } }); await listen(server); try { @@ -151,7 +205,7 @@ test("workbench trace event detail route declares detail-only authority", async facts: durableFacts({ sessionId, traceId, finalText: "sealed final", traceEventText: "detail row" }), outboxRows: [] }); - const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime }); + const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: runtime, env: { ...PROJECTION_REALTIME_ENV } }); await listen(server); try { @@ -170,7 +224,7 @@ test("workbench trace event detail route declares detail-only authority", async }); test("workbench sync rejects unscoped automatic repair requests", async () => { - const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: createRuntime({ facts: durableFacts({}) }) }); + const server = createCloudApiServer({ accessController: createAccessController(), workbenchRuntime: createRuntime({ facts: durableFacts({}) }), env: { ...PROJECTION_REALTIME_ENV } }); await listen(server); try { diff --git a/internal/cloud/workbench-realtime-authority.ts b/internal/cloud/workbench-realtime-authority.ts index b1703d1a..1a23d3da 100644 --- a/internal/cloud/workbench-realtime-authority.ts +++ b/internal/cloud/workbench-realtime-authority.ts @@ -5,6 +5,7 @@ import { safeSessionId, safeTraceId, sendJson } from "./server-http-utils.ts"; import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts"; import { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts"; +import { workbenchRealtimeCapabilities } from "./workbench-realtime-capabilities.ts"; const SYNC_CONTRACT_VERSION = "workbench-sync-v1"; const REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2"; @@ -16,6 +17,18 @@ export async function handleWorkbenchSyncHttp(request, response, url, options = sendJson(response, 405, { ok: false, error: { code: "method_not_allowed", message: "GET required." } }); return; } + const capabilities = workbenchRealtimeCapabilities(options.env ?? process.env); + if (!capabilities.projectionRealtime) { + sendJson(response, 503, { + ok: false, + error: { + code: "workbench_projection_realtime_disabled", + message: "Workbench projection sync/replay capability is disabled.", + valuesRedacted: true + } + }); + return; + } const sessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId")); const traceId = safeTraceId(url.searchParams.get("traceId")); if (!sessionId && !traceId) { diff --git a/internal/cloud/workbench-realtime-capabilities.ts b/internal/cloud/workbench-realtime-capabilities.ts new file mode 100644 index 00000000..c62e3624 --- /dev/null +++ b/internal/cloud/workbench-realtime-capabilities.ts @@ -0,0 +1,34 @@ +// SPEC: PJ2026-0104010803 Workbench composable realtime capabilities. +// Responsibility: validate YAML-owned capability switches without code defaults or implied mode selection. + +export const WORKBENCH_REALTIME_CAPABILITY_ENVS = Object.freeze({ + directPublish: "HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED", + liveKafkaSse: "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED", + transactionalProjector: "HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED", + projectionOutboxRelay: "HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED", + projectionRealtime: "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED" +}); + +export function workbenchRealtimeCapabilities(env = process.env) { + return Object.fromEntries(Object.entries(WORKBENCH_REALTIME_CAPABILITY_ENVS).map(([key, name]) => [key, requiredBooleanEnv(env, name)])); +} + +export function requiredBooleanEnv(env, name) { + const value = String(env?.[name] ?? "").trim().toLowerCase(); + if (value === "true" || value === "1") return true; + if (value === "false" || value === "0") return false; + throw capabilityError( + "hwlab_workbench_realtime_capability_invalid", + `${name} is required and must be explicitly true or false in the owning YAML.`, + name + ); +} + +function capabilityError(code, message, envName) { + return Object.assign(new Error(message), { + code, + envName, + statusCode: 503, + valuesRedacted: true + }); +} diff --git a/internal/cloud/workbench-turn-projection.ts b/internal/cloud/workbench-turn-projection.ts index fc6221ad..d2f6148b 100644 --- a/internal/cloud/workbench-turn-projection.ts +++ b/internal/cloud/workbench-turn-projection.ts @@ -39,8 +39,8 @@ export function createWorkbenchTurnProjection({ turnId = null, traceId = null, r finalResponse: finalText ? { text: finalText, status, traceId: projectionTraceId, valuesPrinted: false } : null, assistantText: finalText, lastProjectedSeq: traceLastSeq(trace), - sourceRunId: agentRun?.runId ?? lastEvent?.runId ?? lastEvent?.payload?.runId ?? null, - sourceCommandId: agentRun?.commandId ?? lastEvent?.commandId ?? lastEvent?.payload?.commandId ?? null, + sourceRunId: lastEvent?.runId ?? lastEvent?.payload?.runId ?? agentRun?.runId ?? null, + sourceCommandId: lastEvent?.commandId ?? lastEvent?.payload?.commandId ?? agentRun?.commandId ?? null, eventCount: normalizedEventCount(trace), updatedAt: trace?.updatedAt ?? result?.updatedAt ?? session?.updatedAt ?? null, timing, @@ -164,7 +164,13 @@ export function traceTerminalEvidence(trace = null) { if (direct) { const status = terminalStatusFromValue(direct.status ?? direct.terminalStatus ?? trace?.status) ?? "completed"; if (status !== "completed" && retryableProviderInterruptionEvidence(direct, trace)) return null; - return { source: "trace-terminal-evidence", status, evidence: direct, valuesRedacted: true }; + return { + source: "trace-terminal-evidence", + status, + finalResponse: direct.finalResponse ?? null, + evidence: direct, + valuesRedacted: true + }; } const events = Array.isArray(trace?.events) ? trace.events : []; return terminalTraceEventEvidence(events); diff --git a/internal/db/runtime-store-core.ts b/internal/db/runtime-store-core.ts index cdbe503c..aa39889c 100644 --- a/internal/db/runtime-store-core.ts +++ b/internal/db/runtime-store-core.ts @@ -16,7 +16,7 @@ import { deriveProtocolActorFromMeta } from "../audit/index.mjs"; import { - CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + CLOUD_CORE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS @@ -62,10 +62,7 @@ export const postgresCountTables = Object.freeze([ "workbench_event_sequences", "workbench_events", "workbench_session_inputs", - "workbench_projection_checkpoints", - "workbench_kafka_inbox", - "workbench_kafka_dlq", - "hwlab_kafka_outbox" + "workbench_projection_checkpoints" ]); export const postgresRuntimeReadIndexes = Object.freeze([ @@ -424,7 +421,7 @@ export function workbenchInputDelivery(value) { export function workbenchInputStatus(value) { const text = textOr(value, "admitted").toLowerCase().replace(/-/gu, "_"); - return ["admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; + return ["admitting", "admitted", "promoted", "blocked", "failed", "canceled", "completed"].includes(text) ? text : "admitted"; } export function indexWorkbenchAggregateEvents(events = []) { @@ -1304,7 +1301,7 @@ export function isRuntimeDbSslError(error) { ].some((pattern) => message.includes(pattern)); } -export function summarizeRuntimeSchema(rows) { +export function summarizeRuntimeSchema(rows, requiredSchema = requiredPostgresSchema) { const columnsByTable = new Map(); for (const row of rows) { const table = row.table_name; @@ -1318,7 +1315,7 @@ export function summarizeRuntimeSchema(rows) { const missingTables = []; const missingColumns = []; - for (const [table, columns] of Object.entries(requiredPostgresSchema)) { + for (const [table, columns] of Object.entries(requiredSchema)) { const observed = columnsByTable.get(table); if (!observed) { missingTables.push(table); @@ -1335,7 +1332,7 @@ export function summarizeRuntimeSchema(rows) { return { ready: missingTables.length === 0 && missingColumns.length === 0, checked: true, - requiredTables: Object.keys(requiredPostgresSchema), + requiredTables: Object.keys(requiredSchema), missingTables, missingColumns }; @@ -1346,7 +1343,7 @@ export function notCheckedRuntimeMigration() { checked: false, ready: false, table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - requiredMigrationId: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + requiredMigrationId: CLOUD_CORE_MIGRATION_ID, requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, appliedMigrationId: null, appliedSchemaVersion: null, diff --git a/internal/db/runtime-store-memory.ts b/internal/db/runtime-store-memory.ts index 40932361..41cf409e 100644 --- a/internal/db/runtime-store-memory.ts +++ b/internal/db/runtime-store-memory.ts @@ -213,6 +213,20 @@ export class CloudRuntimeStore { }); } + async workbenchTransactionalRealtimeReadiness() { + return { + ready: true, + adapter: RUNTIME_STORE_KIND, + durable: false, + testRuntime: true, + valuesRedacted: true + }; + } + + async assertWorkbenchTransactionalRealtimeReady() { + return this.workbenchTransactionalRealtimeReadiness(); + } + registerGatewaySession(params = {}, requestMeta = {}) { const now = this.now(); const input = asObject(params.gatewaySession ?? params, "gatewaySession"); @@ -664,9 +678,15 @@ export class CloudRuntimeStore { const inputFacts = facts.inputs.map((fact) => memoryWorkbenchSessionInputWithSeq(fact, this.workbenchSessionInputs)); for (const fact of inputFacts) this.workbenchSessionInputs.set(fact.inputId, fact); for (const fact of facts.sessions) this.workbenchSessions.set(fact.sessionId, fact); - for (const fact of facts.messages) this.workbenchMessages.set(fact.messageId, fact); + for (const fact of facts.messages) { + const previous = this.workbenchMessages.get(fact.messageId) ?? null; + this.workbenchMessages.set(fact.messageId, previous && (nonNegativeInteger(previous.projectedSeq) > 0 || previous.sealed === true) ? previous : fact); + } for (const fact of facts.parts) this.workbenchParts.set(fact.partId, fact); - for (const fact of facts.turns) this.workbenchTurns.set(fact.turnId, fact); + for (const fact of facts.turns) { + const previous = this.workbenchTurns.get(fact.turnId) ?? null; + this.workbenchTurns.set(fact.turnId, previous && (nonNegativeInteger(previous.projectedSeq) > 0 || previous.sealed === true) ? previous : fact); + } for (const fact of facts.traceEvents) this.workbenchTraceEvents.set(fact.id, fact); for (const fact of facts.checkpoints) { const previous = this.workbenchProjectionCheckpoints.get(fact.traceId) ?? null; @@ -701,6 +721,37 @@ export class CloudRuntimeStore { return { written: true, admissionOnly: true, fact: stored, persistence: this.summary(), valuesPrinted: false }; } + promoteWorkbenchTurnAdmission(params = {}, requestMeta = {}) { + const facts = normalizeWorkbenchFacts(params.facts ?? params, requestMeta, this.now()); + const sessionFact = facts.sessions[0]; + const inputFact = facts.inputs[0]; + const turnFact = facts.turns[0]; + if (!sessionFact?.sessionId || !inputFact?.inputId || !turnFact?.turnId) throw new Error("Workbench turn admission promotion requires session, input, and turn facts."); + const storedSession = this.writeWorkbenchSessionAdmissionFact({ fact: sessionFact }, requestMeta).fact; + const storedInput = memoryWorkbenchSessionInputWithSeq(inputFact, this.workbenchSessionInputs); + this.workbenchSessionInputs.set(storedInput.inputId, storedInput); + for (const fact of facts.messages) { + const previous = this.workbenchMessages.get(fact.messageId) ?? null; + if (!previous || (nonNegativeInteger(previous.projectedSeq) <= 0 && previous.sealed !== true)) this.workbenchMessages.set(fact.messageId, fact); + } + for (const fact of facts.parts) { + const previous = this.workbenchParts.get(fact.partId) ?? null; + if (!previous || (nonNegativeInteger(previous.projectedSeq) <= 0 && previous.sealed !== true)) this.workbenchParts.set(fact.partId, fact); + } + for (const fact of facts.turns) { + const previous = this.workbenchTurns.get(fact.turnId) ?? null; + if (!previous || (nonNegativeInteger(previous.projectedSeq) <= 0 && previous.sealed !== true)) this.workbenchTurns.set(fact.turnId, fact); + } + return { + written: true, + promoted: true, + outboxCommitted: true, + facts: { ...facts, sessions: [storedSession], inputs: [storedInput] }, + persistence: this.summary(), + valuesPrinted: false + }; + } + queryWorkbenchFacts(params = {}) { const families = workbenchFactFamilySet(params); const sessions = [...this.workbenchSessions.values()].filter((record) => matchesWorkbenchSessionFactQuery(record, params)); diff --git a/internal/db/runtime-store-postgres-kafka.test.ts b/internal/db/runtime-store-postgres-kafka.test.ts index d5f64614..f1a6eee7 100644 --- a/internal/db/runtime-store-postgres-kafka.test.ts +++ b/internal/db/runtime-store-postgres-kafka.test.ts @@ -139,7 +139,7 @@ test("terminal Kafka transaction persists sealed turn and immutable SSE snapshot if (sql.startsWith("SELECT pg_advisory_xact_lock")) return { rows: [] }; if (sql.startsWith("SELECT source_topic")) return { rows: [] }; if (sql.startsWith("INSERT INTO workbench_kafka_inbox")) return { rows: [] }; - if (sql.startsWith("SELECT checkpoint_json")) return { rows: [{ checkpoint_json: { traceId: "trc_terminal", sessionId: "ses_terminal", assistantText: "final body", finalResponse: { text: "final body", status: "completed" }, projectedSeq: 2, sourceSeq: 2, terminal: true, sealed: true } }] }; + if (sql.startsWith("SELECT checkpoint_json")) return { rows: [{ checkpoint_json: { traceId: "trc_terminal", sessionId: "ses_terminal", assistantText: "final body", finalResponse: { text: "final body", status: "running" }, projectedSeq: 2, sourceSeq: 2, terminal: false, sealed: false } }] }; if (sql.startsWith("SELECT owner_user_id")) return { rows: [{ owner_user_id: "usr-owner", session_json: { sessionId: "ses_terminal", ownerUserId: "usr-owner" } }] }; if (sql.startsWith("SELECT GREATEST")) return { rows: [{ max_projected_seq: 2 }] }; if (sql.startsWith("INSERT INTO hwlab_kafka_outbox") || sql.startsWith("UPDATE workbench_kafka_inbox")) return { rows: [] }; @@ -172,7 +172,7 @@ test("terminal Kafka transaction persists sealed turn and immutable SSE snapshot assert.equal(sse.payload.turn.finalResponse.text, "final body"); }); -test("late nonterminal Kafka event is acknowledged without advancing sealed projectedSeq", async () => { +test("late second terminal is acknowledged without facts or either projection outbox", async () => { const queries = []; const client = { async query(sql, params) { @@ -196,7 +196,7 @@ test("late nonterminal Kafka event is acknowledged without advancing sealed proj projectedSeq, previousCheckpoint, projectedAt, - event: { traceId: "trc_sealed", sessionId: "ses_sealed", sourceEventId: "evt-late", sourceSeq: 10, status: "running", terminal: false } + event: { traceId: "trc_sealed", sessionId: "ses_sealed", sourceEventId: "evt-late", sourceSeq: 10, type: "result", eventType: "terminal", status: "failed", terminal: true } }), hwlabEvent: { eventId: "hwlab:evt-late" }, hwlabTopic: "hwlab.event.v1", @@ -205,7 +205,9 @@ test("late nonterminal Kafka event is acknowledged without advancing sealed proj }); assert.equal(result.suppressedAfterSeal, true); + assert.equal(result.projectionWritten, false); assert.equal(result.projectedSeq, 9); + assert.equal(queries.some(({ sql }) => sql.startsWith("INSERT INTO hwlab_kafka_outbox")), false); const inboxUpdate = queries.find(({ sql }) => sql.startsWith("UPDATE workbench_kafka_inbox")); assert.equal(inboxUpdate.params[0], 9); }); diff --git a/internal/db/runtime-store-postgres-kafka.ts b/internal/db/runtime-store-postgres-kafka.ts index 690ec4a8..a3744719 100644 --- a/internal/db/runtime-store-postgres-kafka.ts +++ b/internal/db/runtime-store-postgres-kafka.ts @@ -74,17 +74,20 @@ export async function commitAgentRunKafkaProjection(store, params = {}) { const committedProjectedSeq = built?.written === false ? nonNegativeInteger(built?.projectedSeq) || nonNegativeInteger(previousCheckpoint?.projectedSeq) || projectedSeq : projectedSeq; - const hwlabEvent = objectValue(params.hwlabEvent); - const hwlabEventId = requiredText(hwlabEvent.eventId, "HWLAB Kafka eventId"); - await client.query( - "INSERT INTO hwlab_kafka_outbox (event_id, topic, partition_key, payload_json, headers_json, attempt_count, next_attempt_at, lease_owner, lease_expires_at, published_at, last_error, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,0,$6,NULL,NULL,NULL,NULL,$6,$6) ON CONFLICT (event_id) DO NOTHING", - [hwlabEventId, requiredText(params.hwlabTopic, "HWLAB Kafka topic"), requiredText(params.partitionKey, "HWLAB Kafka partition key"), stableJson(hwlabEvent), stableJson(objectValue(params.headers)), now] - ); + const projectionWritten = built?.written !== false; + if (projectionWritten) { + const hwlabEvent = objectValue(params.hwlabEvent); + const hwlabEventId = requiredText(hwlabEvent.eventId, "HWLAB Kafka eventId"); + await client.query( + "INSERT INTO hwlab_kafka_outbox (event_id, topic, partition_key, payload_json, headers_json, attempt_count, next_attempt_at, lease_owner, lease_expires_at, published_at, last_error, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,0,$6,NULL,NULL,NULL,NULL,$6,$6) ON CONFLICT (event_id) DO NOTHING", + [hwlabEventId, requiredText(params.hwlabTopic, "HWLAB Kafka topic"), requiredText(params.partitionKey, "HWLAB Kafka partition key"), stableJson(hwlabEvent), stableJson(objectValue(params.headers)), now] + ); + } await client.query( "UPDATE workbench_kafka_inbox SET status = 'projected', projected_seq = $1, updated_at = $2, projected_at = $2 WHERE source_topic = $3 AND source_partition = $4 AND source_offset = $5", [committedProjectedSeq, now, transport.sourceTopic, transport.sourcePartition, transport.sourceOffset] ); - return { duplicate: false, conflict: false, projectedSeq: committedProjectedSeq, facts, events: persistedEvents, suppressedAfterSeal: built?.suppressedAfterSeal === true }; + return { duplicate: false, conflict: false, projectedSeq: committedProjectedSeq, facts, events: persistedEvents, projectionWritten, suppressedAfterSeal: built?.suppressedAfterSeal === true }; }); if (result.facts) store.memory.writeWorkbenchFacts({ facts: result.facts }, params.requestMeta ?? {}); return { ...result, transport, sourceEventId, traceId, sessionId, sourceSeq, valuesPrinted: false }; diff --git a/internal/db/runtime-store-postgres.ts b/internal/db/runtime-store-postgres.ts index 4a4518ed..395208d0 100644 --- a/internal/db/runtime-store-postgres.ts +++ b/internal/db/runtime-store-postgres.ts @@ -16,10 +16,13 @@ import { deriveProtocolActorFromMeta } from "../audit/index.mjs"; import { - CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + CLOUD_CORE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS + CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS, + CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID, + CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION, + CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS } from "./schema.ts"; @@ -776,6 +779,79 @@ export class PostgresCloudRuntimeStore { return withPersistence({ written: true, admissionOnly: true, fact: memoryResult.fact, valuesPrinted: false }, this.summary()); } + async promoteWorkbenchTurnAdmission(params = {}, requestMeta = {}) { + const readiness = this.summary(); + if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { + await this.assertReadyForWrites(); + } + const facts = normalizeWorkbenchFacts(params.facts ?? params, requestMeta, this.now()); + const sessionFact = facts.sessions[0]; + const inputFact = facts.inputs[0]; + const turnFact = facts.turns[0]; + if (!sessionFact?.sessionId || !inputFact?.inputId || !turnFact?.turnId) throw new Error("Workbench turn admission promotion requires session, input, and turn facts."); + const aggregateEvents = normalizeWorkbenchAggregateEventsForFacts(facts, requestMeta, this.now()); + const transaction = await this.withDurableTransaction("workbench.turn-admission.promote", async (client) => { + await client.query("SELECT pg_advisory_xact_lock(hashtext($1))", [`workbench-projection:${turnFact.traceId}`]); + const existingTurnResult = await client.query( + "SELECT projected_seq, sealed FROM workbench_turns WHERE turn_id = $1 FOR UPDATE", + [turnFact.turnId] + ); + const existingTurn = existingTurnResult.rows?.[0] ?? null; + const supersededByProjection = nonNegativeInteger(existingTurn?.projected_seq) > 0 || existingTurn?.sealed === true; + const persistedEvents = []; + const transactionEvents = supersededByProjection + ? aggregateEvents.filter((event) => event.factFamily === "inputs") + : aggregateEvents; + for (const event of transactionEvents) persistedEvents.push(await this.persistWorkbenchAggregateEvent(event, client)); + const eventIndex = indexWorkbenchAggregateEvents(persistedEvents); + const promotedInput = inputFactWithAggregateSeq(inputFact, eventIndex); + await this.persistWorkbenchSessionInputFact(promotedInput, client); + if (supersededByProjection) { + return { persistedEvents, promotedInput, outbox: null, supersededByProjection: true }; + } + await this.persistWorkbenchSessionAdmissionFact(sessionFact, client); + for (const fact of facts.messages) await this.persistWorkbenchMessageAdmissionFact(fact, client); + for (const fact of facts.parts) await this.persistWorkbenchPartAdmissionFact(fact, client); + for (const fact of facts.turns) await this.persistWorkbenchTurnAdmissionFact(fact, client); + const turnEvent = eventIndex.get(`turn:${turnFact.turnId}`) ?? eventIndex.get(`sourceEvent:${turnFact.sourceEventId}`) ?? null; + const outbox = { + entityFamily: "turns", + entityId: turnFact.turnId, + eventSeq: turnEvent?.eventSeq ?? null, + aggregateId: turnEvent?.aggregateId ?? null, + aggregateSeq: turnEvent?.aggregateSeq ?? 0, + projectionRevision: turnEvent?.projectionRevision ?? turnFact.projectedSeq, + traceId: turnFact.traceId, + sessionId: turnFact.sessionId, + turnId: turnFact.turnId, + messageId: turnFact.messageId, + projectedSeq: turnFact.projectedSeq, + sourceSeq: turnFact.sourceSeq, + sourceEventId: turnFact.sourceEventId, + commitType: "admission", + terminal: false, + sealed: false, + payload: immutableProjectionOutboxPayload("turns", turnFact, { status: turnFact.status, admissionState: "promoted", eventSeq: turnEvent?.eventSeq ?? null, aggregateSeq: turnEvent?.aggregateSeq ?? null }), + createdAt: turnFact.updatedAt ?? this.now() + }; + await this.persistWorkbenchProjectionOutbox(outbox, client); + return { persistedEvents, promotedInput, outbox, supersededByProjection: false }; + }); + const memoryResult = transaction.supersededByProjection + ? this.memory.writeWorkbenchFacts({ facts: { inputs: [transaction.promotedInput] } }, requestMeta) + : this.memory.promoteWorkbenchTurnAdmission({ facts: { ...facts, inputs: [transaction.promotedInput] } }, requestMeta); + return withPersistence({ + written: true, + promoted: true, + outboxCommitted: !transaction.supersededByProjection, + outbox: transaction.outbox, + supersededByProjection: transaction.supersededByProjection, + facts: memoryResult.facts, + events: transaction.persistedEvents, + valuesPrinted: false + }, this.summary()); + } + async commitAgentRunKafkaProjection(params = {}) { await this.assertReadyForWrites(); return commitAgentRunKafkaProjectionOperation(this, params); @@ -1215,6 +1291,13 @@ export class PostgresCloudRuntimeStore { ); } + async persistWorkbenchMessageAdmissionFact(record, client = this) { + await client.query( + "INSERT INTO workbench_messages (message_id, session_id, turn_id, trace_id, role, status, projected_seq, source_seq, source_event_id, terminal, sealed, message_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT (message_id) DO UPDATE SET session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, role = EXCLUDED.role, status = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.status ELSE workbench_messages.status END, source_event_id = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.source_event_id ELSE workbench_messages.source_event_id END, message_json = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.message_json ELSE workbench_messages.message_json END, updated_at = CASE WHEN workbench_messages.projected_seq <= 0 AND NOT workbench_messages.sealed THEN EXCLUDED.updated_at ELSE workbench_messages.updated_at END", + [record.messageId, record.sessionId, record.turnId, record.traceId, record.role, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] + ); + } + async persistWorkbenchPartFact(record, client = this) { await client.query( "INSERT INTO workbench_parts (part_id, message_id, session_id, turn_id, trace_id, part_index, part_type, status, projected_seq, source_seq, source_event_id, terminal, sealed, part_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (part_id) DO UPDATE SET message_id = EXCLUDED.message_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, part_index = EXCLUDED.part_index, part_type = EXCLUDED.part_type, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, part_json = EXCLUDED.part_json, updated_at = EXCLUDED.updated_at", @@ -1222,6 +1305,13 @@ export class PostgresCloudRuntimeStore { ); } + async persistWorkbenchPartAdmissionFact(record, client = this) { + await client.query( + "INSERT INTO workbench_parts (part_id, message_id, session_id, turn_id, trace_id, part_index, part_type, status, projected_seq, source_seq, source_event_id, terminal, sealed, part_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15,$16) ON CONFLICT (part_id) DO UPDATE SET message_id = EXCLUDED.message_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, trace_id = EXCLUDED.trace_id, part_index = EXCLUDED.part_index, part_type = EXCLUDED.part_type, status = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.status ELSE workbench_parts.status END, source_event_id = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.source_event_id ELSE workbench_parts.source_event_id END, part_json = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.part_json ELSE workbench_parts.part_json END, updated_at = CASE WHEN workbench_parts.projected_seq <= 0 AND NOT workbench_parts.sealed THEN EXCLUDED.updated_at ELSE workbench_parts.updated_at END", + [record.partId, record.messageId, record.sessionId, record.turnId, record.traceId, record.partIndex, record.partType, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record), record.createdAt, record.updatedAt] + ); + } + async persistWorkbenchTurnFact(record, client = this) { await client.query( "INSERT INTO workbench_turns (turn_id, session_id, trace_id, message_id, status, projected_seq, source_seq, source_event_id, terminal, sealed, final_response_json, diagnostic_json, turn_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (turn_id) DO UPDATE SET session_id = EXCLUDED.session_id, trace_id = EXCLUDED.trace_id, message_id = EXCLUDED.message_id, status = EXCLUDED.status, projected_seq = EXCLUDED.projected_seq, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, final_response_json = EXCLUDED.final_response_json, diagnostic_json = EXCLUDED.diagnostic_json, turn_json = EXCLUDED.turn_json, updated_at = EXCLUDED.updated_at", @@ -1229,6 +1319,13 @@ export class PostgresCloudRuntimeStore { ); } + async persistWorkbenchTurnAdmissionFact(record, client = this) { + await client.query( + "INSERT INTO workbench_turns (turn_id, session_id, trace_id, message_id, status, projected_seq, source_seq, source_event_id, terminal, sealed, final_response_json, diagnostic_json, turn_json, created_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14,$15) ON CONFLICT (turn_id) DO UPDATE SET session_id = EXCLUDED.session_id, trace_id = EXCLUDED.trace_id, message_id = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.message_id ELSE workbench_turns.message_id END, status = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.status ELSE workbench_turns.status END, source_event_id = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.source_event_id ELSE workbench_turns.source_event_id END, diagnostic_json = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.diagnostic_json ELSE workbench_turns.diagnostic_json END, turn_json = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.turn_json ELSE workbench_turns.turn_json END, updated_at = CASE WHEN workbench_turns.projected_seq <= 0 AND NOT workbench_turns.sealed THEN EXCLUDED.updated_at ELSE workbench_turns.updated_at END", + [record.turnId, record.sessionId, record.traceId, record.messageId, record.status, record.projectedSeq, record.sourceSeq, record.sourceEventId, record.terminal, record.sealed, stableJson(record.finalResponse), stableJson(record.diagnostic), stableJson(record), record.createdAt, record.updatedAt] + ); + } + async persistWorkbenchTraceEventFact(record, client = this) { await client.query( "INSERT INTO workbench_trace_events (id, trace_id, session_id, turn_id, message_id, source_seq, source_event_id, projected_seq, event_type, terminal, sealed, event_json, occurred_at, updated_at) VALUES ($1,$2,$3,$4,$5,$6,$7,$8,$9,$10,$11,$12,$13,$14) ON CONFLICT (id) DO UPDATE SET trace_id = EXCLUDED.trace_id, session_id = EXCLUDED.session_id, turn_id = EXCLUDED.turn_id, message_id = EXCLUDED.message_id, source_seq = EXCLUDED.source_seq, source_event_id = EXCLUDED.source_event_id, projected_seq = EXCLUDED.projected_seq, event_type = EXCLUDED.event_type, terminal = EXCLUDED.terminal, sealed = EXCLUDED.sealed, event_json = EXCLUDED.event_json, occurred_at = EXCLUDED.occurred_at, updated_at = EXCLUDED.updated_at", @@ -1307,17 +1404,17 @@ export class PostgresCloudRuntimeStore { async readMigrationReadiness() { const result = await this.query( `SELECT id, schema_version FROM ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} WHERE id = $1 LIMIT 1`, - [CLOUD_RUNTIME_DURABLE_MIGRATION_ID] + [CLOUD_CORE_MIGRATION_ID] ); const row = result.rows?.[0] ?? null; const ready = - row?.id === CLOUD_RUNTIME_DURABLE_MIGRATION_ID && + row?.id === CLOUD_CORE_MIGRATION_ID && row?.schema_version === CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION; return { checked: true, ready, table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - requiredMigrationId: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + requiredMigrationId: CLOUD_CORE_MIGRATION_ID, requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, appliedMigrationId: row?.id ?? null, appliedSchemaVersion: row?.schema_version ?? null, @@ -1325,6 +1422,45 @@ export class PostgresCloudRuntimeStore { }; } + async workbenchTransactionalRealtimeReadiness() { + const schemaResult = await this.query( + "SELECT table_name, column_name FROM information_schema.columns WHERE table_schema = current_schema() AND table_name = ANY($1::text[])", + [Object.keys(CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS)] + ); + const schema = summarizeRuntimeSchema(schemaResult?.rows ?? [], CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS); + const migrationResult = await this.query( + `SELECT id, schema_version FROM ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} WHERE id = $1 LIMIT 1`, + [CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID] + ); + const row = migrationResult.rows?.[0] ?? null; + const migration = { + checked: true, + ready: row?.id === CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID && row?.schema_version === CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION, + table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + requiredMigrationId: CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID, + requiredSchemaVersion: CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION, + appliedMigrationId: row?.id ?? null, + appliedSchemaVersion: row?.schema_version ?? null, + missing: !row + }; + return { + ready: schema.ready && migration.ready, + schema, + migration, + valuesRedacted: true + }; + } + + async assertWorkbenchTransactionalRealtimeReady() { + const readiness = await this.workbenchTransactionalRealtimeReadiness(); + if (readiness.ready) return readiness; + const error = new Error("Workbench transactional realtime schema or migration is not ready."); + error.code = readiness.schema.ready ? "workbench_transactional_realtime_migration_blocked" : "workbench_transactional_realtime_schema_blocked"; + error.readiness = readiness; + error.valuesRedacted = true; + throw error; + } + async ensureRuntimeReadIndexes() { if (this.runtimeReadIndexesReady) return; if (!this.runtimeReadIndexesReadyPromise) { diff --git a/internal/db/runtime-store.test.ts b/internal/db/runtime-store.test.ts index 346b17e7..51370388 100644 --- a/internal/db/runtime-store.test.ts +++ b/internal/db/runtime-store.test.ts @@ -16,7 +16,7 @@ import { createConfiguredCloudRuntimeStore } from "./runtime-store.ts"; import { - CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + CLOUD_CORE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS } from "./schema.ts"; @@ -1451,8 +1451,8 @@ function createFakePostgresClient({ hwlab_schema_migrations: new Map() }; if (migrationReady) { - state.hwlab_schema_migrations.set(CLOUD_RUNTIME_DURABLE_MIGRATION_ID, { - id: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + state.hwlab_schema_migrations.set(CLOUD_CORE_MIGRATION_ID, { + id: CLOUD_CORE_MIGRATION_ID, schema_version: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION }); } diff --git a/internal/db/schema.test.ts b/internal/db/schema.test.ts index fffcfd60..8814efe8 100644 --- a/internal/db/schema.test.ts +++ b/internal/db/schema.test.ts @@ -7,11 +7,13 @@ import { fileURLToPath } from "node:url"; import { TABLES, assertProtocolRecord, assertProtocolRecords } from "../protocol/index.mjs"; import { CLOUD_CORE_MIGRATIONS, - CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + CLOUD_CORE_MIGRATION_ID, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS, CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS, + CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID, + CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION, FROZEN_CLOUD_CORE_TABLES, assertFrozenCloudCoreTables, requiredRuntimeDurableColumns @@ -62,7 +64,7 @@ test("versioned migration chain exposes columns required by the durable runtime for (const column of CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS) { assert.match(migrationMatch[1], new RegExp(`\\b${column}\\b`), `missing migration ledger column ${column}`); } - assert.match(sql, new RegExp(`'${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}'`, "u")); + assert.match(sql, new RegExp(`'${CLOUD_CORE_MIGRATION_ID}'`, "u")); assert.match(sql, new RegExp(`'${CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION}'`, "u")); assert.equal(CLOUD_CORE_MIGRATIONS[0].runtimeDurableMigrationTable, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE); }); @@ -114,7 +116,8 @@ test("v8 migration upgrades the existing Workbench outbox before enforcing realt assert.match(sql, /entity_family = COALESCE\(entity_family, 'legacy'\)/u); assert.match(sql, /CREATE UNIQUE INDEX idx_workbench_projection_outbox_event_id ON workbench_projection_outbox\(outbox_event_id\)/u); assert.match(sql, /CREATE TRIGGER trg_notify_hwlab_workbench_projection/u); - assert.match(sql, new RegExp(`'${CLOUD_RUNTIME_DURABLE_MIGRATION_ID}'`, "u")); + assert.match(sql, new RegExp(`'${CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID}'`, "u")); + assert.match(sql, new RegExp(`'${CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION}'`, "u")); assert.match(sql, /ON CONFLICT \(id\) DO UPDATE SET\s+schema_version = EXCLUDED\.schema_version/u); }); diff --git a/internal/db/schema.ts b/internal/db/schema.ts index a3b236aa..5cd9f440 100644 --- a/internal/db/schema.ts +++ b/internal/db/schema.ts @@ -5,8 +5,9 @@ import { TABLES } from "../protocol/index.mjs"; export const CLOUD_CORE_MIGRATION_ID = "0001_cloud_core_skeleton"; -export const CLOUD_RUNTIME_DURABLE_MIGRATION_ID = "0008_workbench_kafka_realtime"; -export const CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION = "runtime-durable-postgres-v8"; +export const CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION = "runtime-durable-postgres-v7"; +export const CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID = "0008_workbench_kafka_realtime"; +export const CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION = "runtime-durable-postgres-v8"; export const CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE = "hwlab_schema_migrations"; export const FROZEN_CLOUD_CORE_TABLES = Object.freeze([...TABLES]); @@ -319,9 +320,6 @@ export const CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS = Object.freeze({ ]), workbench_projection_outbox: Object.freeze([ "outbox_seq", - "outbox_event_id", - "entity_family", - "entity_id", "event_seq", "aggregate_id", "aggregate_seq", @@ -338,6 +336,14 @@ export const CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS = Object.freeze({ "sealed", "payload_json", "created_at" + ]) +}); + +export const CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS = Object.freeze({ + workbench_projection_outbox: Object.freeze([ + "outbox_event_id", + "entity_family", + "entity_id" ]), workbench_kafka_inbox: Object.freeze([ "source_topic", @@ -389,6 +395,15 @@ export const CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS = Object.freeze({ ]) }); +export const CLOUD_TRANSACTIONAL_REALTIME_REQUIRED_TABLE_COLUMNS = Object.freeze({ + ...CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS, + ...CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS, + workbench_projection_outbox: Object.freeze([ + ...CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS.workbench_projection_outbox, + ...CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS.workbench_projection_outbox + ]) +}); + export const CLOUD_CORE_MIGRATIONS = Object.freeze([ { id: CLOUD_CORE_MIGRATION_ID, @@ -399,11 +414,11 @@ export const CLOUD_CORE_MIGRATIONS = Object.freeze([ runtimeDurableMigrationTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE }, { - id: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + id: CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID, path: "internal/db/migrations/0008_workbench_kafka_realtime.sql", tables: Object.freeze(["workbench_projection_outbox", "workbench_kafka_inbox", "workbench_kafka_dlq", "hwlab_kafka_outbox"]), connectsToDatabase: false, - runtimeDurableAdapterSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + runtimeDurableAdapterSchemaVersion: CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION, runtimeDurableMigrationTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE } ]); @@ -420,3 +435,9 @@ export function requiredRuntimeDurableColumns() { columns.map((column) => `${table}.${column}`) ); } + +export function requiredTransactionalRealtimeColumns() { + return Object.entries(CLOUD_TRANSACTIONAL_REALTIME_TABLE_COLUMNS).flatMap(([table, columns]) => + columns.map((column) => `${table}.${column}`) + ); +} diff --git a/internal/dev-entrypoint/cloud-web-runtime.mjs b/internal/dev-entrypoint/cloud-web-runtime.mjs index 52e30f80..8fbb4a73 100644 --- a/internal/dev-entrypoint/cloud-web-runtime.mjs +++ b/internal/dev-entrypoint/cloud-web-runtime.mjs @@ -390,7 +390,12 @@ function validateDisplayLocale(locale) { function workbenchRuntimeConfigFromEnv() { const traceTimeline = {}; - const result = {}; + const result = { + realtimeFeatures: { + liveKafkaSse: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED, "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED"), + projectionRealtime: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED, "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED") + } + }; const autoExpandRunning = parseEnvBoolean(process.env.HWLAB_WORKBENCH_TRACE_AUTO_EXPAND_RUNNING); const autoCollapseTerminal = parseEnvBoolean(process.env.HWLAB_WORKBENCH_TRACE_AUTO_COLLAPSE_TERMINAL); const traceExplorerUrlTemplate = traceExplorerUrlTemplateFromEnv(process.env.HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE); @@ -401,6 +406,13 @@ function workbenchRuntimeConfigFromEnv() { return Object.keys(result).length > 0 ? result : null; } +function requiredWorkbenchRealtimeFeature(value, name) { + const normalized = String(value ?? "").trim().toLowerCase(); + if (normalized === "true" || normalized === "1") return true; + if (normalized === "false" || normalized === "0") return false; + throw new Error(`${name} is required and must be explicitly true or false`); +} + function traceExplorerUrlTemplateFromEnv(value) { if (value === undefined || value === null || value === "") return null; const template = String(value).trim(); diff --git a/internal/dev-entrypoint/cloud-web-runtime.test.mjs b/internal/dev-entrypoint/cloud-web-runtime.test.mjs index 5f0512b2..d16bc98c 100644 --- a/internal/dev-entrypoint/cloud-web-runtime.test.mjs +++ b/internal/dev-entrypoint/cloud-web-runtime.test.mjs @@ -429,7 +429,9 @@ test("cloud web serves client deep links through the Vue shell", async () => { const restoreEnv = withEnv({ HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai", HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN", - HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间" + HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" }); const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-")); await writeFile(path.join(root, "index.html"), "
\n", "utf8"); @@ -484,7 +486,9 @@ test("cloud web injects trace explorer runtime config from env", async () => { HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai", HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN", HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", - HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE: "/v1/workbench/traces/{trace_id}/events" + HWLAB_WORKBENCH_TRACE_EXPLORER_URL_TEMPLATE: "/v1/workbench/traces/{trace_id}/events", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" }); const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-")); await writeFile(path.join(root, "index.html"), "
\n", "utf8"); @@ -588,7 +592,9 @@ test("cloud web OpenCode frame-url endpoint mints traced tickets", async () => { HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai", HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN", HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", - HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test" + HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" }); const cloudApiRequests = []; const opencodeRequests = []; @@ -1028,7 +1034,9 @@ test("cloud web OpenCode proxy accepts short-lived tickets minted by the shell", HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai", HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN", HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: "北京时间", - HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test" + HWLAB_CLOUD_WEB_OPENCODE_URL: "https://opencode.example.test", + HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true", + HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" }); const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-cloud-web-runtime-")); await writeFile(path.join(root, "index.html"), "
\n", "utf8"); diff --git a/scripts/gitops-render.mjs b/scripts/gitops-render.mjs index ebf13b4d..078c65fb 100644 --- a/scripts/gitops-render.mjs +++ b/scripts/gitops-render.mjs @@ -18,6 +18,7 @@ import { versionNameForRuntimeLane } from "./src/runtime-lane.ts"; import { readStructuredFile } from "./src/structured-config.mjs"; +import { CLOUD_CORE_MIGRATIONS } from "../internal/db/schema.ts"; const execFileAsync = promisify(execFile); const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -5074,7 +5075,7 @@ function runtimePostgresImageForProfile(deploy, profile) { return typeof image === "string" && image.trim().length > 0 ? image.trim() : "postgres:16-alpine"; } -function v02PostgresManifest({ profile = "v02", migrationSql, source, image = "postgres:16-alpine" }) { +function v02PostgresManifest({ profile = "v02", migrationSources, source, image = "postgres:16-alpine" }) { const namespace = namespaceNameForProfile(profile); const name = `${namespace}-postgres`; const dbName = `hwlab_${profile}`; @@ -5086,6 +5087,9 @@ function v02PostgresManifest({ profile = "v02", migrationSql, source, image = "p "hwlab.pikastech.local/profile": profile, "hwlab.pikastech.local/source-commit": source.full }; + assert.ok(Array.isArray(migrationSources) && migrationSources.length > 0, "runtime Postgres migration chain is required"); + const migrationSql = migrationSources.map((migration) => migration.sql).join("\n"); + const migrationData = Object.fromEntries(migrationSources.map((migration) => [path.basename(migration.path), migration.sql])); const migrationSha256 = createHash("sha256").update(migrationSql).digest("hex"); const templateLabels = { "app.kubernetes.io/name": name, @@ -5104,7 +5108,7 @@ function v02PostgresManifest({ profile = "v02", migrationSql, source, image = "p apiVersion: "v1", kind: "ConfigMap", metadata: { name: `${name}-init`, namespace, labels }, - data: { "0001_cloud_core_skeleton.sql": migrationSql } + data: migrationData }, { apiVersion: "v1", @@ -5907,9 +5911,13 @@ async function plannedFiles(args) { if (!artifactCatalog) artifactCatalog = await readJson(args.catalogPath); const settings = ciLaneSettings(args); const profiles = laneRuntimeProfiles(args); - const migrationSql = isRuntimeLane(args.lane) - ? await readFile(path.join(repoRoot, "internal/db/migrations/0001_cloud_core_skeleton.sql"), "utf8") - : null; + const migrationSources = isRuntimeLane(args.lane) + ? await Promise.all(CLOUD_CORE_MIGRATIONS.map(async (migration) => ({ + id: migration.id, + path: migration.path, + sql: await readFile(path.join(repoRoot, migration.path), "utf8") + }))) + : []; const metricsSidecarScript = isRuntimeLane(args.lane) ? await readFile(path.join(repoRoot, "internal/dev-entrypoint/metrics-sidecar.mjs"), "utf8") : null; @@ -5987,7 +5995,7 @@ async function plannedFiles(args) { putJson(`${runtimePath}/workloads.yaml`, transformWorkloads({ workloads, deploy, catalog: artifactCatalog, source, sourceBranch: args.sourceBranch, gitReadUrl: args.gitReadUrl, registryPrefix: args.registryPrefix, runtimeEndpoint: endpoints.runtimeEndpoint, webEndpoint: endpoints.webEndpoint, profile, nodeId: args.nodeId, useDeployImages: args.useDeployImages, metricsSidecarSha256 })); putJson(`${runtimePath}/deepseek-proxy.yaml`, deepSeekProxyManifest({ profile, source, sourceBranch: args.sourceBranch, sourceRepo: args.sourceRepo, gitReadUrl: args.gitReadUrl, deploy, registryPrefix: args.registryPrefix, catalog: artifactCatalog, useDeployImages: args.useDeployImages, metricsSidecarSha256 })); if (isRuntimeLane(profile) && externalPostgres) putJson(`${runtimePath}/external-postgres.yaml`, externalPostgresManifest({ profile, config: externalPostgres, source })); - if (isRuntimeLane(profile) && !externalPostgres) putJson(`${runtimePath}/postgres.yaml`, v02PostgresManifest({ profile, migrationSql, source, image: runtimePostgresImageForProfile(deploy, profile) })); + if (isRuntimeLane(profile) && !externalPostgres) putJson(`${runtimePath}/postgres.yaml`, v02PostgresManifest({ profile, migrationSources, source, image: runtimePostgresImageForProfile(deploy, profile) })); if (workbenchRedis) putJson(`${runtimePath}/workbench-redis.yaml`, workbenchRuntimeRedisManifest({ profile, config: workbenchRedis, source })); if (isRuntimeLane(profile)) putJson(`${runtimePath}/openfga.yaml`, v02OpenFgaManifest({ profile, source })); if (isRuntimeLane(profile)) putJson(`${runtimePath}/observability.yaml`, observabilityManifest({ deploy, profile, namespace, labels: profileLabels, annotations, metricsSidecarScript })); diff --git a/scripts/gitops-render.test.ts b/scripts/gitops-render.test.ts index 53480965..fd2c45b9 100644 --- a/scripts/gitops-render.test.ts +++ b/scripts/gitops-render.test.ts @@ -1,11 +1,14 @@ import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import test from "node:test"; import vm from "node:vm"; +import { CLOUD_CORE_MIGRATIONS, CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID } from "../internal/db/schema.ts"; + function taskByName(pipeline, name) { const task = pipeline.spec.tasks.find((item) => item.name === name); assert.ok(task, `missing task ${name}`); @@ -499,12 +502,23 @@ test("v02 render follows TypeScript runtime checks and does not self-patch boots assert.equal(frpcDeployment.spec.template.metadata.labels["hwlab.pikastech.local/source-commit"], undefined); const postgres = JSON.parse(await readFile(path.join(outDir, "runtime-v02", "postgres.yaml"), "utf8")); + const postgresMigrationConfig = postgres.items.find((item) => item.kind === "ConfigMap" && item.metadata?.name === "hwlab-v02-postgres-init"); const postgresStatefulSet = postgres.items.find((item) => item.kind === "StatefulSet"); const postgresMigrationLabel = postgresStatefulSet.spec.template.metadata.labels["hwlab.pikastech.local/migration-sha256"]; const postgresMigrationAnnotation = postgresStatefulSet.spec.template.metadata.annotations["hwlab.pikastech.local/migration-sha256"]; + const migrationSources = await Promise.all(CLOUD_CORE_MIGRATIONS.map(async (migration) => ({ + ...migration, + sql: await readFile(path.join(process.cwd(), migration.path), "utf8") + }))); + const migrationData = Object.fromEntries(migrationSources.map((migration) => [path.basename(migration.path), migration.sql])); + const migrationSha256 = createHash("sha256").update(migrationSources.map((migration) => migration.sql).join("\n")).digest("hex"); + assert.deepEqual(postgresMigrationConfig.data, migrationData); + assert.deepEqual(Object.keys(postgresMigrationConfig.data), CLOUD_CORE_MIGRATIONS.map((migration) => path.basename(migration.path))); + assert.match(postgresMigrationConfig.data["0008_workbench_kafka_realtime.sql"], new RegExp(`'${CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID}'`, "u")); assert.ok(postgresMigrationLabel); assert.equal(postgresMigrationLabel.length <= 63, true); assert.match(postgresMigrationAnnotation, /^[a-f0-9]{64}$/u); + assert.equal(postgresMigrationAnnotation, migrationSha256); assert.equal(postgresMigrationAnnotation.startsWith(postgresMigrationLabel), true); assert.equal(postgresStatefulSet.spec.template.metadata.labels["hwlab.pikastech.local/source-commit"], undefined); diff --git a/scripts/src/dev-runtime-migration.mjs b/scripts/src/dev-runtime-migration.mjs index ffda0968..f0362bcc 100644 --- a/scripts/src/dev-runtime-migration.mjs +++ b/scripts/src/dev-runtime-migration.mjs @@ -5,11 +5,11 @@ import { fileURLToPath } from "node:url"; import { CLOUD_CORE_MIGRATIONS, - CLOUD_RUNTIME_DURABLE_MIGRATION_ID, - CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID as CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION as CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS, - CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS + CLOUD_TRANSACTIONAL_REALTIME_REQUIRED_TABLE_COLUMNS as CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS } from "../../internal/db/schema.ts"; import { PostgresCloudRuntimeStore, @@ -267,6 +267,10 @@ async function verifyRuntimeReadiness(report, env, options) { let runtime; try { runtime = await runtimeStore.readiness(); + if (runtime.gates?.schema?.ready === true && runtime.gates?.migration?.ready === true) { + const transactionalReadiness = await runtimeStore.workbenchTransactionalRealtimeReadiness(); + runtime = mergeTransactionalRealtimeReadiness(runtime, transactionalReadiness); + } } finally { if (!options.queryClient && typeof runtimeStore.pool?.end === "function") { await runtimeStore.pool.end(); @@ -290,6 +294,56 @@ async function verifyRuntimeReadiness(report, env, options) { } } +function mergeTransactionalRealtimeReadiness(runtime, readiness = {}) { + const schemaReady = readiness.schema?.ready === true; + const migrationReady = readiness.migration?.ready === true; + const transactionalReady = readiness.ready === true && schemaReady && migrationReady; + const ready = runtime.ready === true && transactionalReady; + const blocker = ready + ? null + : !schemaReady + ? RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED + : !migrationReady + ? RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED + : runtime.blocker; + return { + ...runtime, + ready, + status: ready ? "ready" : "blocked", + blocker, + reason: ready + ? "Postgres transactional Workbench realtime schema and migration are ready" + : !schemaReady + ? "Postgres transactional Workbench realtime schema is incomplete" + : !migrationReady + ? "Postgres transactional Workbench realtime migration is not recorded" + : runtime.reason, + liveRuntimeEvidence: runtime.liveRuntimeEvidence === true && ready, + connection: { + ...runtime.connection, + queryResult: ready + ? "transactional_realtime_readiness_ready" + : !schemaReady + ? "schema_blocked" + : !migrationReady + ? "migration_blocked" + : runtime.connection?.queryResult + }, + schema: readiness.schema, + migration: readiness.migration, + gates: { + ...runtime.gates, + schema: schemaReady ? readyGate() : blockedGate(RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED), + migration: !schemaReady + ? notCheckedGate() + : migrationReady + ? readyGate() + : blockedGate(RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED), + durability: transactionalReady ? runtime.gates?.durability ?? notCheckedGate() : notCheckedGate() + } + }; +} + async function applyMigration(report, sql, env, options) { report.actions.liveDbWriteAttempted = true; let client; diff --git a/scripts/src/dev-runtime-migration.test.mjs b/scripts/src/dev-runtime-migration.test.mjs index 632a2b5e..126a6caa 100644 --- a/scripts/src/dev-runtime-migration.test.mjs +++ b/scripts/src/dev-runtime-migration.test.mjs @@ -9,11 +9,13 @@ import { parseArgs } from "./dev-runtime-migration.mjs"; import { + CLOUD_CORE_MIGRATION_ID, CLOUD_CORE_MIGRATIONS, - CLOUD_RUNTIME_DURABLE_MIGRATION_ID, - CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION as CLOUD_CORE_SCHEMA_VERSION, + CLOUD_TRANSACTIONAL_REALTIME_MIGRATION_ID as CLOUD_RUNTIME_DURABLE_MIGRATION_ID, + CLOUD_TRANSACTIONAL_REALTIME_SCHEMA_VERSION as CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, - CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS + CLOUD_TRANSACTIONAL_REALTIME_REQUIRED_TABLE_COLUMNS as CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS } from "../../internal/db/schema.ts"; const execFileAsync = promisify(execFile); @@ -124,6 +126,7 @@ test("live dry-run separates migration ledger blocker from auth and schema", asy assert.equal(report.gates.readiness.status, "not_checked"); assert.equal(report.blockers[0].scope, "runtime-migration-ledger"); assert.equal(report.runtime.migration.missing, true); + assert.equal(report.runtime.migration.requiredMigrationId, CLOUD_RUNTIME_DURABLE_MIGRATION_ID); }); test("live dry-run separates SSL negotiation blocker from auth and schema", async () => { @@ -312,6 +315,8 @@ test("apply mode records ledger and verifies durable runtime readiness", async ( assert.equal(report.gates.readiness.status, "ready"); assert.equal(report.runtime.migration.appliedMigrationId, CLOUD_RUNTIME_DURABLE_MIGRATION_ID); assert.equal(report.runtime.migration.appliedSchemaVersion, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION); + assert.ok(queryClient.calls.some((call) => call.params[0] === CLOUD_CORE_MIGRATION_ID)); + assert.ok(queryClient.calls.some((call) => call.params[0] === CLOUD_RUNTIME_DURABLE_MIGRATION_ID)); assert.deepEqual(report.applyBoundary.writeScope, [ ...CLOUD_CORE_MIGRATIONS.map((migration) => `DEV Postgres schema objects declared by ${migration.path}`), ...CLOUD_CORE_MIGRATIONS.map((migration) => `DEV ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} ledger row ${migration.id}`) @@ -455,8 +460,14 @@ function createFakeQueryClient({ migrationReady, countErrorCode = null, migratio return { rows: schemaRows() }; } if (sql.startsWith("SELECT id, schema_version FROM hwlab_schema_migrations")) { + const requestedMigrationId = params[0]; + if (requestedMigrationId === CLOUD_CORE_MIGRATION_ID) { + return { + rows: [{ id: CLOUD_CORE_MIGRATION_ID, schema_version: CLOUD_CORE_SCHEMA_VERSION }] + }; + } return { - rows: state.migrationReady + rows: requestedMigrationId === CLOUD_RUNTIME_DURABLE_MIGRATION_ID && state.migrationReady ? [ { id: CLOUD_RUNTIME_DURABLE_MIGRATION_ID, diff --git a/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts b/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts index 036fe4d0..ff99b6ab 100644 --- a/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts +++ b/web/hwlab-cloud-web/scripts/workbench-realtime-runtime.test.ts @@ -6,7 +6,9 @@ import test from "node:test"; import type { ChatMessage } from "../src/types/index.ts"; import { workbenchSessionDetailPathForTest, workbenchSessionMessagesPathForTest } from "../src/api/workbench.ts"; +import { workbenchEventStreamPath, workbenchProjectionEventStreamPath } from "../src/api/workbench-events.ts"; import type { WorkbenchStreamTransportRecovery } from "../src/utils/workbench-realtime-runtime.ts"; +import { workbenchRealtimeTraceIdForCapabilities, workbenchRealtimeTransportEnabled } from "../src/utils/workbench-stream-transport.ts"; import { workbenchRuntimePolicy } from "../src/config/workbench-runtime-policy.ts"; import { AsyncQueue, work } from "../src/utils/scheduler/async-queue.ts"; import { createCoalescedEventQueue } from "../src/utils/scheduler/coalesced-event-queue.ts"; @@ -17,7 +19,8 @@ import { createSafeStorageRuntime, isStorageQuotaError, migrateLegacyStorage, no import { checkWorkbenchHealth, createWorkbenchHealthProbeCache } from "../src/utils/workbench-health.ts"; import { messageDiagnosticView } from "../src/utils/workbench-error-runtime.ts"; import { WORKBENCH_TIMELINE_OPENCODE_PARITY, buildWorkbenchTimelineRows, normalizeWorkbenchTimelineMessages, workbenchTimelineSignature } from "../src/stores/workbench-timeline-model.ts"; -import { reduceWorkbenchRealtimeEvent } from "../src/stores/workbench-event-reducer.ts"; +import { reduceWorkbenchRealtimeEvent, workbenchRealtimeEventIsBusinessActivity } from "../src/stores/workbench-event-reducer.ts"; +import { reduceWorkbenchLiveKafkaMessageState, workbenchLiveKafkaMessageId } from "../src/stores/workbench-live-kafka-event.ts"; import { planWorkbenchRealtimeApply, planWorkbenchRealtimeRecovery } from "../src/stores/workbench-realtime-plan.ts"; import { WORKBENCH_REALTIME_AUTHORITY_VERSION, workbenchRealtimePrimaryAuthorityDecision, workbenchSyncReplayEvents } from "../src/stores/workbench-realtime-authority.ts"; import { cleanupWorkbenchServerStateDroppedSessions, cleanupWorkbenchServerStateSessions, createWorkbenchServerState, reduceWorkbenchServerState } from "../src/stores/workbench-server-state.ts"; @@ -79,6 +82,25 @@ test("Workbench API uses metadata-only session detail and bounded messages paths assert.equal(workbenchSessionMessagesPathForTest("ses_metadata", { limit: 9 }), "/v1/workbench/sessions/ses_metadata/messages?limit=9"); }); +test("live SSE transport never sends an afterSeq cursor while projection transport can", () => { + assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, projectionRealtime: false }, sessionId: "ses_live", traceId: "trc_live", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_live&traceId=trc_live"); + assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: false, projectionRealtime: true }, sessionId: "ses_projection", traceId: "trc_projection", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_projection&traceId=trc_projection&afterSeq=42"); + assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, projectionRealtime: true }, sessionId: "ses_both", traceId: "trc_both", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_both&traceId=trc_both"); + assert.equal(workbenchProjectionEventStreamPath({ sessionId: "ses_both", traceId: "trc_both", afterSeq: 42 }), "/v1/workbench/projection-events?sessionId=ses_both&traceId=trc_both&afterSeq=42"); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, projectionRealtime: false }), false); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, projectionRealtime: false }), true); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, projectionRealtime: true }), true); + assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, projectionRealtime: true }), true); +}); + +test("live Kafka keeps the pre-submit session SSE key when a turn trace becomes active", () => { + const capabilities = { liveKafkaSse: true, projectionRealtime: false }; + const beforeSubmit = workbenchRealtimeScopeKey("ses_live", workbenchRealtimeTraceIdForCapabilities(capabilities, null, null)); + const afterSubmit = workbenchRealtimeScopeKey("ses_live", workbenchRealtimeTraceIdForCapabilities(capabilities, "trc_current_request", "trc_message")); + assert.equal(afterSubmit, beforeSubmit); + assert.equal(workbenchRealtimeTraceIdForCapabilities({ liveKafkaSse: false, projectionRealtime: true }, "trc_current_request", "trc_message"), "trc_current_request"); +}); + test("Error runtime owns Workbench message diagnostic view model", () => { const degraded = messageDiagnosticView(agentMessage({ status: "running", @@ -416,6 +438,53 @@ test("realtime event reducer classifies SSE payloads before store side effects", assert.equal(error.diagnostic.traceId, "trc_2"); }); +test("live transport connected and heartbeat frames do not extend business activity timeouts", () => { + assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "connected" }, "workbench.connected", true), false); + assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "heartbeat" }, "workbench.heartbeat", true), false); + assert.equal(workbenchRealtimeEventIsBusinessActivity({ schema: "hwlab.event.v1", event: { type: "assistant" } }, "hwlab.event.v1", true), true); + assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "trace.event" }, "workbench.trace.event", false), true); +}); + +test("live hwlab envelope projects assistant, tool/output, and terminal without replay or finalizer", () => { + const envelope = (event: Record) => ({ + schema: "hwlab.event.v1", + eventType: "hwlab.trace.event.projected", + eventId: `hwlab:${String(event.sourceEventId ?? event.type)}`, + hwlabSessionId: "ses_live_web", + sessionId: "ses_live_web", + traceId: "trc_live_web", + runId: "run_live_web", + commandId: "cmd_live_web", + context: { runId: "run_live_web", commandId: "cmd_live_web", valuesRedacted: true }, + event + }); + const assistantEnvelope = envelope({ type: "assistant", eventType: "assistant", sourceEventId: "evt_assistant", traceId: "trc_live_web", sessionId: "ses_live_web", status: "running", assistantText: "running increment", terminal: false }); + const assistant = reduceWorkbenchRealtimeEvent(assistantEnvelope, "hwlab.event.v1"); + assert.equal(assistant.action.type, "trace.event"); + let visible = reduceWorkbenchLiveKafkaMessageState({ text: "", status: "running", terminal: false }, assistantEnvelope.event); + assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false }); + + const toolEnvelope = envelope({ type: "tool", eventType: "tool", sourceEventId: "evt_tool", traceId: "trc_live_web", sessionId: "ses_live_web", status: "completed", toolName: "commandExecution", outputSummary: "ok", terminal: false }); + assert.equal(reduceWorkbenchRealtimeEvent(toolEnvelope, "hwlab.event.v1").action.type, "trace.event"); + visible = reduceWorkbenchLiveKafkaMessageState(visible, toolEnvelope.event); + assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false }); + + const outputEnvelope = envelope({ type: "output", eventType: "status", sourceEventId: "evt_output", traceId: "trc_live_web", sessionId: "ses_live_web", status: "running", message: "stdout increment", terminal: false }); + visible = reduceWorkbenchLiveKafkaMessageState(visible, outputEnvelope.event); + assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false }); + + const terminalEnvelope = envelope({ type: "result", eventType: "terminal", sourceEventId: "evt_terminal", traceId: "trc_live_web", sessionId: "ses_live_web", status: "completed", terminal: true }); + visible = reduceWorkbenchLiveKafkaMessageState(visible, terminalEnvelope.event); + assert.deepEqual(visible, { text: "running increment", status: "completed", terminal: true }); + assert.equal(workbenchLiveKafkaMessageId("trc_live_web"), "msg_live_trc_live_web"); + + const failed = reduceWorkbenchLiveKafkaMessageState( + { text: "partial response", status: "running", terminal: false }, + { type: "result", eventType: "terminal", status: "failed", message: "provider stream disconnected", terminal: true } + ); + assert.deepEqual(failed, { text: "provider stream disconnected", status: "failed", terminal: true }); +}); + test("realtime apply planner turns reducer actions into store steps", () => { const reduced = reduceWorkbenchRealtimeEvent(realtimeEvent({ type: "trace.event", traceId: "trc_1", event: { traceId: "trc_1", label: "delta" }, snapshot: { traceId: "trc_1", status: "running" }, entity: { family: "traceEvents", id: "trc_1:1", version: 1, projectionRevision: "prj_1" } }), "workbench.trace.event"); const tracePlan = planWorkbenchRealtimeApply(reduced.action); diff --git a/web/hwlab-cloud-web/src/api/index.ts b/web/hwlab-cloud-web/src/api/index.ts index b8b17ed0..bfabd7a6 100644 --- a/web/hwlab-cloud-web/src/api/index.ts +++ b/web/hwlab-cloud-web/src/api/index.ts @@ -4,7 +4,7 @@ export { fetchJson, fetchText, type ActivityRef, type ActivityRefSource, type ApiRequestOptions } from "./client"; export { authAPI, HWLAB_WEB_SESSION_COOKIE } from "./auth"; export { workbenchAPI } from "./workbench"; -export { workbenchDebugAPI, connectWorkbenchDebugFakeSse, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseResponse, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream } from "./workbench-debug"; +export { workbenchDebugAPI, connectWorkbenchDebugFakeSse, connectWorkbenchKafkaSseDebug, workbenchKafkaSseDebugCorrelationIds, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseResponse, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream, type WorkbenchKafkaSseDebugEvent, type WorkbenchKafkaSseDebugStream, type WorkbenchKafkaSseDebugStreamName } from "./workbench-debug"; export { connectWorkbenchEvents, type WorkbenchEventStream, type WorkbenchRealtimeEvent } from "./workbench-events"; export { agentAPI } from "./agent"; export { hwpodAPI } from "./hwpod"; diff --git a/web/hwlab-cloud-web/src/api/workbench-debug.test.ts b/web/hwlab-cloud-web/src/api/workbench-debug.test.ts new file mode 100644 index 00000000..20991c39 --- /dev/null +++ b/web/hwlab-cloud-web/src/api/workbench-debug.test.ts @@ -0,0 +1,106 @@ +import assert from "node:assert/strict"; +import { test } from "bun:test"; + +import { connectWorkbenchKafkaSseDebug, workbenchKafkaSseDebugCorrelationIds, workbenchKafkaSseDebugPath } from "./workbench-debug"; + +test("Kafka SSE debug path preserves stream and explicit correlation filters", () => { + assert.equal( + workbenchKafkaSseDebugPath({ + stream: "agentrun", + fromBeginning: true, + traceId: "trc_debug", + sessionId: "ses_debug", + runId: "run_debug", + commandId: "cmd_debug" + }), + "/v1/workbench/debug/kafka-sse/events?stream=agentrun&fromBeginning=true&traceId=trc_debug&sessionId=ses_debug&runId=run_debug&commandId=cmd_debug" + ); +}); + +test("Kafka SSE debug correlation ids normalize stdio snake_case metadata", () => { + assert.deepEqual( + workbenchKafkaSseDebugCorrelationIds({ + value: { + trace_id: "trc_stdio_debug", + metadata: { + session_id: "ses_stdio_debug", + run_id: "run_stdio_debug", + command_id: "cmd_stdio_debug" + } + } + }), + ["trc_stdio_debug", "ses_stdio_debug", "run_stdio_debug", "cmd_stdio_debug"] + ); +}); + +test("Kafka SSE debug client reports open from named consumer-ready and error from named Kafka error", () => { + const originalEventSource = globalThis.EventSource; + try { + globalThis.EventSource = FakeEventSource as unknown as typeof EventSource; + const opened: string[] = []; + const errors: string[] = []; + const events: string[] = []; + const stream = connectWorkbenchKafkaSseDebug({ + stream: "hwlab", + onOpen: () => opened.push("open"), + onError: (event) => errors.push(event.type), + onEvent: (_event, eventName) => events.push(eventName) + }); + assert.ok(stream); + const source = FakeEventSource.latest; + assert.ok(source); + + source.emitTransportOpen(); + assert.deepEqual(opened, []); + + source.emitNamed("hwlab.kafka.connected", { ok: true, consumerReady: false, groupId: "not-ready" }); + assert.deepEqual(opened, []); + source.emitNamed("hwlab.kafka.connected", { ok: true, consumerReady: true, groupId: "hwlab-v03-debug-sse-ready" }); + assert.deepEqual(opened, ["open"]); + + source.emitNamed("hwlab.kafka.error", { ok: false, error: { code: "kafka_sse_error", message: "consumer stopped" } }); + assert.deepEqual(errors, ["hwlab.kafka.error"]); + assert.deepEqual(events, ["hwlab.kafka.connected", "hwlab.kafka.connected", "hwlab.kafka.error"]); + stream.close(); + assert.equal(source.closed, true); + } finally { + globalThis.EventSource = originalEventSource; + FakeEventSource.latest = null; + } +}); + +class FakeEventSource { + static latest: FakeEventSource | null = null; + onopen: ((event: Event) => void) | null = null; + onerror: ((event: Event) => void) | null = null; + onmessage: ((event: MessageEvent) => void) | null = null; + closed = false; + private listeners = new Map void>>(); + + constructor(_url: string | URL, _eventSourceInitDict?: EventSourceInit) { + FakeEventSource.latest = this; + } + + addEventListener(name: string, listener: (event: MessageEvent) => void) { + const listeners = this.listeners.get(name) ?? new Set(); + listeners.add(listener); + this.listeners.set(name, listeners); + } + + removeEventListener(name: string, listener: (event: MessageEvent) => void) { + this.listeners.get(name)?.delete(listener); + } + + close() { + this.closed = true; + } + + emitTransportOpen() { + this.onopen?.(new Event("open")); + } + + emitNamed(name: string, payload: Record) { + const event = new MessageEvent(name, { data: JSON.stringify(payload) }); + for (const listener of this.listeners.get(name) ?? []) listener(event); + } +} diff --git a/web/hwlab-cloud-web/src/api/workbench-debug.ts b/web/hwlab-cloud-web/src/api/workbench-debug.ts index 653f256e..a3b61806 100644 --- a/web/hwlab-cloud-web/src/api/workbench-debug.ts +++ b/web/hwlab-cloud-web/src/api/workbench-debug.ts @@ -1,5 +1,5 @@ // SPEC: PJ2026-010401080313 Workbench实时权威 draft-2026-07-09-p1-single-step-debug. -// Responsibility: Cloud Web client for Workbench debug-only fake SSE queues. +// Responsibility: Cloud Web clients for Workbench fake SSE and raw Kafka stream diagnostics. import { fetchJson, type ApiRequestOptions } from "@/api/client"; import type { ApiResult } from "@/types"; @@ -50,6 +50,43 @@ export interface WorkbenchDebugFakeSseStream { close: () => void; } +export type WorkbenchKafkaSseDebugStreamName = "stdio" | "agentrun" | "hwlab"; + +export interface WorkbenchKafkaSseDebugEvent { + ok?: boolean; + contractVersion?: string; + consumerReady?: boolean; + groupId?: string; + stream?: WorkbenchKafkaSseDebugStreamName; + topic?: string; + partition?: number; + offset?: string; + key?: string | null; + timestamp?: string | null; + value?: unknown; + filters?: Record; + resolvedFilters?: Record; + serverSentAt?: string; + valuesPrinted?: boolean; + error?: { code?: string; message?: string; [key: string]: unknown }; +} + +export interface WorkbenchKafkaSseDebugStreamOptions { + stream: WorkbenchKafkaSseDebugStreamName; + fromBeginning?: boolean; + traceId?: string | null; + sessionId?: string | null; + runId?: string | null; + commandId?: string | null; + onEvent: (event: WorkbenchKafkaSseDebugEvent, eventName: string) => void; + onOpen?: () => void; + onError?: (event: Event) => void; +} + +export interface WorkbenchKafkaSseDebugStream { + close: () => void; +} + const DEBUG_FAKE_SSE_EVENTS = [ "workbench.connected", "workbench.trace.snapshot", @@ -60,6 +97,8 @@ const DEBUG_FAKE_SSE_EVENTS = [ "workbench.error" ]; +const DEBUG_KAFKA_SSE_EVENTS = ["hwlab.kafka.connected", "hwlab.kafka.event", "hwlab.kafka.error"]; + export const workbenchDebugAPI = { describe: (queueId = "trace-card", options: ApiRequestOptions = {}): Promise> => fetchJson(debugPath("", { queueId }), { ...options, timeoutName: "workbench debug fake sse" }), reset: (input: { queueId: string; sequenceId: string }, options: ApiRequestOptions = {}): Promise> => fetchJson(debugPath("/reset"), { ...options, method: "POST", body: JSON.stringify(input), timeoutName: "workbench debug fake sse reset" }), @@ -93,6 +132,58 @@ export function connectWorkbenchDebugFakeSse(options: WorkbenchDebugFakeSseStrea }; } +export function connectWorkbenchKafkaSseDebug(options: WorkbenchKafkaSseDebugStreamOptions): WorkbenchKafkaSseDebugStream | null { + if (typeof EventSource === "undefined") return null; + const source = new EventSource(workbenchKafkaSseDebugPath(options), { withCredentials: true }); + source.onerror = (event) => options.onError?.(event); + const listeners = DEBUG_KAFKA_SSE_EVENTS.map((name) => { + const listener = (event: MessageEvent) => { + const payload = parseKafkaSseDebugEvent(event.data); + if (!payload) return; + options.onEvent(payload, name); + if (name === "hwlab.kafka.connected" && payload.consumerReady === true) options.onOpen?.(); + if (name === "hwlab.kafka.error") options.onError?.(event); + }; + source.addEventListener(name, listener); + return { name, listener }; + }); + source.onmessage = (event) => { + const payload = parseKafkaSseDebugEvent(event.data); + if (payload) options.onEvent(payload, "message"); + }; + return { + close() { + for (const { name, listener } of listeners) source.removeEventListener(name, listener); + source.close(); + } + }; +} + +export function workbenchKafkaSseDebugPath(options: Pick): string { + const query = new URLSearchParams({ stream: options.stream }); + if (options.fromBeginning === true) query.set("fromBeginning", "true"); + appendDebugFilter(query, "traceId", options.traceId); + appendDebugFilter(query, "sessionId", options.sessionId); + appendDebugFilter(query, "runId", options.runId); + appendDebugFilter(query, "commandId", options.commandId); + return `/v1/workbench/debug/kafka-sse/events?${query.toString()}`; +} + +export function workbenchKafkaSseDebugCorrelationIds(event: WorkbenchKafkaSseDebugEvent): string[] { + const value = recordValue(event.value); + const context = recordValue(value.context); + const metadata = recordValue(value.metadata); + const run = recordValue(value.run); + const command = recordValue(value.command); + const payload = recordValue(value.payload); + return uniqueStrings([ + value.traceId, value.trace_id, context.traceId, context.trace_id, metadata.traceId, metadata.trace_id, payload.traceId, payload.trace_id, + value.hwlabSessionId, value.sessionId, value.session_id, context.hwlabSessionId, context.sessionId, context.session_id, metadata.sessionId, metadata.session_id, payload.hwlabSessionId, payload.sessionId, payload.session_id, + value.runId, value.run_id, context.runId, context.run_id, metadata.runId, metadata.run_id, run.runId, run.run_id, payload.runId, payload.run_id, + value.commandId, value.command_id, context.commandId, context.command_id, metadata.commandId, metadata.command_id, command.commandId, command.command_id, payload.commandId, payload.command_id + ]); +} + function debugPath(suffix: string, params: Record = {}): string { const query = new URLSearchParams(); for (const [key, value] of Object.entries(params)) { @@ -111,3 +202,25 @@ function parseRealtimeEvent(raw: string): WorkbenchRealtimeEvent | null { return null; } } + +function parseKafkaSseDebugEvent(raw: string): WorkbenchKafkaSseDebugEvent | null { + try { + const value = JSON.parse(raw); + return value && typeof value === "object" && !Array.isArray(value) ? value as WorkbenchKafkaSseDebugEvent : null; + } catch { + return null; + } +} + +function appendDebugFilter(query: URLSearchParams, name: string, value: string | null | undefined): void { + const text = typeof value === "string" ? value.trim() : ""; + if (text) query.set(name, text); +} + +function recordValue(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + +function uniqueStrings(values: unknown[]): string[] { + return [...new Set(values.map((value) => typeof value === "string" ? value.trim() : "").filter(Boolean))]; +} diff --git a/web/hwlab-cloud-web/src/api/workbench-events.test.ts b/web/hwlab-cloud-web/src/api/workbench-events.test.ts index 33273739..7d5af779 100644 --- a/web/hwlab-cloud-web/src/api/workbench-events.test.ts +++ b/web/hwlab-cloud-web/src/api/workbench-events.test.ts @@ -2,6 +2,7 @@ import assert from "node:assert/strict"; import { test } from "bun:test"; import { realtimeCoalesceKey, type WorkbenchRealtimeEvent } from "./workbench-events"; +import { createCoalescedEventQueue } from "../utils/scheduler/coalesced-event-queue"; test("turn snapshot coalescing keys by trace instead of per sequence", () => { const first: WorkbenchRealtimeEvent = { type: "turn.snapshot", turn: { sessionId: "ses_queue", traceId: "trc_queue", status: "running" }, cursor: { traceSeq: 10 } }; @@ -16,3 +17,25 @@ test("trace events keep sequence-specific coalescing keys", () => { assert.notEqual(realtimeCoalesceKey(first, "workbench.trace.event"), realtimeCoalesceKey(next, "workbench.trace.event")); }); + +test("live Kafka envelopes from one trace retain every assistant, tool, output, and terminal event", () => { + const delivered: WorkbenchRealtimeEvent[] = []; + const queue = createCoalescedEventQueue({ + keyOf: (event) => realtimeCoalesceKey(event, "hwlab.event.v1"), + onFlush: (events) => delivered.push(...events) + }); + for (const sourceEventId of ["evt_assistant", "evt_tool", "evt_output", "evt_terminal"]) { + queue.push({ + schema: "hwlab.event.v1", + eventId: `hwlab:${sourceEventId}`, + sourceEventId, + sessionId: "ses_live_queue", + traceId: "trc_live_queue", + event: { sourceEventId, traceId: "trc_live_queue", sessionId: "ses_live_queue" } + }); + } + + queue.flush(); + + assert.deepEqual(delivered.map((event) => event.sourceEventId), ["evt_assistant", "evt_tool", "evt_output", "evt_terminal"]); +}); diff --git a/web/hwlab-cloud-web/src/api/workbench-events.ts b/web/hwlab-cloud-web/src/api/workbench-events.ts index 0922b813..12ba9def 100644 --- a/web/hwlab-cloud-web/src/api/workbench-events.ts +++ b/web/hwlab-cloud-web/src/api/workbench-events.ts @@ -6,6 +6,7 @@ import type { ApiResult, ChatMessage, ProjectionDiagnostic, TraceEvent } from "@ import { createCoalescedEventQueue } from "@/utils/scheduler/coalesced-event-queue"; import { composeWorkbenchScopedKey, firstScopePart } from "@/utils/workbench-key"; import { recordWorkbenchRuntimeDiagnostic, recordWorkbenchSseLifecycle } from "@/utils/workbench-performance"; +import type { WorkbenchRealtimeCapabilities } from "@/config/runtime"; export interface WorkbenchRealtimeTraceSnapshot { traceId?: string | null; @@ -23,15 +24,26 @@ export interface WorkbenchRealtimeTraceSnapshot { } export interface WorkbenchRealtimeEvent { + schema?: string | null; + eventType?: string | null; + eventId?: string | null; + sourceEventId?: string | null; + capabilities?: WorkbenchRealtimeCapabilities | null; + liveOnly?: boolean | null; + replay?: boolean | null; + lossPossible?: boolean | null; type?: string; contractVersion?: string | null; realtimeAuthority?: string | null; status?: string; serverSentAt?: string | null; eventCreatedAt?: string | null; + hwlabSessionId?: string | null; sessionId?: string | null; threadId?: string | null; traceId?: string | null; + runId?: string | null; + commandId?: string | null; event?: TraceEvent | null; snapshot?: WorkbenchRealtimeTraceSnapshot | null; message?: ChatMessage | null; @@ -88,6 +100,7 @@ export interface WorkbenchSyncReplayResponse { } export interface WorkbenchEventStreamOptions { + realtimeCapabilities: WorkbenchRealtimeCapabilities; sessionId?: string | null; traceId?: string | null; afterSeq?: number | null; @@ -109,6 +122,7 @@ interface QueuedRealtimeEvent { } const WORKBENCH_EVENT_NAMES = [ + "hwlab.event.v1", "workbench.connected", "workbench.trace.snapshot", "workbench.trace.event", @@ -121,11 +135,7 @@ const WORKBENCH_EVENT_NAMES = [ export function connectWorkbenchEvents(options: WorkbenchEventStreamOptions): WorkbenchEventStream | null { if (typeof EventSource === "undefined") return null; - const params = new URLSearchParams(); - appendParam(params, "sessionId", options.sessionId); - appendParam(params, "traceId", options.traceId); - appendNumberParam(params, "afterSeq", options.afterSeq); - const eventRoute = `/v1/workbench/events?${params.toString()}`; + const eventRoute = workbenchEventStreamPath(options); const connectStartedAt = typeof performance === "undefined" ? Date.now() : performance.now(); recordWorkbenchSseLifecycle({ state: "connect", route: eventRoute, sessionId: options.sessionId, traceId: options.traceId }); const source = new EventSource(eventRoute, { withCredentials: true }); @@ -192,6 +202,22 @@ export function connectWorkbenchEvents(options: WorkbenchEventStreamOptions): Wo }; } +export function workbenchEventStreamPath(options: Pick): string { + const params = new URLSearchParams(); + appendParam(params, "sessionId", options.sessionId); + appendParam(params, "traceId", options.traceId); + if (!options.realtimeCapabilities.liveKafkaSse && options.realtimeCapabilities.projectionRealtime) appendNumberParam(params, "afterSeq", options.afterSeq); + return `/v1/workbench/events?${params.toString()}`; +} + +export function workbenchProjectionEventStreamPath(options: Pick): string { + const params = new URLSearchParams(); + appendParam(params, "sessionId", options.sessionId); + appendParam(params, "traceId", options.traceId); + appendNumberParam(params, "afterSeq", options.afterSeq); + return `/v1/workbench/projection-events?${params.toString()}`; +} + export async function fetchWorkbenchSyncReplay(input: WorkbenchSyncReplayRequest, options: ApiRequestOptions = {}): Promise> { return fetchJson(workbenchSyncReplayPath(input), { ...options, @@ -238,6 +264,10 @@ function scheduleRealtimeFlushYield(flush: () => void, yieldMs: number | null | } export function realtimeCoalesceKey(event: WorkbenchRealtimeEvent, eventName: string): string | null { + if (eventName === "hwlab.event.v1" || event.schema === "hwlab.event.v1") { + const eventId = firstScopePart(event.eventId, event.sourceEventId, event.event?.sourceEventId); + return eventId ? composeWorkbenchScopedKey("workbench.sse.live-kafka-event", eventId) : null; + } const entity = event.entity; const entityFamily = firstScopePart(entity?.family); const entityId = firstScopePart(entity?.id); diff --git a/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.test.ts b/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.test.ts index 8cfc0ffa..5c1d9765 100644 --- a/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.test.ts +++ b/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.test.ts @@ -47,3 +47,22 @@ test("mergeRunnerTrace can seal terminal metadata without requiring trace rows", assert.equal(merged.status, "completed"); assert.equal(merged.events, previousEvents); }); + +test("mergeRunnerTrace accumulates live Kafka deltas with an honest source and event count", () => { + const previous = trace({ + eventSource: "hwlab-kafka-sse", + eventCount: 1, + events: [{ source: "agentrun.kafka", sourceSeq: 1, type: "assistant", label: "assistant" }] + }); + const next = trace({ + eventSource: "hwlab-kafka-sse", + eventCount: 1, + events: [{ source: "agentrun.kafka", sourceSeq: 2, type: "tool", label: "tool" }] + }); + + const merged = mergeRunnerTrace(previous, next); + + assert.equal(merged.eventSource, "hwlab-kafka-sse"); + assert.equal(merged.eventCount, 2); + assert.deepEqual(merged.events?.map((event) => event.sourceSeq), [1, 2]); +}); diff --git a/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.ts b/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.ts index 32b5073d..80ee4d03 100644 --- a/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.ts +++ b/web/hwlab-cloud-web/src/composables/workbench-trace-snapshot.ts @@ -91,7 +91,9 @@ export function mergeRunnerTrace(previous: ChatMessage["runnerTrace"], next: Non ? previousEvents.length > 0 ? mergeTraceEvents(previousEvents, nextEvents, { previousCursor, nextRange: next.range }) : nextEvents : keepPreviousEvents && previousAuthoritative && next.eventsCompacted === true ? previousEvents : keepPreviousEvents ? mergeTraceEvents(previousEvents, nextEvents, { previousCursor, nextRange: next.range }) : nextEvents.length > previousEvents.length ? mergeTraceEvents(previousEvents, nextEvents, { previousCursor, nextRange: next.range }) : nextEvents; - const eventCount = keepPreviousEvents ? previous.eventCount ?? events.length : next.eventCount ?? previous.eventCount ?? events.length; + const eventCount = next.eventSource === "hwlab-kafka-sse" + ? events.length + : keepPreviousEvents ? previous.eventCount ?? events.length : next.eventCount ?? previous.eventCount ?? events.length; const timing = mergeTraceTimingProjection(previous, next); return { ...previous, diff --git a/web/hwlab-cloud-web/src/config/runtime.ts b/web/hwlab-cloud-web/src/config/runtime.ts index adf9018a..673cb10c 100644 --- a/web/hwlab-cloud-web/src/config/runtime.ts +++ b/web/hwlab-cloud-web/src/config/runtime.ts @@ -10,6 +10,11 @@ export interface OpenCodeFrameConfig { url: string; } +export interface WorkbenchRealtimeCapabilities { + liveKafkaSse: boolean; + projectionRealtime: boolean; +} + const DEFAULT_DISPLAY_DATE_TIME_OPTIONS: Intl.DateTimeFormatOptions = { year: "numeric", month: "2-digit", @@ -80,6 +85,14 @@ export function workbenchTraceTimelinePolicy(): WorkbenchTraceTimelinePolicy { }; } +export function workbenchRealtimeCapabilities(): WorkbenchRealtimeCapabilities { + const features = window.HWLAB_CLOUD_WEB_CONFIG?.workbench?.realtimeFeatures; + if (typeof features?.liveKafkaSse !== "boolean" || typeof features?.projectionRealtime !== "boolean") { + throw new Error("HWLAB Cloud Web workbench.realtimeFeatures are required"); + } + return { liveKafkaSse: features.liveKafkaSse, projectionRealtime: features.projectionRealtime }; +} + export function traceExplorerHref(traceId: string | null | undefined): string | null { const safeTraceId = normalizedTraceId(traceId); if (!safeTraceId) return null; diff --git a/web/hwlab-cloud-web/src/stores/workbench-event-reducer.ts b/web/hwlab-cloud-web/src/stores/workbench-event-reducer.ts index 147611cd..2c81c088 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-event-reducer.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-event-reducer.ts @@ -48,7 +48,15 @@ export function reduceWorkbenchRealtimeEvent(event: WorkbenchRealtimeEvent, even }; } +export function workbenchRealtimeEventIsBusinessActivity(event: WorkbenchRealtimeEvent, eventName: string, liveKafkaSse: boolean): boolean { + if (!liveKafkaSse) return true; + return eventName === "hwlab.event.v1" || event.schema === "hwlab.event.v1"; +} + function reduceRealtimeAction(event: WorkbenchRealtimeEvent, eventName: string): WorkbenchRealtimeAction { + if (event.schema === "hwlab.event.v1" && event.event) { + return { type: "trace.event", traceId: realtimeTraceId(event), event: event.event, snapshot: null, realtimeEvent: event }; + } const authority = primaryAuthority(event); if (authority) return authority; switch (event.type) { @@ -81,5 +89,5 @@ function realtimeTraceId(event: WorkbenchRealtimeEvent): string | null { } function realtimeSessionId(event: WorkbenchRealtimeEvent): string | null { - return firstNonEmptyString(event.sessionId, event.message?.sessionId, event.snapshot?.sessionId, event.event?.sessionId) ?? null; + return firstNonEmptyString(event.hwlabSessionId, event.sessionId, event.message?.sessionId, event.snapshot?.sessionId, event.event?.sessionId) ?? null; } diff --git a/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts b/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts new file mode 100644 index 00000000..baf2343c --- /dev/null +++ b/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts @@ -0,0 +1,50 @@ +// SPEC: PJ2026-0104010803 Workbench live Kafka SSE. +// Responsibility: project one transparent hwlab.event.v1 business event into visible turn state without replay/finalizer/polling. + +import type { TraceEvent } from "@/types"; +import { firstNonEmptyString } from "@/utils"; + +export interface WorkbenchLiveMessageState { + text: string; + status: string; + terminal: boolean; +} + +export function reduceWorkbenchLiveKafkaMessageState(previous: WorkbenchLiveMessageState, event: TraceEvent): WorkbenchLiveMessageState { + const terminal = workbenchLiveKafkaEventIsTerminal(event); + const status = terminal ? firstNonEmptyString(event.status, "completed") ?? "completed" : "running"; + const assistantText = workbenchLiveKafkaAssistantText(event); + return { + text: assistantText ?? (terminal ? workbenchLiveKafkaTerminalText(event, previous.text, status) : previous.text), + status, + terminal + }; +} + +export function workbenchLiveKafkaMessageId(traceId: string): string { + return `msg_live_${traceId}`; +} + +export function workbenchLiveKafkaEventIsTerminal(event: TraceEvent | null | undefined): boolean { + return Boolean(event?.terminal === true || firstNonEmptyString(event?.eventType) === "terminal" || firstNonEmptyString(event?.type) === "result"); +} + +export function workbenchLiveKafkaAssistantText(event: TraceEvent | null | undefined): string | null { + if (!event || firstNonEmptyString(event.type, event.eventType) !== "assistant") return null; + const finalResponse = recordValue(event.finalResponse); + return firstNonEmptyString(finalResponse?.text, event.assistantText, event.text, event.message) ?? null; +} + +export function workbenchLiveKafkaTerminalText(event: TraceEvent, previousText: string, status: string): string { + const finalResponse = recordValue(event.finalResponse); + const error = recordValue(event.error); + const failed = ["failed", "blocked", "timeout", "canceled", "cancelled", "interrupted", "expired"].includes(status); + const text = failed + ? firstNonEmptyString(finalResponse?.text, event.message, error?.message, event.failureKind, event.errorCode, previousText) + : firstNonEmptyString(finalResponse?.text, event.assistantText, event.text, previousText, event.message); + return text ?? `AgentRun ${status}`; +} + +function recordValue(value: unknown): Record | null { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : null; +} diff --git a/web/hwlab-cloud-web/src/stores/workbench-session.test.ts b/web/hwlab-cloud-web/src/stores/workbench-session.test.ts index f7eb4ab5..e24615f0 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-session.test.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-session.test.ts @@ -65,6 +65,17 @@ test("composer treats failed message with terminal body as sealed terminal turn" assert.equal(composer.targetTraceId, null); }); +test("composer blocks turn submission until the live session stream is connected", () => { + const sessionId = "ses_realtime_connecting"; + const connecting = resolveComposerState({ messages: [], activeSessionId: sessionId, chatPending: false, realtimeReady: false }); + const connected = resolveComposerState({ messages: [], activeSessionId: sessionId, chatPending: false, realtimeReady: true }); + + assert.equal(connecting.disabled, true); + assert.equal(connecting.disabledReason, "realtime_connecting"); + assert.equal(connecting.sessionId, sessionId); + assert.equal(connected.disabled, false); +}); + test("cancel target remains available for completed message until final response exists", () => { const traceId = "trc_cancel_completed_without_final"; const sessionId = "ses_cancel_completed_without_final"; diff --git a/web/hwlab-cloud-web/src/stores/workbench-session.ts b/web/hwlab-cloud-web/src/stores/workbench-session.ts index 2568f407..f4c3deca 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-session.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-session.ts @@ -75,7 +75,7 @@ const BUILTIN_PROVIDER_PROFILE_LABELS: Readonly> = Object "minimax-m3": "MiniMax-M3" }); -export function resolveComposerState(input: { messages: ChatMessage[]; sessions?: WorkbenchSessionRecord[]; activeSessionId: string | null; chatPending: boolean; currentRequest?: { traceId?: string | null; sessionId?: string | null; threadId?: string | null; status?: string | null } | null; turnStatusAuthority?: TurnStatusAuthorityMap }): ComposerState { +export function resolveComposerState(input: { messages: ChatMessage[]; sessions?: WorkbenchSessionRecord[]; activeSessionId: string | null; chatPending: boolean; realtimeReady?: boolean; currentRequest?: { traceId?: string | null; sessionId?: string | null; threadId?: string | null; status?: string | null } | null; turnStatusAuthority?: TurnStatusAuthorityMap }): ComposerState { const sessionId = firstNonEmptyString(input.activeSessionId); const latestMessage = latestSessionMessage(input.messages, sessionId); const currentRequest = input.currentRequest && input.currentRequest.sessionId === sessionId ? input.currentRequest : null; @@ -94,6 +94,9 @@ export function resolveComposerState(input: { messages: ChatMessage[]; sessions? const effectiveSessionId = firstNonEmptyString(turn?.sessionId, currentRequest?.sessionId, active?.sessionId, sessionId); const threadId = firstNonEmptyString(turn?.threadId, currentRequest?.threadId, active?.threadId); const canSteer = Boolean(effectiveSessionId && activeTraceId && activeByStatus && !terminal); + if (effectiveSessionId && input.realtimeReady === false) { + return { disabled: true, disabledReason: "realtime_connecting", submitMode: canSteer ? "steer" : "turn", route: canSteer ? "/v1/agent/chat/steer" : "/v1/agent/chat", targetTraceId: canSteer ? activeTraceId : null, sessionId: effectiveSessionId, threadId }; + } if (canSteer) return { disabled: false, disabledReason: null, submitMode: "steer", route: "/v1/agent/chat/steer", targetTraceId: activeTraceId, sessionId: effectiveSessionId, threadId }; if (!effectiveSessionId) return { disabled: true, disabledReason: "session_required", submitMode: "turn", route: "/v1/agent/chat", targetTraceId: null, sessionId: null, threadId }; return { disabled: false, disabledReason: null, submitMode: "turn", route: "/v1/agent/chat", targetTraceId: null, sessionId: effectiveSessionId, threadId }; diff --git a/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts b/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts index 7b9784e6..e2a2e48b 100644 --- a/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts +++ b/web/hwlab-cloud-web/src/stores/workbench-trace-detail.ts @@ -25,7 +25,7 @@ export function terminalSealResultWithoutTraceEvents(result: AgentChatResultResp return { ...rest, runnerTrace: traceRest }; } -export function realtimeSnapshotToTraceSnapshot(traceId: string, snapshot: WorkbenchRealtimeEvent["snapshot"], eventPageEvents?: TraceEvent[]): TraceSnapshot { +export function realtimeSnapshotToTraceSnapshot(traceId: string, snapshot: WorkbenchRealtimeEvent["snapshot"], eventPageEvents?: TraceEvent[], options: { eventSource?: string } = {}): TraceSnapshot { const source = snapshot ?? { traceId }; const events = Array.isArray(eventPageEvents) ? eventPageEvents : Array.isArray(source.events) ? source.events : []; const { traceId: _sourceTraceId, error: _sourceError, ...sourceRest } = source; @@ -53,7 +53,7 @@ export function realtimeSnapshotToTraceSnapshot(traceId: string, snapshot: Workb blocker: recordValue(source.blocker) ?? recordValue(recordValue(source.projection)?.blocker) ?? undefined, ...messageTimingPatch(source), lastEventLabel: text(source.lastEventLabel) ?? undefined, - eventSource: "trace-api", + eventSource: options.eventSource ?? "trace-api", updatedAt: text(source.updatedAt) ?? new Date().toISOString() } as TraceSnapshot; } diff --git a/web/hwlab-cloud-web/src/stores/workbench.ts b/web/hwlab-cloud-web/src/stores/workbench.ts index eaaad5fd..05ca500a 100644 --- a/web/hwlab-cloud-web/src/stores/workbench.ts +++ b/web/hwlab-cloud-web/src/stores/workbench.ts @@ -4,23 +4,25 @@ import { computed, nextTick, ref } from "vue"; import { defineStore } from "pinia"; import { api } from "@/api"; +import { workbenchRealtimeCapabilities } from "@/config/runtime"; import { workbenchRuntimePolicy } from "@/config/workbench-runtime-policy"; import { createKeyedSingleflight } from "@/utils/scheduler/keyed-singleflight"; import { createWorkbenchHealthProbeCache } from "@/utils/workbench-health"; import { agentErrorFromProjection, normalizeApiErrorRecord, normalizeErrorDiagnostic, normalizeProjectionDiagnostic, projectionDiagnosticFromApiFailure, projectionDiagnosticFromFailure } from "@/utils/workbench-error-runtime"; import { readWorkbenchJson, readWorkbenchNumber, readWorkbenchString, removeWorkbenchStorageKey, writeWorkbenchJson, writeWorkbenchString } from "@/utils/workbench-storage-runtime"; -import { createWorkbenchStreamTransportRuntime, type WorkbenchRealtimeEvent, type WorkbenchStreamTransportRecovery } from "@/utils/workbench-realtime-runtime"; +import { createWorkbenchStreamTransportRuntime, workbenchRealtimeTraceIdForCapabilities, type WorkbenchRealtimeEvent, type WorkbenchStreamTransportRecovery } from "@/utils/workbench-realtime-runtime"; import { mergeRunnerTrace, snapshotToRunnerTrace, type TraceSnapshot } from "@/composables/workbench-trace-snapshot"; import type { WorkbenchMessagePageResponse, WorkbenchSessionDetailResponse } from "@/api/workbench"; import type { AgentChatResponse, AgentChatResultResponse, AgentRunProvenance, ApiError, ApiResult, ChatMessage, ErrorDiagnostic, LiveSurface, ProjectionBlocker, ProjectionDiagnostic, ProviderProfile, TraceEvent, WorkbenchSessionRecord, WorkbenchTurnTimingProjection } from "@/types"; import { firstNonEmptyString, nextProtocolId, normalizeWorkbenchSessionId, normalizeWorkbenchSessionRouteId } from "@/utils"; -import { composeWorkbenchScopedKey } from "@/utils/workbench-key"; +import { composeWorkbenchScopedKey, workbenchRealtimeScopeKey } from "@/utils/workbench-key"; import { failWorkbenchSessionSwitch, failWorkbenchSubmitJourney, finishWorkbenchSessionSwitchFullLoad, markWorkbenchSubmitApiAccepted, markWorkbenchTraceEventsReceived, markWorkbenchTraceProjected, recordWorkbenchLoadingState, recordWorkbenchRuntimeDiagnostic, startWorkbenchSessionSwitch, startWorkbenchSubmitJourney } from "@/utils/workbench-performance"; import { RECENT_DRAFTS_STORAGE_KEY, appendSessionPage, defaultProviderProfileOptions, isArchivedSession, mergeSessionIntoList, normalizeChatMessageStatus, normalizeRecentDrafts, normalizeWorkbenchMessageTitle, providerProfileOptionsFromPayload, recordRecentDraft, resolveCancelableAgentMessage, resolveComposerState, selectActiveTurnStatusRefreshTraceIds, shouldReadTerminalTraceDetail, shouldShowSessionListLoading, sortSessionTabs, stableSessionList, type DraftEntry, type ProviderProfileOption, type TurnStatusAuthority } from "./workbench-session"; import { initialWorkbenchSessionIdFromLocation } from "./workbench-projection"; import { cleanupWorkbenchServerStateSessions, selectActiveMessages, selectActiveSession, selectSessionList, selectSessionStatusAuthority, selectTraceAuthorityById, selectTurnStatusAuthority, type WorkbenchServerAction } from "./workbench-server-state"; import { cleanupDroppedWorkbenchSessionCaches, trimWorkbenchSessionCache } from "./workbench-session-cache"; -import { reduceWorkbenchRealtimeEvent, type WorkbenchRealtimeAction } from "./workbench-event-reducer"; +import { reduceWorkbenchRealtimeEvent, workbenchRealtimeEventIsBusinessActivity, type WorkbenchRealtimeAction } from "./workbench-event-reducer"; +import { reduceWorkbenchLiveKafkaMessageState, workbenchLiveKafkaEventIsTerminal, workbenchLiveKafkaMessageId } from "./workbench-live-kafka-event"; import { messageHasSealedTerminalResult, messageIsSealedTerminal, traceAuthorityIsSealed } from "./workbench-terminal-authority"; import { boundedProjectionMessageLimit, mergeBoundedProjectionMessages, selectProjectionMessageWindow, traceProjectionIsTerminalSealed } from "./workbench-message-projection-budget"; import { @@ -106,6 +108,7 @@ interface RealtimeTurnProjectionItem { export const useWorkbenchStore = defineStore("workbench", () => { const runtimePolicy = workbenchRuntimePolicy(); + const realtimeCapabilities = workbenchRealtimeCapabilities(); const workbenchColadaReducer = useWorkbenchColadaReducer(); const workbenchColadaQueries = useWorkbenchColadaQueries(); const workbenchColadaMutations = useWorkbenchColadaMutations(); @@ -131,6 +134,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { const sessionListLoadingMore = ref(false); const sessionListLoadMoreError = ref(null); const chatPending = ref(false); + const liveRealtimeReadySessionId = ref(null); const error = ref(null); const activityRef = ref({ lastActivityAt: Date.now(), lastActivityIso: new Date().toISOString(), waitingFor: "idle", lastEventLabel: null as string | null }); const currentRequest = ref<{ traceId: string; sessionId: string | null; threadId: string | null; status?: string | null } | null>(null); @@ -160,7 +164,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { const sessionListLoadedCount = computed(() => sessionTabs.value.length); const sessionListLoading = computed(() => shouldShowSessionListLoading({ loading: loading.value, sessionsReady: sessionsReady.value })); const sessionDetailLoading = computed(() => Boolean(sessionDetailLoadingId.value || switchingSessionId.value || (loading.value && messages.value.length === 0))); - const composer = computed(() => resolveComposerState({ messages: messages.value, sessions: sessions.value, activeSessionId: activeSessionId.value, chatPending: chatPending.value, currentRequest: currentRequest.value, turnStatusAuthority: turnStatusAuthority.value })); + const composer = computed(() => resolveComposerState({ messages: messages.value, sessions: sessions.value, activeSessionId: activeSessionId.value, chatPending: chatPending.value, realtimeReady: !realtimeCapabilities.liveKafkaSse || liveRealtimeReadySessionId.value === activeSessionId.value, currentRequest: currentRequest.value, turnStatusAuthority: turnStatusAuthority.value })); function recordActivity(label = "user-activity"): void { const now = Date.now(); @@ -375,6 +379,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { } function scheduleSessionListRefresh(includeSessionId: string | null | undefined = activeSessionId.value, delayMs = runtimePolicy.sessionListRealtimeRefreshDelayMs): void { + if (realtimeCapabilities.liveKafkaSse) return; void includeSessionId; if (typeof window === "undefined") { void workbenchColadaQueries.invalidateSessionList(); @@ -754,6 +759,11 @@ export const useWorkbenchStore = defineStore("workbench", () => { const traceId = steerMode && composer.value.targetTraceId ? composer.value.targetTraceId : nextProtocolId("trc"); const steerTraceId = steerMode ? nextProtocolId("trc_steer") : null; const sessionId = composer.value.sessionId; + if (realtimeCapabilities.liveKafkaSse && liveRealtimeReadySessionId.value !== sessionId) { + error.value = "realtime_connecting"; + restartRealtime("submit-readiness"); + return false; + } const threadId = composer.value.threadId; const providerThreadId = providerThreadIdForRequest(threadId); const submittedAt = new Date().toISOString(); @@ -763,6 +773,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { const pending = makeMessage("agent", "", "running", { traceId, sessionId, threadId, title: "Code Agent", retryInput: value, traceAutoLifecycle: "running", createdAt: submittedAt, updatedAt: submittedAt, ...optimisticRunningTimingPatch(submittedAt) }); appendActiveMessages(user, pending); projectOptimisticRunningTurn({ sessionId, threadId, traceId, userText: value }); + currentRequest.value = { traceId, sessionId, threadId, status: "running" }; } startWorkbenchSubmitJourney({ traceId, sessionId, entry: submitEntry, backend: providerProfile.value, transport: "sse" }); chatPending.value = true; @@ -811,6 +822,11 @@ export const useWorkbenchStore = defineStore("workbench", () => { if (canonicalTraceId !== traceId) { reduceServerState({ type: "turn.forget", traceId }); } + if (traceTerminalBodyIsVisible(canonicalTraceId, sessionId)) { + chatPending.value = false; + currentRequest.value = null; + return true; + } currentRequest.value = { traceId: canonicalTraceId, sessionId: firstNonEmptyString(response.data.sessionId, sessionId), @@ -1103,7 +1119,10 @@ export const useWorkbenchStore = defineStore("workbench", () => { const traceId = realtimeTraceId(); const sessionId = selectedSessionId.value ?? null; void reason; + const scopeChanged = realtimeTransport.currentKey() !== workbenchRealtimeScopeKey(sessionId, traceId); + if (realtimeCapabilities.liveKafkaSse && scopeChanged) liveRealtimeReadySessionId.value = null; realtimeTransport.restart({ + realtimeCapabilities, sessionId, traceId, errorRecoveryMinMs: runtimePolicy.workbenchRealtimeErrorSyncReplayMinMs, @@ -1111,8 +1130,23 @@ export const useWorkbenchStore = defineStore("workbench", () => { flushMaxChunkMs: runtimePolicy.workbenchRealtimeFlushMaxChunkMs, flushYieldMs: runtimePolicy.workbenchRealtimeFlushYieldMs, onOpen: () => undefined, + onError: () => { + if (realtimeCapabilities.liveKafkaSse) liveRealtimeReadySessionId.value = null; + }, + onState: (state) => { + if (realtimeCapabilities.liveKafkaSse && ["connecting", "error", "closed", "blocked"].includes(state.phase)) liveRealtimeReadySessionId.value = null; + }, onRecovery: (recovery) => handleRealtimeRecovery(recovery), - onEvent: (event, eventName) => applyRealtimeEvent(event, eventName) + onEvent: (event, eventName) => { + if (realtimeCapabilities.liveKafkaSse && eventName === "workbench.connected") { + const filters = recordValue(event.filters); + const connectedSessionId = normalizeWorkbenchSessionId(filters?.sessionId); + const valid = event.liveOnly === true && event.replay === false && event.capabilities?.liveKafkaSse === true && connectedSessionId === sessionId; + liveRealtimeReadySessionId.value = valid ? connectedSessionId : null; + if (!valid) error.value = "workbench_live_realtime_contract_invalid"; + } + applyRealtimeEvent(event, eventName); + } }); } @@ -1139,6 +1173,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { } async function refreshWorkbenchSyncReplay(sessionId: string | null, traceId: string | null, sinceOutboxSeq: number | null, reason: string): Promise { + if (realtimeCapabilities.liveKafkaSse || !realtimeCapabilities.projectionRealtime) return; let cursor = firstFiniteNumber(sinceOutboxSeq) ?? 0; for (;;) { const result = await workbenchColadaQueries.fetchSyncReplay({ sessionId, traceId, since: cursor }, { timeoutMs: 8000, activityRef: () => activityRef.value }); @@ -1161,11 +1196,16 @@ export const useWorkbenchStore = defineStore("workbench", () => { } function stopRealtime(): void { + liveRealtimeReadySessionId.value = null; realtimeTransport.stop(); } function realtimeTraceId(): string | null { - return firstNonEmptyString(currentRequest.value?.traceId, activeTraceIdFromMessages(messages.value, turnStatusAuthority.value)); + return workbenchRealtimeTraceIdForCapabilities( + realtimeCapabilities, + currentRequest.value?.traceId, + activeTraceIdFromMessages(messages.value, turnStatusAuthority.value) + ); } function normalizeTraceAuthoritySessionId(value: unknown): string | null { @@ -1179,7 +1219,8 @@ export const useWorkbenchStore = defineStore("workbench", () => { const turn = recordValue(event.turn); const snapshot = recordValue(event.snapshot); const traceEvent = recordValue(event.event); - return normalizeTraceAuthoritySessionId(firstNonEmptyString(event.sessionId, turn?.sessionId, snapshot?.sessionId, traceEvent?.sessionId)); + const sessionId = firstNonEmptyString(event.hwlabSessionId, event.sessionId, turn?.sessionId, snapshot?.sessionId, traceEvent?.sessionId); + return event.schema === "hwlab.event.v1" ? normalizeWorkbenchSessionId(sessionId) : normalizeTraceAuthoritySessionId(sessionId); } function traceResultSessionId(result: AgentChatResultResponse | TraceSnapshot | Record | null | undefined): string | null { @@ -1193,8 +1234,8 @@ export const useWorkbenchStore = defineStore("workbench", () => { return normalizeTraceAuthoritySessionId(firstNonEmptyString(message?.sessionId, message?.runnerTrace?.sessionId)); } - function traceOwnerSessionId(traceId: string | null | undefined, authoritySessionId: string | null | undefined): string | null { - const sessionId = normalizeTraceAuthoritySessionId(authoritySessionId); + function traceOwnerSessionId(traceId: string | null | undefined, authoritySessionId: string | null | undefined, canonicalSession = false): string | null { + const sessionId = canonicalSession ? normalizeWorkbenchSessionId(authoritySessionId) : normalizeTraceAuthoritySessionId(authoritySessionId); if (sessionId) return sessionId; const id = firstNonEmptyString(traceId); if (!id) return activeSessionId.value; @@ -1204,21 +1245,22 @@ export const useWorkbenchStore = defineStore("workbench", () => { return activeSessionId.value; } - function messageMatchesTraceAuthority(message: ChatMessage, traceId: string, authoritySessionId: string | null | undefined, ownerSessionId: string | null | undefined): boolean { + function messageMatchesTraceAuthority(message: ChatMessage, traceId: string, authoritySessionId: string | null | undefined, ownerSessionId: string | null | undefined, canonicalSession = false): boolean { if (message.role !== "agent" || firstNonEmptyString(message.traceId, message.runnerTrace?.traceId) !== traceId) return false; - const sessionId = normalizeTraceAuthoritySessionId(authoritySessionId); - const ownerId = normalizeTraceAuthoritySessionId(ownerSessionId); - const messageSessionId = messageSessionAuthority(message); + const normalizeSession = canonicalSession ? normalizeWorkbenchSessionId : normalizeTraceAuthoritySessionId; + const sessionId = normalizeSession(authoritySessionId); + const ownerId = normalizeSession(ownerSessionId); + const messageSessionId = canonicalSession ? normalizeWorkbenchSessionId(firstNonEmptyString(message.sessionId, message.runnerTrace?.sessionId)) : messageSessionAuthority(message); if (sessionId && ownerId && sessionId !== ownerId) return false; if (messageSessionId && ownerId && messageSessionId !== ownerId) return false; if (sessionId && messageSessionId && sessionId !== messageSessionId) return false; return true; } - function shouldApplyActiveTraceAuthority(traceId: string | null | undefined, authoritySessionId: string | null | undefined): boolean { + function shouldApplyActiveTraceAuthority(traceId: string | null | undefined, authoritySessionId: string | null | undefined, canonicalSession = false): boolean { const id = firstNonEmptyString(traceId); const activeId = activeSessionId.value; - const sessionId = normalizeTraceAuthoritySessionId(authoritySessionId); + const sessionId = canonicalSession ? normalizeWorkbenchSessionId(authoritySessionId) : normalizeTraceAuthoritySessionId(authoritySessionId); if (!activeId) return false; if (sessionId && sessionId !== activeId) return false; if (!id) return Boolean(sessionId && sessionId === activeId); @@ -1245,7 +1287,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { function applyRealtimeEvent(event: WorkbenchRealtimeEvent, eventName: string): void { const reduced = reduceWorkbenchRealtimeEvent(event, eventName); - recordActivity(reduced.activityLabel); + if (workbenchRealtimeEventIsBusinessActivity(event, eventName, realtimeCapabilities.liveKafkaSse)) recordActivity(reduced.activityLabel); applyWorkbenchRealtimeAction(reduced.action); } @@ -1301,15 +1343,69 @@ export const useWorkbenchStore = defineStore("workbench", () => { function applyRealtimeTraceEvent(traceId: string | null | undefined, event: WorkbenchRealtimeEvent["event"], snapshot: WorkbenchRealtimeEvent["snapshot"], realtimeEvent?: WorkbenchRealtimeEvent | null): void { const id = firstNonEmptyString(traceId, event?.traceId, snapshot?.traceId); if (!id) return; + const liveKafkaEnvelope = realtimeEvent?.schema === "hwlab.event.v1"; const sessionId = realtimeEvent ? realtimeEventSessionId(realtimeEvent) : traceResultSessionId(snapshot ?? event ?? null); - if (!shouldApplyActiveTraceAuthority(id, sessionId)) return; + if (!shouldApplyActiveTraceAuthority(id, sessionId, liveKafkaEnvelope)) return; if (traceTerminalBodyIsVisible(id, sessionId)) { recordWorkbenchRuntimeDiagnostic({ module: "workbench-terminal-priority", sessionId, traceId: id, outcome: "ok", diagnostic: { code: "terminal_low_priority_sse_trace_skip", source: "realtime-trace-event", valuesRedacted: true } }); return; } const events = event ? [event] : Array.isArray(snapshot?.events) ? snapshot.events : []; markWorkbenchTraceEventsReceived({ traceId: id, events, transport: "sse", serverSentAt: realtimeEvent?.serverSentAt, eventCreatedAt: realtimeEvent?.eventCreatedAt, traceSeq: realtimeEvent?.traceSeq ?? realtimeEvent?.cursor?.traceSeq }); - applyTraceSnapshot(id, realtimeSnapshotToTraceSnapshot(id, snapshot ?? { traceId: id, status: event?.status, events }, events)); + const liveTerminal = liveKafkaEnvelope && workbenchLiveKafkaEventIsTerminal(event); + const eventSnapshot = snapshot ?? { traceId: id, status: liveKafkaEnvelope && !liveTerminal ? "running" : event?.status, events }; + if (liveKafkaEnvelope && event) applyLiveKafkaBusinessEvent(id, sessionId, event); + applyTraceSnapshot(id, realtimeSnapshotToTraceSnapshot(id, eventSnapshot, events, liveKafkaEnvelope ? { eventSource: "hwlab-kafka-sse" } : undefined), liveKafkaEnvelope); + } + + function applyLiveKafkaBusinessEvent(traceId: string, authoritySessionId: string | null, event: TraceEvent): void { + const eventSessionId = firstNonEmptyString(event.sessionId); + const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId ?? eventSessionId, true); + if (!ownerSessionId) return; + const previousMessage = (serverState.value.messagesBySessionId[ownerSessionId] ?? []).find((message) => messageMatchesTraceAuthority(message, traceId, authoritySessionId ?? eventSessionId, ownerSessionId, true)) ?? null; + const liveState = reduceWorkbenchLiveKafkaMessageState({ text: previousMessage?.text ?? "", status: previousMessage?.status ?? "running", terminal: false }, event); + const terminal = liveState.terminal; + const status = liveState.status; + const assistantText = liveState.text; + const updatedAt = firstNonEmptyString(event.createdAt, event.updatedAt) ?? new Date().toISOString(); + const messageId = workbenchLiveKafkaMessageId(traceId); + const message = previousMessage ?? makeMessage("agent", "", "running", { + id: messageId, + messageId, + traceId, + sessionId: ownerSessionId, + title: "Code Agent", + traceAutoLifecycle: "running", + createdAt: updatedAt, + updatedAt + }); + reduceServerState({ + type: "message.upsert", + sessionId: ownerSessionId, + message: { + ...message, + ...(assistantText ? { text: assistantText } : {}), + status: status as ChatMessage["status"], + traceAutoLifecycle: terminal ? "terminal" : "running", + ...(terminal && assistantText ? { finalResponse: { text: assistantText, status, traceId } } : {}), + updatedAt + } + }); + rememberTurnStatus(traceId, { + traceId, + sessionId: ownerSessionId, + status, + running: !terminal, + terminal, + finalResponse: terminal && assistantText ? { text: assistantText, status, traceId } : undefined, + updatedAt + } as AgentChatResultResponse); + const existing = sessions.value.find((session) => session.sessionId === ownerSessionId) ?? null; + if (existing) rememberSessionList(mergeSessionIntoList(sessions.value, { ...existing, status, lastTraceId: traceId, updatedAt: firstNonEmptyString(event.createdAt, event.updatedAt) ?? new Date().toISOString() })); + if (terminal && currentRequest.value?.traceId === traceId) { + chatPending.value = false; + currentRequest.value = null; + } } function applyRealtimeTurnSnapshot(turn: Record): void { @@ -1414,6 +1510,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { } function publishWorkbenchProjectionSignal(sessionId: string | null | undefined, traceId: string | null | undefined, reason: string): void { + if (realtimeCapabilities.liveKafkaSse || !realtimeCapabilities.projectionRealtime) return; if (typeof window === "undefined") return; const id = normalizeWorkbenchSessionId(sessionId); if (!id) return; @@ -1423,6 +1520,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { } function handleWorkbenchProjectionSignal(value: unknown): void { + if (realtimeCapabilities.liveKafkaSse || !realtimeCapabilities.projectionRealtime) return; const record = recordValue(value); if (!record || record.sourceId === workbenchProjectionSignalSourceId) return; if (firstNonEmptyString(record.type) !== "session-projection") return; @@ -1432,16 +1530,18 @@ export const useWorkbenchStore = defineStore("workbench", () => { void refreshCrossTabSessionFromSyncReplay(sessionId, `cross-tab-session-projection:${firstNonEmptyString(record.reason, traceId, "session") ?? "session"}`); } - function applyTraceSnapshot(traceId: string, snapshot: TraceSnapshot): void { + function applyTraceSnapshot(traceId: string, snapshot: TraceSnapshot, canonicalSession = false): void { const trace = snapshotToRunnerTrace({ ...snapshot, traceStatus: firstNonEmptyString(snapshot.traceStatus, snapshot.status) ?? undefined }); - const authoritySessionId = traceResultSessionId(trace) ?? traceResultSessionId(snapshot); - const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId); + const authoritySessionId = canonicalSession + ? normalizeWorkbenchSessionId(firstNonEmptyString(trace.sessionId, snapshot.sessionId)) + : traceResultSessionId(trace) ?? traceResultSessionId(snapshot); + const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId, canonicalSession); if (!ownerSessionId) return; updateSessionMessages(ownerSessionId, (source) => source.map((message) => { - if (!messageMatchesTraceAuthority(message, traceId, authoritySessionId, ownerSessionId)) return message; + if (!messageMatchesTraceAuthority(message, traceId, authoritySessionId, ownerSessionId, canonicalSession)) return message; const mergedRunnerTrace = mergeRunnerTrace(message.runnerTrace, trace); const traceStatus = normalizedStatusText(trace.status ?? snapshot.status) ?? null; const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(traceStatus, null); diff --git a/web/hwlab-cloud-web/src/types/global.d.ts b/web/hwlab-cloud-web/src/types/global.d.ts index 1b38ba95..8bbded04 100644 --- a/web/hwlab-cloud-web/src/types/global.d.ts +++ b/web/hwlab-cloud-web/src/types/global.d.ts @@ -8,6 +8,10 @@ declare global { label?: string; }; workbench?: { + realtimeFeatures?: { + liveKafkaSse?: boolean; + projectionRealtime?: boolean; + }; traceTimeline?: { autoExpandRunning?: boolean; autoCollapseTerminal?: boolean; diff --git a/web/hwlab-cloud-web/src/utils/workbench-realtime-runtime.ts b/web/hwlab-cloud-web/src/utils/workbench-realtime-runtime.ts index a0ec201c..b1eddb6a 100644 --- a/web/hwlab-cloud-web/src/utils/workbench-realtime-runtime.ts +++ b/web/hwlab-cloud-web/src/utils/workbench-realtime-runtime.ts @@ -1,4 +1,4 @@ // SPEC: PJ2026-0106050514 Workbench实时运行面 draft-2026-06-30-p0-1297-spec-first. // Responsibility: Production Workbench realtime runtime entry point for SSE transport, event coalescing, and scoped cursor ownership. -export { createWorkbenchStreamTransportRuntime, WorkbenchStreamTransportRuntime, type WorkbenchStreamTransportRecovery, type WorkbenchStreamTransportRestartInput, type WorkbenchStreamTransportRestartResult, type WorkbenchStreamTransportState, type WorkbenchRealtimeEvent } from "@/utils/workbench-stream-transport"; +export { createWorkbenchStreamTransportRuntime, WorkbenchStreamTransportRuntime, workbenchRealtimeTraceIdForCapabilities, type WorkbenchStreamTransportRecovery, type WorkbenchStreamTransportRestartInput, type WorkbenchStreamTransportRestartResult, type WorkbenchStreamTransportState, type WorkbenchRealtimeEvent } from "@/utils/workbench-stream-transport"; diff --git a/web/hwlab-cloud-web/src/utils/workbench-stream-transport.ts b/web/hwlab-cloud-web/src/utils/workbench-stream-transport.ts index 8d11d039..7a558a1e 100644 --- a/web/hwlab-cloud-web/src/utils/workbench-stream-transport.ts +++ b/web/hwlab-cloud-web/src/utils/workbench-stream-transport.ts @@ -8,11 +8,13 @@ // - OpenCode stream.transport.ts:504-530 reduce output/trace. import { connectWorkbenchEvents, type WorkbenchEventStream, type WorkbenchRealtimeEvent } from "@/api/workbench-events"; +import type { WorkbenchRealtimeCapabilities } from "@/config/runtime"; import { workbenchRealtimeScopeKey } from "@/utils/workbench-key"; export type { WorkbenchRealtimeEvent }; export interface WorkbenchStreamTransportRestartInput { + realtimeCapabilities: WorkbenchRealtimeCapabilities; sessionId?: string | null; traceId?: string | null; afterSeq?: number | null; @@ -74,6 +76,22 @@ interface WorkbenchTransportCursor { traceSeq: number | null; } +export function workbenchRealtimeTraceIdForCapabilities( + capabilities: WorkbenchRealtimeCapabilities, + ...candidates: Array +): string | null { + if (capabilities.liveKafkaSse) return null; + for (const candidate of candidates) { + const traceId = typeof candidate === "string" ? candidate.trim() : ""; + if (traceId) return traceId; + } + return null; +} + +export function workbenchRealtimeTransportEnabled(capabilities: WorkbenchRealtimeCapabilities): boolean { + return capabilities.liveKafkaSse || capabilities.projectionRealtime; +} + export class WorkbenchStreamTransportRuntime { private stream: WorkbenchEventStream | null = null; private key = ""; @@ -88,12 +106,14 @@ export class WorkbenchStreamTransportRuntime { this.stop(); this.key = key; this.armWait(key); + if (!workbenchRealtimeTransportEnabled(input.realtimeCapabilities)) return { key, changed: true, streamStarted: false }; if (!input.sessionId && !input.traceId) return { key, changed: true, streamStarted: false }; this.emitState(input, "connecting", null); this.stream = connectWorkbenchEvents({ + realtimeCapabilities: input.realtimeCapabilities, sessionId: input.sessionId ?? null, traceId: input.traceId ?? null, - afterSeq: input.afterSeq ?? this.cursorByKey.get(key)?.outboxSeq ?? null, + afterSeq: !input.realtimeCapabilities.liveKafkaSse && input.realtimeCapabilities.projectionRealtime ? input.afterSeq ?? this.cursorByKey.get(key)?.outboxSeq ?? null : null, flushMaxItemsPerChunk: input.flushMaxItemsPerChunk, flushMaxChunkMs: input.flushMaxChunkMs, flushYieldMs: input.flushYieldMs, @@ -109,7 +129,7 @@ export class WorkbenchStreamTransportRuntime { this.requestRecovery(input, event.type || "eventsource-error"); }, onEvent: (event, eventName) => { - this.rememberCursor(key, event); + if (!input.realtimeCapabilities.liveKafkaSse && input.realtimeCapabilities.projectionRealtime) this.rememberCursor(key, event); this.markLive(key); this.emitState(input, "event", eventName); input.onEvent(event, eventName); @@ -180,7 +200,7 @@ export class WorkbenchStreamTransportRuntime { const sessionId = input.sessionId ?? null; const traceId = input.traceId ?? null; const cursor = this.currentCursor(key); - const actions: WorkbenchStreamTransportRecoveryAction[] = sessionId || traceId ? ["sync-replay"] : []; + const actions: WorkbenchStreamTransportRecoveryAction[] = !input.realtimeCapabilities.liveKafkaSse && input.realtimeCapabilities.projectionRealtime && (sessionId || traceId) ? ["sync-replay"] : []; const diagnostic = this.diagnosticEnvelope("workbench_sse_recovery", reason, key, actions); input.onRecovery?.({ key, sessionId, traceId, tick: this.wait?.tick ?? this.tick, reason, actions, outboxSeq: cursor.outboxSeq, traceSeq: cursor.traceSeq, diagnostic }); } diff --git a/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue b/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue index b6ae82fb..68fa1a13 100644 --- a/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue +++ b/web/hwlab-cloud-web/src/views/workbench/WorkbenchDebugView.vue @@ -3,26 +3,57 @@ // Responsibility: Workbench debug route for backend-driven fake SSE single-step Trace card inspection. import { computed, onBeforeUnmount, onMounted, ref } from "vue"; -import { connectWorkbenchDebugFakeSse, workbenchDebugAPI, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream } from "@/api"; +import { connectWorkbenchDebugFakeSse, connectWorkbenchKafkaSseDebug, workbenchDebugAPI, workbenchKafkaSseDebugCorrelationIds, type WorkbenchDebugFakeSseQueue, type WorkbenchDebugFakeSseSequence, type WorkbenchDebugFakeSseStream, type WorkbenchKafkaSseDebugEvent, type WorkbenchKafkaSseDebugStream, type WorkbenchKafkaSseDebugStreamName } from "@/api"; import StatusBadge from "@/components/common/StatusBadge.vue"; import WorkbenchMessageCard from "@/components/workbench/WorkbenchMessageCard.vue"; import type { WorkbenchRealtimeEvent } from "@/api/workbench-events"; import { applyWorkbenchDebugFakeSseEvent, createWorkbenchDebugFakeSseState } from "@/stores/workbench-debug-fake-sse"; const queueId = "trace-card"; +const activeTab = ref<"fake" | "kafka">("fake"); const selectedSequenceId = ref("trace-card-basic"); const sequences = ref([]); const queue = ref(null); const projection = ref(createWorkbenchDebugFakeSseState()); const streamStatus = ref<"connecting" | "open" | "error" | "closed">("connecting"); +const kafkaStatus = ref<"idle" | "connecting" | "open" | "error" | "closed">("idle"); +const kafkaStreamName = ref("hwlab"); +const kafkaFromBeginning = ref(false); +const kafkaFilters = ref({ traceId: "", sessionId: "", runId: "", commandId: "" }); +const kafkaEvents = ref([]); +const kafkaConnection = ref(null); +const kafkaTraceStatuses = ref>(createKafkaStatusMap()); +const kafkaTraceEvents = ref>(createKafkaEventMap()); +const kafkaTraceConnections = ref>(createKafkaConnectionMap()); const busy = ref(false); const error = ref(null); const appendDraft = ref(defaultAppendDraft()); let stream: WorkbenchDebugFakeSseStream | null = null; +let kafkaStream: WorkbenchKafkaSseDebugStream | null = null; +const kafkaTraceStreams: Partial> = {}; + +const KAFKA_TRACE_STREAMS: Array<{ name: WorkbenchKafkaSseDebugStreamName; label: string; topic: string }> = [ + { name: "stdio", label: "codex stdio", topic: "codex-stdio.raw.v1" }, + { name: "agentrun", label: "AgentRun event", topic: "agentrun.event.v1" }, + { name: "hwlab", label: "HWLAB event", topic: "hwlab.event.v1" } +]; + +type KafkaStreamStatus = "idle" | "connecting" | "open" | "error" | "closed"; + +interface KafkaRawRow { + id: string; + eventName: string; + receivedAt: string; + payload: WorkbenchKafkaSseDebugEvent; +} + const message = computed(() => projection.value.message); const logs = computed(() => projection.value.logs); const queueSummary = computed(() => queue.value ? `${queue.value.cursor}/${queue.value.eventCount}` : "-"); const streamBadgeStatus = computed(() => streamStatus.value === "open" ? "completed" : streamStatus.value === "error" ? "failed" : "running"); +const kafkaBadgeStatus = computed(() => kafkaStatus.value === "open" ? "completed" : kafkaStatus.value === "error" ? "failed" : "running"); +const kafkaTraceEventCount = computed(() => Object.values(kafkaTraceEvents.value).reduce((total, rows) => total + rows.length, 0)); +const kafkaTraceConnectedCount = computed(() => Object.values(kafkaTraceStatuses.value).filter((status) => status === "open").length); const canStep = computed(() => !busy.value && (queue.value?.remaining ?? 0) > 0); onMounted(async () => { @@ -34,7 +65,11 @@ onMounted(async () => { onBeforeUnmount(() => { stream?.close(); stream = null; + kafkaStream?.close(); + kafkaStream = null; + closeKafkaTraceStreams(); streamStatus.value = "closed"; + kafkaStatus.value = "closed"; }); async function refreshDescription(): Promise { @@ -64,6 +99,108 @@ function openStream(): void { if (!stream) streamStatus.value = "error"; } +function openKafkaStream(): void { + kafkaStream?.close(); + kafkaStatus.value = "connecting"; + kafkaConnection.value = null; + kafkaStream = connectWorkbenchKafkaSseDebug({ + stream: kafkaStreamName.value, + fromBeginning: kafkaFromBeginning.value, + traceId: cleanFilter(kafkaFilters.value.traceId), + sessionId: cleanFilter(kafkaFilters.value.sessionId), + runId: cleanFilter(kafkaFilters.value.runId), + commandId: cleanFilter(kafkaFilters.value.commandId), + onOpen: () => { kafkaStatus.value = "open"; }, + onError: () => { kafkaStatus.value = "error"; }, + onEvent: (event, eventName) => { + if (eventName === "hwlab.kafka.connected") { + kafkaConnection.value = event; + return; + } + if (eventName === "hwlab.kafka.error") { + kafkaStatus.value = "error"; + error.value = event.error?.message || "Kafka SSE 事件流错误"; + } + kafkaEvents.value = [{ id: kafkaRowId(event, eventName), eventName, receivedAt: new Date().toISOString(), payload: event }, ...kafkaEvents.value].slice(0, 120); + } + }); + if (!kafkaStream) kafkaStatus.value = "error"; +} + +function closeKafkaStream(): void { + kafkaStream?.close(); + kafkaStream = null; + kafkaStatus.value = "closed"; +} + +function clearKafkaEvents(): void { + kafkaEvents.value = []; +} + +function openKafkaTraceStreams(): void { + closeKafkaTraceStreams(); + kafkaTraceEvents.value = createKafkaEventMap(); + kafkaTraceConnections.value = createKafkaConnectionMap(); + kafkaTraceStatuses.value = createKafkaStatusMap("connecting"); + for (const streamName of KAFKA_TRACE_STREAMS.map((entry) => entry.name)) { + const opened = connectWorkbenchKafkaSseDebug({ + stream: streamName, + fromBeginning: true, + traceId: cleanFilter(kafkaFilters.value.traceId), + sessionId: cleanFilter(kafkaFilters.value.sessionId), + runId: cleanFilter(kafkaFilters.value.runId), + commandId: cleanFilter(kafkaFilters.value.commandId), + onOpen: () => setKafkaTraceStatus(streamName, "open"), + onError: () => setKafkaTraceStatus(streamName, "error"), + onEvent: (event, eventName) => { + if (eventName === "hwlab.kafka.connected") { + kafkaTraceConnections.value = { ...kafkaTraceConnections.value, [streamName]: event }; + return; + } + if (eventName === "hwlab.kafka.error") { + setKafkaTraceStatus(streamName, "error"); + error.value = event.error?.message || "Kafka SSE 三流订阅错误"; + } + pushKafkaTraceEvent(streamName, { id: kafkaRowId(event, eventName), eventName, receivedAt: new Date().toISOString(), payload: event }); + } + }); + if (opened) kafkaTraceStreams[streamName] = opened; + else setKafkaTraceStatus(streamName, "error"); + } +} + +function closeKafkaTraceStreams(): void { + for (const streamName of KAFKA_TRACE_STREAMS.map((entry) => entry.name)) { + kafkaTraceStreams[streamName]?.close(); + delete kafkaTraceStreams[streamName]; + } + kafkaTraceStatuses.value = Object.fromEntries(KAFKA_TRACE_STREAMS.map((entry) => [entry.name, kafkaTraceStatuses.value[entry.name] === "idle" ? "idle" : "closed"])) as Record; +} + +function clearKafkaTraceEvents(): void { + kafkaTraceEvents.value = createKafkaEventMap(); +} + +function setKafkaTraceStatus(streamName: WorkbenchKafkaSseDebugStreamName, status: KafkaStreamStatus): void { + kafkaTraceStatuses.value = { ...kafkaTraceStatuses.value, [streamName]: status }; +} + +function pushKafkaTraceEvent(streamName: WorkbenchKafkaSseDebugStreamName, row: KafkaRawRow): void { + kafkaTraceEvents.value = { ...kafkaTraceEvents.value, [streamName]: [row, ...kafkaTraceEvents.value[streamName]].slice(0, 80) }; +} + +function createKafkaStatusMap(status: KafkaStreamStatus = "idle"): Record { + return { stdio: status, agentrun: status, hwlab: status }; +} + +function createKafkaEventMap(): Record { + return { stdio: [], agentrun: [], hwlab: [] }; +} + +function createKafkaConnectionMap(): Record { + return { stdio: null, agentrun: null, hwlab: null }; +} + async function resetQueue(): Promise { busy.value = true; error.value = null; @@ -122,6 +259,36 @@ function parseAppendDraft(raw: string): { ok: true; events: WorkbenchRealtimeEve } } +function cleanFilter(value: string): string | undefined { + const text = value.trim(); + return text ? text : undefined; +} + +function kafkaRowId(event: WorkbenchKafkaSseDebugEvent, eventName: string): string { + const topic = event.topic || "topic"; + const partition = event.partition ?? "p"; + const offset = event.offset || Date.now(); + return `${eventName}:${topic}:${partition}:${offset}:${Math.random().toString(16).slice(2)}`; +} + +function kafkaEventSummary(event: WorkbenchKafkaSseDebugEvent): string { + const value = recordValue(event.value); + const nestedEvent = recordValue(value.event); + return String(value.eventType || value.type || nestedEvent.type || event.error?.code || "event"); +} + +function kafkaEventIds(event: WorkbenchKafkaSseDebugEvent): string { + return workbenchKafkaSseDebugCorrelationIds(event).join(" · ") || "-"; +} + +function kafkaEventJson(event: WorkbenchKafkaSseDebugEvent): string { + return JSON.stringify(event.value ?? event, null, 2); +} + +function recordValue(value: unknown): Record { + return value && typeof value === "object" && !Array.isArray(value) ? value as Record : {}; +} + function defaultAppendDraft(): string { return JSON.stringify({ type: "trace.event", @@ -150,12 +317,18 @@ function defaultAppendDraft(): string {

实时调试台

- - Queue {{ queueSummary }} + + + {{ activeTab === 'fake' ? `Queue ${queueSummary}` : `${kafkaTraceConnectedCount}/3 streams · ${kafkaTraceEventCount + kafkaEvents.length} events` }}
-
+ + +
+ + + + + + + + + + + +
+

{{ error }}

-
+
等待第一条 fake SSE message.snapshot
@@ -200,6 +410,64 @@ function defaultAppendDraft(): string {
+
+
+
+ Trace Streams + {{ kafkaTraceConnectedCount }}/3 open · {{ kafkaTraceEventCount }} events +
+
+
+
+
+ {{ streamEntry.label }} + {{ streamEntry.topic }} + consumer ready · {{ kafkaTraceConnections[streamEntry.name]?.groupId || '-' }} +
+ +
+
等待匹配 traceId 的 {{ streamEntry.topic }} 事件。
+
    +
  1. +
    + {{ row.eventName }} + {{ kafkaEventSummary(row.payload) }} + #{{ row.payload.offset || '-' }} +
    +
    + {{ kafkaEventIds(row.payload) }} + +
    +
    {{ kafkaEventJson(row.payload) }}
    +
  2. +
+
+
+
+ +
+
+ Raw SSE Events + consumer ready · {{ kafkaConnection.groupId || '-' }} · {{ kafkaEvents.length }} events + {{ kafkaEvents.length }} events +
+
等待后端从 Kafka 订阅并透传 hwlab event。
+
    +
  1. +
    + {{ row.eventName }} + {{ kafkaEventSummary(row.payload) }} + {{ row.payload.topic || '-' }} #{{ row.payload.offset || '-' }} +
    +
    + {{ kafkaEventIds(row.payload) }} + +
    +
    {{ kafkaEventJson(row.payload) }}
    +
  2. +
+
+
@@ -247,6 +515,27 @@ function defaultAppendDraft(): string { font-weight: 750; } +.workbench-debug-tabs { + display: flex; + gap: 8px; + border-bottom: 1px solid #d8e1eb; +} + +.debug-tab { + border: 0; + border-bottom: 2px solid transparent; + background: transparent; + padding: 10px 12px; + color: #475569; + font-size: 13px; + font-weight: 800; +} + +.debug-tab[data-active="true"] { + border-bottom-color: #0e7490; + color: #0f172a; +} + .workbench-debug-toolbar { flex-wrap: wrap; gap: 10px; @@ -261,6 +550,24 @@ function defaultAppendDraft(): string { font-weight: 750; } +.workbench-debug-toolbar .compact-label { + width: min(220px, 100%); +} + +.kafka-toolbar { + align-items: end; +} + +.debug-check { + display: flex !important; + width: auto !important; + grid-auto-flow: column; + align-items: center; + gap: 7px !important; + padding-bottom: 8px; + color: #334155 !important; +} + .workbench-debug-grid { display: grid; min-height: 0; @@ -383,11 +690,173 @@ function defaultAppendDraft(): string { white-space: nowrap; } +.kafka-debug-layout { + display: grid; + min-height: 0; + grid-template-rows: minmax(280px, 0.9fr) minmax(220px, 0.7fr); + gap: 12px; +} + +.kafka-trace-panel { + display: grid; + min-height: 0; + grid-template-rows: auto minmax(0, 1fr); + gap: 10px; +} + +.kafka-stream-grid { + display: grid; + min-height: 0; + grid-template-columns: repeat(3, minmax(0, 1fr)); + gap: 10px; +} + +.kafka-stream-column { + display: grid; + min-width: 0; + min-height: 0; + grid-template-rows: auto minmax(0, 1fr); + gap: 8px; + border: 1px solid #d8e1eb; + border-radius: 8px; + background: white; + padding: 10px; +} + +.kafka-stream-head { + display: flex; + align-items: start; + justify-content: space-between; + gap: 10px; +} + +.kafka-stream-head div { + display: grid; + min-width: 0; + gap: 3px; +} + +.kafka-stream-head strong { + color: #0f172a; + font-size: 13px; +} + +.kafka-stream-head code { + overflow: hidden; + color: #0f766e; + font-size: 11px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.kafka-consumer-ready { + overflow: hidden; + color: #047857; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 10px; + text-overflow: ellipsis; + white-space: nowrap; +} + +.kafka-debug-panel { + display: grid; + min-height: 0; + grid-template-rows: auto minmax(0, 1fr); + gap: 10px; + border: 1px solid #d8e1eb; + border-radius: 8px; + background: white; + padding: 10px; +} + +.kafka-event-list { + display: grid; + align-content: start; + gap: 8px; + min-height: 0; + margin: 0; + overflow: auto; + padding: 0; + list-style: none; +} + +.kafka-event-list li { + display: grid; + gap: 6px; + border: 1px solid #d8e1eb; + border-left: 3px solid #0e7490; + border-radius: 8px; + background: #f8fafc; + padding: 9px; +} + +.kafka-event-head, +.kafka-event-meta { + display: grid; + grid-template-columns: minmax(150px, 0.5fr) minmax(160px, 0.7fr) minmax(0, 1fr); + gap: 10px; + align-items: center; + color: #334155; + font-size: 12px; +} + +.kafka-event-head code { + color: #0f766e; +} + +.kafka-event-head span, +.kafka-event-meta { + color: #64748b; +} + +.kafka-event-meta { + grid-template-columns: minmax(0, 1fr) auto; +} + +.kafka-event-list pre { + max-height: 220px; + margin: 0; + overflow: auto; + border-radius: 7px; + background: #0f172a; + padding: 9px; + color: #d1fae5; + font-family: ui-monospace, SFMono-Regular, Menlo, Monaco, Consolas, monospace; + font-size: 11px; + line-height: 1.45; + white-space: pre-wrap; + overflow-wrap: anywhere; +} + +.compact-kafka-event-list .kafka-event-head { + grid-template-columns: minmax(92px, 0.6fr) minmax(110px, 0.8fr) minmax(44px, 0.4fr); +} + +.compact-kafka-event-list .kafka-event-meta { + grid-template-columns: minmax(0, 1fr); +} + +.compact-kafka-event-list .kafka-event-meta time { + display: none; +} + +.compact-kafka-event-list pre { + max-height: 160px; +} + @media (max-width: 980px) { .workbench-debug-grid { grid-template-columns: minmax(0, 1fr); } + .kafka-debug-layout { + grid-template-rows: auto auto; + } + + .kafka-stream-grid { + grid-template-columns: minmax(0, 1fr); + } + .debug-side { grid-template-rows: 280px 220px; }