Merge pull request #2540 from pikasTech/fix/2538-fixed-transactional-projection
Pipelines as Code CI / hwlab-nc01-v03-ci-poll- Success

fix(workbench): 固定 transactional 投影与 canonical duration
This commit is contained in:
Lyon
2026-07-14 14:25:17 +08:00
committed by GitHub
41 changed files with 465 additions and 1957 deletions
+44
View File
@@ -0,0 +1,44 @@
{
"$schema": "https://json-schema.org/draft/2020-12/schema",
"$id": "https://github.com/pikasTech/HWLAB/config/feature-config.schema.json",
"title": "HWLAB product feature configuration",
"type": "object",
"additionalProperties": false,
"properties": {
"tracePanel": {
"x-unidesk-feature": "workbench.trace-panel",
"type": "object",
"additionalProperties": false,
"properties": {
"autoExpandRunning": { "type": "boolean" },
"autoCollapseTerminal": { "type": "boolean" }
}
},
"rawEvents": {
"x-unidesk-feature": "workbench.raw-events",
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": { "type": "boolean" },
"maxEntries": { "type": "integer", "minimum": 1 },
"maxRetainedBytes": { "type": "integer", "minimum": 1 }
}
},
"debugReplay": {
"x-unidesk-feature": "workbench.debug-replay",
"type": "object",
"additionalProperties": false,
"properties": {
"enabled": { "type": "boolean" }
}
},
"views": {
"x-unidesk-feature": "workbench.views",
"type": "object",
"additionalProperties": false,
"properties": {
"traceExplorerUrlTemplate": { "type": "string", "minLength": 1 }
}
}
}
}
-8
View File
@@ -621,9 +621,6 @@ lanes:
HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: Asia/Shanghai
HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: zh-CN
HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: 北京时间
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true"
HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "true"
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED: "true"
HWLAB_CLOUD_WEB_OPENCODE_UPSTREAM_URL: http://opencode-server.hwlab-v03.svc.cluster.local:4096
HWLAB_CLOUD_WEB_OPENCODE_USERNAME: secretRef:hwlab-opencode-server-auth/username
@@ -850,11 +847,6 @@ lanes:
HWLAB_CODE_AGENT_AGENTRUN_REPO_URL: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-HWLAB.git
HWLAB_KAFKA_ENABLED: "true"
HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true"
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true"
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true"
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false"
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false"
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED: "true"
HWLAB_KAFKA_BOOTSTRAP_SERVERS: platform-infra-kafka-kafka-bootstrap.platform-infra.svc.cluster.local:9092
HWLAB_KAFKA_STDIO_TOPIC: codex-stdio.raw.v1
+31 -118
View File
@@ -6,7 +6,7 @@ import { test } from "bun:test";
import { createCloudApiBunServer } from "./bun-server.ts";
import { buildCloudApiReadiness } from "./health-contract.ts";
import { decodeCanonicalAgentRunKafkaMessage, kafkaEventBridgeConfig, liveKafkaOtelSpanAttributes, projectAgentRunKafkaEventToHwlabEvent, publishAgentRunKafkaMessageLive, relayHwlabKafkaOutboxOnce, shouldEmitLiveKafkaOtelSpan, startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts";
import { decodeCanonicalAgentRunKafkaMessage, kafkaEventBridgeConfig, projectAgentRunKafkaEventToHwlabEvent, relayHwlabKafkaOutboxOnce, startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts";
const PROJECTOR_ENV = Object.freeze({
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false",
@@ -31,23 +31,6 @@ const PROJECTOR_ENV = Object.freeze({
HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS: "1000"
});
const LIVE_ENV = Object.freeze({
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false",
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false",
HWLAB_KAFKA_ENABLED: "true",
HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true",
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092",
HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC: "agentrun.event.v1",
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1",
HWLAB_KAFKA_CLIENT_ID: "hwlab-live-test",
HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-live-bridge-fixed",
HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID: "hwlab-live-fanout-fixed"
});
test("projects AgentRun assistant_message Kafka event into HWLAB trace event", () => {
const projected = projectAgentRunKafkaEventToHwlabEvent({
schema: "agentrun.event.v1",
@@ -322,110 +305,40 @@ test("projects AgentRun terminal_status Kafka event into terminal HWLAB event",
assert.equal(projected.event.message, "AgentRun completed");
});
test("live Kafka publisher has no PG inbox, facts, checkpoint, or outbox dependency", async () => {
const sent = [];
const runtimeStore = new Proxy({}, { get() { throw new Error("unexpected DB access"); } });
void runtimeStore;
const result = await publishAgentRunKafkaMessageLive({ topic: "agentrun.event.v1", partition: 3, message: canonicalMessage({ offset: "71" }) }, {
producer: { async send(input) { sent.push(input); } },
config: { clientId: "hwlab-live-test", hwlabTopic: "hwlab.event.v1" },
logger: null
test("Workbench bridge ignores obsolete architecture env and fixes transactional authority", () => {
const config = kafkaEventBridgeConfig({
...PROJECTOR_ENV,
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "true",
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
});
assert.equal(result.published, true);
assert.equal(sent.length, 1);
assert.equal(JSON.parse(sent[0].messages[0].value).sourceEvent.offset, "71");
assert.equal(config?.projectorGroupId, "hwlab-projector-test-v1");
assert.deepEqual(config?.relay, {
intervalMs: 60000,
batchSize: 1,
leaseMs: 30000,
sendTimeoutMs: 10000,
retryBackoffMs: 1000
});
assert.deepEqual(Object.keys(config ?? {}).sort(), ["agentRunTopic", "brokers", "clientId", "hwlabTopic", "projectorGroupId", "projectorHeartbeatIntervalMs", "relay"]);
});
test("live Kafka OTel sampling is deterministic and bounded without payload attributes", () => {
const envelope = (agentRunEventType, sourceSeq, event = {}) => ({
schema: "hwlab.event.v1",
eventId: `hwlab:evt_otel_${sourceSeq}`,
sourceEventId: `evt_otel_${sourceSeq}`,
traceId: "trc_live_otel",
hwlabSessionId: "ses_live_otel",
runId: "run_live_otel",
commandId: "cmd_live_otel",
context: { agentRunEventType, sourceSeq, valuesRedacted: true },
event: { type: "backend", eventType: "backend", terminal: false, message: "must-not-leak", payload: { password: "must-not-leak" }, ...event }
});
test("Workbench bridge does not require the six obsolete architecture env", () => {
const env = { ...PROJECTOR_ENV };
for (const name of [
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
]) delete env[name];
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("assistant_message", 1, { type: "assistant" })), true);
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("tool_call", 2, { type: "tool" })), true);
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("terminal_status", 3, { type: "result", terminal: true })), true);
const decisions = Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" })));
assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => (index + 1) % 32 === 0));
assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" }))));
const attributes = liveKafkaOtelSpanAttributes(envelope("assistant_message", 1, { type: "assistant" }), {
topic: "hwlab.event.v1",
partition: 2,
offset: "14"
});
assert.deepEqual(attributes, {
businessTraceId: "trc_live_otel",
hwlabSessionId: "ses_live_otel",
runId: "run_live_otel",
commandId: "cmd_live_otel",
topic: "hwlab.event.v1",
partition: 2,
offset: "14",
sourceTopic: null,
sourcePartition: null,
sourceOffset: null,
eventType: "assistant_message",
terminal: false,
valuesRedacted: true
});
assert.equal(JSON.stringify(attributes).includes("must-not-leak"), false);
});
test("composable Kafka capabilities reject consumer group collisions", () => {
assert.throws(
() => kafkaEventBridgeConfig({
...LIVE_ENV,
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true",
HWLAB_KAFKA_PROJECTOR_GROUP_ID: LIVE_ENV.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID,
HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "1000"
}),
(error) => error?.code === "hwlab_kafka_consumer_groups_not_distinct"
);
});
test("Workbench bridge fails closed unless transactional projection authority is complete", () => {
const invalidAuthorities = [
{ name: "direct/live-only", env: LIVE_ENV },
{
name: "projection realtime only",
env: {
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false",
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false",
HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false",
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true"
}
},
{ name: "projector without outbox relay", env: { ...PROJECTOR_ENV, HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false" } },
{ name: "projector without projection realtime", env: { ...PROJECTOR_ENV, HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" } },
{
name: "direct and transactional dual authority",
env: {
...PROJECTOR_ENV,
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-direct-authority-test",
HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID: "hwlab-live-authority-test"
}
}
];
for (const authority of invalidAuthorities) {
assert.throws(
() => startHwlabKafkaEventBridge({ env: authority.env, runtimeStore: {} }),
(error: any) => error?.code === "hwlab_workbench_transactional_authority_required",
authority.name
);
}
assert.doesNotThrow(() => kafkaEventBridgeConfig(env));
});
test("projects AgentRun run-created envelope even when no source event is present", () => {
+38 -366
View File
@@ -4,12 +4,7 @@
import { createHash, randomUUID } from "node:crypto";
import { Kafka, logLevel } from "kafkajs";
import { emitCodeAgentOtelSpan } from "./otel-trace.ts";
import { buildWorkbenchProjectionEventFacts } from "./workbench-projection-writer.ts";
import {
WORKBENCH_REALTIME_CAPABILITY_ENVS,
workbenchRealtimeCapabilities
} from "./workbench-realtime-capabilities.ts";
const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
export const DEFAULT_AGENTRUN_EVENT_TOPIC = "agentrun.event.v1";
@@ -22,7 +17,6 @@ export const AGENTRUN_STDIO_RECONSTRUCTION_KIND = "stdio-derived partial reconst
const DEFAULT_CLIENT_ID = "hwlab-v03-cloud-api";
const DEFAULT_QUERY_TIMEOUT_MS = 5000;
const DEFAULT_QUERY_LIMIT = 50;
const LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS = 32;
const REQUIRED_KAFKA_ENV = Object.freeze([
"HWLAB_KAFKA_BOOTSTRAP_SERVERS",
"HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC",
@@ -31,35 +25,19 @@ const REQUIRED_KAFKA_ENV = Object.freeze([
]);
export function kafkaEventBridgeConfig(env = process.env) {
const featureEnvPresent = Object.values(WORKBENCH_REALTIME_CAPABILITY_ENVS).some((name) => stringValue(env[name]));
const legacyKafkaEnabled = truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED);
if (!featureEnvPresent && !legacyKafkaEnabled) return null;
const capabilities = workbenchRealtimeCapabilities(env);
if (capabilities.kafkaRefreshReplay && !capabilities.liveKafkaSse) {
throw contractError("hwlab_kafka_refresh_live_dependency_missing", "Kafka refresh replay requires the independently enabled live Kafka SSE capability for handoff.");
}
const kafkaEnabled = capabilities.directPublish || capabilities.liveKafkaSse || capabilities.kafkaRefreshReplay || capabilities.transactionalProjector || capabilities.projectionOutboxRelay;
const transactionalEnabled = capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime;
if (!kafkaEnabled && !transactionalEnabled) return null;
const required = [...REQUIRED_KAFKA_ENV];
if (!kafkaEnabled) required.length = 0;
if (capabilities.directPublish) required.push("HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID");
if (capabilities.liveKafkaSse) required.push("HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID");
if (capabilities.kafkaRefreshReplay) required.push(
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT"
);
if (capabilities.transactionalProjector) required.push("HWLAB_KAFKA_PROJECTOR_GROUP_ID", "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS");
if (capabilities.projectionOutboxRelay) required.push(
const kafkaConfigured = REQUIRED_KAFKA_ENV.every((name) => stringValue(env[name]));
if (!legacyKafkaEnabled && !kafkaConfigured) return null;
const required = [
...REQUIRED_KAFKA_ENV,
"HWLAB_KAFKA_PROJECTOR_GROUP_ID",
"HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS",
"HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS",
"HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE",
"HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS",
"HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS",
"HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS"
);
];
const missing = required.filter((name) => !stringValue(env[name]));
if (missing.length > 0) {
const error = new Error(`HWLAB Kafka capability configuration is incomplete: ${missing.join(", ")}`);
@@ -71,77 +49,37 @@ export function kafkaEventBridgeConfig(env = process.env) {
const agentRunTopic = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC);
const hwlabTopic = stringValue(env.HWLAB_KAFKA_EVENT_TOPIC);
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID);
const relay = capabilities.projectionOutboxRelay ? {
const relay = {
intervalMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS, "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS"),
batchSize: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE, "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE"),
leaseMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS, "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS"),
sendTimeoutMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS, "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS"),
retryBackoffMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS, "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS")
} : null;
const refreshReplay = capabilities.kafkaRefreshReplay ? {
groupIdPrefix: stringValue(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX),
timeoutMs: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS"),
scanLimit: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT"),
matchedEventLimit: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT"),
liveBufferLimit: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT")
} : null;
};
if (relay && relay.sendTimeoutMs + 5000 >= relay.leaseMs) {
const error = new Error("HWLAB Kafka relay lease must exceed producer send timeout by at least 5000ms.");
error.code = "hwlab_kafka_relay_lease_invalid";
throw error;
}
const enabledConsumerGroups = [
capabilities.directPublish ? stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID) : null,
capabilities.transactionalProjector ? stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID) : null,
capabilities.liveKafkaSse ? stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID) : null
].filter(Boolean);
if (new Set(enabledConsumerGroups).size !== enabledConsumerGroups.length) {
const error = new Error("HWLAB Kafka direct publisher, projector, and live SSE consumer groups must be distinct when enabled together.");
error.code = "hwlab_kafka_consumer_groups_not_distinct";
error.groupIds = enabledConsumerGroups;
throw error;
}
return {
capabilities,
brokers,
agentRunTopic,
hwlabTopic,
clientId,
directPublishGroupId: stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID),
projectorGroupId: stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID),
hwlabEventGroupId: stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID),
refreshReplay,
projectorHeartbeatIntervalMs: capabilities.transactionalProjector ? strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS") : null,
projectorHeartbeatIntervalMs: strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS"),
relay
};
}
export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString(), otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString() } = {}) {
const config = kafkaEventBridgeConfig(env);
if (!config) return { started: false, reason: "disabled_or_unconfigured", stop() {}, subscribeProjectionCommits() { return () => {}; }, valuesPrinted: false };
assertTransactionalWorkbenchRealtimeAuthority(config.capabilities);
const components = [];
if (config.capabilities.directPublish || config.capabilities.liveKafkaSse) {
components.push(startLiveHwlabKafkaEventBridge({ config, env, logger, kafkaFactory, otelSpanEmitter }));
}
if (config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay || config.capabilities.projectionRealtime) {
components.push(startTransactionalHwlabKafkaEventBridge({ config, logger, kafkaFactory, runtimeStore, now }));
}
return components.length === 1 ? components[0] : combineKafkaEventBridgeComponents(config, components);
}
function assertTransactionalWorkbenchRealtimeAuthority(capabilities = {}) {
const directAuthority = capabilities.directPublish || capabilities.liveKafkaSse || capabilities.kafkaRefreshReplay;
const completeProjectionAuthority = capabilities.transactionalProjector && capabilities.projectionOutboxRelay && capabilities.projectionRealtime;
if (!directAuthority && completeProjectionAuthority) return;
throw contractError(
"hwlab_workbench_transactional_authority_required",
"Workbench Kafka bridge requires transactionalProjector, projectionOutboxRelay, and projectionRealtime together, with directPublish, liveKafkaSse, and kafkaRefreshReplay disabled."
);
return startTransactionalHwlabKafkaEventBridge({ config, logger, kafkaFactory, runtimeStore, now });
}
function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString() } = {}) {
requireKafkaProjectorStore(runtimeStore, config.capabilities);
requireKafkaProjectorStore(runtimeStore);
let stopped = false;
let consumer = null;
@@ -160,16 +98,14 @@ function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kaf
let startupReady = false;
const ready = (async () => {
await runtimeStore.assertWorkbenchTransactionalRealtimeReady();
const kafka = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay ? kafkaFactory(config) : null;
if (config.capabilities.transactionalProjector) consumer = kafka.consumer({ groupId: config.projectorGroupId, allowAutoTopicCreation: false });
if (config.capabilities.projectionOutboxRelay) producer = kafka.producer({ allowAutoTopicCreation: false });
if (producer) await producer.connect();
if (consumer) await consumer.connect();
if (config.capabilities.projectionRealtime && typeof runtimeStore.subscribeWorkbenchProjectionCommits === "function") {
stopProjectionNotifications = await runtimeStore.subscribeWorkbenchProjectionCommits(publishProjectionCommit);
}
if (consumer) await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
if (consumer) await consumer.run({
const kafka = kafkaFactory(config);
consumer = kafka.consumer({ groupId: config.projectorGroupId, allowAutoTopicCreation: false });
producer = kafka.producer({ allowAutoTopicCreation: false });
await producer.connect();
await consumer.connect();
stopProjectionNotifications = await runtimeStore.subscribeWorkbenchProjectionCommits(publishProjectionCommit);
await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
await consumer.run({
autoCommit: false,
eachBatchAutoResolve: false,
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => {
@@ -196,7 +132,6 @@ function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kaf
}
});
const runRelay = async () => {
if (!config.capabilities.projectionOutboxRelay) return;
if (stopped || relayRunning) return;
relayRunning = true;
try {
@@ -205,18 +140,15 @@ function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kaf
relayRunning = false;
}
};
if (config.capabilities.projectionOutboxRelay) {
relayTimer = setInterval(() => { void runRelay().catch((error) => logWarn(logger, "hwlab-kafka-outbox-relay-failed", { message: errorMessage(error), valuesPrinted: false })); }, config.relay.intervalMs);
relayTimer.unref?.();
await runRelay();
}
relayTimer = setInterval(() => { void runRelay().catch((error) => logWarn(logger, "hwlab-kafka-outbox-relay-failed", { message: errorMessage(error), valuesPrinted: false })); }, config.relay.intervalMs);
relayTimer.unref?.();
await runRelay();
startupReady = true;
logInfo(logger, "hwlab-kafka-event-bridge-started", {
agentRunTopic: config.agentRunTopic,
hwlabTopic: config.hwlabTopic,
clientId: config.clientId,
projectorGroupId: config.projectorGroupId,
capabilities: config.capabilities,
valuesPrinted: false
});
})();
@@ -249,10 +181,8 @@ function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kaf
const transactionalReadiness = typeof runtimeStore.workbenchTransactionalRealtimeReadiness === "function"
? await runtimeStore.workbenchTransactionalRealtimeReadiness()
: { ready: true, valuesRedacted: true };
const projectorStatus = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay
? await runtimeStore.hwlabKafkaProjectorStatus()
: {};
return { status: transactionalReadiness.ready === false ? "blocked" : "ready", capabilities: config.capabilities, transactionalReadiness, ...projectorStatus, valuesRedacted: true };
const projectorStatus = await runtimeStore.hwlabKafkaProjectorStatus();
return { status: transactionalReadiness.ready === false ? "blocked" : "ready", transactionalReadiness, ...projectorStatus, valuesRedacted: true };
},
startupStatus() {
if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true };
@@ -267,269 +197,6 @@ function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kaf
};
}
function startLiveHwlabKafkaEventBridge({ config, env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
let stopped = false;
let bridgeConsumer = null;
let fanoutConsumer = null;
let producer = null;
let startupError = null;
let startupReady = false;
const subscribers = new Set();
const lagByPartition = new Map();
const publishLiveEnvelope = (envelope, transport) => {
for (const listener of [...subscribers]) {
try {
listener(envelope, transport);
} catch (error) {
logWarn(logger, "hwlab-live-kafka-subscriber-failed", { message: errorMessage(error), valuesPrinted: false });
}
}
};
const ready = (async () => {
const kafka = kafkaFactory(config);
if (config.capabilities.directPublish) {
bridgeConsumer = kafka.consumer({ groupId: config.directPublishGroupId, allowAutoTopicCreation: false });
producer = kafka.producer({ allowAutoTopicCreation: false });
}
if (config.capabilities.liveKafkaSse) fanoutConsumer = kafka.consumer({ groupId: config.hwlabEventGroupId, allowAutoTopicCreation: false });
if (producer) await producer.connect();
if (bridgeConsumer) await bridgeConsumer.connect();
if (fanoutConsumer) await fanoutConsumer.connect();
if (bridgeConsumer) await bridgeConsumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
if (fanoutConsumer) await fanoutConsumer.subscribe({ topic: config.hwlabTopic, fromBeginning: false });
if (fanoutConsumer) await fanoutConsumer.run({
eachBatchAutoResolve: false,
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => {
let lastOffset = null;
for (const message of batch.messages) {
if (stopped || !isRunning() || isStale()) break;
const envelope = parseJson(message?.value ? Buffer.from(message.value).toString("utf8") : "");
if (!envelope || envelope.schema !== "hwlab.event.v1") {
logWarn(logger, "hwlab-live-kafka-envelope-ignored", { reason: "schema-invalid", valuesPrinted: false });
} else {
const transport = { topic: batch.topic, partition: batch.partition, offset: message.offset };
emitLiveKafkaOtelSpan("hwlab.kafka.live.fanout_receive", envelope, transport, { env, otelSpanEmitter });
publishLiveEnvelope(envelope, transport);
}
resolveOffset(message.offset);
lastOffset = message.offset;
await heartbeat();
}
if (lastOffset !== null) lagByPartition.set(batch.partition, kafkaPartitionLag(batch.highWatermark, lastOffset));
}
});
if (bridgeConsumer) await bridgeConsumer.run({
eachMessage: async ({ topic, partition, message }) => {
if (stopped) return;
await publishAgentRunKafkaMessageLive({ topic, partition, message }, { producer, config, env, logger, otelSpanEmitter });
}
});
startupReady = true;
logInfo(logger, "hwlab-live-kafka-event-bridge-started", {
capabilities: config.capabilities,
agentRunTopic: config.agentRunTopic,
hwlabTopic: config.hwlabTopic,
clientId: config.clientId,
directPublishGroupId: config.directPublishGroupId,
hwlabEventGroupId: config.hwlabEventGroupId,
valuesPrinted: false
});
})();
void ready.catch((error) => {
startupError = error;
logWarn(logger, "hwlab-live-kafka-event-bridge-start-failed", {
message: errorMessage(error),
agentRunTopic: config.agentRunTopic,
hwlabTopic: config.hwlabTopic,
valuesPrinted: false
});
});
return {
started: true,
...config,
ready,
liveReady: ready,
async stop() {
stopped = true;
subscribers.clear();
await Promise.allSettled([bridgeConsumer?.stop?.(), fanoutConsumer?.stop?.()]);
await Promise.allSettled([bridgeConsumer?.disconnect?.(), fanoutConsumer?.disconnect?.(), producer?.disconnect?.()]);
},
async status() {
if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true };
return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, subscriberCount: subscribers.size, consumerLag: liveKafkaLagSummary(lagByPartition), lossPossible: true, valuesRedacted: true };
},
startupStatus() {
if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true };
return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, valuesRedacted: true };
},
subscribeProjectionCommits() { return () => {}; },
subscribeLiveHwlabEvents(listener) {
if (typeof listener !== "function") return () => {};
subscribers.add(listener);
return () => subscribers.delete(listener);
},
queryHwlabEventRetention(params = {}) {
return queryKafkaEventStream({ ...params, env, stream: "hwlab", topic: config.hwlabTopic, kafkaFactory });
},
liveSubscriberCount() { return subscribers.size; },
valuesPrinted: false
};
}
function kafkaPartitionLag(highWatermark, lastOffset) {
try {
const lag = BigInt(String(highWatermark ?? "0")) - (BigInt(String(lastOffset ?? "0")) + 1n);
return Number(lag > 0n ? lag : 0n);
} catch {
return null;
}
}
function liveKafkaLagSummary(lagByPartition) {
const partitions = [...lagByPartition.entries()].map(([partition, lag]) => ({ partition, lag })).sort((left, right) => left.partition - right.partition);
const known = partitions.map((item) => item.lag).filter((lag) => Number.isFinite(lag));
return {
known: known.length > 0,
total: known.length > 0 ? known.reduce((sum, lag) => sum + lag, 0) : null,
partitions,
valuesRedacted: true
};
}
function combineKafkaEventBridgeComponents(config, components) {
const ready = Promise.all(components.map((component) => component.ready));
const liveOwner = components.find((component) => typeof component.subscribeLiveHwlabEvents === "function");
const componentStartup = () => components.map((component) => component.startupStatus?.() ?? { status: "initializing" });
return {
started: true,
...config,
ready,
liveReady: liveOwner?.liveReady ?? liveOwner?.ready ?? ready,
async stop() { await Promise.allSettled(components.map((component) => component.stop())); },
async status() {
const statuses = await Promise.all(components.map((component) => component.status()));
const blocked = statuses.find((status) => status.status === "blocked");
const initializing = statuses.some((status) => status.status !== "ready");
return {
status: blocked ? "blocked" : initializing ? "initializing" : "ready",
capabilities: config.capabilities,
components: statuses,
errorCode: blocked?.errorCode ?? null,
message: blocked?.message ?? null,
valuesRedacted: true
};
},
startupStatus() {
const statuses = componentStartup();
const blocked = statuses.find((status) => status.status === "blocked");
return {
status: blocked ? "blocked" : statuses.every((status) => status.status === "ready") ? "ready" : "initializing",
capabilities: config.capabilities,
components: statuses,
errorCode: blocked?.errorCode ?? null,
message: blocked?.message ?? null,
valuesRedacted: true
};
},
subscribeProjectionCommits(listener) {
const stops = components.map((component) => component.subscribeProjectionCommits?.(listener)).filter((stop) => typeof stop === "function");
return () => stops.forEach((stop) => stop());
},
subscribeLiveHwlabEvents(listener) {
return liveOwner?.subscribeLiveHwlabEvents(listener) ?? (() => {});
},
queryHwlabEventRetention(params = {}) {
if (typeof liveOwner?.queryHwlabEventRetention !== "function") throw contractError("hwlab_kafka_refresh_query_unconfigured", "Kafka refresh retention query is not configured.");
return liveOwner.queryHwlabEventRetention(params);
},
liveSubscriberCount() {
const owner = components.find((component) => typeof component.liveSubscriberCount === "function");
return owner?.liveSubscriberCount() ?? 0;
},
valuesPrinted: false
};
}
export async function publishAgentRunKafkaMessageLive(kafkaMessage = {}, { producer, config, env = process.env, logger = console, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
const transport = kafkaTransport(kafkaMessage);
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
if (!decoded.ok) {
logWarn(logger, "hwlab-live-kafka-message-invalid", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false });
return { handled: true, invalid: true, published: false, valuesPrinted: false };
}
if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) {
return { handled: true, ignored: true, published: false, valuesPrinted: false };
}
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, {
source: config.clientId,
sourceTopic: transport.sourceTopic,
sourcePartition: transport.sourcePartition,
sourceOffset: transport.sourceOffset,
sourceKey: transport.sourceKey,
inputSha256: transport.inputSha256
});
const publishMetadata = await producer.send({ topic: config.hwlabTopic, messages: [buildHwlabKafkaProducerMessage(projected)] });
const publishedRecord = Array.isArray(publishMetadata) ? publishMetadata[0] : null;
emitLiveKafkaOtelSpan("hwlab.kafka.live.direct_publish", projected, {
topic: config.hwlabTopic,
partition: publishedRecord?.partition,
offset: publishedRecord?.baseOffset ?? publishedRecord?.offset,
sourceTopic: transport.sourceTopic,
sourcePartition: transport.sourcePartition,
sourceOffset: transport.sourceOffset
}, { env, otelSpanEmitter });
return { handled: true, published: true, envelope: projected, sessionId: projected.sessionId, traceId: projected.traceId, valuesPrinted: false };
}
export function shouldEmitLiveKafkaOtelSpan(envelope = {}) {
const event = objectValue(envelope.event);
const context = objectValue(envelope.context);
const eventType = firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType)?.toLowerCase();
if (event.terminal === true || eventType === "terminal" || eventType === "terminal_status") return true;
if (["assistant", "assistant_progress", "assistant_message", "tool", "tool_call"].includes(eventType)) return true;
if (eventType !== "command_output" && event.type !== "output") return true;
const sourceSeq = integerValue(context.sourceSeq ?? event.sourceSeq);
if (Number.isInteger(sourceSeq) && sourceSeq > 0) return sourceSeq % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0;
const identity = firstText(envelope.sourceEventId, envelope.eventId, envelope.traceId, envelope.hwlabSessionId, envelope.sessionId);
if (!identity) return false;
return createHash("sha256").update(identity).digest().readUInt32BE(0) % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0;
}
export function liveKafkaOtelSpanAttributes(envelope = {}, transport = {}) {
const event = objectValue(envelope.event);
const context = objectValue(envelope.context);
const sourceEvent = objectValue(envelope.sourceEvent);
return {
businessTraceId: firstText(envelope.traceId, event.traceId),
hwlabSessionId: firstText(envelope.hwlabSessionId, envelope.sessionId, event.sessionId),
runId: firstText(envelope.runId, context.runId, event.runId),
commandId: firstText(envelope.commandId, context.commandId, event.commandId),
topic: firstText(transport.topic, sourceEvent.topic),
partition: integerValue(transport.partition ?? sourceEvent.partition),
offset: firstText(transport.offset, sourceEvent.offset),
sourceTopic: firstText(transport.sourceTopic),
sourcePartition: integerValue(transport.sourcePartition),
sourceOffset: firstText(transport.sourceOffset),
eventType: firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType),
terminal: event.terminal === true,
valuesRedacted: true
};
}
export function emitLiveKafkaOtelSpan(name, envelope = {}, transport = {}, { env = process.env, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
if (typeof otelSpanEmitter !== "function" || !shouldEmitLiveKafkaOtelSpan(envelope)) return { emitted: false, valuesRedacted: true };
const attributes = liveKafkaOtelSpanAttributes(envelope, transport);
if (!attributes.businessTraceId) return { emitted: false, valuesRedacted: true };
try {
const pending = otelSpanEmitter(name, attributes.businessTraceId, env, { attributes });
void Promise.resolve(pending).catch(() => undefined);
} catch {
// OTel is best effort and must never change the live Kafka delivery path.
}
return { emitted: true, valuesRedacted: true };
}
export async function projectAgentRunKafkaMessage(kafkaMessage = {}, { runtimeStore, config, logger = console } = {}) {
const transport = kafkaTransport(kafkaMessage);
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
@@ -953,13 +620,18 @@ function kafkaTransport(kafkaMessage = {}) {
};
}
function requireKafkaProjectorStore(runtimeStore, capabilities = {}) {
const required = [];
if (capabilities.transactionalProjector) required.push("commitAgentRunKafkaProjection", "recordFailedAgentRunKafkaMessage", "recordIgnoredAgentRunKafkaMessage");
if (capabilities.projectionOutboxRelay) required.push("claimHwlabKafkaOutbox", "completeHwlabKafkaOutbox", "retryHwlabKafkaOutbox");
if (capabilities.projectionRealtime) required.push("subscribeWorkbenchProjectionCommits");
if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay) required.push("hwlabKafkaProjectorStatus");
if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime) required.push("assertWorkbenchTransactionalRealtimeReady");
function requireKafkaProjectorStore(runtimeStore) {
const required = [
"commitAgentRunKafkaProjection",
"recordFailedAgentRunKafkaMessage",
"recordIgnoredAgentRunKafkaMessage",
"claimHwlabKafkaOutbox",
"completeHwlabKafkaOutbox",
"retryHwlabKafkaOutbox",
"subscribeWorkbenchProjectionCommits",
"hwlabKafkaProjectorStatus",
"assertWorkbenchTransactionalRealtimeReady"
];
const missing = required.filter((name) => typeof runtimeStore?.[name] !== "function");
if (missing.length > 0) throw contractError("hwlab_kafka_projector_store_invalid", `Kafka durable capabilities require a runtime store: ${missing.join(", ")}`);
}
@@ -20,9 +20,6 @@ import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle
import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts";
import { promoteWorkbenchTurnAdmission as persistWorkbenchTurnAdmissionPromotion, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts";
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
import {
workbenchRealtimeCapabilities
} from "./workbench-realtime-capabilities.ts";
import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts";
import {
firstHeaderValue,
@@ -245,7 +242,6 @@ export async function handleCodeAgentChatHttp(request, response, options) {
lastEventAt: submitted?.lastEventAt ?? admittedTiming.lastEventAt,
finishedAt: submitted?.finishedAt ?? admittedTiming.finishedAt,
durationMs: submitted?.durationMs ?? admittedTiming.durationMs,
realtimeCapabilities: workbenchRealtimeCapabilities(options.env ?? process.env),
admission: submitted?.agentRun ? {
state: submitted.admissionState ?? "promoted",
runId: submitted.agentRun.runId ?? null,
@@ -1046,8 +1042,6 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) {
}
});
if (adapterEnabled) {
const realtimeCapabilities = workbenchRealtimeCapabilities(runtimeEnv);
const transactionalAdmission = realtimeCapabilities.transactionalProjector;
const pendingPromotion = results.get(traceId);
if (pendingPromotion?.admissionState === "promotion-pending" && pendingPromotion?.agentRun?.durableDispatch === true) {
const retryParams = { ...params, userBillingReservation: pendingPromotion.userBillingReservation ?? null };
@@ -1062,15 +1056,13 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) {
}
const lifecycle = codeAgentTurnLifecycleFields(traceId, params);
if (transactionalAdmission) {
await persistCodeAgentAdmissionInputState({
params,
options,
traceId,
status: "admitting",
messageId: lifecycle.userMessageId
});
}
await persistCodeAgentAdmissionInputState({
params,
options,
traceId,
status: "admitting",
messageId: lifecycle.userMessageId
});
traceStore.append(traceId, {
type: "request",
@@ -1091,16 +1083,14 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) {
error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null
});
if (billingPreflight?.blocked) {
if (transactionalAdmission) {
await persistCodeAgentAdmissionInputState({
params,
options,
traceId,
status: "blocked",
messageId: lifecycle.userMessageId,
error: billingPreflight.payload?.error ?? billingPreflight.blocker ?? { code: "billing_preflight_blocked" }
});
}
await persistCodeAgentAdmissionInputState({
params,
options,
traceId,
status: "blocked",
messageId: lifecycle.userMessageId,
error: billingPreflight.payload?.error ?? billingPreflight.blocker ?? { code: "billing_preflight_blocked" }
});
traceStore.append(traceId, {
type: "billing",
status: "blocked",
@@ -1126,16 +1116,14 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) {
const failure = codeAgentDispatchFailure(error);
const rejected = codeAgentSubmissionUnavailablePayload(failure, { params: dispatchParams, options: executionOptions, traceId });
await finalizeCodeAgentBillingUsage({ payload: rejected, params: dispatchParams, options: executionOptions });
if (transactionalAdmission) {
await persistCodeAgentAdmissionInputState({
params: dispatchParams,
options: executionOptions,
traceId,
status: "failed",
messageId: lifecycle.userMessageId,
error: failure.payload?.error ?? error
});
}
await persistCodeAgentAdmissionInputState({
params: dispatchParams,
options: executionOptions,
traceId,
status: "failed",
messageId: lifecycle.userMessageId,
error: failure.payload?.error ?? error
});
void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, {
startTimeMs: dispatchStartedAt,
status: "error",
@@ -1156,17 +1144,15 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) {
}, dispatchParams);
results.set(traceId, pending);
try {
if (transactionalAdmission) {
await persistCodeAgentAdmissionInputState({
params: dispatchParams,
options: executionOptions,
traceId,
status: "admitting",
messageId: lifecycle.userMessageId,
commandId: payload.agentRun?.commandId ?? null,
agentRun: payload.agentRun ?? null
});
}
await persistCodeAgentAdmissionInputState({
params: dispatchParams,
options: executionOptions,
traceId,
status: "admitting",
messageId: lifecycle.userMessageId,
commandId: payload.agentRun?.commandId ?? null,
agentRun: payload.agentRun ?? null
});
await promoteCodeAgentTurnAdmission({ payload: pending, params: dispatchParams, options: executionOptions, traceId });
} catch (error) {
throw codeAgentPromotionUnavailableError(error, { payload: pending, params: dispatchParams, options: executionOptions, traceId });
@@ -1324,30 +1310,6 @@ async function promoteCodeAgentTurnAdmission({ payload = {}, params = {}, option
throw error;
}
const lifecycle = codeAgentTurnLifecycleFields(traceId, params);
const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env);
if (!realtimeCapabilities.transactionalProjector) {
const runningOwner = await recordCodeAgentSessionOwner({ payload: { ...payload, status: "running" }, params, options, status: "running" });
if (!runningOwner) {
const error = new Error("Code Agent live turn did not durably promote the session owner to running.");
error.code = "workbench_admission_owner_not_promoted";
error.valuesRedacted = true;
throw error;
}
traceStore.append(traceId, {
type: "request",
status: "admitted",
label: "turn:admitted-live-kafka",
message: "Code Agent turn is live after AgentRun durable dispatch; lifecycle visibility is owned only by hwlab.event.v1.",
sessionId: safeSessionId(params.sessionId) || null,
runId: payload.agentRun?.runId ?? null,
commandId: payload.agentRun?.commandId ?? null,
dispatchIntentId: payload.agentRun?.dispatchIntentId ?? null,
waitingFor: "hwlab-live-kafka-sse",
terminal: false,
valuesPrinted: false
});
return { written: false, promoted: true, outboxCommitted: false, realtimeCapabilities, liveOnly: realtimeCapabilities.liveKafkaSse, valuesPrinted: false };
}
const inputFact = buildCodeAgentSessionInputFact({
params,
options,
-1
View File
@@ -16,7 +16,6 @@ import {
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts";
import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts";
import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts";
import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts";
@@ -46,29 +46,10 @@ const TRANSACTIONAL_REALTIME_ENV = Object.freeze({
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true"
});
const REFRESH_REALTIME_ENV = Object.freeze({
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "true",
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
});
function projectionRealtimeBridge(capabilities = {}) {
function projectionRealtimeBridge() {
return {
started: true,
capabilities: {
directPublish: false,
liveKafkaSse: false,
kafkaRefreshReplay: false,
transactionalProjector: false,
projectionOutboxRelay: false,
projectionRealtime: true,
...capabilities
},
ready: Promise.resolve(),
subscribeLiveHwlabEvents() { return () => {}; },
subscribeProjectionCommits() { return () => {}; },
async stop() {}
};
@@ -90,22 +71,6 @@ test("projection outbox emits immutable assistant versions in row order", () =>
assert.deepEqual(events.map((item) => item.payload.cursor.outboxSeq), [10, 11]);
});
test("workbench sync repair endpoint is removed from the product API", async () => {
const server = createCloudApiServer({
accessController: realtimeAccessController(),
workbenchRuntime: {},
kafkaEventBridge: projectionRealtimeBridge(),
env: { ...TRANSACTIONAL_REALTIME_ENV }
});
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
try {
const response = await getJson(server.address().port, "/v1/workbench/sync?sessionId=ses_removed_sync_repair");
assert.equal(response.status, 404);
} finally {
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
}
});
test("workbench realtime initial connection emits current snapshot without replaying historical outbox", async () => {
const sessionId = "ses_realtime_snapshot_only";
const traceId = "trc_realtime_snapshot_only";
@@ -146,7 +111,7 @@ test("workbench realtime initial connection emits current snapshot without repla
}
});
test("product realtime uses projection outbox even when obsolete live capability values remain present", async () => {
test("product realtime uses the projection outbox authority", async () => {
const sessionId = "ses_composable_realtime";
const traceId = "trc_composable_realtime";
let projectionReads = 0;
@@ -169,7 +134,7 @@ test("product realtime uses projection outbox even when obsolete live capability
const server = createCloudApiServer({
accessController: realtimeAccessController(),
workbenchRuntime: runtime,
kafkaEventBridge: projectionRealtimeBridge({ directPublish: true, liveKafkaSse: true }),
kafkaEventBridge: projectionRealtimeBridge(),
env: {
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
@@ -827,57 +792,6 @@ test("workbench trace event page keeps per-trace terminal status after later ses
}
});
function refreshReplayBridge({ records = [], endOffset = "0", onQuery = null, beforeQueryResult = null, ready = Promise.resolve(), liveReady = Promise.resolve() } = {}) {
let liveListener = null;
return {
capabilities: { directPublish: true, liveKafkaSse: true, kafkaRefreshReplay: true, transactionalProjector: false, projectionOutboxRelay: false, projectionRealtime: false },
refreshReplay: {
groupIdPrefix: "hwlab-v03-refresh-test",
timeoutMs: 2500,
scanLimit: 500,
matchedEventLimit: 50,
liveBufferLimit: 20
},
ready,
liveReady,
subscribeLiveHwlabEvents(listener) { liveListener = listener; return () => { if (liveListener === listener) liveListener = null; }; },
async queryHwlabEventRetention(params) {
onQuery?.(params);
beforeQueryResult?.(liveListener);
return {
topic: "hwlab.event.v1",
events: records,
completionReason: "end-offset",
reachedEndOffsets: true,
endOffsetsAvailable: true,
endOffsets: [{ partition: 0, startOffset: "0", endOffset }],
completion: { reason: "end-offset", complete: true, barrierReached: true, retentionStartVerified: true }
};
},
async stop() {}
};
}
function refreshRecord(offset, sessionId, traceId, type, extra = {}) {
const sourceEventId = `evt_refresh_${offset}`;
return {
topic: "hwlab.event.v1",
partition: 0,
offset: String(offset),
value: {
schema: "hwlab.event.v1",
eventType: "hwlab.trace.event.projected",
eventId: `hwlab:${sourceEventId}`,
sourceEventId,
traceId,
hwlabSessionId: sessionId,
sessionId,
event: { type, eventType: type === "result" ? "terminal" : type, traceId, sessionId, sourceEventId, status: "running", ...extra },
valuesPrinted: false
}
};
}
function realtimeAccessController({ actor = ACTOR, sessions = [] } = {}) {
const bySessionId = new Map(sessions.map((session) => [session.id, session]));
const byTraceId = new Map(sessions.filter((session) => session.lastTraceId).map((session) => [session.lastTraceId, session]));
@@ -14,16 +14,10 @@ import {
} from "./server-http-utils.ts";
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
import { emitLiveKafkaOtelSpan } from "./kafka-event-bridge.ts";
import { createWorkbenchKafkaRefreshHandoff, workbenchKafkaRefreshErrorPayload } from "./workbench-kafka-refresh-handoff.ts";
import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts";
import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts";
import { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts";
import { durableTraceStatus, RUNNING_STATUSES, TERMINAL_STATUSES } from "./workbench-turn-projection.ts";
import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts";
import {
workbenchRealtimeCapabilities
} from "./workbench-realtime-capabilities.ts";
import * as workbenchFacts from "./server-workbench-facts.ts";
const DEFAULT_PAGE_LIMIT = 50;
@@ -183,16 +177,11 @@ export async function drainWorkbenchRealtimeConnections(options = {}) {
export async function handleWorkbenchRealtimeHttp(request, response, url, options = {}) {
try {
if (request.method !== "GET") return methodNotAllowed(response, "GET");
const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env);
const projectionOnly = url.pathname === "/v1/workbench/projection-events";
if (projectionOnly) {
sendJson(response, 410, workbenchError("workbench_projection_events_route_removed", "Use /v1/workbench/events for the single transactional projection authority."));
return;
}
if (!realtimeCapabilities.projectionRealtime) {
sendJson(response, 503, workbenchError("workbench_projection_realtime_disabled", "Workbench product realtime requires the transactional projection outbox authority."));
return;
}
const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId"));
const requestedTraceId = safeTraceId(url.searchParams.get("traceId"));
const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS);
@@ -453,405 +442,6 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option
}
}
async function handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, options = {}) {
const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId"));
const requestedTraceId = safeTraceId(url.searchParams.get("traceId"));
const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env);
const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS);
attachWorkbenchRealtimeOtelContext(request, {
sessionId: requestedSessionId,
traceId: requestedTraceId,
heartbeatMs,
realtimeSource: "live-kafka-sse",
realtimeCapabilities
});
const perf = options.backendPerformance;
const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options);
if (!auth) return;
if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) {
sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench realtime is keyed by sessionId/traceId only."));
return;
}
if (!requestedSessionId && !requestedTraceId) {
sendJson(response, 400, workbenchError("workbench_realtime_scope_required", "Workbench realtime requires sessionId or traceId."));
return;
}
const authorization = await authorizeLiveKafkaWorkbenchRealtimeScope(options, auth.actor, {
sessionId: requestedSessionId,
traceId: requestedTraceId
});
if (!authorization.ok) {
sendJson(response, authorization.status, authorization.body);
return;
}
const authorizedSessionId = authorization.sessionId;
const bridge = options.kafkaEventBridge;
if (bridge?.capabilities?.liveKafkaSse !== true || typeof bridge?.subscribeLiveHwlabEvents !== "function") {
sendJson(response, 503, workbenchError("workbench_live_kafka_unconfigured", "Workbench live Kafka fanout is not configured."));
return;
}
if (realtimeCapabilities.kafkaRefreshReplay) {
await handleKafkaRefreshReplayWorkbenchRealtimeHttp(request, response, options, {
bridge,
heartbeatMs,
realtimeCapabilities,
requestedSessionId: authorizedSessionId,
requestedTraceId
});
return;
}
let closed = false;
let realtimeCloseReason = "client_close";
let realtimeCloseSignal = null;
const realtimeStartedAtMs = Date.now();
const cleanup = [];
const isActive = () => !closed && !response.destroyed && !response.writableEnded;
const closeConnection = () => {
if (closed) return;
closed = true;
emitWorkbenchRealtimeClosedOtelSpan(request, options.env, {
reason: realtimeCloseReason,
signal: realtimeCloseSignal,
startedAtMs: realtimeStartedAtMs,
sessionId: authorizedSessionId,
threadId: null,
traceId: requestedTraceId,
activeConnectionCount: activeWorkbenchRealtimeConnections.size
});
for (const item of cleanup.splice(0)) item();
};
response.once("close", closeConnection);
request.once?.("aborted", closeConnection);
request.socket?.once?.("close", closeConnection);
cleanup.push(() => request.off?.("aborted", closeConnection));
cleanup.push(() => request.socket?.off?.("close", closeConnection));
const bufferedEnvelopes = [];
let liveDeliveryStarted = false;
let enqueueEvent = null;
const unsubscribe = bridge.subscribeLiveHwlabEvents((envelope, transport) => {
if (!liveKafkaEnvelopeMatches(envelope, authorizedSessionId, requestedTraceId)) return;
if (!liveDeliveryStarted || typeof enqueueEvent !== "function") {
bufferedEnvelopes.push({ envelope, transport });
return;
}
void enqueueEvent("hwlab.event.v1", envelope, transport);
});
if (typeof unsubscribe === "function") cleanup.push(unsubscribe);
try {
await (bridge.liveReady ?? bridge.ready);
} catch (error) {
closeConnection();
throw error;
}
if (!isActive()) {
closeConnection();
return;
}
response.writeHead(200, {
"content-type": "text/event-stream; charset=utf-8",
"cache-control": "no-store, no-transform",
connection: "keep-alive",
"x-accel-buffering": "no",
"x-content-type-options": "nosniff"
});
if (typeof response.flushHeaders === "function") response.flushHeaders();
const writeEvent = async (name, payload, transport = null) => {
if (!isActive()) return false;
const block = `event: ${name}\ndata: ${JSON.stringify(payload)}\n\n`;
const writable = response.write(block);
if (!writable) await waitForResponseDrain(response, isActive);
const active = isActive();
if (active && name === "hwlab.event.v1") {
emitLiveKafkaOtelSpan("hwlab.workbench.live_sse.business_event_write", payload, transport, {
env: options.env,
otelSpanEmitter: options.otelSpanEmitter ?? emitCodeAgentOtelSpan
});
}
return active;
};
let writeChain = Promise.resolve(true);
enqueueEvent = (name, payload, transport = null) => {
writeChain = writeChain.then(() => writeEvent(name, payload, transport));
return writeChain;
};
const realtimeConnection = {
close(fields = {}) {
if (!isActive()) return false;
realtimeCloseReason = textValue(fields.reason) || "server_shutdown";
realtimeCloseSignal = textValue(fields.signal) || null;
void enqueueEvent("workbench.server_draining", {
type: "server.draining",
status: "closing",
reason: realtimeCloseReason,
signal: realtimeCloseSignal,
capabilities: realtimeCapabilities,
lossPossible: true
}).finally(() => {
if (!response.writableEnded) response.end();
});
return true;
},
isActive
};
activeWorkbenchRealtimeConnections.add(realtimeConnection);
cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection));
await enqueueEvent("workbench.connected", {
type: "connected",
status: "connected",
capabilities: realtimeCapabilities,
realtimeSource: "hwlab.event.v1",
deliverySemantics: "live-only",
liveOnly: true,
replay: false,
replaySupported: false,
lossPossible: true,
filters: { sessionId: authorizedSessionId, traceId: requestedTraceId }
});
liveDeliveryStarted = true;
const readyBufferedEnvelopes = bufferedEnvelopes.splice(0);
for (const { envelope, transport } of readyBufferedEnvelopes) void enqueueEvent("hwlab.event.v1", envelope, transport);
await writeChain;
if (!isActive()) {
closeConnection();
return;
}
const heartbeatTimer = setInterval(() => {
void enqueueEvent("workbench.heartbeat", {
type: "heartbeat",
status: "connected",
realtimeSource: "hwlab.event.v1",
liveOnly: true,
replay: false,
lossPossible: true,
serverSentAt: new Date().toISOString(),
valuesPrinted: false
});
}, heartbeatMs);
heartbeatTimer.unref?.();
cleanup.push(() => clearInterval(heartbeatTimer));
emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env);
}
async function handleKafkaRefreshReplayWorkbenchRealtimeHttp(request, response, options, input) {
const {
bridge,
heartbeatMs,
realtimeCapabilities,
requestedSessionId,
requestedTraceId
} = input;
const refreshReplay = bridge?.refreshReplay;
if (!refreshReplay || typeof bridge?.queryHwlabEventRetention !== "function") {
sendJson(response, 503, workbenchError("workbench_kafka_refresh_unconfigured", "Workbench Kafka refresh replay is enabled without its retention query runtime."));
return;
}
let closed = false;
let realtimeCloseReason = "client_close";
let realtimeCloseSignal = null;
let failureClosing = false;
const realtimeStartedAtMs = Date.now();
const cleanup = [];
const isActive = () => !closed && !response.destroyed && !response.writableEnded;
const closeConnection = () => {
if (closed) return;
closed = true;
emitWorkbenchRealtimeClosedOtelSpan(request, options.env, {
reason: realtimeCloseReason,
signal: realtimeCloseSignal,
startedAtMs: realtimeStartedAtMs,
sessionId: requestedSessionId,
threadId: null,
traceId: requestedTraceId,
activeConnectionCount: activeWorkbenchRealtimeConnections.size
});
for (const item of cleanup.splice(0)) item();
};
response.once("close", closeConnection);
request.once?.("aborted", closeConnection);
request.socket?.once?.("close", closeConnection);
cleanup.push(() => request.off?.("aborted", closeConnection));
cleanup.push(() => request.socket?.off?.("close", closeConnection));
try {
await (bridge.liveReady ?? bridge.ready);
} catch (error) {
for (const item of cleanup.splice(0)) item();
throw error;
}
if (!isActive()) {
closeConnection();
return;
}
response.writeHead(200, {
"content-type": "text/event-stream; charset=utf-8",
"cache-control": "no-store, no-transform",
connection: "keep-alive",
"x-accel-buffering": "no",
"x-content-type-options": "nosniff"
});
if (typeof response.flushHeaders === "function") response.flushHeaders();
const writeEvent = async (name, payload, transport = null) => {
if (!isActive()) return false;
const block = `event: ${name}\ndata: ${JSON.stringify(payload)}\n\n`;
const writable = response.write(block);
if (!writable) await waitForResponseDrain(response, isActive);
const active = isActive();
if (active && name === "hwlab.event.v1") {
emitLiveKafkaOtelSpan("hwlab.workbench.live_sse.business_event_write", payload, transport, {
env: options.env,
otelSpanEmitter: options.otelSpanEmitter ?? emitCodeAgentOtelSpan
});
}
return active;
};
let writeChain = Promise.resolve(true);
const enqueueEvent = (name, payload, transport = null) => {
writeChain = writeChain.then(() => writeEvent(name, payload, transport));
return writeChain;
};
const closeWithRefreshFailure = async (error) => {
if (failureClosing || !isActive()) return;
failureClosing = true;
await enqueueEvent("workbench.error", workbenchKafkaRefreshErrorPayload(error, {
sessionId: requestedSessionId,
traceId: requestedTraceId
}));
await writeChain;
if (!response.writableEnded) response.end();
};
const realtimeConnection = {
close(fields = {}) {
if (!isActive()) return false;
realtimeCloseReason = textValue(fields.reason) || "server_shutdown";
realtimeCloseSignal = textValue(fields.signal) || null;
void enqueueEvent("workbench.server_draining", {
type: "server.draining",
status: "closing",
reason: realtimeCloseReason,
signal: realtimeCloseSignal,
capabilities: realtimeCapabilities,
lossPossible: false
}).finally(() => {
if (!response.writableEnded) response.end();
});
return true;
},
isActive
};
activeWorkbenchRealtimeConnections.add(realtimeConnection);
cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection));
const handoff = createWorkbenchKafkaRefreshHandoff({
sessionId: requestedSessionId,
traceId: requestedTraceId,
liveBufferLimit: refreshReplay.liveBufferLimit,
subscribeLive: (listener) => bridge.subscribeLiveHwlabEvents(listener),
queryRetention: ({ signal }) => bridge.queryHwlabEventRetention({
sessionId: requestedSessionId,
traceId: requestedTraceId,
limit: refreshReplay.matchedEventLimit,
scanLimit: refreshReplay.scanLimit,
timeoutMs: refreshReplay.timeoutMs,
groupIdPrefix: refreshReplay.groupIdPrefix,
fromBeginning: true,
signal
}),
deliverEvent: (envelope, transport) => enqueueEvent("hwlab.event.v1", envelope, transport),
deliverConnected: (summary) => enqueueEvent("workbench.connected", {
type: "connected",
status: "connected",
capabilities: realtimeCapabilities,
realtimeSource: "hwlab.event.v1",
deliverySemantics: "kafka-retention-then-live",
liveOnly: false,
replay: true,
replaySupported: true,
lossPossible: false,
filters: { sessionId: requestedSessionId, traceId: requestedTraceId },
refreshReplay: summary
}),
onFailure: closeWithRefreshFailure
});
cleanup.push(() => handoff.stop("connection-closed"));
try {
await handoff.start();
} catch (error) {
if (isActive()) await closeWithRefreshFailure(error);
return;
}
if (!isActive()) return;
const heartbeatTimer = setInterval(() => {
void enqueueEvent("workbench.heartbeat", {
type: "heartbeat",
status: "connected",
realtimeSource: "hwlab.event.v1",
deliverySemantics: "kafka-retention-then-live",
liveOnly: false,
replay: true,
lossPossible: false,
serverSentAt: new Date().toISOString(),
valuesPrinted: false
});
}, heartbeatMs);
heartbeatTimer.unref?.();
cleanup.push(() => clearInterval(heartbeatTimer));
emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env);
}
async function authorizeLiveKafkaWorkbenchRealtimeScope(options, actor, { sessionId, traceId }) {
const access = options.accessController;
if ((sessionId && typeof access?.getAgentSession !== "function") || (traceId && typeof access?.getAgentSessionByTraceId !== "function")) {
return {
ok: false,
status: 503,
body: workbenchError("workbench_realtime_authorization_unconfigured", "Workbench live realtime ownership authorization is not configured.")
};
}
const [sessionById, sessionByTrace] = await Promise.all([
sessionId ? access.getAgentSession(sessionId) : null,
traceId ? access.getAgentSessionByTraceId(traceId) : null
]);
const sessionByIdId = sessionById ? safeSessionId(sessionById.id ?? sessionById.sessionId) : null;
const sessionByTraceId = sessionByTrace ? safeSessionId(sessionByTrace.id ?? sessionByTrace.sessionId) : null;
const scopeMissing = (sessionId && sessionByIdId !== sessionId)
|| (traceId && !sessionByTraceId)
|| (sessionId && traceId && sessionByIdId !== sessionByTraceId);
const resolvedSessions = [sessionById, sessionByTrace].filter(Boolean);
const ownedByActor = actor?.role === "admin" || (Boolean(actor?.id)
&& resolvedSessions.length > 0
&& resolvedSessions.every((session) => textValue(session.ownerUserId) === actor.id));
if (scopeMissing || !ownedByActor) {
return {
ok: false,
status: 404,
body: workbenchError("workbench_realtime_scope_not_found", "Workbench realtime scope is not visible to the current actor.")
};
}
return { ok: true, sessionId: sessionByIdId ?? sessionByTraceId };
}
function liveKafkaEnvelopeMatches(envelope, sessionId, traceId) {
if (!envelope || envelope.schema !== "hwlab.event.v1") return false;
if (sessionId && safeSessionId(envelope.hwlabSessionId ?? envelope.sessionId) !== sessionId) return false;
if (traceId && safeTraceId(envelope.traceId) !== traceId) return false;
return true;
}
export function attachWorkbenchRealtimeOtelContext(request, fields = {}) {
const context = request?.hwlabHttpRequestContext;
if (!context) return;
@@ -865,9 +455,9 @@ export function attachWorkbenchRealtimeOtelContext(request, fields = {}) {
"workbench.thread_id": fields.threadId ?? null,
"workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null,
"workbench.sse.realtime_source": fields.realtimeSource ?? null,
"workbench.sse.live_kafka_enabled": fields.realtimeCapabilities?.liveKafkaSse ?? null,
"workbench.sse.projection_realtime_enabled": fields.realtimeCapabilities?.projectionRealtime ?? null,
"workbench.sse.outbox_mode": fields.realtimeCapabilities?.liveKafkaSse === true ? false : true
"workbench.sse.live_kafka_enabled": false,
"workbench.sse.projection_realtime_enabled": true,
"workbench.sse.outbox_mode": true
};
}
-3
View File
@@ -372,10 +372,7 @@ export async function buildHealthPayload(options = {}) {
async function kafkaProjectorHealth(projector, { liveProbe = false } = {}) {
if (!projector?.started) return { started: false, reason: projector?.reason ?? "disabled", valuesRedacted: true };
const identity = {
capabilities: projector.capabilities ?? null,
directPublishGroupId: projector.directPublishGroupId ?? null,
projectorGroupId: projector.projectorGroupId ?? null,
hwlabEventGroupId: projector.hwlabEventGroupId ?? null,
agentRunTopic: projector.agentRunTopic,
hwlabTopic: projector.hwlabTopic
};
@@ -144,6 +144,89 @@ test("authoritative assistant final seals terminal with a final response", () =>
assert.equal(built.facts.checkpoints[0].projectionStatus, "caught_up");
});
test("canonical event time owns running, terminal, replay, and duration", () => {
const running = buildWorkbenchProjectionEventFacts({
projectedSeq: 20,
projectedAt: "2026-07-10T10:17:00.000Z",
event: {
traceId: "trc_canonical_duration",
sessionId: "ses_canonical_duration",
sourceEventId: "evt_canonical_running",
sourceSeq: 41,
type: "assistant",
eventType: "assistant_progress",
status: "running",
createdAt: "2026-07-10T10:00:01.000Z",
startedAt: "2026-07-10T09:43:00.000Z",
durationMs: 1020000
}
});
const runningCheckpoint = running.facts.checkpoints[0];
assert.equal(runningCheckpoint.startedAt, "2026-07-10T10:00:01.000Z");
assert.equal(runningCheckpoint.lastEventAt, "2026-07-10T10:00:01.000Z");
assert.equal(runningCheckpoint.durationMs, null);
const terminal = buildWorkbenchProjectionEventFacts({
projectedSeq: 21,
projectedAt: "2026-07-10T10:18:00.000Z",
previousCheckpoint: runningCheckpoint,
event: {
traceId: "trc_canonical_duration",
sessionId: "ses_canonical_duration",
sourceEventId: "evt_canonical_terminal",
sourceSeq: 42,
type: "assistant",
eventType: "assistant",
status: "completed",
terminal: true,
replyAuthority: true,
text: "canonical final",
createdAt: "2026-07-10T10:00:04.000Z",
finishedAt: "2026-07-10T10:18:00.000Z",
durationMs: 1020000
}
});
const terminalCheckpoint = terminal.facts.checkpoints[0];
assert.equal(terminalCheckpoint.startedAt, "2026-07-10T10:00:01.000Z");
assert.equal(terminalCheckpoint.finishedAt, "2026-07-10T10:00:04.000Z");
assert.equal(terminalCheckpoint.durationMs, 3000);
const [replayed] = projectionOutboxRealtimeEvents({
events: [{
outboxSeq: 72,
entityFamily: "traceEvents",
entityId: terminal.facts.traceEvents[0].id,
projectedSeq: 21,
projectionRevision: 1,
traceId: "trc_canonical_duration",
sessionId: "ses_canonical_duration",
commitType: "event",
terminal: true,
sealed: true,
payload: { family: "traceEvents", fact: terminal.facts.traceEvents[0] }
}]
});
assert.deepEqual(replayed.payload.cursor, { outboxSeq: 72, traceSeq: 21 });
assert.equal(replayed.payload.event.durationMs, 3000);
const [refreshed] = projectionOutboxRealtimeEvents({
events: [{
outboxSeq: 72,
entityFamily: "traceEvents",
entityId: terminal.facts.traceEvents[0].id,
projectedSeq: 21,
projectionRevision: 1,
traceId: "trc_canonical_duration",
sessionId: "ses_canonical_duration",
commitType: "event",
terminal: true,
sealed: true,
payload: { family: "traceEvents", fact: terminal.facts.traceEvents[0] }
}]
});
assert.deepEqual(refreshed.payload.cursor, replayed.payload.cursor);
assert.equal(refreshed.payload.event.durationMs, replayed.payload.event.durationMs);
});
test("completed terminal without final response remains visibly unsealed", () => {
const built = buildWorkbenchProjectionEventFacts({
projectedSeq: 1,
+4 -14
View File
@@ -205,12 +205,9 @@ export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = {
valuesPrinted: false
};
}
const explicitEventTiming = normalizeTimingProjection(event.timing) ?? normalizeTimingProjection(event);
const explicitDurationMs = durationValue(event.durationMs ?? explicitEventTiming?.durationMs);
const explicitStartedAt = optionalTimestampValue(event.startedAt ?? event.traceStartedAt ?? event.runnerStartedAt ?? explicitEventTiming?.startedAt);
const provisionalLastEventAt = latestTimestamp(previousTiming?.lastEventAt, explicitEventTiming?.lastEventAt, occurredAt);
const provisionalFinishedAt = sourceTerminal ? latestTimestamp(explicitEventTiming?.finishedAt, optionalTimestampValue(event.finishedAt), provisionalLastEventAt, occurredAt) : null;
const startedAt = previousTiming?.startedAt ?? explicitStartedAt ?? startedAtFromDuration(provisionalFinishedAt, explicitDurationMs) ?? (sourceSeq <= 1 ? occurredAt : null);
const provisionalLastEventAt = latestTimestamp(previousTiming?.lastEventAt, occurredAt);
const provisionalFinishedAt = sourceTerminal ? provisionalLastEventAt : null;
const startedAt = previousTiming?.startedAt ?? occurredAt;
const lastEventAt = provisionalLastEventAt;
const finishedAt = sourceTerminal ? latestTimestamp(provisionalFinishedAt, lastEventAt) : null;
const sourceEventId = workbenchSourceEventId(event, { traceId, sourceSeq, occurredAt });
@@ -226,7 +223,7 @@ export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = {
const terminalSealBlocked = sourceTerminal && sourceStatus === "completed" && !messageFinalText;
const effectiveTerminal = sourceTerminal && !terminalSealBlocked;
const effectiveStatus = effectiveTerminal ? sourceStatus : "running";
const timing = eventTimingProjection({ startedAt, lastEventAt, finishedAt, terminal: effectiveTerminal, durationMs: explicitDurationMs ?? previousTiming?.durationMs });
const timing = eventTimingProjection({ startedAt, lastEventAt, finishedAt, terminal: effectiveTerminal });
const timingAuthorityIssue = terminalTimingAuthorityIssue(timing, { terminal: effectiveTerminal, traceId, source: "kafka", status: effectiveStatus, label: event.label, sourceSeq });
const projectionDiagnostic = { timingAuthorityIssue, terminalSealBlocked, sourceTerminal, sourceStatus, authority: "agentrun-kafka-projector", valuesRedacted: true };
const userMessageFact = sessionId && previousCheckpoint?.userMessage ? normalizeMessageFact(previousCheckpoint.userMessage, 0, { traceId, sessionId, turnId, terminal: false, terminalStatus: "sent", finalText: null, timestamp: projectedAt, timing }) : null;
@@ -499,13 +496,6 @@ function elapsedBetween(startedAt, endedAt) {
return Math.trunc(end - start);
}
function startedAtFromDuration(finishedAt, durationMs) {
const end = Date.parse(String(finishedAt ?? ""));
const duration = durationValue(durationMs);
if (!Number.isFinite(end) || duration === null) return null;
return new Date(Math.max(0, end - duration)).toISOString();
}
function terminalTimingAuthorityIssue(timing, { terminal = false, traceId = null, source = null, status = null, label = null, sourceSeq = null } = {}) {
if (!terminal) return null;
const startedAt = optionalTimestampValue(timing?.startedAt);
@@ -1,35 +0,0 @@
// SPEC: PJ2026-0104010803 Workbench composable realtime capabilities.
// Responsibility: validate YAML-owned capability switches without code defaults or implied mode selection.
export const WORKBENCH_REALTIME_CAPABILITY_ENVS = Object.freeze({
directPublish: "HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
liveKafkaSse: "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
kafkaRefreshReplay: "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
transactionalProjector: "HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
projectionOutboxRelay: "HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
projectionRealtime: "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
});
export function workbenchRealtimeCapabilities(env = process.env) {
return Object.fromEntries(Object.entries(WORKBENCH_REALTIME_CAPABILITY_ENVS).map(([key, name]) => [key, requiredBooleanEnv(env, name)]));
}
export function requiredBooleanEnv(env, name) {
const value = String(env?.[name] ?? "").trim().toLowerCase();
if (value === "true" || value === "1") return true;
if (value === "false" || value === "0") return false;
throw capabilityError(
"hwlab_workbench_realtime_capability_invalid",
`${name} is required and must be explicitly true or false in the owning YAML.`,
name
);
}
function capabilityError(code, message, envName) {
return Object.assign(new Error(message), {
code,
envName,
statusCode: 503,
valuesRedacted: true
});
}
@@ -400,11 +400,6 @@ function validateDisplayLocale(locale) {
function workbenchRuntimeConfigFromEnv() {
const traceTimeline = {};
const result = {
realtimeFeatures: {
liveKafkaSse: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED, "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED"),
kafkaRefreshReplay: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED"),
projectionRealtime: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED, "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED")
},
debugCapabilities: {
isolatedKafka: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED, "HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED"),
rawHwlabEventWindow: {
@@ -1,5 +1,5 @@
import assert from "node:assert/strict";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { createServer } from "node:http";
import { connect } from "node:net";
import os from "node:os";
@@ -16,7 +16,25 @@ test("cloud web classifies Bun client disconnect ECONNRESET as non-fatal", () =>
assert.equal(isClientDisconnectError(new Error("database migration failed")), false);
});
test("serveCloudWeb rejects a missing refresh replay capability without opening health or SPA routes", async () => {
test("repo feature schema owns one unique key per product feature", async () => {
const schema = JSON.parse(await readFile(new URL("../../config/feature-config.schema.json", import.meta.url), "utf8"));
assert.equal(schema.$schema, "https://json-schema.org/draft/2020-12/schema");
assert.deepEqual(Object.keys(schema.properties), ["tracePanel", "rawEvents", "debugReplay", "views"]);
const features = Object.values(schema.properties).map((property) => property["x-unidesk-feature"]);
assert.equal(features.every((feature) => typeof feature === "string" && feature.length > 0), true);
assert.equal(new Set(features).size, features.length);
const encoded = JSON.stringify(schema);
for (const architectureEnv of [
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
]) assert.equal(encoded.includes(architectureEnv), false);
});
test("serveCloudWeb starts without obsolete architecture capabilities", async () => {
const restoreEnv = withEnv({
HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai",
HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN",
@@ -29,19 +47,21 @@ test("serveCloudWeb rejects a missing refresh replay capability without opening
HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_MAX_ENTRIES: "200",
HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_MAX_RETAINED_BYTES: "1048576"
});
delete process.env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED;
const port = await unusedPort();
let server = null;
try {
await assert.rejects(serveCloudWeb({
server = await serveCloudWeb({
port,
serviceId: "hwlab-cloud-web",
healthPayload: () => ({ status: "ok" }),
sendJson() {
assert.fail("missing runtime configuration must fail before serving any response");
sendJson(response, statusCode, body) {
response.writeHead(statusCode, { "content-type": "application/json" });
response.end(JSON.stringify(body));
}
}), /HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED/u);
assert.equal(await canConnect(port), false, "serveCloudWeb must not bind before runtime configuration is valid");
});
assert.equal(await canConnect(port), true);
} finally {
if (server) await close(server);
restoreEnv();
}
});
@@ -497,7 +517,7 @@ test("cloud web serves client deep links through the Vue shell", async () => {
const html = await response.text();
assert.match(html, /<div id="root"><\/div>/u, route);
assert.match(html, /HWLAB_CLOUD_WEB_CONFIG/u, route);
assert.match(html, /"realtimeFeatures":\{"liveKafkaSse":true,"kafkaRefreshReplay":true,"projectionRealtime":false\}/u, route);
assert.doesNotMatch(html, /realtimeFeatures|liveKafkaSse|kafkaRefreshReplay|projectionRealtime/u, route);
assert.match(html, /"debugCapabilities":\{"isolatedKafka":true,"rawHwlabEventWindow":\{"enabled":true,"maxEntries":200,"maxRetainedBytes":1048576\}\}/u, route);
}
+16 -7
View File
@@ -687,11 +687,14 @@ test("v03 render keeps node identity as data instead of generated structure", as
assert.equal(cloudApiEnv.get("HWLAB_CLOUD_DB_SSL_MODE"), "require");
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_ENABLED"), "true");
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED"), "true");
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED"), "true");
assert.equal(cloudApiEnv.get("HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED"), "true");
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED"), "false");
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED"), "false");
assert.equal(cloudApiEnv.get("HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"), "false");
for (const name of [
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
]) assert.equal(cloudApiEnv.has(name), false);
assert.equal(cloudApiEnv.get("HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED"), "true");
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_BOOTSTRAP_SERVERS"), "platform-infra-kafka-kafka-bootstrap.platform-infra.svc.cluster.local:9092");
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_STDIO_TOPIC"), "codex-stdio.raw.v1");
@@ -709,8 +712,14 @@ test("v03 render keeps node identity as data instead of generated structure", as
const cloudWeb = (workloadsJson.items ?? []).find((item) => item.kind === "Deployment" && item.metadata?.name === "hwlab-cloud-web");
const cloudWebContainer = collectContainersFromItem(cloudWeb).find((container) => container.name === "hwlab-cloud-web");
const cloudWebEnv = new Map((cloudWebContainer?.env ?? []).map((entry) => [entry.name, entry.value]));
assert.equal(cloudWebEnv.get("HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED"), "true");
assert.equal(cloudWebEnv.get("HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"), "false");
for (const name of [
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
]) assert.equal(cloudWebEnv.has(name), false);
assert.equal(cloudWebEnv.get("HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED"), "true");
assert.equal(cloudWebEnv.get("HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_ENABLED"), "1");
assert.equal(cloudWebEnv.get("HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_MAX_ENTRIES"), "200");
@@ -3,13 +3,14 @@ import test from "node:test";
import { readStructuredFile } from "./src/structured-config.mjs";
const PRODUCT_CAPABILITIES = Object.freeze({
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
});
const OBSOLETE_ARCHITECTURE_ENV = Object.freeze([
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
]);
const LIVE_KAFKA_CONTRACT = Object.freeze({
HWLAB_KAFKA_STDIO_TOPIC: "codex-stdio.raw.v1",
@@ -21,7 +22,6 @@ const LIVE_KAFKA_CONTRACT = Object.freeze({
});
const UNIDESK_OWNED_REFRESH_REPLAY_ENV = Object.freeze([
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT",
@@ -37,15 +37,16 @@ const ISOLATED_DEBUG_CONTRACT = Object.freeze({
HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_TIMEOUT_MS: "15000"
});
test("v03 YAML keeps pure Kafka realtime capabilities independently composable", async () => {
test("v03 YAML fixes one transactional projection architecture without architecture env", async () => {
const deploy = await readStructuredFile(process.cwd(), "deploy/deploy.yaml");
const cloudApi = deploy.lanes?.v03?.services?.find((service) => service.serviceId === "hwlab-cloud-api");
const cloudWeb = deploy.lanes?.v03?.serviceDeclarations?.["hwlab-cloud-web"];
assert.ok(cloudApi, "deploy.lanes.v03.services must declare hwlab-cloud-api");
assert.ok(cloudWeb, "deploy.lanes.v03.serviceDeclarations must declare hwlab-cloud-web");
for (const [name, expected] of Object.entries(PRODUCT_CAPABILITIES)) {
assert.equal(cloudApi.env?.[name], expected, `${name} must be explicitly YAML-owned`);
for (const name of OBSOLETE_ARCHITECTURE_ENV) {
assert.equal(cloudApi.env?.[name], undefined, `${name} must not be rendered into cloud-api`);
assert.equal(cloudWeb.env?.[name], undefined, `${name} must not be rendered into cloud-web`);
}
for (const [name, expected] of Object.entries(LIVE_KAFKA_CONTRACT)) {
assert.equal(cloudApi.env?.[name], expected, `${name} must preserve the pure Kafka live contract`);
@@ -59,8 +60,5 @@ test("v03 YAML keeps pure Kafka realtime capabilities independently composable",
assert.equal(cloudApi.env?.HWLAB_SESSION_COOKIE_DOMAIN, undefined, "internal and public origins require host-scoped session cookies");
assert.notEqual(cloudApi.env?.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID, "hwlab-v03-agentrun-event-bridge");
assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED, "true");
assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED, "true");
assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED, "false");
assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED, "true");
});
-1
View File
@@ -199,7 +199,6 @@ assertIncludes(workbenchRealtimePlanSource, "afterOutboxSeq: finiteNumber(recove
assert.doesNotMatch(workbenchRealtimePlanSource, /force:\s*true/u, "Realtime recovery planner must not turn transport recovery into force-refresh work");
assertIncludes(workbenchColadaSource, "const state = await queryCache.refresh(entry);", "Workbench reads must preserve Colada staleTime/min-interval governance");
assert.doesNotMatch(workbenchColadaSource, /queryCache\.fetch\(entry/u, "Workbench reads must not call queryCache.fetch(entry), which bypasses Colada freshness governance");
assertIncludes(workbenchStoreSource, "runtimePolicy.sessionListRealtimeRefreshDelayMs", "Realtime recovery delay must come from runtime policy instead of store constants");
assertIncludes(workbenchStoreSource, "workbenchColadaQueries.fetchSession", "Realtime session detail recovery must enter Colada query facade");
assertIncludes(workbenchStoreSource, "runtimePolicy.workbenchSessionDetailMinRefreshMs", "Realtime session detail recovery budget must come from runtime policy");
assert.doesNotMatch(workbenchStoreSource, /refreshRealtimeSessionMessages[\s\S]{0,900}refreshSessionMessageProjectionPage\(id, \{ force: true \}\)/u, "Realtime session message recovery must not force-bypass the message projection refresh budget");
@@ -0,0 +1,93 @@
import assert from "node:assert/strict";
import { readFile } from "node:fs/promises";
import test from "node:test";
import { fileURLToPath } from "node:url";
const repositoryRoot = fileURLToPath(new URL("../../..", import.meta.url));
const productionFiles = [
"internal/cloud/kafka-event-bridge.ts",
"internal/cloud/server-workbench-realtime-http.ts",
"internal/cloud/server-workbench-facts.ts",
"internal/cloud/server-code-agent-admission-http.ts",
"internal/cloud/server.ts",
"web/hwlab-cloud-web/src/config/runtime.ts",
"web/hwlab-cloud-web/src/config/workbench-runtime-policy.ts",
"web/hwlab-cloud-web/src/types/global.d.ts",
"web/hwlab-cloud-web/src/api/workbench-events.ts",
"web/hwlab-cloud-web/src/utils/workbench-stream-transport.ts",
"web/hwlab-cloud-web/src/utils/workbench-realtime-runtime.ts",
"web/hwlab-cloud-web/src/stores/workbench.ts",
"web/hwlab-cloud-web/src/stores/workbench-colada-queries.ts",
"web/hwlab-cloud-web/src/stores/workbench-event-reducer.ts",
"web/hwlab-cloud-web/src/stores/workbench-realtime-plan.ts",
"web/hwlab-cloud-web/src/stores/workbench-session.ts",
"web/hwlab-cloud-web/src/stores/workbench-session-messages-read-budget.ts"
];
const forbiddenArchitectureIdentifiers = [
"directPublish",
"liveKafkaSse",
"kafkaRefreshReplay",
"transactionalProjector",
"projectionOutboxRelay",
"projectionRealtime",
"startLiveHwlabKafkaEventBridge",
"handleLiveKafkaWorkbenchRealtimeHttp",
"handleKafkaRefreshReplayWorkbenchRealtimeHttp",
"subscribeLiveHwlabEvents",
"queryHwlabEventRetention",
"WorkbenchRealtimeCapabilities",
"workbenchRealtimeCapabilities",
"workbenchHistoryAuthorityPolicy",
"workbenchRealtimeTransportEnabled",
"workbenchRealtimeTraceIdForCapabilities",
"workbenchProjectionEventStreamPath",
"scheduleSessionListRefresh",
"invalidateSessionList",
"fetchWorkbenchTurnStatus",
"refreshTurnStatusByTraceId",
"refreshMessageProjectionForTrace",
"selectActiveTurnStatusRefreshTraceIds",
"sessionDetailAutoReadDecision",
"projectionMergeCommitSummary",
"forceRead",
"sealRestoredActiveTurnMessages",
"messageNeedsRestoredTurnSeal",
"refreshSessionStatusAuthority",
"readTerminalTraceDetailGaps",
"workbenchTraceMessagesWindowLimit",
"handleWorkbenchSyncHttp",
"/v1/workbench/sync"
];
const obsoleteArchitectureEnv = [
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
];
test("Workbench production source has one fixed transactional projection architecture", async () => {
const violations: string[] = [];
for (const relativePath of productionFiles) {
const source = await readFile(`${repositoryRoot}/${relativePath}`, "utf8");
for (const identifier of forbiddenArchitectureIdentifiers) {
if (source.includes(identifier)) violations.push(`${relativePath}: ${identifier}`);
}
}
assert.deepEqual(violations, []);
});
test("production deploy does not render architecture selection env", async () => {
const deploySource = await readFile(`${repositoryRoot}/deploy/deploy.yaml`, "utf8");
const violations = obsoleteArchitectureEnv.filter((name) => deploySource.includes(name));
assert.deepEqual(violations, []);
});
test("raw hwlab Kafka fixtures remain isolated to the explicit debug path", async () => {
const fixtureSource = await readFile(`${repositoryRoot}/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts`, "utf8");
const debugSource = await readFile(`${repositoryRoot}/web/hwlab-cloud-web/src/stores/workbench-isolated-kafka-debug.ts`, "utf8");
assert.match(fixtureSource, /historical hwlab\.event\.debug\.v1 fixtures/u);
assert.match(fixtureSource, /never imported by the product Workbench SSE path/u);
assert.match(debugSource, /workbench-live-kafka-event/u);
});
@@ -6,9 +6,9 @@ import test from "node:test";
import type { ChatMessage } from "../src/types/index.ts";
import { workbenchSessionDetailPathForTest, workbenchSessionMessagesPathForTest } from "../src/api/workbench.ts";
import { workbenchEventStreamPath, workbenchProjectionEventStreamPath } from "../src/api/workbench-events.ts";
import { workbenchEventStreamPath } from "../src/api/workbench-events.ts";
import type { WorkbenchStreamTransportRecovery } from "../src/utils/workbench-realtime-runtime.ts";
import { workbenchRealtimeTraceIdForCapabilities, workbenchRealtimeTransportEnabled } from "../src/utils/workbench-stream-transport.ts";
import { workbenchRealtimeTraceId } from "../src/utils/workbench-stream-transport.ts";
import { workbenchRuntimePolicy } from "../src/config/workbench-runtime-policy.ts";
import { AsyncQueue, work } from "../src/utils/scheduler/async-queue.ts";
import { createCoalescedEventQueue } from "../src/utils/scheduler/coalesced-event-queue.ts";
@@ -20,13 +20,11 @@ import { checkWorkbenchHealth, createWorkbenchHealthProbeCache } from "../src/ut
import { messageDiagnosticView } from "../src/utils/workbench-error-runtime.ts";
import { projectRejectedWorkbenchAdmission } from "../src/stores/workbench-admission-failure.ts";
import { WORKBENCH_TIMELINE_OPENCODE_PARITY, buildWorkbenchTimelineRows, normalizeWorkbenchTimelineMessages, workbenchTimelineSignature } from "../src/stores/workbench-timeline-model.ts";
import { reduceWorkbenchRealtimeEvent, workbenchRealtimeEventIsBusinessActivity } from "../src/stores/workbench-event-reducer.ts";
import { reduceWorkbenchLiveKafkaMessageState, workbenchAgentMessageIdForTrace } from "../src/stores/workbench-live-kafka-event.ts";
import { reduceWorkbenchRealtimeEvent } from "../src/stores/workbench-event-reducer.ts";
import { planWorkbenchRealtimeApply, planWorkbenchRealtimeRecovery } from "../src/stores/workbench-realtime-plan.ts";
import { WORKBENCH_REALTIME_AUTHORITY_VERSION, workbenchRealtimePrimaryAuthorityDecision } from "../src/stores/workbench-realtime-authority.ts";
import { cleanupWorkbenchServerStateDroppedSessions, cleanupWorkbenchServerStateSessions, createWorkbenchServerState, reduceWorkbenchServerState } from "../src/stores/workbench-server-state.ts";
import { cleanupDroppedWorkbenchSessionCaches, trimWorkbenchSessionCache } from "../src/stores/workbench-session-cache.ts";
import { selectActiveTurnStatusRefreshTraceIds } from "../src/stores/workbench-session.ts";
test("Workbench scoped keys encode delimiter characters", () => {
assert.deepEqual(splitWorkbenchScopedKey(workbenchRealtimeScopeKey("ses|one", "trc/two")), ["workbench.realtime", "ses|one", "trc/two"]);
@@ -41,7 +39,6 @@ test("Workbench runtime policy reads injected config while preserving defaults",
traceDetailMaxPages: 2,
workbenchSessionDetailMinRefreshMs: 1234,
workbenchSessionMessagesWindowLimit: 9,
workbenchTraceMessagesWindowLimit: 4,
workbenchRealtimeErrorSyncReplayMinMs: 0,
workbenchRealtimeFlushMaxItemsPerChunk: 2,
workbenchRealtimeFlushMaxChunkMs: 6,
@@ -53,7 +50,6 @@ test("Workbench runtime policy reads injected config while preserving defaults",
assert.equal(policy.traceDetailMaxPages, 2);
assert.equal(policy.workbenchSessionDetailMinRefreshMs, 1234);
assert.equal(policy.workbenchSessionMessagesWindowLimit, 9);
assert.equal(policy.workbenchTraceMessagesWindowLimit, 4);
assert.equal(policy.workbenchRealtimeErrorSyncReplayMinMs, 0);
assert.equal(policy.workbenchRealtimeFlushMaxItemsPerChunk, 2);
assert.equal(policy.workbenchRealtimeFlushMaxChunkMs, 6);
@@ -83,24 +79,16 @@ test("Workbench API uses metadata-only session detail and bounded messages paths
assert.equal(workbenchSessionMessagesPathForTest("ses_metadata", { limit: 9 }), "/v1/workbench/sessions/ses_metadata/messages?limit=9");
});
test("only the projection authority sends the durable afterSeq cursor", () => {
assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, kafkaRefreshReplay: false, projectionRealtime: false }, sessionId: "ses_live", traceId: "trc_live", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_live&traceId=trc_live");
assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: true }, sessionId: "ses_projection", traceId: "trc_projection", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_projection&traceId=trc_projection&afterSeq=42");
assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: true }, sessionId: "ses_both", traceId: "trc_both", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_both&traceId=trc_both&afterSeq=42");
assert.equal(workbenchProjectionEventStreamPath({ sessionId: "ses_both", traceId: "trc_both", afterSeq: 42 }), "/v1/workbench/projection-events?sessionId=ses_both&traceId=trc_both&afterSeq=42");
assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: false }), false);
assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, kafkaRefreshReplay: false, projectionRealtime: false }), false);
assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: true }), true);
assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: true }), true);
test("the product EventSource always sends the durable outbox cursor", () => {
assert.equal(workbenchEventStreamPath({ sessionId: "ses_projection", traceId: "trc_projection", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_projection&traceId=trc_projection&afterSeq=42");
});
test("obsolete live flags cannot erase the active projection trace scope", () => {
const capabilities = { liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: false };
const beforeSubmit = workbenchRealtimeScopeKey("ses_live", workbenchRealtimeTraceIdForCapabilities(capabilities, null, null));
const afterSubmit = workbenchRealtimeScopeKey("ses_live", workbenchRealtimeTraceIdForCapabilities(capabilities, "trc_current_request", "trc_message"));
test("the active projection trace scope follows the current request", () => {
const beforeSubmit = workbenchRealtimeScopeKey("ses_projection", workbenchRealtimeTraceId(null, null));
const afterSubmit = workbenchRealtimeScopeKey("ses_projection", workbenchRealtimeTraceId("trc_current_request", "trc_message"));
assert.notEqual(afterSubmit, beforeSubmit);
assert.equal(afterSubmit, workbenchRealtimeScopeKey("ses_live", "trc_current_request"));
assert.equal(workbenchRealtimeTraceIdForCapabilities({ liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: true }, "trc_current_request", "trc_message"), "trc_current_request");
assert.equal(afterSubmit, workbenchRealtimeScopeKey("ses_projection", "trc_current_request"));
assert.equal(workbenchRealtimeTraceId(null, "trc_message"), "trc_message");
});
test("Error runtime owns Workbench message diagnostic view model", () => {
@@ -480,42 +468,6 @@ test("server-state cleanup removes dropped session trace and turn authority", ()
assert.deepEqual(Object.keys(cleanedById.traceById), []);
});
test("passive message projection selects active turn status refresh without local request", () => {
const traceIds = selectActiveTurnStatusRefreshTraceIds({
messages: [
agentMessage({ id: "m_old", traceId: "trc_old", status: "completed", text: "done" }),
agentMessage({ id: "m_new", traceId: "trc_new", status: "running", text: "thinking" })
],
currentRequestTraceId: null,
turnStatusAuthority: {}
});
assert.deepEqual(traceIds, ["trc_new"]);
});
test("passive message projection skips sealed terminal turn status refresh", () => {
const traceIds = selectActiveTurnStatusRefreshTraceIds({
messages: [agentMessage({ id: "m_done", traceId: "trc_done", status: "completed", text: "done" })],
turnStatusAuthority: { trc_done: { traceId: "trc_done", status: "completed", running: false, terminal: true, sessionId: "ses_1" } }
});
assert.deepEqual(traceIds, []);
});
test("turn status refresh keeps local request trace priority", () => {
const traceIds = selectActiveTurnStatusRefreshTraceIds({
messages: [
agentMessage({ id: "m_request", traceId: "trc_request", status: "pending", text: "" }),
agentMessage({ id: "m_observed", traceId: "trc_observed", status: "running", text: "thinking" })
],
currentRequestTraceId: "trc_request",
turnStatusAuthority: {},
limit: 2
});
assert.deepEqual(traceIds, ["trc_request", "trc_observed"]);
});
test("realtime event reducer classifies SSE payloads before store side effects", () => {
const trace = reduceWorkbenchRealtimeEvent(realtimeEvent({ type: "trace.event", traceId: "trc_1", event: { traceId: "trc_1", label: "delta" }, snapshot: { traceId: "trc_1", status: "running" }, entity: { family: "traceEvents", id: "trc_1:1", version: 1, projectionRevision: "prj_1" } }), "workbench.trace.event");
assert.equal(trace.activityLabel, "realtime:trace.event");
@@ -533,53 +485,6 @@ test("realtime event reducer classifies SSE payloads before store side effects",
assert.equal(error.diagnostic.traceId, "trc_2");
});
test("live transport connected and heartbeat frames do not extend business activity timeouts", () => {
assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "connected" }, "workbench.connected", true), false);
assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "heartbeat" }, "workbench.heartbeat", true), false);
assert.equal(workbenchRealtimeEventIsBusinessActivity({ schema: "hwlab.event.v1", event: { type: "assistant" } }, "hwlab.event.v1", true), true);
assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "trace.event" }, "workbench.trace.event", false), true);
});
test("live hwlab envelope projects assistant, tool/output, and terminal without replay or finalizer", () => {
const envelope = (event: Record<string, unknown>) => ({
schema: "hwlab.event.v1",
eventType: "hwlab.trace.event.projected",
eventId: `hwlab:${String(event.sourceEventId ?? event.type)}`,
hwlabSessionId: "ses_live_web",
sessionId: "ses_live_web",
traceId: "trc_live_web",
runId: "run_live_web",
commandId: "cmd_live_web",
context: { runId: "run_live_web", commandId: "cmd_live_web", valuesRedacted: true },
event
});
const assistantEnvelope = envelope({ type: "assistant", eventType: "assistant", sourceEventId: "evt_assistant", traceId: "trc_live_web", sessionId: "ses_live_web", status: "running", assistantText: "running increment", terminal: false });
const assistant = reduceWorkbenchRealtimeEvent(assistantEnvelope, "hwlab.event.v1");
assert.equal(assistant.action.type, "trace.event");
let visible = reduceWorkbenchLiveKafkaMessageState({ text: "", status: "running", terminal: false }, assistantEnvelope.event);
assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false });
const toolEnvelope = envelope({ type: "tool", eventType: "tool", sourceEventId: "evt_tool", traceId: "trc_live_web", sessionId: "ses_live_web", status: "completed", toolName: "commandExecution", outputSummary: "ok", terminal: false });
assert.equal(reduceWorkbenchRealtimeEvent(toolEnvelope, "hwlab.event.v1").action.type, "trace.event");
visible = reduceWorkbenchLiveKafkaMessageState(visible, toolEnvelope.event);
assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false });
const outputEnvelope = envelope({ type: "output", eventType: "status", sourceEventId: "evt_output", traceId: "trc_live_web", sessionId: "ses_live_web", status: "running", message: "stdout increment", terminal: false });
visible = reduceWorkbenchLiveKafkaMessageState(visible, outputEnvelope.event);
assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false });
const terminalEnvelope = envelope({ type: "result", eventType: "terminal", sourceEventId: "evt_terminal", traceId: "trc_live_web", sessionId: "ses_live_web", status: "completed", terminal: true });
visible = reduceWorkbenchLiveKafkaMessageState(visible, terminalEnvelope.event);
assert.deepEqual(visible, { text: "running increment", status: "completed", terminal: true });
assert.equal(workbenchAgentMessageIdForTrace("trc_live_web"), "msg_live_web_agent");
const failed = reduceWorkbenchLiveKafkaMessageState(
{ text: "partial response", status: "running", terminal: false },
{ type: "result", eventType: "terminal", status: "failed", message: "provider stream disconnected", terminal: true }
);
assert.deepEqual(failed, { text: "provider stream disconnected", status: "failed", terminal: true });
});
test("realtime apply planner turns reducer actions into store steps", () => {
const reduced = reduceWorkbenchRealtimeEvent(realtimeEvent({ type: "trace.event", traceId: "trc_1", event: { traceId: "trc_1", label: "delta" }, snapshot: { traceId: "trc_1", status: "running" }, entity: { family: "traceEvents", id: "trc_1:1", version: 1, projectionRevision: "prj_1" } }), "workbench.trace.event");
const tracePlan = planWorkbenchRealtimeApply(reduced.action);
@@ -270,7 +270,7 @@ test("Workbench sealed completed final body survives stale running session messa
assert.equal(selectActiveSession(state, "ses_final_seal")?.status, "completed");
});
test("Workbench session messages repair terminal zero timing from running projection", () => {
test("Workbench session messages preserve canonical terminal zero timing", () => {
let state = createWorkbenchServerState();
state = reduceWorkbenchServerState(state, {
type: "session.messages",
@@ -288,8 +288,8 @@ test("Workbench session messages repair terminal zero timing from running projec
const agent = selectActiveMessages(state, "ses_terminal_zero_repair").find((message) => message.role === "agent");
assert.equal(agent?.status, "completed");
assert.equal(agent?.durationMs, 6_000);
assert.equal(agent?.timing?.durationMs, 6_000);
assert.equal(agent?.durationMs, 0);
assert.equal(agent?.timing?.durationMs, 0);
});
test("Workbench session messages can replace sealed terminal zero with positive timing", () => {
@@ -2,7 +2,6 @@ import assert from "node:assert/strict";
import { test } from "bun:test";
import { connectWorkbenchEvents, realtimeCoalesceKey, workbenchEventStreamPath, type WorkbenchRealtimeEvent, type WorkbenchSseIngressFrame } from "./workbench-events";
import { createCoalescedEventQueue } from "../utils/scheduler/coalesced-event-queue";
test("turn snapshot coalescing keys by trace instead of per sequence", () => {
const first: WorkbenchRealtimeEvent = { type: "turn.snapshot", turn: { sessionId: "ses_queue", traceId: "trc_queue", status: "running" }, cursor: { traceSeq: 10 } };
@@ -21,7 +20,6 @@ test("trace events keep sequence-specific coalescing keys", () => {
test("product EventSource resumes the transactional projection authority with one durable outbox cursor", () => {
assert.equal(
workbenchEventStreamPath({
realtimeCapabilities: { liveKafkaSse: true, kafkaRefreshReplay: false, projectionRealtime: true },
sessionId: "ses_projection_authority",
traceId: "trc_projection_authority",
afterSeq: 42
@@ -30,29 +28,7 @@ test("product EventSource resumes the transactional projection authority with on
);
});
test("live Kafka envelopes from one trace retain every assistant, tool, output, and terminal event", () => {
const delivered: WorkbenchRealtimeEvent[] = [];
const queue = createCoalescedEventQueue<WorkbenchRealtimeEvent>({
keyOf: (event) => realtimeCoalesceKey(event, "hwlab.event.v1"),
onFlush: (events) => delivered.push(...events)
});
for (const sourceEventId of ["evt_assistant", "evt_tool", "evt_output", "evt_terminal"]) {
queue.push({
schema: "hwlab.event.v1",
eventId: `hwlab:${sourceEventId}`,
sourceEventId,
sessionId: "ses_live_queue",
traceId: "trc_live_queue",
event: { sourceEventId, traceId: "trc_live_queue", sessionId: "ses_live_queue" }
});
}
queue.flush();
assert.deepEqual(delivered.map((event) => event.sourceEventId), ["evt_assistant", "evt_tool", "evt_output", "evt_terminal"]);
});
test("product EventSource tees every raw HWLAB frame once before decode and reducer delivery", async () => {
test("product EventSource tees projection frames once before reducer delivery", async () => {
const originalEventSource = globalThis.EventSource;
const ingress: WorkbenchSseIngressFrame[] = [];
const delivered: WorkbenchRealtimeEvent[] = [];
@@ -60,7 +36,6 @@ test("product EventSource tees every raw HWLAB frame once before decode and redu
globalThis.EventSource = FakeProductEventSource as unknown as typeof EventSource;
try {
const stream = connectWorkbenchEvents({
realtimeCapabilities: { liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: false },
sessionId: "ses_raw_tee",
flushYieldMs: 1,
onIngress: (frame) => ingress.push(frame),
@@ -69,17 +44,17 @@ test("product EventSource tees every raw HWLAB frame once before decode and redu
assert.ok(stream);
const source = FakeProductEventSource.instances[0];
assert.ok(source);
source.emitRaw("hwlab.event.v1", '{"schema":"hwlab.event.v1","eventId":"evt_raw_tee"}');
source.emitRaw("hwlab.event.v1", "{invalid-json");
source.emitRaw("workbench.trace.event", '{"type":"trace.event","traceId":"trc_raw_tee","event":{"traceId":"trc_raw_tee","projectedSeq":1}}');
source.emitRaw("workbench.trace.event", "{invalid-json");
await new Promise((resolve) => setTimeout(resolve, 10));
assert.equal(FakeProductEventSource.instances.length, 1);
assert.deepEqual(ingress.map((frame) => [frame.eventName, frame.decodeStatus]), [
["hwlab.event.v1", "accepted"],
["hwlab.event.v1", "invalid-json"]
["workbench.trace.event", "accepted"],
["workbench.trace.event", "invalid-json"]
]);
assert.equal(delivered.length, 1);
assert.equal(delivered[0]?.eventId, "evt_raw_tee");
assert.equal(delivered[0]?.traceId, "trc_raw_tee");
stream.close();
} finally {
globalThis.EventSource = originalEventSource;
@@ -6,7 +6,6 @@ import type { ApiResult, ChatMessage, ProjectionDiagnostic, TraceEvent } from "@
import { createCoalescedEventQueue } from "@/utils/scheduler/coalesced-event-queue";
import { composeWorkbenchScopedKey, firstScopePart } from "@/utils/workbench-key";
import { recordWorkbenchRuntimeDiagnostic, recordWorkbenchSseLifecycle } from "@/utils/workbench-performance";
import type { WorkbenchRealtimeCapabilities } from "@/config/runtime";
import { decodeWorkbenchRealtimeEventFrame } from "./workbench-realtime-codec";
export interface WorkbenchRealtimeTraceSnapshot {
@@ -29,11 +28,6 @@ export interface WorkbenchRealtimeEvent {
eventType?: string | null;
eventId?: string | null;
sourceEventId?: string | null;
capabilities?: WorkbenchRealtimeCapabilities | null;
liveOnly?: boolean | null;
replay?: boolean | null;
deliverySemantics?: string | null;
lossPossible?: boolean | null;
type?: string;
contractVersion?: string | null;
realtimeAuthority?: string | null;
@@ -76,7 +70,6 @@ export interface WorkbenchRealtimeEvent {
}
export interface WorkbenchEventStreamOptions {
realtimeCapabilities: WorkbenchRealtimeCapabilities;
sessionId?: string | null;
traceId?: string | null;
afterSeq?: number | null;
@@ -105,7 +98,6 @@ interface QueuedRealtimeEvent {
}
const WORKBENCH_EVENT_NAMES = [
"hwlab.event.v1",
"workbench.connected",
"workbench.trace.snapshot",
"workbench.trace.event",
@@ -183,20 +175,12 @@ export function connectWorkbenchEvents(options: WorkbenchEventStreamOptions): Wo
};
}
export function workbenchEventStreamPath(options: Pick<WorkbenchEventStreamOptions, "realtimeCapabilities" | "sessionId" | "traceId" | "afterSeq">): string {
const params = new URLSearchParams();
appendParam(params, "sessionId", options.sessionId);
appendParam(params, "traceId", options.traceId);
if (options.realtimeCapabilities.projectionRealtime) appendNumberParam(params, "afterSeq", options.afterSeq);
return `/v1/workbench/events?${params.toString()}`;
}
export function workbenchProjectionEventStreamPath(options: Pick<WorkbenchEventStreamOptions, "sessionId" | "traceId" | "afterSeq">): string {
export function workbenchEventStreamPath(options: Pick<WorkbenchEventStreamOptions, "sessionId" | "traceId" | "afterSeq">): string {
const params = new URLSearchParams();
appendParam(params, "sessionId", options.sessionId);
appendParam(params, "traceId", options.traceId);
appendNumberParam(params, "afterSeq", options.afterSeq);
return `/v1/workbench/projection-events?${params.toString()}`;
return `/v1/workbench/events?${params.toString()}`;
}
function appendParam(params: URLSearchParams, key: string, value: string | null | undefined): void {
@@ -228,10 +212,6 @@ function scheduleRealtimeFlushYield(flush: () => void, yieldMs: number | null |
}
export function realtimeCoalesceKey(event: WorkbenchRealtimeEvent, eventName: string): string | null {
if (eventName === "hwlab.event.v1" || event.schema === "hwlab.event.v1") {
const eventId = firstScopePart(event.eventId, event.sourceEventId, event.event?.sourceEventId);
return eventId ? composeWorkbenchScopedKey("workbench.sse.live-kafka-event", eventId) : null;
}
const entity = event.entity;
const entityFamily = firstScopePart(entity?.family);
const entityId = firstScopePart(entity?.id);
-44
View File
@@ -129,7 +129,6 @@ export const workbenchAPI = {
return fetchJson(sessionDetailPath(sessionId, detailOptions), { timeoutMs: requestTimeoutMs(detailOptions.timeoutMs, SESSION_DETAIL_TIMEOUT_MS), timeoutName: "workbench session" });
},
sessionMessages: (sessionId: string, options: SessionMessageOptions = {}): Promise<ApiResult<WorkbenchMessagePageResponse>> => fetchJson(sessionMessagesPath(sessionId, options), { timeoutMs: requestTimeoutMs(options.timeoutMs, SESSION_DETAIL_TIMEOUT_MS), timeoutName: "workbench session messages" }),
turn: async (traceId: string, timeoutMs = 8000, activityRef?: ApiRequestOptions["activityRef"]): Promise<ApiResult<AgentChatResultResponse>> => normalizeWorkbenchTurnResult(await fetchJson<Record<string, unknown>>(`/v1/workbench/turns/${encodeURIComponent(traceId)}`, { timeoutMs, timeoutName: "workbench turn", activityRef }), traceId),
traceEvents: async (traceId: string, timeoutMs = 8000, activityRef?: ApiRequestOptions["activityRef"], options?: WorkbenchTraceRequestOptions): Promise<ApiResult<AgentChatResultResponse>> => normalizeWorkbenchTraceResult(await fetchJson<Record<string, unknown>>(workbenchTracePath(traceId, options), { timeoutMs, timeoutName: "workbench trace", activityRef }), traceId)
};
@@ -186,42 +185,6 @@ function workbenchTracePath(traceId: string, options: WorkbenchTraceRequestOptio
return `/v1/workbench/traces/${encodeURIComponent(traceId)}/events${query ? `?${query}` : ""}`;
}
function normalizeWorkbenchTurnResult(result: ApiResult<Record<string, unknown>>, traceId: string): ApiResult<AgentChatResultResponse> {
if (!result.ok || !result.data) return result as ApiResult<AgentChatResultResponse>;
const turn = recordValue(result.data.turn) ?? result.data;
const trace = recordValue(turn.trace);
const status = normalizeStatus(textValue(turn.status)) ?? "unknown";
const projection = normalizeProjectionDiagnostic(result.data.projection, turn.projection, result.data);
const timing = normalizeTimingProjection(turn);
return {
...result,
data: {
...turn,
projection,
timing,
startedAt: timing?.startedAt ?? undefined,
lastEventAt: timing?.lastEventAt ?? undefined,
finishedAt: timing?.finishedAt ?? undefined,
durationMs: timing?.durationMs ?? undefined,
projectionStatus: projection?.projectionStatus ?? textValue(result.data.projectionStatus) ?? undefined,
projectionHealth: projection?.projectionHealth ?? textValue(result.data.projectionHealth) ?? undefined,
lastProjectedSeq: Number.isFinite(Number(result.data.lastProjectedSeq)) ? Number(result.data.lastProjectedSeq) : undefined,
sourceRunId: projection?.sourceRunId ?? textValue(result.data.sourceRunId) ?? undefined,
sourceCommandId: projection?.sourceCommandId ?? textValue(result.data.sourceCommandId) ?? undefined,
staleMs: projection?.staleMs ?? numberValue(result.data.staleMs) ?? undefined,
blocker: projection?.blocker ?? recordValue(result.data.blocker) ?? undefined,
traceId: textValue(turn.traceId) ?? traceId,
status,
running: typeof turn.running === "boolean" ? turn.running : undefined,
terminal: typeof turn.terminal === "boolean" ? turn.terminal : undefined,
sessionId: textValue(turn.sessionId) ?? undefined,
threadId: textValue(turn.threadId) ?? undefined,
agentRun: recordValue(turn.agentRun) ?? undefined,
updatedAt: textValue(turn.updatedAt) ?? textValue(result.data.observedAt) ?? undefined
} as AgentChatResultResponse
};
}
function normalizeWorkbenchTraceResult(result: ApiResult<Record<string, unknown>>, traceId: string): ApiResult<AgentChatResultResponse> {
if (!result.ok || !result.data) return result as ApiResult<AgentChatResultResponse>;
const events = Array.isArray(result.data.events) ? result.data.events : [];
@@ -308,13 +271,6 @@ function normalizeTimingProjection(value: unknown): WorkbenchTurnTimingProjectio
} as WorkbenchTurnTimingProjection;
}
function normalizeStatus(value: unknown): string | null {
const text = textValue(value);
if (!text) return null;
const normalized = text.toLowerCase().replace(/_/gu, "-");
return normalized === "cancelled" ? "canceled" : normalized;
}
function recordValue(value: unknown): Record<string, unknown> | null {
return value && typeof value === "object" ? value as Record<string, unknown> : null;
}
-14
View File
@@ -10,12 +10,6 @@ export interface OpenCodeFrameConfig {
url: string;
}
export interface WorkbenchRealtimeCapabilities {
liveKafkaSse: boolean;
kafkaRefreshReplay: boolean;
projectionRealtime: boolean;
}
export interface WorkbenchDebugCapabilities {
isolatedKafka: boolean;
rawHwlabEventWindow: WorkbenchRawHwlabEventWindowCapability;
@@ -97,14 +91,6 @@ export function workbenchTraceTimelinePolicy(): WorkbenchTraceTimelinePolicy {
};
}
export function workbenchRealtimeCapabilities(): WorkbenchRealtimeCapabilities {
const features = window.HWLAB_CLOUD_WEB_CONFIG?.workbench?.realtimeFeatures;
if (typeof features?.liveKafkaSse !== "boolean" || typeof features?.kafkaRefreshReplay !== "boolean" || typeof features?.projectionRealtime !== "boolean") {
throw new Error("HWLAB Cloud Web workbench.realtimeFeatures are required");
}
return { liveKafkaSse: features.liveKafkaSse, kafkaRefreshReplay: features.kafkaRefreshReplay, projectionRealtime: features.projectionRealtime };
}
export function workbenchDebugCapabilities(): WorkbenchDebugCapabilities {
const features = window.HWLAB_CLOUD_WEB_CONFIG?.workbench?.debugCapabilities;
const rawWindow = features?.rawHwlabEventWindow;
@@ -14,16 +14,12 @@ export interface WorkbenchRuntimePolicy {
sessionListPageLimit: number;
workbenchDetailReadConcurrency: number;
workbenchReadFailureCooldownMs: number;
workbenchTurnStatusMinRefreshMs: number;
workbenchTraceEventsMinRefreshMs: number;
workbenchSessionDetailMinRefreshMs: number;
workbenchSessionMessagesMinRefreshMs: number;
workbenchSessionMessagesWindowLimit: number;
workbenchTraceMessagesWindowLimit: number;
workbenchRealtimeSessionMessagesMinRefreshMs: number;
workbenchTraceEventsTimeoutMs: number;
sessionListRealtimeRefreshDelayMs: number;
sessionListTerminalRefreshDelayMs: number;
sessionListMinRefreshIntervalMs: number;
workbenchRealtimeErrorSyncReplayMinMs: number;
workbenchRealtimeFlushMaxItemsPerChunk: number;
@@ -44,16 +40,12 @@ const DEFAULT_WORKBENCH_RUNTIME_POLICY: WorkbenchRuntimePolicy = Object.freeze({
sessionListPageLimit: 20,
workbenchDetailReadConcurrency: 3,
workbenchReadFailureCooldownMs: 5_000,
workbenchTurnStatusMinRefreshMs: 2_000,
workbenchTraceEventsMinRefreshMs: 4_000,
workbenchSessionDetailMinRefreshMs: 5_000,
workbenchSessionMessagesMinRefreshMs: 5_000,
workbenchSessionMessagesWindowLimit: 20,
workbenchTraceMessagesWindowLimit: 8,
workbenchRealtimeSessionMessagesMinRefreshMs: 1_000,
workbenchTraceEventsTimeoutMs: 5_000,
sessionListRealtimeRefreshDelayMs: 5_000,
sessionListTerminalRefreshDelayMs: 1_500,
sessionListMinRefreshIntervalMs: 15_000,
workbenchRealtimeErrorSyncReplayMinMs: 2_000,
workbenchRealtimeFlushMaxItemsPerChunk: 4,
@@ -76,16 +68,12 @@ export function workbenchRuntimePolicy(input: unknown = runtimePolicyConfig()):
sessionListPageLimit: positiveInteger(source.sessionListPageLimit, DEFAULT_WORKBENCH_RUNTIME_POLICY.sessionListPageLimit),
workbenchDetailReadConcurrency: positiveInteger(source.workbenchDetailReadConcurrency ?? source.workbenchReadHydrationConcurrency, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchDetailReadConcurrency),
workbenchReadFailureCooldownMs: positiveNumber(source.workbenchReadFailureCooldownMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchReadFailureCooldownMs),
workbenchTurnStatusMinRefreshMs: nonNegativeNumber(source.workbenchTurnStatusMinRefreshMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchTurnStatusMinRefreshMs),
workbenchTraceEventsMinRefreshMs: nonNegativeNumber(source.workbenchTraceEventsMinRefreshMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchTraceEventsMinRefreshMs),
workbenchSessionDetailMinRefreshMs: nonNegativeNumber(source.workbenchSessionDetailMinRefreshMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchSessionDetailMinRefreshMs),
workbenchSessionMessagesMinRefreshMs: nonNegativeNumber(source.workbenchSessionMessagesMinRefreshMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchSessionMessagesMinRefreshMs),
workbenchSessionMessagesWindowLimit: positiveInteger(source.workbenchSessionMessagesWindowLimit, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchSessionMessagesWindowLimit),
workbenchTraceMessagesWindowLimit: positiveInteger(source.workbenchTraceMessagesWindowLimit, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchTraceMessagesWindowLimit),
workbenchRealtimeSessionMessagesMinRefreshMs: nonNegativeNumber(source.workbenchRealtimeSessionMessagesMinRefreshMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeSessionMessagesMinRefreshMs),
workbenchTraceEventsTimeoutMs: positiveNumber(source.workbenchTraceEventsTimeoutMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchTraceEventsTimeoutMs),
sessionListRealtimeRefreshDelayMs: nonNegativeNumber(source.sessionListRealtimeRefreshDelayMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.sessionListRealtimeRefreshDelayMs),
sessionListTerminalRefreshDelayMs: nonNegativeNumber(source.sessionListTerminalRefreshDelayMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.sessionListTerminalRefreshDelayMs),
sessionListMinRefreshIntervalMs: nonNegativeNumber(source.sessionListMinRefreshIntervalMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.sessionListMinRefreshIntervalMs),
workbenchRealtimeErrorSyncReplayMinMs: nonNegativeNumber(source.workbenchRealtimeErrorSyncReplayMinMs ?? source.workbenchRealtimeErrorGapFillMinMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeErrorSyncReplayMinMs),
workbenchRealtimeFlushMaxItemsPerChunk: positiveInteger(source.workbenchRealtimeFlushMaxItemsPerChunk, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeFlushMaxItemsPerChunk),
@@ -31,8 +31,6 @@ export const workbenchColadaKeys = {
historySessionMessagesRoot: (): EntryKey => ["workbench", "history", "session-messages"],
historySessionMessages: (sessionId: string, input: WorkbenchSessionMessagesKeyInput = {}): EntryKey => ["workbench", "history", "session-messages", sessionId, normalizeKeyObject({ cursor: input.cursor ?? null, limit: finiteNumber(input.limit) })],
detailRoot: (): EntryKey => ["workbench", "detail"],
detailTurnRoot: (): EntryKey => ["workbench", "detail", "turn"],
detailTurn: (traceId: string): EntryKey => ["workbench", "detail", "turn", traceId],
detailTraceEventsRoot: (): EntryKey => ["workbench", "detail", "trace-events"],
detailTraceEvents: (traceId: string, input: WorkbenchTraceEventsKeyInput = {}): EntryKey => ["workbench", "detail", "trace-events", traceId, normalizeKeyObject({ afterProjectedSeq: finiteNumber(input.afterProjectedSeq), limit: finiteNumber(input.limit) })],
sessionsRoot: (): EntryKey => workbenchColadaKeys.snapshotSessionsRoot(),
@@ -41,8 +39,6 @@ export const workbenchColadaKeys = {
session: (sessionId: string): EntryKey => workbenchColadaKeys.snapshotSession(sessionId),
sessionMessagesRoot: (): EntryKey => workbenchColadaKeys.historySessionMessagesRoot(),
sessionMessages: (sessionId: string, input: WorkbenchSessionMessagesKeyInput = {}): EntryKey => workbenchColadaKeys.historySessionMessages(sessionId, input),
turnRoot: (): EntryKey => workbenchColadaKeys.detailTurnRoot(),
turn: (traceId: string): EntryKey => workbenchColadaKeys.detailTurn(traceId),
traceEventsRoot: (): EntryKey => workbenchColadaKeys.detailTraceEventsRoot(),
traceEvents: (traceId: string, input: WorkbenchTraceEventsKeyInput = {}): EntryKey => workbenchColadaKeys.detailTraceEvents(traceId, input),
providerProfiles: (): EntryKey => ["workbench", "provider-profiles"],
@@ -3,7 +3,6 @@
import { useQueryCache, type EntryKey, type QueryCache } from "@pinia/colada";
import { api } from "@/api";
import type { ApiRequestOptions } from "@/api/client";
import type { AgentChatResultResponse, ApiResult } from "@/types";
import type { SessionListOptions, SessionMessageOptions, WorkbenchMessagePageResponse, WorkbenchSessionDetailResponse, WorkbenchSessionListResponse, WorkbenchTraceRequestOptions } from "@/api/workbench";
import { workbenchColadaKeys } from "./workbench-colada-keys";
@@ -13,14 +12,9 @@ interface QueryRunOptions {
minIntervalMs?: number | null;
}
export interface WorkbenchTurnQueryOptions extends QueryRunOptions {
timeoutMs?: number;
activityRef?: ApiRequestOptions["activityRef"];
}
export interface WorkbenchTraceEventsQueryOptions extends QueryRunOptions, WorkbenchTraceRequestOptions {
timeoutMs?: number;
activityRef?: ApiRequestOptions["activityRef"];
activityRef?: import("@/api/client").ApiRequestOptions["activityRef"];
}
export interface WorkbenchSessionDetailQueryOptions extends QueryRunOptions {
@@ -33,13 +27,10 @@ export interface WorkbenchColadaQueries {
fetchSessions: (options?: SessionListOptions & QueryRunOptions) => Promise<ApiResult<WorkbenchSessionListResponse>>;
fetchSession: (sessionId: string, options?: WorkbenchSessionDetailQueryOptions) => Promise<ApiResult<WorkbenchSessionDetailResponse>>;
fetchSessionMessages: (sessionId: string, options?: SessionMessageOptions & QueryRunOptions) => Promise<ApiResult<WorkbenchMessagePageResponse>>;
fetchTurn: (traceId: string, options?: WorkbenchTurnQueryOptions) => Promise<ApiResult<AgentChatResultResponse>>;
fetchTraceEvents: (traceId: string, options?: WorkbenchTraceEventsQueryOptions) => Promise<ApiResult<AgentChatResultResponse>>;
fetchProviderProfiles: (options?: QueryRunOptions) => Promise<ApiResult<unknown>>;
invalidateSessionList: () => Promise<unknown>;
invalidateSession: (sessionId: string | null | undefined) => Promise<unknown>;
invalidateSessionMessages: (sessionId: string | null | undefined) => Promise<unknown>;
invalidateTurn: (traceId: string | null | undefined) => Promise<unknown>;
invalidateTraceEvents: (traceId: string | null | undefined) => Promise<unknown>;
invalidateTraceScope: (input: { sessionId?: string | null; traceId?: string | null }) => Promise<unknown[]>;
}
@@ -52,13 +43,10 @@ export function useWorkbenchColadaQueries(): WorkbenchColadaQueries {
fetchSessions: (options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.sessions(options), () => api.workbench.sessions(options), options),
fetchSession: (sessionId, options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.session(sessionId), () => api.workbench.session(sessionId, { timeoutMs: options.timeoutMs ?? null, includeMessages: false }), options),
fetchSessionMessages: (sessionId, options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.sessionMessages(sessionId, options), () => api.workbench.sessionMessages(sessionId, options), options),
fetchTurn: (traceId, options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.turn(traceId), () => api.workbench.turn(traceId, options.timeoutMs ?? 8000, options.activityRef), options),
fetchTraceEvents: (traceId, options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.traceEvents(traceId, options), () => api.workbench.traceEvents(traceId, options.timeoutMs ?? 8000, options.activityRef, { afterProjectedSeq: options.afterProjectedSeq, limit: options.limit }), options),
fetchProviderProfiles: (options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.providerProfiles(), () => api.providerProfiles.catalog(), options),
invalidateSessionList: () => queryCache.invalidateQueries({ key: workbenchColadaKeys.sessionsRoot() }),
invalidateSession: (sessionId) => sessionId ? queryCache.invalidateQueries({ key: workbenchColadaKeys.session(sessionId) }) : Promise.resolve(),
invalidateSessionMessages: (sessionId) => sessionId ? queryCache.invalidateQueries({ key: workbenchColadaKeys.sessionMessagesRoot().concat(sessionId) }) : Promise.resolve(),
invalidateTurn: (traceId) => traceId ? queryCache.invalidateQueries({ key: workbenchColadaKeys.turn(traceId) }) : Promise.resolve(),
invalidateTraceEvents: (traceId) => traceId ? queryCache.invalidateQueries({ key: workbenchColadaKeys.traceEventsRoot().concat(traceId) }) : Promise.resolve(),
invalidateTraceScope: async (input) => Promise.all([
queryCache.invalidateQueries({ key: workbenchColadaKeys.snapshotRoot() }),
@@ -48,15 +48,7 @@ export function reduceWorkbenchRealtimeEvent(event: WorkbenchRealtimeEvent, even
};
}
export function workbenchRealtimeEventIsBusinessActivity(event: WorkbenchRealtimeEvent, eventName: string, liveKafkaSse: boolean): boolean {
if (!liveKafkaSse) return true;
return eventName === "hwlab.event.v1" || event.schema === "hwlab.event.v1";
}
function reduceRealtimeAction(event: WorkbenchRealtimeEvent, eventName: string): WorkbenchRealtimeAction {
if (event.schema === "hwlab.event.v1" && event.event) {
return { type: "trace.event", traceId: realtimeTraceId(event), event: event.event, snapshot: null, realtimeEvent: event };
}
const authority = primaryAuthority(event);
if (authority) return authority;
switch (event.type) {
@@ -5,9 +5,7 @@ import type { WorkbenchKafkaSseDebugEvent } from "@/api/workbench-debug";
import type { WorkbenchRealtimeEvent } from "@/api/workbench-events";
import type { ChatMessage } from "@/types";
import { firstNonEmptyString } from "@/utils";
import { reduceWorkbenchRealtimeEvent } from "./workbench-event-reducer";
import { projectWorkbenchLiveKafkaMessage } from "./workbench-live-kafka-event";
import { planWorkbenchRealtimeApply } from "./workbench-realtime-plan";
export interface WorkbenchIsolatedKafkaDebugLog {
id: string;
@@ -48,22 +46,17 @@ export function applyWorkbenchIsolatedKafkaDebugEvent(
if (!traceId) return rejected(base, debugEvent, null, "debug-envelope-trace-missing");
if (expected && traceId !== expected) return rejected(base, debugEvent, traceId, "debug-envelope-trace-mismatch");
const event = { ...debugEvent, schema: "hwlab.event.v1" } as WorkbenchRealtimeEvent;
const reduced = reduceWorkbenchRealtimeEvent(event, "hwlab.event.v1");
const plan = planWorkbenchRealtimeApply(reduced.action);
const traceStep = plan.steps.find((step) => step.type === "apply-trace-event");
if (!traceStep || traceStep.type !== "apply-trace-event" || !traceStep.event) {
return withLog(base, event, traceId, reduced.action.type, plan.steps.map((step) => step.type), false, reduced.action.type === "ignore" ? reduced.action.reason : "debug-envelope-not-a-trace-event");
}
const traceEvent = debugEvent.event;
if (!traceEvent) return withLog(base, debugEvent, traceId, "ignore", [], false, "debug-envelope-not-a-trace-event");
const message = projectWorkbenchLiveKafkaMessage({
previous: state.message,
traceId,
sessionId: firstNonEmptyString(event.hwlabSessionId, event.sessionId, traceStep.event.sessionId, state.message?.sessionId) ?? "ses_workbench_isolated_debug",
event: traceStep.event,
sessionId: firstNonEmptyString(debugEvent.hwlabSessionId, debugEvent.sessionId, traceEvent.sessionId, state.message?.sessionId) ?? "ses_workbench_isolated_debug",
event: traceEvent,
receivedAt: new Date().toISOString(),
title: "Code Agent · 隔离调试"
});
return withLog({ ...base, message, appliedCount: state.appliedCount + 1, error: null }, event, traceId, reduced.action.type, plan.steps.map((step) => step.type), true, null);
return withLog({ ...base, message, appliedCount: state.appliedCount + 1, error: null }, debugEvent, traceId, "debug.trace.event", ["debug-project-message"], true, null);
}
export function workbenchCurrentDebugTraceId(messages: ChatMessage[], sessionLastTraceId?: string | null): string | null {
@@ -1,28 +0,0 @@
import assert from "node:assert/strict";
import { test } from "bun:test";
import { workbenchHistoryAuthorityPolicy } from "./workbench-kafka-refresh-policy";
test("every capability combination keeps automatic HTTP terminal and history readers disabled", () => {
for (const liveKafkaSse of [false, true]) {
for (const kafkaRefreshReplay of [false, true]) {
for (const projectionRealtime of [false, true]) {
const policy = workbenchHistoryAuthorityPolicy({ liveKafkaSse, kafkaRefreshReplay, projectionRealtime });
const calls = { fetchSessionMessagesPage: 0, fetchTurn: 0, fetchTraceEvents: 0 };
if (policy.sessionMessagesHydrate) calls.fetchSessionMessagesPage += 1;
if (policy.turnStatusHydrate) calls.fetchTurn += 1;
if (policy.traceEventsHydrate) calls.fetchTraceEvents += 1;
assert.deepEqual(policy, {
kafkaRetention: false,
sessionMetadataRead: true,
sessionMessagesHydrate: false,
turnStatusHydrate: false,
traceEventsHydrate: false,
syncReplay: projectionRealtime
});
assert.deepEqual(calls, { fetchSessionMessagesPage: 0, fetchTurn: 0, fetchTraceEvents: 0 });
}
}
}
});
@@ -1,21 +0,0 @@
import type { WorkbenchRealtimeCapabilities } from "@/config/runtime";
export interface WorkbenchHistoryAuthorityPolicy {
kafkaRetention: boolean;
sessionMetadataRead: true;
sessionMessagesHydrate: false;
turnStatusHydrate: false;
traceEventsHydrate: false;
syncReplay: boolean;
}
export function workbenchHistoryAuthorityPolicy(capabilities: WorkbenchRealtimeCapabilities): WorkbenchHistoryAuthorityPolicy {
return {
kafkaRetention: false,
sessionMetadataRead: true,
sessionMessagesHydrate: false,
turnStatusHydrate: false,
traceEventsHydrate: false,
syncReplay: capabilities.projectionRealtime
};
}
@@ -1,5 +1,5 @@
// SPEC: PJ2026-0104010803 Workbench live Kafka SSE.
// Responsibility: project one transparent hwlab.event.v1 business event into visible turn state without replay/finalizer/polling.
// SPEC: PJ2026-0104010803 isolated Kafka debug fixture.
// Responsibility: project historical hwlab.event.debug.v1 fixtures for the admin-only isolated debugger; never imported by the product Workbench SSE path.
import { mergeRunnerTrace } from "../composables/workbench-trace-snapshot";
import type { ChatMessage, TraceEvent, WorkbenchTurnTimingProjection } from "../types";
@@ -137,17 +137,17 @@ test("workbench active terminal paths seal final response from turn authority",
const projectBlock = source.slice(source.indexOf("function projectTurnAuthorityToMessages"), source.indexOf("async function submitMessage"));
const realtimeTurnBlock = source.slice(source.indexOf("function applyRealtimeTurnSnapshot"), source.indexOf("function scheduleRealtimeTurnProjection"));
const realtimeTurnProjectionBlock = source.slice(source.indexOf("function flushRealtimeTurnProjection"), source.indexOf("function installRealtimeVisibilityHandler"));
const completeBlock = source.slice(source.indexOf("function completeTrace"), source.indexOf("async function hydrateTerminalMessageDiagnostics"));
const completeBlock = source.slice(source.indexOf("function completeTrace"), source.indexOf("function applyTerminalResultDiagnostics"));
const terminalDiagnosticsBlock = source.slice(source.indexOf("function applyTerminalResultDiagnostics"), source.indexOf("function applyRealtimeProjectionError"));
const crossTabSyncBlock = source.slice(source.indexOf("async function refreshCrossTabSessionFromSyncReplay"), source.indexOf("function completeTrace"));
const sessionDetailReadBlock = source.slice(source.indexOf("function fetchSessionDetailPage"), source.indexOf("function sessionMessageProjectionWindowLimit"));
const traceDetailReadBlock = source.slice(source.indexOf("async function readTraceEventsForExplicitDetailPages"), source.indexOf("async function fetchTraceDetailEventsPage"));
const loadBlock = source.slice(source.indexOf("async function loadWorkbenchSession"), source.indexOf("async function sealRestoredActiveTurnMessages"));
const restoreSealBlock = source.slice(source.indexOf("async function sealRestoredActiveTurnMessages"), source.indexOf("function reattachRestoredActiveTrace"));
const loadBlock = source.slice(source.indexOf("async function loadWorkbenchSession"), source.indexOf("function reattachRestoredActiveTrace"));
assert.match(projectBlock, /terminalAuthorityMessageFromTurnResult\(result\)/u);
assert.match(projectBlock, /type:\s*"message\.upsert"/u);
assert.match(projectBlock, /\.\.\.terminalPatch/u);
assert.match(source, /createKeyedSingleflight<ApiResult<AgentChatResultResponse>>/u);
assert.match(source, /const traceDetailReadSingleflight = createKeyedSingleflight<void>\(\)/u);
assert.match(source, /traceDetailReadSingleflight\.run/u);
assert.doesNotMatch(source, new RegExp(["refresh", "RealtimeSessionMessages"].join(""), "u"));
assert.match(realtimeTurnBlock, /rememberTurnStatus\(traceId, result\)[\s\S]*scheduleRealtimeTurnProjection\(\{ traceId, result, terminalTurn \}\)/u);
@@ -156,7 +156,8 @@ test("workbench active terminal paths seal final response from turn authority",
assert.match(realtimeTurnProjectionBlock, /workbench_realtime_turn_projection_budget/u);
assert.doesNotMatch(realtimeTurnProjectionBlock, /refreshWorkbenchSyncReplay|refreshTerminalTraceFromSyncReplay/u);
assert.match(completeBlock, /projectTurnAuthorityToMessages\(traceId, result, "complete-trace"\)/u);
assert.match(completeBlock, /options\.forceRead[\s\S]*refreshMessageProjectionForTrace\(ownerSessionId, traceId, \{ force: true \}\)/u);
assert.doesNotMatch(completeBlock, /forceRead|refreshMessageProjectionForTrace|fetchWorkbenchTurnStatus|refreshTurnStatusByTraceId/u);
assert.doesNotMatch(terminalDiagnosticsBlock, /readTraceEventsForMessages|fetchSessionMessagesPage|fetchWorkbenchTurnStatus/u);
assert.doesNotMatch(completeBlock, /scheduleActiveTraceSyncReplay|refreshWorkbenchSyncReplay/u);
assert.match(source.slice(source.indexOf("function handleWorkbenchProjectionSignal"), source.indexOf("function applyTraceSnapshot")), /restartRealtime\("cross-tab-session-projection", true\)/u);
assert.doesNotMatch(source, /scheduleActiveTraceSyncReplay|refreshActiveTraceFromSyncReplay/u);
@@ -169,10 +170,10 @@ test("workbench active terminal paths seal final response from turn authority",
assert.match(loadBlock, /fetchSessionDetailPage\(requestId, \{ reason: "load-session:detail", force: true \}\)/u);
assert.match(loadBlock, /sessionFromWorkbenchSession\(detail\.data\?\.session, \{ includeMessages: false \}\)/u);
assert.match(loadBlock, /const fallbackMessages = seed\?\.sessionId === id \? seed\.messages \?\? \[\] : \[\];/u);
assert.match(loadBlock, /fetchSessionMessagesPage\(normalizedRequestId, \{ limit: messageLimit, reason: "load-session:messages", force: true \}\)/u);
assert.doesNotMatch(loadBlock, /messages:\s*\[\]/u);
assert.doesNotMatch(loadBlock, /limit:\s*100/u);
assert.match(restoreSealBlock, /workbench_restored_turn_auto_read_disabled/u);
assert.doesNotMatch(restoreSealBlock, /fetchWorkbenchTurnStatus|Promise\.all/u);
assert.doesNotMatch(restoreSealBlock, /force:\s*true/u);
assert.doesNotMatch(source, /sealRestoredActiveTurnMessages|messageNeedsRestoredTurnSeal|refreshSessionStatusAuthority|readTerminalTraceDetailGaps/u);
});
test("workbench active SSE path only recovers by reconnecting the events cursor", () => {
@@ -181,8 +182,7 @@ test("workbench active SSE path only recovers by reconnecting the events cursor"
source.slice(source.indexOf("async function submitMessage"), source.indexOf("async function cancelAgentMessage")),
source.slice(source.indexOf("function reattachTrace"), source.indexOf("function restartRealtime")),
source.slice(source.indexOf("function flushRealtimeTurnProjection"), source.indexOf("function installRealtimeVisibilityHandler")),
source.slice(source.indexOf("function completeTrace"), source.indexOf("async function hydrateTerminalMessageDiagnostics")),
source.slice(source.indexOf("async function sealRestoredActiveTurnMessages"), source.indexOf("function reattachRestoredActiveTrace"))
source.slice(source.indexOf("function completeTrace"), source.indexOf("function applyTerminalResultDiagnostics"))
].join("\n");
assert.doesNotMatch(activeBlocks, /refreshWorkbenchSyncReplay|scheduleActiveTraceSyncReplay|setTimeout|setInterval/u);
@@ -1,13 +1,7 @@
import assert from "node:assert/strict";
import { test } from "bun:test";
import type { ChatMessage } from "@/types";
import { projectionMergeCommitSummary, sessionDetailAutoReadDecision, traceEventsAutoReadDecision, workbenchSessionDetailReadKey, workbenchSessionMessagesReadKey, workbenchTraceEventsReadKey } from "./workbench-session-messages-read-budget";
import { selectActiveTurnStatusRefreshTraceIds } from "./workbench-session";
function message(id: string): ChatMessage {
return { id, role: "agent", text: "", status: "running", createdAt: "2026-07-02T00:00:00.000Z" } as ChatMessage;
}
import { traceEventsAutoReadDecision, workbenchSessionDetailReadKey, workbenchSessionMessagesReadKey, workbenchTraceEventsReadKey } from "./workbench-session-messages-read-budget";
test("session messages singleflight key separates limit and force class", () => {
const limit8 = workbenchSessionMessagesReadKey({ sessionId: "ses_key", limit: 8, force: false });
@@ -18,7 +12,6 @@ test("session messages singleflight key separates limit and force class", () =>
assert.notEqual(limit20, force20);
assert.equal(limit20, workbenchSessionMessagesReadKey({ sessionId: "ses_key", limit: 20, force: false }));
});
test("session detail singleflight key separates force freshness class", () => {
const fresh = workbenchSessionDetailReadKey({ sessionId: "ses_key", force: false });
const forced = workbenchSessionDetailReadKey({ sessionId: "ses_key", force: true });
@@ -26,7 +19,6 @@ test("session detail singleflight key separates force freshness class", () => {
assert.notEqual(fresh, forced);
assert.equal(fresh, workbenchSessionDetailReadKey({ sessionId: "ses_key" }));
});
test("trace events auto read skips initial and completed equivalent ranges", () => {
const key = workbenchTraceEventsReadKey({ traceId: "trc_range", afterProjectedSeq: 12, limit: 50 });
@@ -35,25 +27,3 @@ test("trace events auto read skips initial and completed equivalent ranges", ()
assert.equal(traceEventsAutoReadDecision({ traceId: "trc_range", afterProjectedSeq: 12, limit: 50, cachedRange: { traceId: "trc_range", afterProjectedSeq: 12, limit: 50, nextProjectedSeq: 40, hasMore: true } }).read, true);
assert.equal(key, workbenchTraceEventsReadKey({ traceId: "trc_range", afterProjectedSeq: 12, limit: 50 }));
});
test("session detail auto read defers trace scoped realtime refresh to turn authority", () => {
assert.deepEqual(sessionDetailAutoReadDecision({ traceId: "trc_turn", nowMs: 10_000, minIntervalMs: 5_000 }), { read: false, reason: "turn-authority-first" });
assert.deepEqual(sessionDetailAutoReadDecision({ nowMs: 10_000, lastReadAtMs: 9_000, minIntervalMs: 5_000 }), { read: false, reason: "min-interval" });
assert.deepEqual(sessionDetailAutoReadDecision({ nowMs: 10_000, lastReadAtMs: 1_000, minIntervalMs: 5_000 }), { read: true, reason: "metadata-only" });
});
test("projection merge summary reports no write without blocking follow-up hydrate", () => {
const existing = [{ ...message("msg_same"), traceId: "trc_current", sessionId: "ses_same" }];
const summary = projectionMergeCommitSummary(existing, existing);
const hydrateTraceIds = selectActiveTurnStatusRefreshTraceIds({ messages: existing, currentRequestTraceId: "trc_current", limit: 1 });
assert.deepEqual(summary, { changed: false, changedCount: 0 });
assert.deepEqual(hydrateTraceIds, ["trc_current"]);
});
test("projection merge summary reports changed message references", () => {
const existing = [message("msg_old")];
const merged = [message("msg_new")];
assert.deepEqual(projectionMergeCommitSummary(existing, merged), { changed: true, changedCount: 1 });
});
@@ -1,7 +1,6 @@
// SPEC: pikasTech/HWLAB#2356 Workbench bounded request storm follow-up.
// Responsibility: Pure budgeting helpers for Workbench detail read de-duplication and projection merge writes.
import type { ChatMessage } from "@/types";
import { firstNonEmptyString } from "@/utils";
import { composeWorkbenchScopedKey } from "@/utils/workbench-key";
@@ -16,11 +15,6 @@ export interface SessionDetailReadKeyInput {
force?: boolean | null;
}
export interface ProjectionMergeCommitSummary {
changed: boolean;
changedCount: number;
}
export interface TraceEventsReadRangeInput {
traceId: string | null | undefined;
afterProjectedSeq: number | null | undefined;
@@ -38,15 +32,6 @@ export interface TraceEventsAutoReadDecisionInput extends TraceEventsReadRangeIn
cachedRange?: TraceEventsReadRangeRecord | null;
}
export interface SessionDetailAutoReadDecisionInput {
force?: boolean | null;
traceId?: string | null;
terminalBodyVisible?: boolean | null;
lastReadAtMs?: number | null;
nowMs: number;
minIntervalMs: number;
}
export interface ReadDetailDecision {
read: boolean;
reason: string;
@@ -89,25 +74,6 @@ export function traceEventsAutoReadDecision(input: TraceEventsAutoReadDecisionIn
return { read: true, reason: "delta-range" };
}
export function sessionDetailAutoReadDecision(input: SessionDetailAutoReadDecisionInput): ReadDetailDecision {
if (input.force === true) return { read: true, reason: "force" };
if (input.terminalBodyVisible === true) return { read: false, reason: "terminal-visible" };
if (firstNonEmptyString(input.traceId)) return { read: false, reason: "turn-authority-first" };
const lastReadAtMs = Number(input.lastReadAtMs);
const minIntervalMs = Math.max(0, Math.trunc(Number(input.minIntervalMs)));
if (Number.isFinite(lastReadAtMs) && input.nowMs - lastReadAtMs < minIntervalMs) return { read: false, reason: "min-interval" };
return { read: true, reason: "metadata-only" };
}
export function projectionMergeCommitSummary(existing: ChatMessage[], merged: ChatMessage[]): ProjectionMergeCommitSummary {
let changedCount = Math.max(0, merged.length - existing.length);
const sharedLength = Math.min(existing.length, merged.length);
for (let index = 0; index < sharedLength; index += 1) {
if (existing[index] !== merged[index]) changedCount += 1;
}
return { changed: changedCount > 0, changedCount };
}
function sessionMessagesReadLimitPart(value: number | null | undefined): number {
const number = Number(value);
return Number.isFinite(number) && number > 0 ? Math.trunc(number) : 1;
@@ -2,7 +2,7 @@ import assert from "node:assert/strict";
import { test } from "bun:test";
import type { ChatMessage, WorkbenchSessionRecord } from "../types";
import { resolveCancelableAgentMessage, resolveComposerState, selectActiveTurnStatusRefreshTraceIds, sessionToSessionTab, shouldReadTerminalTraceDetail } from "./workbench-session";
import { resolveCancelableAgentMessage, resolveComposerState, sessionToSessionTab, shouldReadTerminalTraceDetail } from "./workbench-session";
import type { TurnStatusAuthority } from "./workbench-session";
function agentMessage(input: Partial<ChatMessage> & Pick<ChatMessage, "id" | "traceId" | "sessionId">): ChatMessage {
@@ -96,22 +96,6 @@ test("cancel target remains available for failed message until terminal body exi
assert.equal(resolveCancelableAgentMessage({ messages: [sealed], targetTraceId: traceId, targetSessionId: sessionId, turnStatusAuthority: { [traceId]: turnStatus({ traceId, sessionId, status: "running", running: true, terminal: false }) } }), null);
});
test("turn status refresh selector keeps automatic hydrate bounded to the active trace", () => {
const sessionId = "ses_turn_hydrate_budget";
const older = agentMessage({ id: "msg_older", traceId: "trc_older", sessionId, status: "running" });
const current = agentMessage({ id: "msg_current", traceId: "trc_current", sessionId, status: "running" });
assert.deepEqual(selectActiveTurnStatusRefreshTraceIds({ messages: [older, current], currentRequestTraceId: current.traceId, limit: 1 }), [current.traceId]);
});
test("turn status refresh selector skips sealed terminal traces", () => {
const sessionId = "ses_turn_detail_sealed";
const traceId = "trc_sealed";
const sealed = agentMessage({ id: "msg_sealed", traceId, sessionId, status: "completed", text: "final" });
assert.deepEqual(selectActiveTurnStatusRefreshTraceIds({ messages: [sealed], currentRequestTraceId: traceId, turnStatusAuthority: { [traceId]: turnStatus({ traceId, sessionId, status: "completed", running: false, terminal: true }) }, limit: 1 }), []);
});
test("terminal detail read skips sealed terminal trace authority", () => {
const sessionId = "ses_terminal_detail_sealed";
const traceId = "trc_terminal_detail_sealed";
@@ -117,22 +117,6 @@ export function resolveCancelableAgentMessage(input: { messages: ChatMessage[];
return null;
}
export function selectActiveTurnStatusRefreshTraceIds(input: { messages: ChatMessage[]; currentRequestTraceId?: string | null; turnStatusAuthority?: TurnStatusAuthorityMap; limit?: number }): string[] {
const traceIds = new Set<string>();
const limit = Math.max(1, Math.trunc(input.limit ?? 3));
const currentRequestTraceId = firstNonEmptyString(input.currentRequestTraceId);
if (currentRequestTraceId && traceNeedsTurnStatusRefresh(currentRequestTraceId, input.messages, input.turnStatusAuthority)) traceIds.add(currentRequestTraceId);
if (traceIds.size >= limit) return [...traceIds];
for (const message of [...input.messages].reverse()) {
const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId);
if (!traceId || traceIds.has(traceId)) continue;
if (!messageHasActiveTurnStatusEvidence(message, traceId, currentRequestTraceId)) continue;
if (traceNeedsTurnStatusRefresh(traceId, input.messages, input.turnStatusAuthority)) traceIds.add(traceId);
if (traceIds.size >= limit) break;
}
return [...traceIds];
}
export function shouldReadTerminalTraceDetail(input: { traceId?: string | null; messages: ChatMessage[]; turnStatusAuthority?: TurnStatusAuthorityMap }): boolean {
const traceId = firstNonEmptyString(input.traceId);
if (!traceId) return false;
@@ -317,21 +301,6 @@ function latestMessageForTrace(messages: ChatMessage[], traceId: string): ChatMe
return [...messages].reverse().find((message) => firstNonEmptyString(message.traceId, message.runnerTrace?.traceId) === traceId) ?? null;
}
function messageHasActiveTurnStatusEvidence(message: ChatMessage, traceId: string, currentRequestTraceId: string | null): boolean {
if (currentRequestTraceId === traceId) return true;
return isActiveStatus(message.status) || message.traceAutoLifecycle === "running";
}
function traceNeedsTurnStatusRefresh(traceId: string | null | undefined, messages: ChatMessage[], turnStatusAuthority: TurnStatusAuthorityMap | undefined): boolean {
const id = firstNonEmptyString(traceId);
if (!id) return false;
const turn = turnStatusAuthority?.[id];
const message = latestMessageForTrace(messages, id);
if (turnStatusAuthorityIsSealed(turn, message)) return false;
if (messageIsSealedTerminal(message)) return false;
return true;
}
function resolveSessionTabStatus(session: WorkbenchSessionRecord, authority: SessionStatusAuthority | null | undefined, latestAgent: ChatMessage | null = latestAgentMessage(session.messages)): string {
const latestTraceId = firstNonEmptyString(latestAgent?.traceId, latestAgent?.runnerTrace?.traceId);
const authorityTraceId = firstNonEmptyString(authority?.lastTraceId);
+33 -341
View File
@@ -4,29 +4,27 @@
import { computed, nextTick, ref } from "vue";
import { defineStore } from "pinia";
import { api } from "@/api";
import { workbenchDebugCapabilities, workbenchRealtimeCapabilities } from "@/config/runtime";
import { workbenchDebugCapabilities } from "@/config/runtime";
import { workbenchRuntimePolicy } from "@/config/workbench-runtime-policy";
import { createKeyedSingleflight } from "@/utils/scheduler/keyed-singleflight";
import { createWorkbenchHealthProbeCache } from "@/utils/workbench-health";
import { agentErrorFromProjection, normalizeApiErrorRecord, normalizeErrorDiagnostic, normalizeProjectionDiagnostic, projectionDiagnosticFromApiFailure, projectionDiagnosticFromFailure } from "@/utils/workbench-error-runtime";
import { readWorkbenchJson, readWorkbenchNumber, readWorkbenchString, removeWorkbenchStorageKey, writeWorkbenchJson, writeWorkbenchString } from "@/utils/workbench-storage-runtime";
import { createWorkbenchStreamTransportRuntime, workbenchRealtimeTraceIdForCapabilities, type WorkbenchRealtimeEvent, type WorkbenchStreamTransportRecovery } from "@/utils/workbench-realtime-runtime";
import { createWorkbenchStreamTransportRuntime, workbenchRealtimeTraceId, type WorkbenchRealtimeEvent, type WorkbenchStreamTransportRecovery } from "@/utils/workbench-realtime-runtime";
import { mergeRunnerTrace, snapshotToRunnerTrace, type TraceSnapshot } from "@/composables/workbench-trace-snapshot";
import type { WorkbenchMessagePageResponse, WorkbenchSessionDetailResponse } from "@/api/workbench";
import type { AgentChatResponse, AgentChatResultResponse, AgentRunProvenance, ApiError, ApiResult, ChatMessage, ErrorDiagnostic, LiveSurface, ProjectionBlocker, ProjectionDiagnostic, ProviderProfile, TraceEvent, WorkbenchSessionRecord, WorkbenchTurnTimingProjection } from "@/types";
import { firstNonEmptyString, nextProtocolId, normalizeWorkbenchSessionId, normalizeWorkbenchSessionRouteId } from "@/utils";
import { composeWorkbenchScopedKey, workbenchRealtimeScopeKey } from "@/utils/workbench-key";
import { failWorkbenchSessionSwitch, failWorkbenchSubmitJourney, finishWorkbenchSessionSwitchFullLoad, markWorkbenchSubmitApiAccepted, markWorkbenchTraceEventsReceived, markWorkbenchTraceProjected, recordWorkbenchLoadingState, recordWorkbenchRuntimeDiagnostic, startWorkbenchSessionSwitch, startWorkbenchSubmitJourney } from "@/utils/workbench-performance";
import { RECENT_DRAFTS_STORAGE_KEY, appendSessionPage, defaultProviderProfileOptions, isArchivedSession, mergeSessionIntoList, normalizeChatMessageStatus, normalizeRecentDrafts, normalizeWorkbenchMessageTitle, providerProfileOptionsFromPayload, recordRecentDraft, resolveCancelableAgentMessage, resolveComposerState, selectActiveTurnStatusRefreshTraceIds, shouldReadTerminalTraceDetail, shouldShowSessionListLoading, sortSessionTabs, stableSessionList, type DraftEntry, type ProviderProfileOption, type TurnStatusAuthority } from "./workbench-session";
import { RECENT_DRAFTS_STORAGE_KEY, appendSessionPage, defaultProviderProfileOptions, isArchivedSession, mergeSessionIntoList, normalizeChatMessageStatus, normalizeRecentDrafts, normalizeWorkbenchMessageTitle, providerProfileOptionsFromPayload, recordRecentDraft, resolveCancelableAgentMessage, resolveComposerState, shouldReadTerminalTraceDetail, shouldShowSessionListLoading, sortSessionTabs, stableSessionList, type DraftEntry, type ProviderProfileOption, type TurnStatusAuthority } from "./workbench-session";
import { initialWorkbenchSessionIdFromLocation } from "./workbench-projection";
import { cleanupWorkbenchServerStateSessions, selectActiveMessages, selectActiveSession, selectSessionList, selectSessionStatusAuthority, selectTraceAuthorityById, selectTurnStatusAuthority, type WorkbenchServerAction } from "./workbench-server-state";
import { cleanupDroppedWorkbenchSessionCaches, trimWorkbenchSessionCache } from "./workbench-session-cache";
import { reduceWorkbenchRealtimeEvent, workbenchRealtimeEventIsBusinessActivity, type WorkbenchRealtimeAction } from "./workbench-event-reducer";
import { projectWorkbenchLiveKafkaMessage, projectWorkbenchLiveKafkaUserMessage, workbenchAgentMessageIdForTrace, workbenchLiveKafkaEnvelope, workbenchLiveKafkaProjectionTarget, workbenchUserMessageIdForTrace } from "./workbench-live-kafka-event";
import { reduceWorkbenchRealtimeEvent, type WorkbenchRealtimeAction } from "./workbench-event-reducer";
import { projectRejectedWorkbenchAdmission } from "./workbench-admission-failure";
import { workbenchHistoryAuthorityPolicy } from "./workbench-kafka-refresh-policy";
import { messageHasSealedTerminalResult, messageIsSealedTerminal, traceAuthorityIsSealed } from "./workbench-terminal-authority";
import { boundedProjectionMessageLimit, mergeBoundedProjectionMessages, selectProjectionMessageWindow, traceProjectionIsTerminalSealed } from "./workbench-message-projection-budget";
import { boundedProjectionMessageLimit } from "./workbench-message-projection-budget";
import {
agentErrorDisplayText,
agentErrorFromApiFailure,
@@ -39,7 +37,6 @@ import {
isTerminalMessageStatus,
isTraceActiveStatus,
messageHasTerminalResponse,
messageNeedsTerminalDiagnostics,
messageNeedsTraceDetailRead,
messageStatusPatchForTerminalMerge,
terminalAuthorityMessageFromTurnResult,
@@ -48,7 +45,6 @@ import {
messageTimingPatch,
messageTimingPatchForMerge,
messageTimingPatchFromProjection,
messageTimingSealPatchForProjectionMerge,
mergeTerminalResultTrace,
nonBlockingProjection,
normalizeAgentError,
@@ -65,7 +61,6 @@ import {
terminalMessageTimingPatchForNormalize,
turnResultIsTerminalForMerge,
turnResultStatusForMerge,
traceHasTerminalResponse,
traceHasEvents,
traceResultHasTerminalEvidence,
} from "./workbench-message-projection-runtime";
@@ -73,7 +68,7 @@ import { planWorkbenchRealtimeApply, planWorkbenchRealtimeRecovery, type Workben
import { useWorkbenchColadaMutations } from "./workbench-colada-mutations";
import { useWorkbenchColadaQueries } from "./workbench-colada-queries";
import { useWorkbenchColadaReducer } from "./workbench-colada-reducer";
import { projectionMergeCommitSummary, traceEventsAutoReadDecision, workbenchSessionDetailReadKey, workbenchSessionMessagesReadKey, workbenchTraceEventsReadKey, type TraceEventsReadRangeRecord } from "./workbench-session-messages-read-budget";
import { traceEventsAutoReadDecision, workbenchSessionDetailReadKey, workbenchSessionMessagesReadKey, workbenchTraceEventsReadKey, type TraceEventsReadRangeRecord } from "./workbench-session-messages-read-budget";
import { realtimeSnapshotToTraceSnapshot, terminalSealResultWithoutTraceEvents, traceDetailProjectedSeq, traceNextProjectedSeq } from "./workbench-trace-detail";
import { appendRawHwlabIngressFrame, createRawHwlabIngressState } from "./workbench-raw-hwlab-ingress";
@@ -108,15 +103,21 @@ interface RealtimeTurnProjectionItem {
terminalTurn: boolean;
}
function workbenchMessageIdForTrace(traceId: string, role: "user" | "agent"): string {
const suffix = firstNonEmptyString(traceId)
?.replace(/^trc_/u, "")
.replace(/[^A-Za-z0-9_.:-]/gu, "_")
.slice(0, 48);
if (!suffix) throw new Error("traceId must produce a stable Workbench message identity");
return `msg_${suffix}_${role}`;
}
export const useWorkbenchStore = defineStore("workbench", () => {
const runtimePolicy = workbenchRuntimePolicy();
const realtimeCapabilities = workbenchRealtimeCapabilities();
const historyAuthorityPolicy = workbenchHistoryAuthorityPolicy(realtimeCapabilities);
const debugCapabilities = workbenchDebugCapabilities();
const workbenchColadaReducer = useWorkbenchColadaReducer();
const workbenchColadaQueries = useWorkbenchColadaQueries();
const workbenchColadaMutations = useWorkbenchColadaMutations();
const turnStatusReadSingleflight = createKeyedSingleflight<ApiResult<AgentChatResultResponse>>();
const traceDetailReadSingleflight = createKeyedSingleflight<void>();
const sessionMessagesReadSingleflight = createKeyedSingleflight<ApiResult<WorkbenchMessagePageResponse>>();
const sessionDetailReadSingleflight = createKeyedSingleflight<ApiResult<WorkbenchSessionDetailResponse>>();
@@ -138,7 +139,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
const sessionListLoadingMore = ref(false);
const sessionListLoadMoreError = ref<string | null>(null);
const chatPending = ref(false);
const liveRealtimeReadySessionId = ref<string | null>(null);
const rawHwlabIngress = ref(createRawHwlabIngressState());
const error = ref<string | null>(null);
const activityRef = ref({ lastActivityAt: Date.now(), lastActivityIso: new Date().toISOString(), waitingFor: "idle", lastEventLabel: null as string | null });
@@ -169,7 +169,7 @@ export const useWorkbenchStore = defineStore("workbench", () => {
const sessionListLoadedCount = computed(() => sessionTabs.value.length);
const sessionListLoading = computed(() => shouldShowSessionListLoading({ loading: loading.value, sessionsReady: sessionsReady.value }));
const sessionDetailLoading = computed(() => Boolean(sessionDetailLoadingId.value || switchingSessionId.value || (loading.value && messages.value.length === 0)));
const composer = computed(() => resolveComposerState({ messages: messages.value, sessions: sessions.value, activeSessionId: activeSessionId.value, chatPending: chatPending.value, realtimeReady: !realtimeCapabilities.liveKafkaSse || liveRealtimeReadySessionId.value === activeSessionId.value, currentRequest: currentRequest.value, turnStatusAuthority: turnStatusAuthority.value }));
const composer = computed(() => resolveComposerState({ messages: messages.value, sessions: sessions.value, activeSessionId: activeSessionId.value, chatPending: chatPending.value, realtimeReady: true, currentRequest: currentRequest.value, turnStatusAuthority: turnStatusAuthority.value }));
function recordActivity(label = "user-activity"): void {
const now = Date.now();
@@ -228,7 +228,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
const nextSessions = stableSessionList(sessions.value, listedSessions, stableSelected?.sessionId ?? selected?.sessionId ?? routeSessionId ?? includeSessionId, stableSelected);
rememberSessionList(nextSessions);
sessionsReady.value = sessionsResult.ok || nextSessions.length > 0;
void refreshSessionStatusAuthority(nextSessions);
clearSessionDetailLoading(targetSessionId);
loading.value = false;
recordWorkbenchLoadingState({ scope: "session_detail", active: false, reason: "hydrate", sessionId: targetSessionId });
@@ -376,23 +375,12 @@ export const useWorkbenchStore = defineStore("workbench", () => {
rememberSessionList(next);
applySessionPagination(response.data);
sessionsReady.value = true;
if (historyAuthorityPolicy.turnStatusHydrate) await refreshSessionStatusAuthority(next);
return;
}
sessionsReady.value = sessions.value.length > 0;
if (sessions.value.length === 0) error.value = response.error ?? "session list unavailable";
}
function scheduleSessionListRefresh(includeSessionId: string | null | undefined = activeSessionId.value, delayMs = runtimePolicy.sessionListRealtimeRefreshDelayMs): void {
if (realtimeCapabilities.liveKafkaSse) return;
void includeSessionId;
if (typeof window === "undefined") {
void workbenchColadaQueries.invalidateSessionList();
return;
}
window.setTimeout(() => { void workbenchColadaQueries.invalidateSessionList(); }, delayMs);
}
async function loadMoreSessions(): Promise<void> {
if (sessionListLoadingMore.value || !sessionListHasMore.value) return;
const cursor = sessionListNextCursor.value;
@@ -412,7 +400,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
rememberSessionList(next);
applySessionPagination(response.data);
sessionsReady.value = true;
if (historyAuthorityPolicy.turnStatusHydrate) await refreshSessionStatusAuthority(next);
}
function currentSessionListLimit(): number {
@@ -511,15 +498,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
return projectedActiveSession.value ?? null;
}
async function refreshSessionStatusAuthority(source: WorkbenchSessionRecord[] = sessions.value): Promise<void> {
void source;
}
function fetchWorkbenchTurnStatus(traceId: string, useActivityTimeout = shouldUseActivityTimeoutForTrace(traceId), options: { force?: boolean } = {}): Promise<ApiResult<AgentChatResultResponse>> {
const activitySource = useActivityTimeout ? () => activityRef.value : null;
return turnStatusReadSingleflight.run(traceId, () => workbenchColadaQueries.fetchTurn(traceId, { timeoutMs: 8000, activityRef: activitySource, minIntervalMs: runtimePolicy.workbenchTurnStatusMinRefreshMs, force: options.force }), { reason: options.force ? "force-turn-status" : "turn-status" });
}
function fetchWorkbenchTraceEvents(traceId: string, afterProjectedSeq: number, useActivityTimeout = shouldUseActivityTimeoutForTrace(traceId), options: { force?: boolean } = {}): Promise<ApiResult<AgentChatResultResponse>> {
const activitySource = useActivityTimeout ? () => activityRef.value : null;
return workbenchColadaQueries.fetchTraceEvents(traceId, { timeoutMs: runtimePolicy.workbenchTraceEventsTimeoutMs, activityRef: activitySource, afterProjectedSeq, limit: runtimePolicy.traceDetailPageLimit, minIntervalMs: runtimePolicy.workbenchTraceEventsMinRefreshMs, force: options.force });
@@ -536,25 +514,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
return isTraceActiveStatus(turn?.status) || isTraceActiveStatus(message?.status);
}
async function refreshMessageProjectionForTrace(sessionId: string | null | undefined, traceId: string, options: { force?: boolean } = {}): Promise<void> {
const id = normalizeWorkbenchSessionId(sessionId);
if (!id) return;
const existing = serverState.value.messagesBySessionId[id] ?? [];
if (traceProjectionIsTerminalSealed(traceId, existing)) return;
const response = await fetchSessionMessagesPage(id, { limit: traceMessageProjectionWindowLimit(), reason: `trace-message-page:${traceId}`, force: options.force });
if (!response.ok || !response.data) {
if (shouldSuppressTransientWorkbenchReadFailure(response)) return;
if (traceHasTerminalResponse(traceId, messages.value)) return;
applyProjectionDiagnostic(traceId, projectionDiagnosticFromApiFailure(response, { code: "message_projection_refresh_failed", message: response.error ?? "消息投影刷新失败,主消息正文保持上一份 canonical projection。", health: response.status === 0 ? "unavailable" : "degraded" }));
return;
}
const pageMessages = Array.isArray(response.data.messages) ? response.data.messages.map((message) => normalizeChatMessage(message as ChatMessage)) : [];
const merged = mergeMessageProjectionPage(id, pageMessages, { traceId, limit: traceMessageProjectionWindowLimit() });
commitMergedSessionMessages(id, pageMessages, merged, { traceId, limit: traceMessageProjectionWindowLimit(), reason: "trace-message-page" });
await hydrateTurnStatusAuthority(merged, { traceId, limit: 1, reason: "trace-message-page" });
if (!traceProjectionIsTerminalSealed(traceId, merged)) readTerminalTraceDetailGaps(merged, `trace-message-page:${traceId}`);
}
function fetchSessionMessagesPage(sessionId: string, options: SessionMessagesReadOptions): Promise<ApiResult<WorkbenchMessagePageResponse>> {
const key = workbenchSessionMessagesReadKey({ sessionId, limit: options.limit, force: options.force });
return sessionMessagesReadSingleflight.run(key, () => workbenchColadaQueries.fetchSessionMessages(sessionId, { limit: options.limit, minIntervalMs: runtimePolicy.workbenchSessionMessagesMinRefreshMs, force: options.force }), { reason: options.reason });
@@ -569,121 +528,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
return boundedProjectionMessageLimit(runtimePolicy.workbenchSessionMessagesWindowLimit, runtimePolicy.sessionListPageLimit);
}
function traceMessageProjectionWindowLimit(): number {
return boundedProjectionMessageLimit(runtimePolicy.workbenchTraceMessagesWindowLimit, runtimePolicy.traceDetailAutoQueueLimit);
}
function mergeMessageProjectionPage(sessionId: string, pageMessages: ChatMessage[], options: { traceId?: string | null; limit: number }): ChatMessage[] {
const existing = serverState.value.messagesBySessionId[sessionId] ?? [];
const windowMessages = selectProjectionMessageWindow(pageMessages, options);
const incoming = windowMessages.map((message) => mergeMessageProjectionMessage(message, existing));
return mergeBoundedProjectionMessages(existing, incoming);
}
function commitMergedSessionMessages(sessionId: string, incoming: ChatMessage[], merged: ChatMessage[], options: { traceId?: string | null; limit: number; reason: string }): void {
const existing = serverState.value.messagesBySessionId[sessionId] ?? [];
const summary = projectionMergeCommitSummary(existing, merged);
recordWorkbenchRuntimeDiagnostic({
module: "workbench-message-projection",
sessionId,
traceId: firstNonEmptyString(options.traceId) ?? null,
outcome: "ok",
diagnostic: {
code: "workbench_message_projection_merge",
reason: options.reason,
source: "session-messages-page",
inputCount: incoming.length,
existingCount: existing.length,
mergedCount: merged.length,
changedCount: summary.changedCount,
limit: options.limit,
valuesRedacted: true
}
});
if (summary.changed) rememberSessionMessages(sessionId, merged);
}
function mergeMessageProjectionMessage(message: ChatMessage, existing: ChatMessage[]): ChatMessage {
const previous = findExistingProjectionMessage(message, existing);
const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId, previous?.traceId, previous?.runnerTrace?.traceId);
const authority = traceId ? traceAuthorityById.value[traceId] ?? null : null;
const preservedTrace = authority ?? previous?.runnerTrace ?? null;
const timingSealPatch = messageTimingSealPatchForProjectionMerge(message, previous);
if (!preservedTrace) return { ...message, ...timingSealPatch };
const runnerTrace = message.runnerTrace ? mergeRunnerTrace(message.runnerTrace, preservedTrace) : preservedTrace;
return { ...message, ...timingSealPatch, runnerTrace };
}
function findExistingProjectionMessage(message: ChatMessage, existing: ChatMessage[]): ChatMessage | null {
const messageId = firstNonEmptyString(message.messageId, message.id);
if (messageId) {
const byId = existing.find((item) => firstNonEmptyString(item.messageId, item.id) === messageId);
if (byId) return byId;
}
const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId);
if (!traceId) return null;
return existing.find((item) => item.role === message.role && firstNonEmptyString(item.traceId, item.runnerTrace?.traceId) === traceId) ?? null;
}
async function hydrateTurnStatusAuthority(source: ChatMessage[] = messages.value, options: { traceId?: string | null; limit?: number; reason?: string; force?: boolean } = {}): Promise<void> {
const traceIds = activeTurnStatusRefreshTraceIds(source, options);
if (options.force !== true) {
if (traceIds.length > 0) {
recordWorkbenchRuntimeDiagnostic({
module: "workbench-turn-status",
traceId: options.traceId ?? traceIds[0] ?? null,
outcome: "ok",
diagnostic: { code: "workbench_turn_status_auto_read_disabled", reason: options.reason ?? "auto-turn-status-hydrate", source: "projection-sse-authority", requestedCount: traceIds.length, valuesRedacted: true }
});
}
return;
}
for (const traceId of traceIds) {
await refreshTurnStatusByTraceId(traceId);
}
}
function activeTurnStatusRefreshTraceIds(source: ChatMessage[] = messages.value, options: { traceId?: string | null; limit?: number; reason?: string; force?: boolean } = {}): string[] {
const requestedTraceId = firstNonEmptyString(options.traceId, currentRequest.value?.traceId);
const traceIds = selectActiveTurnStatusRefreshTraceIds({ messages: source, currentRequestTraceId: requestedTraceId, turnStatusAuthority: turnStatusAuthority.value, limit: options.limit ?? 1 });
recordWorkbenchRuntimeDiagnostic({
module: "workbench-turn-status",
traceId: requestedTraceId ?? traceIds[0] ?? null,
outcome: "ok",
diagnostic: {
code: "workbench_turn_status_hydrate_budget",
reason: options.reason ?? "auto-turn-status-hydrate",
source: "session-message-projection",
requestedCount: traceIds.length,
limit: options.limit ?? 1,
valuesRedacted: true
}
});
return traceIds;
}
async function refreshTurnStatusByTraceId(traceId: string | null | undefined, options: { force?: boolean } = {}): Promise<void> {
const id = firstNonEmptyString(traceId);
if (!id) return;
if (options.force !== true) {
recordWorkbenchRuntimeDiagnostic({ module: "workbench-turn-status", traceId: id, outcome: "ok", diagnostic: { code: "workbench_turn_status_auto_read_disabled", source: "projection-sse-authority", valuesRedacted: true } });
return;
}
if (traceTerminalBodyIsVisible(id)) {
recordWorkbenchRuntimeDiagnostic({ module: "workbench-terminal-priority", traceId: id, outcome: "ok", diagnostic: { code: "terminal_low_priority_turn_skip", source: "turn-status", valuesRedacted: true } });
return;
}
const response = await fetchWorkbenchTurnStatus(id, shouldUseActivityTimeoutForTrace(id), { force: options.force });
if (response.ok && response.data) {
applyTurnStatusSnapshot(id, response.data);
if (turnResultIsTerminalForMerge(response.data as AgentChatResultResponse)) completeTrace(id, response.data, { forceRead: options.force });
return;
}
if (shouldSuppressTransientWorkbenchReadFailure(response)) return;
if (traceHasTerminalResponse(id, messages.value)) return;
applyProjectionDiagnostic(id, projectionDiagnosticFromApiFailure(response, { code: "turn_status_poll_failed", message: response.error ?? "状态更新失败,当前 turn 状态暂不可见。", health: response.status === 0 ? "unavailable" : "degraded" }));
}
function applyTurnStatusSnapshot(traceId: string, result: AgentChatResultResponse | TraceSnapshot): void {
const ownerSessionId = traceOwnerSessionId(traceId, traceResultSessionId(result));
if (!ownerSessionId) return;
@@ -763,14 +607,9 @@ export const useWorkbenchStore = defineStore("workbench", () => {
const submitEntry = steerMode ? "steer" : "existing";
const traceId = steerMode && composer.value.targetTraceId ? composer.value.targetTraceId : nextProtocolId("trc");
const steerTraceId = steerMode ? nextProtocolId("trc_steer") : null;
const userMessageId = workbenchUserMessageIdForTrace(steerTraceId ?? traceId);
const agentMessageId = workbenchAgentMessageIdForTrace(traceId);
const userMessageId = workbenchMessageIdForTrace(steerTraceId ?? traceId, "user");
const agentMessageId = workbenchMessageIdForTrace(traceId, "agent");
const sessionId = composer.value.sessionId;
if (realtimeCapabilities.liveKafkaSse && liveRealtimeReadySessionId.value !== sessionId) {
error.value = "realtime_connecting";
restartRealtime("submit-readiness");
return false;
}
const threadId = composer.value.threadId;
const providerThreadId = providerThreadIdForRequest(threadId);
const submittedAt = new Date().toISOString();
@@ -820,7 +659,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
status: "running"
};
void readTraceEventsForMessages(messages.value);
scheduleSessionListRefresh(sessionId, runtimePolicy.sessionListTerminalRefreshDelayMs);
restartRealtime("steer");
return true;
}
@@ -842,7 +680,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
};
applyTurnStatusSnapshot(canonicalTraceId, response.data);
publishWorkbenchProjectionSignal(sessionId, canonicalTraceId, "submit-admitted");
scheduleSessionListRefresh(sessionId, runtimePolicy.sessionListTerminalRefreshDelayMs);
if (turnResultIsTerminalForMerge(response.data as AgentChatResultResponse)) {
completeTrace(canonicalTraceId, response.data as AgentChatResultResponse);
return true;
@@ -864,7 +701,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
if (message.status === "running") chatPending.value = false;
currentRequest.value = null;
void clearActiveTrace(traceId, "cancel-agent-message");
scheduleSessionListRefresh(sessionId, runtimePolicy.sessionListTerminalRefreshDelayMs);
}
async function cancelRunningTrace(): Promise<void> {
@@ -966,13 +802,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
return fetchWorkbenchTraceEvents(traceId, afterProjectedSeq, shouldUseActivityTimeoutForTrace(traceId), { force: options.force });
}
function readTerminalTraceDetailGaps(source: ChatMessage[], reason: string): void {
// Terminal trace rows are loaded on explicit force/detail paths. Auto-filling
// historical terminal traces makes multi-turn Workbench sessions main-thread bound.
void source;
void reason;
}
async function readTraceEventsForMessages(source: ChatMessage[] = messages.value, options: { force?: boolean } = {}): Promise<void> {
const candidates = traceDetailReadCandidates(source);
if (options.force !== true) {
@@ -1128,10 +957,8 @@ export const useWorkbenchStore = defineStore("workbench", () => {
void reason;
const realtimeScopeKey = workbenchRealtimeScopeKey(sessionId, traceId);
const scopeChanged = realtimeTransport.currentKey() !== realtimeScopeKey;
if (realtimeCapabilities.liveKafkaSse && scopeChanged) liveRealtimeReadySessionId.value = null;
if (debugCapabilities.rawHwlabEventWindow.enabled && scopeChanged) rawHwlabIngress.value = createRawHwlabIngressState(realtimeScopeKey);
realtimeTransport.restart({
realtimeCapabilities,
sessionId,
traceId,
forceReconnect,
@@ -1142,31 +969,8 @@ export const useWorkbenchStore = defineStore("workbench", () => {
onIngress: debugCapabilities.rawHwlabEventWindow.enabled
? (frame) => { rawHwlabIngress.value = appendRawHwlabIngressFrame(rawHwlabIngress.value, frame, debugCapabilities.rawHwlabEventWindow); }
: undefined,
onOpen: () => undefined,
onError: () => {
if (realtimeCapabilities.liveKafkaSse) liveRealtimeReadySessionId.value = null;
},
onState: (state) => {
if (realtimeCapabilities.liveKafkaSse && ["connecting", "error", "closed", "blocked"].includes(state.phase)) liveRealtimeReadySessionId.value = null;
},
onRecovery: (recovery) => handleRealtimeRecovery(recovery),
onEvent: (event, eventName) => {
if (realtimeCapabilities.liveKafkaSse && eventName === "workbench.connected") {
const filters = recordValue(event.filters);
const connectedSessionId = normalizeWorkbenchSessionId(filters?.sessionId);
const refreshReplay = realtimeCapabilities.kafkaRefreshReplay;
const deliveryValid = refreshReplay
? event.deliverySemantics === "kafka-retention-then-live" && event.liveOnly === false && event.replay === true && event.lossPossible === false
: event.deliverySemantics === "live-only" && event.liveOnly === true && event.replay === false;
const valid = deliveryValid
&& event.capabilities?.liveKafkaSse === true
&& event.capabilities?.kafkaRefreshReplay === refreshReplay
&& connectedSessionId === sessionId;
liveRealtimeReadySessionId.value = valid ? connectedSessionId : null;
if (!valid) error.value = "workbench_live_realtime_contract_invalid";
}
applyRealtimeEvent(event, eventName);
}
onEvent: (event, eventName) => applyRealtimeEvent(event, eventName)
});
}
@@ -1193,13 +997,11 @@ export const useWorkbenchStore = defineStore("workbench", () => {
}
function stopRealtime(): void {
liveRealtimeReadySessionId.value = null;
realtimeTransport.stop();
}
function realtimeTraceId(): string | null {
return workbenchRealtimeTraceIdForCapabilities(
realtimeCapabilities,
return workbenchRealtimeTraceId(
currentRequest.value?.traceId,
activeTraceIdFromMessages(messages.value, turnStatusAuthority.value)
);
@@ -1217,7 +1019,7 @@ export const useWorkbenchStore = defineStore("workbench", () => {
const snapshot = recordValue(event.snapshot);
const traceEvent = recordValue(event.event);
const sessionId = firstNonEmptyString(event.hwlabSessionId, event.sessionId, turn?.sessionId, snapshot?.sessionId, traceEvent?.sessionId);
return event.schema === "hwlab.event.v1" ? normalizeWorkbenchSessionId(sessionId) : normalizeTraceAuthoritySessionId(sessionId);
return normalizeTraceAuthoritySessionId(sessionId);
}
function traceResultSessionId(result: AgentChatResultResponse | TraceSnapshot | Record<string, unknown> | null | undefined): string | null {
@@ -1284,7 +1086,7 @@ export const useWorkbenchStore = defineStore("workbench", () => {
function applyRealtimeEvent(event: WorkbenchRealtimeEvent, eventName: string): void {
const reduced = reduceWorkbenchRealtimeEvent(event, eventName);
if (workbenchRealtimeEventIsBusinessActivity(event, eventName, realtimeCapabilities.liveKafkaSse)) recordActivity(reduced.activityLabel);
recordActivity(reduced.activityLabel);
applyWorkbenchRealtimeAction(reduced.action);
}
@@ -1340,77 +1142,18 @@ export const useWorkbenchStore = defineStore("workbench", () => {
function applyRealtimeTraceEvent(traceId: string | null | undefined, event: WorkbenchRealtimeEvent["event"], snapshot: WorkbenchRealtimeEvent["snapshot"], realtimeEvent?: WorkbenchRealtimeEvent | null): void {
const id = firstNonEmptyString(traceId, event?.traceId, snapshot?.traceId);
if (!id) return;
const liveKafkaEnvelope = workbenchLiveKafkaEnvelope(realtimeEvent?.schema);
const sessionId = realtimeEvent ? realtimeEventSessionId(realtimeEvent) : traceResultSessionId(snapshot ?? event ?? null);
if (!shouldApplyActiveTraceAuthority(id, sessionId, liveKafkaEnvelope)) return;
if (traceTerminalBodyIsVisible(id, sessionId) && !liveKafkaEnvelope) {
if (!shouldApplyActiveTraceAuthority(id, sessionId)) return;
if (traceTerminalBodyIsVisible(id, sessionId)) {
recordWorkbenchRuntimeDiagnostic({ module: "workbench-terminal-priority", sessionId, traceId: id, outcome: "ok", diagnostic: { code: "terminal_low_priority_sse_trace_skip", source: "realtime-trace-event", valuesRedacted: true } });
return;
}
const events = event ? [event] : Array.isArray(snapshot?.events) ? snapshot.events : [];
markWorkbenchTraceEventsReceived({ traceId: id, events, transport: "sse", serverSentAt: realtimeEvent?.serverSentAt, eventCreatedAt: realtimeEvent?.eventCreatedAt, traceSeq: realtimeEvent?.traceSeq ?? realtimeEvent?.cursor?.traceSeq });
if (liveKafkaEnvelope && event) {
applyLiveKafkaBusinessEvent(id, sessionId, event, new Date().toISOString());
return;
}
const eventSnapshot = snapshot ?? { traceId: id, status: event?.status, events };
applyTraceSnapshot(id, realtimeSnapshotToTraceSnapshot(id, eventSnapshot, events));
}
function applyLiveKafkaBusinessEvent(traceId: string, authoritySessionId: string | null, event: TraceEvent, receivedAt: string): void {
const eventSessionId = firstNonEmptyString(event.sessionId);
const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId ?? eventSessionId, true);
if (!ownerSessionId) return;
if (workbenchLiveKafkaProjectionTarget(event) === "user") {
const messageId = firstNonEmptyString(event.userMessageId, event.messageId);
const previousUserMessage = messageId
? (serverState.value.messagesBySessionId[ownerSessionId] ?? []).find((message) => (message.messageId ?? message.id) === messageId) ?? null
: null;
const userMessage = projectWorkbenchLiveKafkaUserMessage({ previous: previousUserMessage, traceId, sessionId: ownerSessionId, event, receivedAt });
if (userMessage) reduceServerState({ type: "message.upsert", sessionId: ownerSessionId, message: userMessage });
else error.value = "workbench_live_user_message_invalid";
return;
}
const previousMessage = (serverState.value.messagesBySessionId[ownerSessionId] ?? []).find((message) => messageMatchesTraceAuthority(message, traceId, authoritySessionId ?? eventSessionId, ownerSessionId, true)) ?? null;
const message = projectWorkbenchLiveKafkaMessage({
previous: previousMessage,
traceId,
sessionId: ownerSessionId,
event,
receivedAt
});
reduceServerState({
type: "message.upsert",
sessionId: ownerSessionId,
message
});
if (message.runnerTrace) rememberTraceAuthority(message.runnerTrace);
markWorkbenchTraceProjected(traceId);
const terminal = message.traceAutoLifecycle === "terminal";
const status = firstNonEmptyString(message.status) ?? "running";
const finalResponse = terminal && message.text ? { text: message.text, status, traceId } : undefined;
rememberTurnStatus(traceId, {
traceId,
sessionId: ownerSessionId,
status,
running: !terminal,
terminal,
finalResponse,
timing: message.timing,
startedAt: message.timing?.startedAt,
lastEventAt: message.timing?.lastEventAt,
finishedAt: message.timing?.finishedAt,
durationMs: message.timing?.durationMs,
updatedAt: receivedAt
} as AgentChatResultResponse);
const existing = sessions.value.find((session) => session.sessionId === ownerSessionId) ?? null;
if (existing) rememberSessionList(mergeSessionIntoList(sessions.value, { ...existing, status, lastTraceId: traceId, updatedAt: receivedAt }));
if (terminal && currentRequest.value?.traceId === traceId) {
chatPending.value = false;
currentRequest.value = null;
}
}
function applyRealtimeTurnSnapshot(turn: Record<string, unknown>): void {
const traceId = firstNonEmptyString(turn.traceId);
if (!traceId) return;
@@ -1513,7 +1256,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
}
function publishWorkbenchProjectionSignal(sessionId: string | null | undefined, traceId: string | null | undefined, reason: string): void {
if (!realtimeCapabilities.projectionRealtime) return;
if (typeof window === "undefined") return;
const id = normalizeWorkbenchSessionId(sessionId);
if (!id) return;
@@ -1523,7 +1265,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
}
function handleWorkbenchProjectionSignal(value: unknown): void {
if (!realtimeCapabilities.projectionRealtime) return;
const record = recordValue(value);
if (!record || record.sourceId === workbenchProjectionSignalSourceId) return;
if (firstNonEmptyString(record.type) !== "session-projection") return;
@@ -1557,10 +1298,9 @@ export const useWorkbenchStore = defineStore("workbench", () => {
return { ...message, ...messageTimingPatchForMerge(message, trace), ...messageStatusPatchForTerminalMerge(message, traceStatus, terminal), runnerTrace, error: clearCompletedDiagnostics ? null : error ?? message.error ?? null, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, updatedAt: new Date().toISOString() };
}));
markWorkbenchTraceProjected(traceId);
if (!traceProjectionIsTerminalSealed(traceId, serverState.value.messagesBySessionId[ownerSessionId] ?? [])) scheduleSessionListRefresh(ownerSessionId, runtimePolicy.sessionListRealtimeRefreshDelayMs);
}
function completeTrace(traceId: string, result: AgentChatResultResponse, options: { forceRead?: boolean } = {}): void {
function completeTrace(traceId: string, result: AgentChatResultResponse): void {
const authoritySessionId = traceResultSessionId(result);
const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId);
if (!ownerSessionId) return;
@@ -1568,25 +1308,12 @@ export const useWorkbenchStore = defineStore("workbench", () => {
projectTurnAuthorityToMessages(traceId, result, "complete-trace");
rememberTurnStatus(traceId, result);
markWorkbenchTraceProjected(traceId);
const ownerMessages = serverState.value.messagesBySessionId[ownerSessionId] ?? [];
const terminalMessage = latestMessageForTrace(traceId, ownerMessages);
if (!traceProjectionIsTerminalSealed(traceId, ownerMessages)) {
if (options.forceRead) void refreshMessageProjectionForTrace(ownerSessionId, traceId, { force: true });
}
if (options.forceRead && terminalMessage && !traceProjectionIsTerminalSealed(traceId, ownerMessages)) void readTraceEventsForMessage(terminalMessage, { force: true });
if (ownerSessionId === activeSessionId.value) {
chatPending.value = false;
currentRequest.value = null;
void clearActiveTrace(traceId, "trace-terminal");
restartRealtime("trace-terminal");
}
if (!traceProjectionIsTerminalSealed(traceId, serverState.value.messagesBySessionId[ownerSessionId] ?? [])) scheduleSessionListRefresh(ownerSessionId, runtimePolicy.sessionListTerminalRefreshDelayMs);
}
async function hydrateTerminalMessageDiagnostics(): Promise<void> {
const targets = messages.value.filter(messageNeedsTerminalDiagnostics).slice(-6);
void targets;
void readTraceEventsForMessages(messages.value);
}
function applyTerminalResultDiagnostics(traceId: string, result: AgentChatResultResponse): void {
@@ -1608,8 +1335,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
const projection = clearCompletedDiagnostics ? nonBlockingProjection(resultProjection) : resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null);
return { ...message, ...messageTimingPatchForMerge(message, result), ...messageStatusPatchForTerminalMerge(message, resultStatus, terminal), title: normalizeWorkbenchMessageTitle(message.role, message.title), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() };
}));
void readTraceEventsForMessages(serverState.value.messagesBySessionId[ownerSessionId] ?? []);
scheduleSessionListRefresh(ownerSessionId, runtimePolicy.sessionListTerminalRefreshDelayMs);
}
function applyRealtimeProjectionError(event: WorkbenchRealtimeEvent): void {
@@ -1619,7 +1344,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
const code = firstNonEmptyString(errorRecord.code, "workbench_realtime_error") ?? "workbench_realtime_error";
const phase = firstNonEmptyString(event.phase, errorRecord.phase, "realtime") ?? "realtime";
const message = firstNonEmptyString(errorRecord.message, errorRecord.summary, "Workbench 实时状态更新异常,最新运行记录暂不可见。") ?? "Workbench 实时状态更新异常,最新运行记录暂不可见。";
liveRealtimeReadySessionId.value = null;
error.value = `${phase} · ${code} · ${message}`;
return;
}
@@ -1651,7 +1375,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
chatPending.value = false;
currentRequest.value = null;
void clearActiveTrace(traceId, "trace-infrastructure-error");
scheduleSessionListRefresh(selectedSessionId.value, runtimePolicy.sessionListTerminalRefreshDelayMs);
}
function projectRejectedAdmissionFailure(input: { traceId: string; message: string; error: ChatMessage["error"]; projection: ProjectionDiagnostic; submittedAt: string }): void {
@@ -1793,12 +1516,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
rememberSessionDetail({ ...session, messages: projectedMessages, messageCount: session.messageCount ?? projectedMessages.length });
sessionsReady.value = true;
currentRequest.value = null;
if (historyAuthorityPolicy.kafkaRetention) {
restartRealtime("apply-selected-session:kafka-retention");
return;
}
void hydrateTurnStatusAuthority(messages.value);
void hydrateTerminalMessageDiagnostics();
void readTraceEventsForMessages(messages.value);
reattachRestoredActiveTrace();
restartRealtime("apply-selected-session");
@@ -1825,8 +1542,10 @@ export const useWorkbenchStore = defineStore("workbench", () => {
if (!requestId) return null;
const normalizedRequestId = normalizeWorkbenchSessionId(requestId);
const messageLimit = sessionMessageProjectionWindowLimit();
const eagerMessages = historyAuthorityPolicy.sessionMessagesHydrate && normalizedRequestId ? fetchSessionMessagesPage(normalizedRequestId, { limit: messageLimit, reason: "load-session:eager", force: true }) : null;
const detail = await fetchSessionDetailPage(requestId, { reason: "load-session:detail", force: true });
const [detail, messagePage] = await Promise.all([
fetchSessionDetailPage(requestId, { reason: "load-session:detail", force: true }),
normalizedRequestId ? fetchSessionMessagesPage(normalizedRequestId, { limit: messageLimit, reason: "load-session:messages", force: true }) : Promise.resolve(null)
]);
if (!detail.ok) return null;
const detailSession = sessionFromWorkbenchSession(detail.data?.session, { includeMessages: false });
const id = detailSession?.sessionId ?? normalizedRequestId ?? seed?.sessionId;
@@ -1834,32 +1553,13 @@ export const useWorkbenchStore = defineStore("workbench", () => {
const fallbackMessages = seed?.sessionId === id ? seed.messages ?? [] : [];
const base = detailSession ? { ...detailSession, messages: fallbackMessages } : seed;
if (!base) return null;
if (!historyAuthorityPolicy.sessionMessagesHydrate) return { ...base, sessionId: id, messages: [], messageCount: base.messageCount ?? 0 };
const messages = eagerMessages && id === normalizedRequestId ? await eagerMessages : await fetchSessionMessagesPage(id, { limit: messageLimit, reason: "load-session", force: true });
const page = messages.ok ? messages.data : null;
const pageMessages = Array.isArray(page?.messages) ? await sealRestoredActiveTurnMessages(page.messages.map((message) => normalizeChatMessage(message as ChatMessage))) : fallbackMessages;
return { ...base, sessionId: id, messages: pageMessages, messageCount: page?.total ?? pageMessages?.length ?? base.messageCount };
}
async function sealRestoredActiveTurnMessages(source: ChatMessage[]): Promise<ChatMessage[]> {
const targets = source.filter(messageNeedsRestoredTurnSeal).slice(-1);
if (targets.length === 0) return source;
for (const message of targets) {
const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId);
if (!traceId || traceProjectionIsTerminalSealed(traceId, source)) continue;
recordWorkbenchRuntimeDiagnostic({
module: "workbench-turn-status",
sessionId: message.sessionId ?? message.runnerTrace?.sessionId ?? null,
traceId,
outcome: "ok",
diagnostic: { code: "workbench_restored_turn_auto_read_disabled", reason: "load-session:restored-turn", source: "projection-sse-authority", valuesRedacted: true }
});
}
return source;
const hydratedMessages = messagePage?.ok && Array.isArray(messagePage.data?.messages)
? messagePage.data.messages.map((message) => normalizeChatMessage(message as ChatMessage))
: fallbackMessages;
return { ...base, sessionId: id, messages: hydratedMessages, messageCount: base.messageCount ?? hydratedMessages.length };
}
function reattachRestoredActiveTrace(): void {
void readTraceEventsForMessages(messages.value);
const traceId = activeTraceIdFromMessages(messages.value, turnStatusAuthority.value);
if (traceId) reattachTrace(traceId);
}
@@ -1936,14 +1636,6 @@ function routeSelectedSessionRecord(sessionId: string | null | undefined, loadin
} as WorkbenchSessionRecord;
}
function messageNeedsRestoredTurnSeal(message: ChatMessage): boolean {
if (message.role !== "agent") return false;
if (messageHasTerminalResponse(message)) return false;
const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId);
if (!traceId) return false;
return isTraceActiveStatus(message.status) || isTerminalMessageStatus(message.status) || message.traceAutoLifecycle === "running";
}
function providerThreadIdForRequest(threadId: string | null | undefined): string | null {
const value = firstNonEmptyString(threadId) ?? null;
if (!value) return null;
-9
View File
@@ -8,11 +8,6 @@ declare global {
label?: string;
};
workbench?: {
realtimeFeatures?: {
liveKafkaSse?: boolean;
kafkaRefreshReplay?: boolean;
projectionRealtime?: boolean;
};
debugCapabilities?: {
isolatedKafka?: boolean;
rawHwlabEventWindow?: {
@@ -55,16 +50,12 @@ declare global {
/** @deprecated Use workbenchDetailReadConcurrency. */
workbenchReadHydrationConcurrency?: number;
workbenchReadFailureCooldownMs?: number;
workbenchTurnStatusMinRefreshMs?: number;
workbenchTraceEventsMinRefreshMs?: number;
workbenchSessionDetailMinRefreshMs?: number;
workbenchSessionMessagesMinRefreshMs?: number;
workbenchSessionMessagesWindowLimit?: number;
workbenchTraceMessagesWindowLimit?: number;
workbenchRealtimeSessionMessagesMinRefreshMs?: number;
workbenchTraceEventsTimeoutMs?: number;
sessionListRealtimeRefreshDelayMs?: number;
sessionListTerminalRefreshDelayMs?: number;
sessionListMinRefreshIntervalMs?: number;
workbenchRealtimeErrorSyncReplayMinMs?: number;
/** @deprecated Use workbenchRealtimeErrorSyncReplayMinMs. */
@@ -1,4 +1,4 @@
// SPEC: PJ2026-0106050514 Workbench实时运行面 draft-2026-06-30-p0-1297-spec-first.
// Responsibility: Production Workbench realtime runtime entry point for SSE transport, event coalescing, and scoped cursor ownership.
export { createWorkbenchStreamTransportRuntime, WorkbenchStreamTransportRuntime, workbenchRealtimeTraceIdForCapabilities, type WorkbenchStreamTransportRecovery, type WorkbenchStreamTransportRestartInput, type WorkbenchStreamTransportRestartResult, type WorkbenchStreamTransportState, type WorkbenchRealtimeEvent } from "@/utils/workbench-stream-transport";
export { createWorkbenchStreamTransportRuntime, WorkbenchStreamTransportRuntime, workbenchRealtimeTraceId, type WorkbenchStreamTransportRecovery, type WorkbenchStreamTransportRestartInput, type WorkbenchStreamTransportRestartResult, type WorkbenchStreamTransportState, type WorkbenchRealtimeEvent } from "@/utils/workbench-stream-transport";
@@ -8,13 +8,11 @@
// - OpenCode stream.transport.ts:504-530 reduce output/trace.
import { connectWorkbenchEvents, type WorkbenchEventStream, type WorkbenchRealtimeEvent, type WorkbenchSseIngressFrame } from "@/api/workbench-events";
import type { WorkbenchRealtimeCapabilities } from "@/config/runtime";
import { workbenchRealtimeScopeKey } from "@/utils/workbench-key";
export type { WorkbenchRealtimeEvent };
export interface WorkbenchStreamTransportRestartInput {
realtimeCapabilities: WorkbenchRealtimeCapabilities;
sessionId?: string | null;
traceId?: string | null;
afterSeq?: number | null;
@@ -78,8 +76,7 @@ interface WorkbenchTransportCursor {
traceSeq: number | null;
}
export function workbenchRealtimeTraceIdForCapabilities(
_capabilities: WorkbenchRealtimeCapabilities,
export function workbenchRealtimeTraceId(
...candidates: Array<string | null | undefined>
): string | null {
for (const candidate of candidates) {
@@ -89,10 +86,6 @@ export function workbenchRealtimeTraceIdForCapabilities(
return null;
}
export function workbenchRealtimeTransportEnabled(capabilities: WorkbenchRealtimeCapabilities): boolean {
return capabilities.projectionRealtime;
}
export class WorkbenchStreamTransportRuntime {
private stream: WorkbenchEventStream | null = null;
private key = "";
@@ -107,11 +100,9 @@ export class WorkbenchStreamTransportRuntime {
this.stop();
this.key = key;
this.armWait(key);
if (!workbenchRealtimeTransportEnabled(input.realtimeCapabilities)) return { key, changed: true, streamStarted: false };
if (!input.sessionId && !input.traceId) return { key, changed: true, streamStarted: false };
this.emitState(input, "connecting", null);
this.stream = connectWorkbenchEvents({
realtimeCapabilities: input.realtimeCapabilities,
sessionId: input.sessionId ?? null,
traceId: input.traceId ?? null,
afterSeq: input.afterSeq ?? this.cursorByKey.get(key)?.outboxSeq ?? null,
@@ -202,7 +193,7 @@ export class WorkbenchStreamTransportRuntime {
const sessionId = input.sessionId ?? null;
const traceId = input.traceId ?? null;
const cursor = this.currentCursor(key);
const actions: WorkbenchStreamTransportRecoveryAction[] = input.realtimeCapabilities.projectionRealtime && (sessionId || traceId) ? ["events-reconnect"] : [];
const actions: WorkbenchStreamTransportRecoveryAction[] = sessionId || traceId ? ["events-reconnect"] : [];
const diagnostic = this.diagnosticEnvelope("workbench_sse_recovery", reason, key, actions);
input.onRecovery?.({ key, sessionId, traceId, tick: this.wait?.tick ?? this.tick, reason, actions, outboxSeq: cursor.outboxSeq, traceSeq: cursor.traceSeq, diagnostic });
}