Merge pull request #2540 from pikasTech/fix/2538-fixed-transactional-projection
Pipelines as Code CI / hwlab-nc01-v03-ci-poll- Success
Pipelines as Code CI / hwlab-nc01-v03-ci-poll- Success
fix(workbench): 固定 transactional 投影与 canonical duration
This commit is contained in:
@@ -0,0 +1,44 @@
|
||||
{
|
||||
"$schema": "https://json-schema.org/draft/2020-12/schema",
|
||||
"$id": "https://github.com/pikasTech/HWLAB/config/feature-config.schema.json",
|
||||
"title": "HWLAB product feature configuration",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"tracePanel": {
|
||||
"x-unidesk-feature": "workbench.trace-panel",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"autoExpandRunning": { "type": "boolean" },
|
||||
"autoCollapseTerminal": { "type": "boolean" }
|
||||
}
|
||||
},
|
||||
"rawEvents": {
|
||||
"x-unidesk-feature": "workbench.raw-events",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": { "type": "boolean" },
|
||||
"maxEntries": { "type": "integer", "minimum": 1 },
|
||||
"maxRetainedBytes": { "type": "integer", "minimum": 1 }
|
||||
}
|
||||
},
|
||||
"debugReplay": {
|
||||
"x-unidesk-feature": "workbench.debug-replay",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"enabled": { "type": "boolean" }
|
||||
}
|
||||
},
|
||||
"views": {
|
||||
"x-unidesk-feature": "workbench.views",
|
||||
"type": "object",
|
||||
"additionalProperties": false,
|
||||
"properties": {
|
||||
"traceExplorerUrlTemplate": { "type": "string", "minLength": 1 }
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -621,9 +621,6 @@ lanes:
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: Asia/Shanghai
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: zh-CN
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LABEL: 北京时间
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true"
|
||||
HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "true"
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED: "true"
|
||||
HWLAB_CLOUD_WEB_OPENCODE_UPSTREAM_URL: http://opencode-server.hwlab-v03.svc.cluster.local:4096
|
||||
HWLAB_CLOUD_WEB_OPENCODE_USERNAME: secretRef:hwlab-opencode-server-auth/username
|
||||
@@ -850,11 +847,6 @@ lanes:
|
||||
HWLAB_CODE_AGENT_AGENTRUN_REPO_URL: http://gitea-http.devops-infra.svc.cluster.local:3000/mirrors/pikasTech-HWLAB.git
|
||||
HWLAB_KAFKA_ENABLED: "true"
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true"
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true"
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true"
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false"
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false"
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED: "true"
|
||||
HWLAB_KAFKA_BOOTSTRAP_SERVERS: platform-infra-kafka-kafka-bootstrap.platform-infra.svc.cluster.local:9092
|
||||
HWLAB_KAFKA_STDIO_TOPIC: codex-stdio.raw.v1
|
||||
|
||||
@@ -6,7 +6,7 @@ import { test } from "bun:test";
|
||||
|
||||
import { createCloudApiBunServer } from "./bun-server.ts";
|
||||
import { buildCloudApiReadiness } from "./health-contract.ts";
|
||||
import { decodeCanonicalAgentRunKafkaMessage, kafkaEventBridgeConfig, liveKafkaOtelSpanAttributes, projectAgentRunKafkaEventToHwlabEvent, publishAgentRunKafkaMessageLive, relayHwlabKafkaOutboxOnce, shouldEmitLiveKafkaOtelSpan, startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts";
|
||||
import { decodeCanonicalAgentRunKafkaMessage, kafkaEventBridgeConfig, projectAgentRunKafkaEventToHwlabEvent, relayHwlabKafkaOutboxOnce, startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts";
|
||||
|
||||
const PROJECTOR_ENV = Object.freeze({
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false",
|
||||
@@ -31,23 +31,6 @@ const PROJECTOR_ENV = Object.freeze({
|
||||
HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS: "1000"
|
||||
});
|
||||
|
||||
const LIVE_ENV = Object.freeze({
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false",
|
||||
HWLAB_KAFKA_ENABLED: "true",
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true",
|
||||
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092",
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC: "agentrun.event.v1",
|
||||
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1",
|
||||
HWLAB_KAFKA_CLIENT_ID: "hwlab-live-test",
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-live-bridge-fixed",
|
||||
HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID: "hwlab-live-fanout-fixed"
|
||||
});
|
||||
|
||||
test("projects AgentRun assistant_message Kafka event into HWLAB trace event", () => {
|
||||
const projected = projectAgentRunKafkaEventToHwlabEvent({
|
||||
schema: "agentrun.event.v1",
|
||||
@@ -322,110 +305,40 @@ test("projects AgentRun terminal_status Kafka event into terminal HWLAB event",
|
||||
assert.equal(projected.event.message, "AgentRun completed");
|
||||
});
|
||||
|
||||
test("live Kafka publisher has no PG inbox, facts, checkpoint, or outbox dependency", async () => {
|
||||
const sent = [];
|
||||
const runtimeStore = new Proxy({}, { get() { throw new Error("unexpected DB access"); } });
|
||||
void runtimeStore;
|
||||
const result = await publishAgentRunKafkaMessageLive({ topic: "agentrun.event.v1", partition: 3, message: canonicalMessage({ offset: "71" }) }, {
|
||||
producer: { async send(input) { sent.push(input); } },
|
||||
config: { clientId: "hwlab-live-test", hwlabTopic: "hwlab.event.v1" },
|
||||
logger: null
|
||||
test("Workbench bridge ignores obsolete architecture env and fixes transactional authority", () => {
|
||||
const config = kafkaEventBridgeConfig({
|
||||
...PROJECTOR_ENV,
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "true",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
});
|
||||
assert.equal(result.published, true);
|
||||
assert.equal(sent.length, 1);
|
||||
assert.equal(JSON.parse(sent[0].messages[0].value).sourceEvent.offset, "71");
|
||||
|
||||
assert.equal(config?.projectorGroupId, "hwlab-projector-test-v1");
|
||||
assert.deepEqual(config?.relay, {
|
||||
intervalMs: 60000,
|
||||
batchSize: 1,
|
||||
leaseMs: 30000,
|
||||
sendTimeoutMs: 10000,
|
||||
retryBackoffMs: 1000
|
||||
});
|
||||
assert.deepEqual(Object.keys(config ?? {}).sort(), ["agentRunTopic", "brokers", "clientId", "hwlabTopic", "projectorGroupId", "projectorHeartbeatIntervalMs", "relay"]);
|
||||
});
|
||||
|
||||
test("live Kafka OTel sampling is deterministic and bounded without payload attributes", () => {
|
||||
const envelope = (agentRunEventType, sourceSeq, event = {}) => ({
|
||||
schema: "hwlab.event.v1",
|
||||
eventId: `hwlab:evt_otel_${sourceSeq}`,
|
||||
sourceEventId: `evt_otel_${sourceSeq}`,
|
||||
traceId: "trc_live_otel",
|
||||
hwlabSessionId: "ses_live_otel",
|
||||
runId: "run_live_otel",
|
||||
commandId: "cmd_live_otel",
|
||||
context: { agentRunEventType, sourceSeq, valuesRedacted: true },
|
||||
event: { type: "backend", eventType: "backend", terminal: false, message: "must-not-leak", payload: { password: "must-not-leak" }, ...event }
|
||||
});
|
||||
test("Workbench bridge does not require the six obsolete architecture env", () => {
|
||||
const env = { ...PROJECTOR_ENV };
|
||||
for (const name of [
|
||||
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
|
||||
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
|
||||
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
|
||||
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
|
||||
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
|
||||
]) delete env[name];
|
||||
|
||||
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("assistant_message", 1, { type: "assistant" })), true);
|
||||
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("tool_call", 2, { type: "tool" })), true);
|
||||
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("terminal_status", 3, { type: "result", terminal: true })), true);
|
||||
const decisions = Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" })));
|
||||
assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => (index + 1) % 32 === 0));
|
||||
assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" }))));
|
||||
|
||||
const attributes = liveKafkaOtelSpanAttributes(envelope("assistant_message", 1, { type: "assistant" }), {
|
||||
topic: "hwlab.event.v1",
|
||||
partition: 2,
|
||||
offset: "14"
|
||||
});
|
||||
assert.deepEqual(attributes, {
|
||||
businessTraceId: "trc_live_otel",
|
||||
hwlabSessionId: "ses_live_otel",
|
||||
runId: "run_live_otel",
|
||||
commandId: "cmd_live_otel",
|
||||
topic: "hwlab.event.v1",
|
||||
partition: 2,
|
||||
offset: "14",
|
||||
sourceTopic: null,
|
||||
sourcePartition: null,
|
||||
sourceOffset: null,
|
||||
eventType: "assistant_message",
|
||||
terminal: false,
|
||||
valuesRedacted: true
|
||||
});
|
||||
assert.equal(JSON.stringify(attributes).includes("must-not-leak"), false);
|
||||
});
|
||||
|
||||
test("composable Kafka capabilities reject consumer group collisions", () => {
|
||||
assert.throws(
|
||||
() => kafkaEventBridgeConfig({
|
||||
...LIVE_ENV,
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true",
|
||||
HWLAB_KAFKA_PROJECTOR_GROUP_ID: LIVE_ENV.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID,
|
||||
HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "1000"
|
||||
}),
|
||||
(error) => error?.code === "hwlab_kafka_consumer_groups_not_distinct"
|
||||
);
|
||||
});
|
||||
|
||||
test("Workbench bridge fails closed unless transactional projection authority is complete", () => {
|
||||
const invalidAuthorities = [
|
||||
{ name: "direct/live-only", env: LIVE_ENV },
|
||||
{
|
||||
name: "projection realtime only",
|
||||
env: {
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "false",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true"
|
||||
}
|
||||
},
|
||||
{ name: "projector without outbox relay", env: { ...PROJECTOR_ENV, HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false" } },
|
||||
{ name: "projector without projection realtime", env: { ...PROJECTOR_ENV, HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false" } },
|
||||
{
|
||||
name: "direct and transactional dual authority",
|
||||
env: {
|
||||
...PROJECTOR_ENV,
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-direct-authority-test",
|
||||
HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID: "hwlab-live-authority-test"
|
||||
}
|
||||
}
|
||||
];
|
||||
|
||||
for (const authority of invalidAuthorities) {
|
||||
assert.throws(
|
||||
() => startHwlabKafkaEventBridge({ env: authority.env, runtimeStore: {} }),
|
||||
(error: any) => error?.code === "hwlab_workbench_transactional_authority_required",
|
||||
authority.name
|
||||
);
|
||||
}
|
||||
assert.doesNotThrow(() => kafkaEventBridgeConfig(env));
|
||||
});
|
||||
|
||||
test("projects AgentRun run-created envelope even when no source event is present", () => {
|
||||
|
||||
@@ -4,12 +4,7 @@
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
|
||||
import { Kafka, logLevel } from "kafkajs";
|
||||
import { emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
||||
import { buildWorkbenchProjectionEventFacts } from "./workbench-projection-writer.ts";
|
||||
import {
|
||||
WORKBENCH_REALTIME_CAPABILITY_ENVS,
|
||||
workbenchRealtimeCapabilities
|
||||
} from "./workbench-realtime-capabilities.ts";
|
||||
|
||||
const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
|
||||
export const DEFAULT_AGENTRUN_EVENT_TOPIC = "agentrun.event.v1";
|
||||
@@ -22,7 +17,6 @@ export const AGENTRUN_STDIO_RECONSTRUCTION_KIND = "stdio-derived partial reconst
|
||||
const DEFAULT_CLIENT_ID = "hwlab-v03-cloud-api";
|
||||
const DEFAULT_QUERY_TIMEOUT_MS = 5000;
|
||||
const DEFAULT_QUERY_LIMIT = 50;
|
||||
const LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS = 32;
|
||||
const REQUIRED_KAFKA_ENV = Object.freeze([
|
||||
"HWLAB_KAFKA_BOOTSTRAP_SERVERS",
|
||||
"HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC",
|
||||
@@ -31,35 +25,19 @@ const REQUIRED_KAFKA_ENV = Object.freeze([
|
||||
]);
|
||||
|
||||
export function kafkaEventBridgeConfig(env = process.env) {
|
||||
const featureEnvPresent = Object.values(WORKBENCH_REALTIME_CAPABILITY_ENVS).some((name) => stringValue(env[name]));
|
||||
const legacyKafkaEnabled = truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED);
|
||||
if (!featureEnvPresent && !legacyKafkaEnabled) return null;
|
||||
const capabilities = workbenchRealtimeCapabilities(env);
|
||||
if (capabilities.kafkaRefreshReplay && !capabilities.liveKafkaSse) {
|
||||
throw contractError("hwlab_kafka_refresh_live_dependency_missing", "Kafka refresh replay requires the independently enabled live Kafka SSE capability for handoff.");
|
||||
}
|
||||
const kafkaEnabled = capabilities.directPublish || capabilities.liveKafkaSse || capabilities.kafkaRefreshReplay || capabilities.transactionalProjector || capabilities.projectionOutboxRelay;
|
||||
const transactionalEnabled = capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime;
|
||||
if (!kafkaEnabled && !transactionalEnabled) return null;
|
||||
const required = [...REQUIRED_KAFKA_ENV];
|
||||
if (!kafkaEnabled) required.length = 0;
|
||||
if (capabilities.directPublish) required.push("HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID");
|
||||
if (capabilities.liveKafkaSse) required.push("HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID");
|
||||
if (capabilities.kafkaRefreshReplay) required.push(
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT"
|
||||
);
|
||||
if (capabilities.transactionalProjector) required.push("HWLAB_KAFKA_PROJECTOR_GROUP_ID", "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS");
|
||||
if (capabilities.projectionOutboxRelay) required.push(
|
||||
const kafkaConfigured = REQUIRED_KAFKA_ENV.every((name) => stringValue(env[name]));
|
||||
if (!legacyKafkaEnabled && !kafkaConfigured) return null;
|
||||
const required = [
|
||||
...REQUIRED_KAFKA_ENV,
|
||||
"HWLAB_KAFKA_PROJECTOR_GROUP_ID",
|
||||
"HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS",
|
||||
"HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS"
|
||||
);
|
||||
];
|
||||
const missing = required.filter((name) => !stringValue(env[name]));
|
||||
if (missing.length > 0) {
|
||||
const error = new Error(`HWLAB Kafka capability configuration is incomplete: ${missing.join(", ")}`);
|
||||
@@ -71,77 +49,37 @@ export function kafkaEventBridgeConfig(env = process.env) {
|
||||
const agentRunTopic = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC);
|
||||
const hwlabTopic = stringValue(env.HWLAB_KAFKA_EVENT_TOPIC);
|
||||
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID);
|
||||
const relay = capabilities.projectionOutboxRelay ? {
|
||||
const relay = {
|
||||
intervalMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS, "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS"),
|
||||
batchSize: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE, "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE"),
|
||||
leaseMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS, "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS"),
|
||||
sendTimeoutMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS, "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS"),
|
||||
retryBackoffMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS, "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS")
|
||||
} : null;
|
||||
const refreshReplay = capabilities.kafkaRefreshReplay ? {
|
||||
groupIdPrefix: stringValue(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX),
|
||||
timeoutMs: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS"),
|
||||
scanLimit: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT"),
|
||||
matchedEventLimit: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_MATCHED_EVENT_LIMIT"),
|
||||
liveBufferLimit: strictPositiveInteger(env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_LIVE_BUFFER_LIMIT")
|
||||
} : null;
|
||||
};
|
||||
if (relay && relay.sendTimeoutMs + 5000 >= relay.leaseMs) {
|
||||
const error = new Error("HWLAB Kafka relay lease must exceed producer send timeout by at least 5000ms.");
|
||||
error.code = "hwlab_kafka_relay_lease_invalid";
|
||||
throw error;
|
||||
}
|
||||
const enabledConsumerGroups = [
|
||||
capabilities.directPublish ? stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID) : null,
|
||||
capabilities.transactionalProjector ? stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID) : null,
|
||||
capabilities.liveKafkaSse ? stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID) : null
|
||||
].filter(Boolean);
|
||||
if (new Set(enabledConsumerGroups).size !== enabledConsumerGroups.length) {
|
||||
const error = new Error("HWLAB Kafka direct publisher, projector, and live SSE consumer groups must be distinct when enabled together.");
|
||||
error.code = "hwlab_kafka_consumer_groups_not_distinct";
|
||||
error.groupIds = enabledConsumerGroups;
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
capabilities,
|
||||
brokers,
|
||||
agentRunTopic,
|
||||
hwlabTopic,
|
||||
clientId,
|
||||
directPublishGroupId: stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID),
|
||||
projectorGroupId: stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID),
|
||||
hwlabEventGroupId: stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID),
|
||||
refreshReplay,
|
||||
projectorHeartbeatIntervalMs: capabilities.transactionalProjector ? strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS") : null,
|
||||
projectorHeartbeatIntervalMs: strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS"),
|
||||
relay
|
||||
};
|
||||
}
|
||||
|
||||
export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString(), otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
||||
export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString() } = {}) {
|
||||
const config = kafkaEventBridgeConfig(env);
|
||||
if (!config) return { started: false, reason: "disabled_or_unconfigured", stop() {}, subscribeProjectionCommits() { return () => {}; }, valuesPrinted: false };
|
||||
assertTransactionalWorkbenchRealtimeAuthority(config.capabilities);
|
||||
const components = [];
|
||||
if (config.capabilities.directPublish || config.capabilities.liveKafkaSse) {
|
||||
components.push(startLiveHwlabKafkaEventBridge({ config, env, logger, kafkaFactory, otelSpanEmitter }));
|
||||
}
|
||||
if (config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay || config.capabilities.projectionRealtime) {
|
||||
components.push(startTransactionalHwlabKafkaEventBridge({ config, logger, kafkaFactory, runtimeStore, now }));
|
||||
}
|
||||
return components.length === 1 ? components[0] : combineKafkaEventBridgeComponents(config, components);
|
||||
}
|
||||
|
||||
function assertTransactionalWorkbenchRealtimeAuthority(capabilities = {}) {
|
||||
const directAuthority = capabilities.directPublish || capabilities.liveKafkaSse || capabilities.kafkaRefreshReplay;
|
||||
const completeProjectionAuthority = capabilities.transactionalProjector && capabilities.projectionOutboxRelay && capabilities.projectionRealtime;
|
||||
if (!directAuthority && completeProjectionAuthority) return;
|
||||
throw contractError(
|
||||
"hwlab_workbench_transactional_authority_required",
|
||||
"Workbench Kafka bridge requires transactionalProjector, projectionOutboxRelay, and projectionRealtime together, with directPublish, liveKafkaSse, and kafkaRefreshReplay disabled."
|
||||
);
|
||||
return startTransactionalHwlabKafkaEventBridge({ config, logger, kafkaFactory, runtimeStore, now });
|
||||
}
|
||||
|
||||
function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString() } = {}) {
|
||||
requireKafkaProjectorStore(runtimeStore, config.capabilities);
|
||||
requireKafkaProjectorStore(runtimeStore);
|
||||
|
||||
let stopped = false;
|
||||
let consumer = null;
|
||||
@@ -160,16 +98,14 @@ function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kaf
|
||||
let startupReady = false;
|
||||
const ready = (async () => {
|
||||
await runtimeStore.assertWorkbenchTransactionalRealtimeReady();
|
||||
const kafka = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay ? kafkaFactory(config) : null;
|
||||
if (config.capabilities.transactionalProjector) consumer = kafka.consumer({ groupId: config.projectorGroupId, allowAutoTopicCreation: false });
|
||||
if (config.capabilities.projectionOutboxRelay) producer = kafka.producer({ allowAutoTopicCreation: false });
|
||||
if (producer) await producer.connect();
|
||||
if (consumer) await consumer.connect();
|
||||
if (config.capabilities.projectionRealtime && typeof runtimeStore.subscribeWorkbenchProjectionCommits === "function") {
|
||||
stopProjectionNotifications = await runtimeStore.subscribeWorkbenchProjectionCommits(publishProjectionCommit);
|
||||
}
|
||||
if (consumer) await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
|
||||
if (consumer) await consumer.run({
|
||||
const kafka = kafkaFactory(config);
|
||||
consumer = kafka.consumer({ groupId: config.projectorGroupId, allowAutoTopicCreation: false });
|
||||
producer = kafka.producer({ allowAutoTopicCreation: false });
|
||||
await producer.connect();
|
||||
await consumer.connect();
|
||||
stopProjectionNotifications = await runtimeStore.subscribeWorkbenchProjectionCommits(publishProjectionCommit);
|
||||
await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
|
||||
await consumer.run({
|
||||
autoCommit: false,
|
||||
eachBatchAutoResolve: false,
|
||||
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => {
|
||||
@@ -196,7 +132,6 @@ function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kaf
|
||||
}
|
||||
});
|
||||
const runRelay = async () => {
|
||||
if (!config.capabilities.projectionOutboxRelay) return;
|
||||
if (stopped || relayRunning) return;
|
||||
relayRunning = true;
|
||||
try {
|
||||
@@ -205,18 +140,15 @@ function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kaf
|
||||
relayRunning = false;
|
||||
}
|
||||
};
|
||||
if (config.capabilities.projectionOutboxRelay) {
|
||||
relayTimer = setInterval(() => { void runRelay().catch((error) => logWarn(logger, "hwlab-kafka-outbox-relay-failed", { message: errorMessage(error), valuesPrinted: false })); }, config.relay.intervalMs);
|
||||
relayTimer.unref?.();
|
||||
await runRelay();
|
||||
}
|
||||
relayTimer = setInterval(() => { void runRelay().catch((error) => logWarn(logger, "hwlab-kafka-outbox-relay-failed", { message: errorMessage(error), valuesPrinted: false })); }, config.relay.intervalMs);
|
||||
relayTimer.unref?.();
|
||||
await runRelay();
|
||||
startupReady = true;
|
||||
logInfo(logger, "hwlab-kafka-event-bridge-started", {
|
||||
agentRunTopic: config.agentRunTopic,
|
||||
hwlabTopic: config.hwlabTopic,
|
||||
clientId: config.clientId,
|
||||
projectorGroupId: config.projectorGroupId,
|
||||
capabilities: config.capabilities,
|
||||
valuesPrinted: false
|
||||
});
|
||||
})();
|
||||
@@ -249,10 +181,8 @@ function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kaf
|
||||
const transactionalReadiness = typeof runtimeStore.workbenchTransactionalRealtimeReadiness === "function"
|
||||
? await runtimeStore.workbenchTransactionalRealtimeReadiness()
|
||||
: { ready: true, valuesRedacted: true };
|
||||
const projectorStatus = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay
|
||||
? await runtimeStore.hwlabKafkaProjectorStatus()
|
||||
: {};
|
||||
return { status: transactionalReadiness.ready === false ? "blocked" : "ready", capabilities: config.capabilities, transactionalReadiness, ...projectorStatus, valuesRedacted: true };
|
||||
const projectorStatus = await runtimeStore.hwlabKafkaProjectorStatus();
|
||||
return { status: transactionalReadiness.ready === false ? "blocked" : "ready", transactionalReadiness, ...projectorStatus, valuesRedacted: true };
|
||||
},
|
||||
startupStatus() {
|
||||
if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
||||
@@ -267,269 +197,6 @@ function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kaf
|
||||
};
|
||||
}
|
||||
|
||||
function startLiveHwlabKafkaEventBridge({ config, env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
||||
let stopped = false;
|
||||
let bridgeConsumer = null;
|
||||
let fanoutConsumer = null;
|
||||
let producer = null;
|
||||
let startupError = null;
|
||||
let startupReady = false;
|
||||
const subscribers = new Set();
|
||||
const lagByPartition = new Map();
|
||||
const publishLiveEnvelope = (envelope, transport) => {
|
||||
for (const listener of [...subscribers]) {
|
||||
try {
|
||||
listener(envelope, transport);
|
||||
} catch (error) {
|
||||
logWarn(logger, "hwlab-live-kafka-subscriber-failed", { message: errorMessage(error), valuesPrinted: false });
|
||||
}
|
||||
}
|
||||
};
|
||||
const ready = (async () => {
|
||||
const kafka = kafkaFactory(config);
|
||||
if (config.capabilities.directPublish) {
|
||||
bridgeConsumer = kafka.consumer({ groupId: config.directPublishGroupId, allowAutoTopicCreation: false });
|
||||
producer = kafka.producer({ allowAutoTopicCreation: false });
|
||||
}
|
||||
if (config.capabilities.liveKafkaSse) fanoutConsumer = kafka.consumer({ groupId: config.hwlabEventGroupId, allowAutoTopicCreation: false });
|
||||
if (producer) await producer.connect();
|
||||
if (bridgeConsumer) await bridgeConsumer.connect();
|
||||
if (fanoutConsumer) await fanoutConsumer.connect();
|
||||
if (bridgeConsumer) await bridgeConsumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
|
||||
if (fanoutConsumer) await fanoutConsumer.subscribe({ topic: config.hwlabTopic, fromBeginning: false });
|
||||
if (fanoutConsumer) await fanoutConsumer.run({
|
||||
eachBatchAutoResolve: false,
|
||||
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => {
|
||||
let lastOffset = null;
|
||||
for (const message of batch.messages) {
|
||||
if (stopped || !isRunning() || isStale()) break;
|
||||
const envelope = parseJson(message?.value ? Buffer.from(message.value).toString("utf8") : "");
|
||||
if (!envelope || envelope.schema !== "hwlab.event.v1") {
|
||||
logWarn(logger, "hwlab-live-kafka-envelope-ignored", { reason: "schema-invalid", valuesPrinted: false });
|
||||
} else {
|
||||
const transport = { topic: batch.topic, partition: batch.partition, offset: message.offset };
|
||||
emitLiveKafkaOtelSpan("hwlab.kafka.live.fanout_receive", envelope, transport, { env, otelSpanEmitter });
|
||||
publishLiveEnvelope(envelope, transport);
|
||||
}
|
||||
resolveOffset(message.offset);
|
||||
lastOffset = message.offset;
|
||||
await heartbeat();
|
||||
}
|
||||
if (lastOffset !== null) lagByPartition.set(batch.partition, kafkaPartitionLag(batch.highWatermark, lastOffset));
|
||||
}
|
||||
});
|
||||
if (bridgeConsumer) await bridgeConsumer.run({
|
||||
eachMessage: async ({ topic, partition, message }) => {
|
||||
if (stopped) return;
|
||||
await publishAgentRunKafkaMessageLive({ topic, partition, message }, { producer, config, env, logger, otelSpanEmitter });
|
||||
}
|
||||
});
|
||||
startupReady = true;
|
||||
logInfo(logger, "hwlab-live-kafka-event-bridge-started", {
|
||||
capabilities: config.capabilities,
|
||||
agentRunTopic: config.agentRunTopic,
|
||||
hwlabTopic: config.hwlabTopic,
|
||||
clientId: config.clientId,
|
||||
directPublishGroupId: config.directPublishGroupId,
|
||||
hwlabEventGroupId: config.hwlabEventGroupId,
|
||||
valuesPrinted: false
|
||||
});
|
||||
})();
|
||||
void ready.catch((error) => {
|
||||
startupError = error;
|
||||
logWarn(logger, "hwlab-live-kafka-event-bridge-start-failed", {
|
||||
message: errorMessage(error),
|
||||
agentRunTopic: config.agentRunTopic,
|
||||
hwlabTopic: config.hwlabTopic,
|
||||
valuesPrinted: false
|
||||
});
|
||||
});
|
||||
return {
|
||||
started: true,
|
||||
...config,
|
||||
ready,
|
||||
liveReady: ready,
|
||||
async stop() {
|
||||
stopped = true;
|
||||
subscribers.clear();
|
||||
await Promise.allSettled([bridgeConsumer?.stop?.(), fanoutConsumer?.stop?.()]);
|
||||
await Promise.allSettled([bridgeConsumer?.disconnect?.(), fanoutConsumer?.disconnect?.(), producer?.disconnect?.()]);
|
||||
},
|
||||
async status() {
|
||||
if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
||||
return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, subscriberCount: subscribers.size, consumerLag: liveKafkaLagSummary(lagByPartition), lossPossible: true, valuesRedacted: true };
|
||||
},
|
||||
startupStatus() {
|
||||
if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
||||
return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, valuesRedacted: true };
|
||||
},
|
||||
subscribeProjectionCommits() { return () => {}; },
|
||||
subscribeLiveHwlabEvents(listener) {
|
||||
if (typeof listener !== "function") return () => {};
|
||||
subscribers.add(listener);
|
||||
return () => subscribers.delete(listener);
|
||||
},
|
||||
queryHwlabEventRetention(params = {}) {
|
||||
return queryKafkaEventStream({ ...params, env, stream: "hwlab", topic: config.hwlabTopic, kafkaFactory });
|
||||
},
|
||||
liveSubscriberCount() { return subscribers.size; },
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
function kafkaPartitionLag(highWatermark, lastOffset) {
|
||||
try {
|
||||
const lag = BigInt(String(highWatermark ?? "0")) - (BigInt(String(lastOffset ?? "0")) + 1n);
|
||||
return Number(lag > 0n ? lag : 0n);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function liveKafkaLagSummary(lagByPartition) {
|
||||
const partitions = [...lagByPartition.entries()].map(([partition, lag]) => ({ partition, lag })).sort((left, right) => left.partition - right.partition);
|
||||
const known = partitions.map((item) => item.lag).filter((lag) => Number.isFinite(lag));
|
||||
return {
|
||||
known: known.length > 0,
|
||||
total: known.length > 0 ? known.reduce((sum, lag) => sum + lag, 0) : null,
|
||||
partitions,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function combineKafkaEventBridgeComponents(config, components) {
|
||||
const ready = Promise.all(components.map((component) => component.ready));
|
||||
const liveOwner = components.find((component) => typeof component.subscribeLiveHwlabEvents === "function");
|
||||
const componentStartup = () => components.map((component) => component.startupStatus?.() ?? { status: "initializing" });
|
||||
return {
|
||||
started: true,
|
||||
...config,
|
||||
ready,
|
||||
liveReady: liveOwner?.liveReady ?? liveOwner?.ready ?? ready,
|
||||
async stop() { await Promise.allSettled(components.map((component) => component.stop())); },
|
||||
async status() {
|
||||
const statuses = await Promise.all(components.map((component) => component.status()));
|
||||
const blocked = statuses.find((status) => status.status === "blocked");
|
||||
const initializing = statuses.some((status) => status.status !== "ready");
|
||||
return {
|
||||
status: blocked ? "blocked" : initializing ? "initializing" : "ready",
|
||||
capabilities: config.capabilities,
|
||||
components: statuses,
|
||||
errorCode: blocked?.errorCode ?? null,
|
||||
message: blocked?.message ?? null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
},
|
||||
startupStatus() {
|
||||
const statuses = componentStartup();
|
||||
const blocked = statuses.find((status) => status.status === "blocked");
|
||||
return {
|
||||
status: blocked ? "blocked" : statuses.every((status) => status.status === "ready") ? "ready" : "initializing",
|
||||
capabilities: config.capabilities,
|
||||
components: statuses,
|
||||
errorCode: blocked?.errorCode ?? null,
|
||||
message: blocked?.message ?? null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
},
|
||||
subscribeProjectionCommits(listener) {
|
||||
const stops = components.map((component) => component.subscribeProjectionCommits?.(listener)).filter((stop) => typeof stop === "function");
|
||||
return () => stops.forEach((stop) => stop());
|
||||
},
|
||||
subscribeLiveHwlabEvents(listener) {
|
||||
return liveOwner?.subscribeLiveHwlabEvents(listener) ?? (() => {});
|
||||
},
|
||||
queryHwlabEventRetention(params = {}) {
|
||||
if (typeof liveOwner?.queryHwlabEventRetention !== "function") throw contractError("hwlab_kafka_refresh_query_unconfigured", "Kafka refresh retention query is not configured.");
|
||||
return liveOwner.queryHwlabEventRetention(params);
|
||||
},
|
||||
liveSubscriberCount() {
|
||||
const owner = components.find((component) => typeof component.liveSubscriberCount === "function");
|
||||
return owner?.liveSubscriberCount() ?? 0;
|
||||
},
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
export async function publishAgentRunKafkaMessageLive(kafkaMessage = {}, { producer, config, env = process.env, logger = console, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
||||
const transport = kafkaTransport(kafkaMessage);
|
||||
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
|
||||
if (!decoded.ok) {
|
||||
logWarn(logger, "hwlab-live-kafka-message-invalid", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false });
|
||||
return { handled: true, invalid: true, published: false, valuesPrinted: false };
|
||||
}
|
||||
if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) {
|
||||
return { handled: true, ignored: true, published: false, valuesPrinted: false };
|
||||
}
|
||||
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, {
|
||||
source: config.clientId,
|
||||
sourceTopic: transport.sourceTopic,
|
||||
sourcePartition: transport.sourcePartition,
|
||||
sourceOffset: transport.sourceOffset,
|
||||
sourceKey: transport.sourceKey,
|
||||
inputSha256: transport.inputSha256
|
||||
});
|
||||
const publishMetadata = await producer.send({ topic: config.hwlabTopic, messages: [buildHwlabKafkaProducerMessage(projected)] });
|
||||
const publishedRecord = Array.isArray(publishMetadata) ? publishMetadata[0] : null;
|
||||
emitLiveKafkaOtelSpan("hwlab.kafka.live.direct_publish", projected, {
|
||||
topic: config.hwlabTopic,
|
||||
partition: publishedRecord?.partition,
|
||||
offset: publishedRecord?.baseOffset ?? publishedRecord?.offset,
|
||||
sourceTopic: transport.sourceTopic,
|
||||
sourcePartition: transport.sourcePartition,
|
||||
sourceOffset: transport.sourceOffset
|
||||
}, { env, otelSpanEmitter });
|
||||
return { handled: true, published: true, envelope: projected, sessionId: projected.sessionId, traceId: projected.traceId, valuesPrinted: false };
|
||||
}
|
||||
|
||||
export function shouldEmitLiveKafkaOtelSpan(envelope = {}) {
|
||||
const event = objectValue(envelope.event);
|
||||
const context = objectValue(envelope.context);
|
||||
const eventType = firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType)?.toLowerCase();
|
||||
if (event.terminal === true || eventType === "terminal" || eventType === "terminal_status") return true;
|
||||
if (["assistant", "assistant_progress", "assistant_message", "tool", "tool_call"].includes(eventType)) return true;
|
||||
if (eventType !== "command_output" && event.type !== "output") return true;
|
||||
const sourceSeq = integerValue(context.sourceSeq ?? event.sourceSeq);
|
||||
if (Number.isInteger(sourceSeq) && sourceSeq > 0) return sourceSeq % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0;
|
||||
const identity = firstText(envelope.sourceEventId, envelope.eventId, envelope.traceId, envelope.hwlabSessionId, envelope.sessionId);
|
||||
if (!identity) return false;
|
||||
return createHash("sha256").update(identity).digest().readUInt32BE(0) % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0;
|
||||
}
|
||||
|
||||
export function liveKafkaOtelSpanAttributes(envelope = {}, transport = {}) {
|
||||
const event = objectValue(envelope.event);
|
||||
const context = objectValue(envelope.context);
|
||||
const sourceEvent = objectValue(envelope.sourceEvent);
|
||||
return {
|
||||
businessTraceId: firstText(envelope.traceId, event.traceId),
|
||||
hwlabSessionId: firstText(envelope.hwlabSessionId, envelope.sessionId, event.sessionId),
|
||||
runId: firstText(envelope.runId, context.runId, event.runId),
|
||||
commandId: firstText(envelope.commandId, context.commandId, event.commandId),
|
||||
topic: firstText(transport.topic, sourceEvent.topic),
|
||||
partition: integerValue(transport.partition ?? sourceEvent.partition),
|
||||
offset: firstText(transport.offset, sourceEvent.offset),
|
||||
sourceTopic: firstText(transport.sourceTopic),
|
||||
sourcePartition: integerValue(transport.sourcePartition),
|
||||
sourceOffset: firstText(transport.sourceOffset),
|
||||
eventType: firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType),
|
||||
terminal: event.terminal === true,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
export function emitLiveKafkaOtelSpan(name, envelope = {}, transport = {}, { env = process.env, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
||||
if (typeof otelSpanEmitter !== "function" || !shouldEmitLiveKafkaOtelSpan(envelope)) return { emitted: false, valuesRedacted: true };
|
||||
const attributes = liveKafkaOtelSpanAttributes(envelope, transport);
|
||||
if (!attributes.businessTraceId) return { emitted: false, valuesRedacted: true };
|
||||
try {
|
||||
const pending = otelSpanEmitter(name, attributes.businessTraceId, env, { attributes });
|
||||
void Promise.resolve(pending).catch(() => undefined);
|
||||
} catch {
|
||||
// OTel is best effort and must never change the live Kafka delivery path.
|
||||
}
|
||||
return { emitted: true, valuesRedacted: true };
|
||||
}
|
||||
|
||||
export async function projectAgentRunKafkaMessage(kafkaMessage = {}, { runtimeStore, config, logger = console } = {}) {
|
||||
const transport = kafkaTransport(kafkaMessage);
|
||||
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
|
||||
@@ -953,13 +620,18 @@ function kafkaTransport(kafkaMessage = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function requireKafkaProjectorStore(runtimeStore, capabilities = {}) {
|
||||
const required = [];
|
||||
if (capabilities.transactionalProjector) required.push("commitAgentRunKafkaProjection", "recordFailedAgentRunKafkaMessage", "recordIgnoredAgentRunKafkaMessage");
|
||||
if (capabilities.projectionOutboxRelay) required.push("claimHwlabKafkaOutbox", "completeHwlabKafkaOutbox", "retryHwlabKafkaOutbox");
|
||||
if (capabilities.projectionRealtime) required.push("subscribeWorkbenchProjectionCommits");
|
||||
if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay) required.push("hwlabKafkaProjectorStatus");
|
||||
if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime) required.push("assertWorkbenchTransactionalRealtimeReady");
|
||||
function requireKafkaProjectorStore(runtimeStore) {
|
||||
const required = [
|
||||
"commitAgentRunKafkaProjection",
|
||||
"recordFailedAgentRunKafkaMessage",
|
||||
"recordIgnoredAgentRunKafkaMessage",
|
||||
"claimHwlabKafkaOutbox",
|
||||
"completeHwlabKafkaOutbox",
|
||||
"retryHwlabKafkaOutbox",
|
||||
"subscribeWorkbenchProjectionCommits",
|
||||
"hwlabKafkaProjectorStatus",
|
||||
"assertWorkbenchTransactionalRealtimeReady"
|
||||
];
|
||||
const missing = required.filter((name) => typeof runtimeStore?.[name] !== "function");
|
||||
if (missing.length > 0) throw contractError("hwlab_kafka_projector_store_invalid", `Kafka durable capabilities require a runtime store: ${missing.join(", ")}`);
|
||||
}
|
||||
|
||||
@@ -20,9 +20,6 @@ import { codeAgentSessionLifecycleSummary } from "./code-agent-session-lifecycle
|
||||
import { messageAuthorityTextValue } from "./code-agent-agentrun-prompt.ts";
|
||||
import { promoteWorkbenchTurnAdmission as persistWorkbenchTurnAdmissionPromotion, writeWorkbenchSessionAdmissionFact } from "./workbench-projection-writer.ts";
|
||||
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
||||
import {
|
||||
workbenchRealtimeCapabilities
|
||||
} from "./workbench-realtime-capabilities.ts";
|
||||
import { codeAgentOtelTraceFields, emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
||||
import {
|
||||
firstHeaderValue,
|
||||
@@ -245,7 +242,6 @@ export async function handleCodeAgentChatHttp(request, response, options) {
|
||||
lastEventAt: submitted?.lastEventAt ?? admittedTiming.lastEventAt,
|
||||
finishedAt: submitted?.finishedAt ?? admittedTiming.finishedAt,
|
||||
durationMs: submitted?.durationMs ?? admittedTiming.durationMs,
|
||||
realtimeCapabilities: workbenchRealtimeCapabilities(options.env ?? process.env),
|
||||
admission: submitted?.agentRun ? {
|
||||
state: submitted.admissionState ?? "promoted",
|
||||
runId: submitted.agentRun.runId ?? null,
|
||||
@@ -1046,8 +1042,6 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) {
|
||||
}
|
||||
});
|
||||
if (adapterEnabled) {
|
||||
const realtimeCapabilities = workbenchRealtimeCapabilities(runtimeEnv);
|
||||
const transactionalAdmission = realtimeCapabilities.transactionalProjector;
|
||||
const pendingPromotion = results.get(traceId);
|
||||
if (pendingPromotion?.admissionState === "promotion-pending" && pendingPromotion?.agentRun?.durableDispatch === true) {
|
||||
const retryParams = { ...params, userBillingReservation: pendingPromotion.userBillingReservation ?? null };
|
||||
@@ -1062,15 +1056,13 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) {
|
||||
}
|
||||
|
||||
const lifecycle = codeAgentTurnLifecycleFields(traceId, params);
|
||||
if (transactionalAdmission) {
|
||||
await persistCodeAgentAdmissionInputState({
|
||||
params,
|
||||
options,
|
||||
traceId,
|
||||
status: "admitting",
|
||||
messageId: lifecycle.userMessageId
|
||||
});
|
||||
}
|
||||
await persistCodeAgentAdmissionInputState({
|
||||
params,
|
||||
options,
|
||||
traceId,
|
||||
status: "admitting",
|
||||
messageId: lifecycle.userMessageId
|
||||
});
|
||||
|
||||
traceStore.append(traceId, {
|
||||
type: "request",
|
||||
@@ -1091,16 +1083,14 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) {
|
||||
error: billingPreflight?.blocked ? billingPreflight?.payload?.error?.message ?? billingPreflight?.blocker?.summary ?? "billing preflight blocked" : null
|
||||
});
|
||||
if (billingPreflight?.blocked) {
|
||||
if (transactionalAdmission) {
|
||||
await persistCodeAgentAdmissionInputState({
|
||||
params,
|
||||
options,
|
||||
traceId,
|
||||
status: "blocked",
|
||||
messageId: lifecycle.userMessageId,
|
||||
error: billingPreflight.payload?.error ?? billingPreflight.blocker ?? { code: "billing_preflight_blocked" }
|
||||
});
|
||||
}
|
||||
await persistCodeAgentAdmissionInputState({
|
||||
params,
|
||||
options,
|
||||
traceId,
|
||||
status: "blocked",
|
||||
messageId: lifecycle.userMessageId,
|
||||
error: billingPreflight.payload?.error ?? billingPreflight.blocker ?? { code: "billing_preflight_blocked" }
|
||||
});
|
||||
traceStore.append(traceId, {
|
||||
type: "billing",
|
||||
status: "blocked",
|
||||
@@ -1126,16 +1116,14 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) {
|
||||
const failure = codeAgentDispatchFailure(error);
|
||||
const rejected = codeAgentSubmissionUnavailablePayload(failure, { params: dispatchParams, options: executionOptions, traceId });
|
||||
await finalizeCodeAgentBillingUsage({ payload: rejected, params: dispatchParams, options: executionOptions });
|
||||
if (transactionalAdmission) {
|
||||
await persistCodeAgentAdmissionInputState({
|
||||
params: dispatchParams,
|
||||
options: executionOptions,
|
||||
traceId,
|
||||
status: "failed",
|
||||
messageId: lifecycle.userMessageId,
|
||||
error: failure.payload?.error ?? error
|
||||
});
|
||||
}
|
||||
await persistCodeAgentAdmissionInputState({
|
||||
params: dispatchParams,
|
||||
options: executionOptions,
|
||||
traceId,
|
||||
status: "failed",
|
||||
messageId: lifecycle.userMessageId,
|
||||
error: failure.payload?.error ?? error
|
||||
});
|
||||
void emitCodeAgentOtelSpan("agentrun_dispatch", traceId, executionOptions.env ?? process.env, {
|
||||
startTimeMs: dispatchStartedAt,
|
||||
status: "error",
|
||||
@@ -1156,17 +1144,15 @@ async function submitCodeAgentChatTurn({ params, options, traceId }) {
|
||||
}, dispatchParams);
|
||||
results.set(traceId, pending);
|
||||
try {
|
||||
if (transactionalAdmission) {
|
||||
await persistCodeAgentAdmissionInputState({
|
||||
params: dispatchParams,
|
||||
options: executionOptions,
|
||||
traceId,
|
||||
status: "admitting",
|
||||
messageId: lifecycle.userMessageId,
|
||||
commandId: payload.agentRun?.commandId ?? null,
|
||||
agentRun: payload.agentRun ?? null
|
||||
});
|
||||
}
|
||||
await persistCodeAgentAdmissionInputState({
|
||||
params: dispatchParams,
|
||||
options: executionOptions,
|
||||
traceId,
|
||||
status: "admitting",
|
||||
messageId: lifecycle.userMessageId,
|
||||
commandId: payload.agentRun?.commandId ?? null,
|
||||
agentRun: payload.agentRun ?? null
|
||||
});
|
||||
await promoteCodeAgentTurnAdmission({ payload: pending, params: dispatchParams, options: executionOptions, traceId });
|
||||
} catch (error) {
|
||||
throw codeAgentPromotionUnavailableError(error, { payload: pending, params: dispatchParams, options: executionOptions, traceId });
|
||||
@@ -1324,30 +1310,6 @@ async function promoteCodeAgentTurnAdmission({ payload = {}, params = {}, option
|
||||
throw error;
|
||||
}
|
||||
const lifecycle = codeAgentTurnLifecycleFields(traceId, params);
|
||||
const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env);
|
||||
if (!realtimeCapabilities.transactionalProjector) {
|
||||
const runningOwner = await recordCodeAgentSessionOwner({ payload: { ...payload, status: "running" }, params, options, status: "running" });
|
||||
if (!runningOwner) {
|
||||
const error = new Error("Code Agent live turn did not durably promote the session owner to running.");
|
||||
error.code = "workbench_admission_owner_not_promoted";
|
||||
error.valuesRedacted = true;
|
||||
throw error;
|
||||
}
|
||||
traceStore.append(traceId, {
|
||||
type: "request",
|
||||
status: "admitted",
|
||||
label: "turn:admitted-live-kafka",
|
||||
message: "Code Agent turn is live after AgentRun durable dispatch; lifecycle visibility is owned only by hwlab.event.v1.",
|
||||
sessionId: safeSessionId(params.sessionId) || null,
|
||||
runId: payload.agentRun?.runId ?? null,
|
||||
commandId: payload.agentRun?.commandId ?? null,
|
||||
dispatchIntentId: payload.agentRun?.dispatchIntentId ?? null,
|
||||
waitingFor: "hwlab-live-kafka-sse",
|
||||
terminal: false,
|
||||
valuesPrinted: false
|
||||
});
|
||||
return { written: false, promoted: true, outboxCommitted: false, realtimeCapabilities, liveOnly: realtimeCapabilities.liveKafkaSse, valuesPrinted: false };
|
||||
}
|
||||
const inputFact = buildCodeAgentSessionInputFact({
|
||||
params,
|
||||
options,
|
||||
|
||||
@@ -16,7 +16,6 @@ import {
|
||||
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
||||
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
|
||||
import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts";
|
||||
import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts";
|
||||
import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts";
|
||||
import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts";
|
||||
|
||||
|
||||
@@ -46,29 +46,10 @@ const TRANSACTIONAL_REALTIME_ENV = Object.freeze({
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true"
|
||||
});
|
||||
|
||||
const REFRESH_REALTIME_ENV = Object.freeze({
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED: "true",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
});
|
||||
|
||||
function projectionRealtimeBridge(capabilities = {}) {
|
||||
function projectionRealtimeBridge() {
|
||||
return {
|
||||
started: true,
|
||||
capabilities: {
|
||||
directPublish: false,
|
||||
liveKafkaSse: false,
|
||||
kafkaRefreshReplay: false,
|
||||
transactionalProjector: false,
|
||||
projectionOutboxRelay: false,
|
||||
projectionRealtime: true,
|
||||
...capabilities
|
||||
},
|
||||
ready: Promise.resolve(),
|
||||
subscribeLiveHwlabEvents() { return () => {}; },
|
||||
subscribeProjectionCommits() { return () => {}; },
|
||||
async stop() {}
|
||||
};
|
||||
@@ -90,22 +71,6 @@ test("projection outbox emits immutable assistant versions in row order", () =>
|
||||
assert.deepEqual(events.map((item) => item.payload.cursor.outboxSeq), [10, 11]);
|
||||
});
|
||||
|
||||
test("workbench sync repair endpoint is removed from the product API", async () => {
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController(),
|
||||
workbenchRuntime: {},
|
||||
kafkaEventBridge: projectionRealtimeBridge(),
|
||||
env: { ...TRANSACTIONAL_REALTIME_ENV }
|
||||
});
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
try {
|
||||
const response = await getJson(server.address().port, "/v1/workbench/sync?sessionId=ses_removed_sync_repair");
|
||||
assert.equal(response.status, 404);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench realtime initial connection emits current snapshot without replaying historical outbox", async () => {
|
||||
const sessionId = "ses_realtime_snapshot_only";
|
||||
const traceId = "trc_realtime_snapshot_only";
|
||||
@@ -146,7 +111,7 @@ test("workbench realtime initial connection emits current snapshot without repla
|
||||
}
|
||||
});
|
||||
|
||||
test("product realtime uses projection outbox even when obsolete live capability values remain present", async () => {
|
||||
test("product realtime uses the projection outbox authority", async () => {
|
||||
const sessionId = "ses_composable_realtime";
|
||||
const traceId = "trc_composable_realtime";
|
||||
let projectionReads = 0;
|
||||
@@ -169,7 +134,7 @@ test("product realtime uses projection outbox even when obsolete live capability
|
||||
const server = createCloudApiServer({
|
||||
accessController: realtimeAccessController(),
|
||||
workbenchRuntime: runtime,
|
||||
kafkaEventBridge: projectionRealtimeBridge({ directPublish: true, liveKafkaSse: true }),
|
||||
kafkaEventBridge: projectionRealtimeBridge(),
|
||||
env: {
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
@@ -827,57 +792,6 @@ test("workbench trace event page keeps per-trace terminal status after later ses
|
||||
}
|
||||
});
|
||||
|
||||
function refreshReplayBridge({ records = [], endOffset = "0", onQuery = null, beforeQueryResult = null, ready = Promise.resolve(), liveReady = Promise.resolve() } = {}) {
|
||||
let liveListener = null;
|
||||
return {
|
||||
capabilities: { directPublish: true, liveKafkaSse: true, kafkaRefreshReplay: true, transactionalProjector: false, projectionOutboxRelay: false, projectionRealtime: false },
|
||||
refreshReplay: {
|
||||
groupIdPrefix: "hwlab-v03-refresh-test",
|
||||
timeoutMs: 2500,
|
||||
scanLimit: 500,
|
||||
matchedEventLimit: 50,
|
||||
liveBufferLimit: 20
|
||||
},
|
||||
ready,
|
||||
liveReady,
|
||||
subscribeLiveHwlabEvents(listener) { liveListener = listener; return () => { if (liveListener === listener) liveListener = null; }; },
|
||||
async queryHwlabEventRetention(params) {
|
||||
onQuery?.(params);
|
||||
beforeQueryResult?.(liveListener);
|
||||
return {
|
||||
topic: "hwlab.event.v1",
|
||||
events: records,
|
||||
completionReason: "end-offset",
|
||||
reachedEndOffsets: true,
|
||||
endOffsetsAvailable: true,
|
||||
endOffsets: [{ partition: 0, startOffset: "0", endOffset }],
|
||||
completion: { reason: "end-offset", complete: true, barrierReached: true, retentionStartVerified: true }
|
||||
};
|
||||
},
|
||||
async stop() {}
|
||||
};
|
||||
}
|
||||
|
||||
function refreshRecord(offset, sessionId, traceId, type, extra = {}) {
|
||||
const sourceEventId = `evt_refresh_${offset}`;
|
||||
return {
|
||||
topic: "hwlab.event.v1",
|
||||
partition: 0,
|
||||
offset: String(offset),
|
||||
value: {
|
||||
schema: "hwlab.event.v1",
|
||||
eventType: "hwlab.trace.event.projected",
|
||||
eventId: `hwlab:${sourceEventId}`,
|
||||
sourceEventId,
|
||||
traceId,
|
||||
hwlabSessionId: sessionId,
|
||||
sessionId,
|
||||
event: { type, eventType: type === "result" ? "terminal" : type, traceId, sessionId, sourceEventId, status: "running", ...extra },
|
||||
valuesPrinted: false
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function realtimeAccessController({ actor = ACTOR, sessions = [] } = {}) {
|
||||
const bySessionId = new Map(sessions.map((session) => [session.id, session]));
|
||||
const byTraceId = new Map(sessions.filter((session) => session.lastTraceId).map((session) => [session.lastTraceId, session]));
|
||||
|
||||
@@ -14,16 +14,10 @@ import {
|
||||
} from "./server-http-utils.ts";
|
||||
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
||||
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
|
||||
import { emitLiveKafkaOtelSpan } from "./kafka-event-bridge.ts";
|
||||
import { createWorkbenchKafkaRefreshHandoff, workbenchKafkaRefreshErrorPayload } from "./workbench-kafka-refresh-handoff.ts";
|
||||
import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts";
|
||||
import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts";
|
||||
import { projectionOutboxRealtimeEvents } from "./workbench-projection-outbox-events.ts";
|
||||
import { durableTraceStatus, RUNNING_STATUSES, TERMINAL_STATUSES } from "./workbench-turn-projection.ts";
|
||||
import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts";
|
||||
import {
|
||||
workbenchRealtimeCapabilities
|
||||
} from "./workbench-realtime-capabilities.ts";
|
||||
import * as workbenchFacts from "./server-workbench-facts.ts";
|
||||
|
||||
const DEFAULT_PAGE_LIMIT = 50;
|
||||
@@ -183,16 +177,11 @@ export async function drainWorkbenchRealtimeConnections(options = {}) {
|
||||
export async function handleWorkbenchRealtimeHttp(request, response, url, options = {}) {
|
||||
try {
|
||||
if (request.method !== "GET") return methodNotAllowed(response, "GET");
|
||||
const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env);
|
||||
const projectionOnly = url.pathname === "/v1/workbench/projection-events";
|
||||
if (projectionOnly) {
|
||||
sendJson(response, 410, workbenchError("workbench_projection_events_route_removed", "Use /v1/workbench/events for the single transactional projection authority."));
|
||||
return;
|
||||
}
|
||||
if (!realtimeCapabilities.projectionRealtime) {
|
||||
sendJson(response, 503, workbenchError("workbench_projection_realtime_disabled", "Workbench product realtime requires the transactional projection outbox authority."));
|
||||
return;
|
||||
}
|
||||
const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId"));
|
||||
const requestedTraceId = safeTraceId(url.searchParams.get("traceId"));
|
||||
const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS);
|
||||
@@ -453,405 +442,6 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option
|
||||
}
|
||||
}
|
||||
|
||||
async function handleLiveKafkaWorkbenchRealtimeHttp(request, response, url, options = {}) {
|
||||
const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId"));
|
||||
const requestedTraceId = safeTraceId(url.searchParams.get("traceId"));
|
||||
const realtimeCapabilities = workbenchRealtimeCapabilities(options.env ?? process.env);
|
||||
const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS);
|
||||
attachWorkbenchRealtimeOtelContext(request, {
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId,
|
||||
heartbeatMs,
|
||||
realtimeSource: "live-kafka-sse",
|
||||
realtimeCapabilities
|
||||
});
|
||||
const perf = options.backendPerformance;
|
||||
const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options);
|
||||
if (!auth) return;
|
||||
if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) {
|
||||
sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench realtime is keyed by sessionId/traceId only."));
|
||||
return;
|
||||
}
|
||||
if (!requestedSessionId && !requestedTraceId) {
|
||||
sendJson(response, 400, workbenchError("workbench_realtime_scope_required", "Workbench realtime requires sessionId or traceId."));
|
||||
return;
|
||||
}
|
||||
const authorization = await authorizeLiveKafkaWorkbenchRealtimeScope(options, auth.actor, {
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId
|
||||
});
|
||||
if (!authorization.ok) {
|
||||
sendJson(response, authorization.status, authorization.body);
|
||||
return;
|
||||
}
|
||||
const authorizedSessionId = authorization.sessionId;
|
||||
const bridge = options.kafkaEventBridge;
|
||||
if (bridge?.capabilities?.liveKafkaSse !== true || typeof bridge?.subscribeLiveHwlabEvents !== "function") {
|
||||
sendJson(response, 503, workbenchError("workbench_live_kafka_unconfigured", "Workbench live Kafka fanout is not configured."));
|
||||
return;
|
||||
}
|
||||
if (realtimeCapabilities.kafkaRefreshReplay) {
|
||||
await handleKafkaRefreshReplayWorkbenchRealtimeHttp(request, response, options, {
|
||||
bridge,
|
||||
heartbeatMs,
|
||||
realtimeCapabilities,
|
||||
requestedSessionId: authorizedSessionId,
|
||||
requestedTraceId
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
let closed = false;
|
||||
let realtimeCloseReason = "client_close";
|
||||
let realtimeCloseSignal = null;
|
||||
const realtimeStartedAtMs = Date.now();
|
||||
const cleanup = [];
|
||||
const isActive = () => !closed && !response.destroyed && !response.writableEnded;
|
||||
const closeConnection = () => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
emitWorkbenchRealtimeClosedOtelSpan(request, options.env, {
|
||||
reason: realtimeCloseReason,
|
||||
signal: realtimeCloseSignal,
|
||||
startedAtMs: realtimeStartedAtMs,
|
||||
sessionId: authorizedSessionId,
|
||||
threadId: null,
|
||||
traceId: requestedTraceId,
|
||||
activeConnectionCount: activeWorkbenchRealtimeConnections.size
|
||||
});
|
||||
for (const item of cleanup.splice(0)) item();
|
||||
};
|
||||
|
||||
response.once("close", closeConnection);
|
||||
request.once?.("aborted", closeConnection);
|
||||
request.socket?.once?.("close", closeConnection);
|
||||
cleanup.push(() => request.off?.("aborted", closeConnection));
|
||||
cleanup.push(() => request.socket?.off?.("close", closeConnection));
|
||||
|
||||
const bufferedEnvelopes = [];
|
||||
let liveDeliveryStarted = false;
|
||||
let enqueueEvent = null;
|
||||
const unsubscribe = bridge.subscribeLiveHwlabEvents((envelope, transport) => {
|
||||
if (!liveKafkaEnvelopeMatches(envelope, authorizedSessionId, requestedTraceId)) return;
|
||||
if (!liveDeliveryStarted || typeof enqueueEvent !== "function") {
|
||||
bufferedEnvelopes.push({ envelope, transport });
|
||||
return;
|
||||
}
|
||||
void enqueueEvent("hwlab.event.v1", envelope, transport);
|
||||
});
|
||||
if (typeof unsubscribe === "function") cleanup.push(unsubscribe);
|
||||
|
||||
try {
|
||||
await (bridge.liveReady ?? bridge.ready);
|
||||
} catch (error) {
|
||||
closeConnection();
|
||||
throw error;
|
||||
}
|
||||
if (!isActive()) {
|
||||
closeConnection();
|
||||
return;
|
||||
}
|
||||
|
||||
response.writeHead(200, {
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"cache-control": "no-store, no-transform",
|
||||
connection: "keep-alive",
|
||||
"x-accel-buffering": "no",
|
||||
"x-content-type-options": "nosniff"
|
||||
});
|
||||
if (typeof response.flushHeaders === "function") response.flushHeaders();
|
||||
|
||||
const writeEvent = async (name, payload, transport = null) => {
|
||||
if (!isActive()) return false;
|
||||
const block = `event: ${name}\ndata: ${JSON.stringify(payload)}\n\n`;
|
||||
const writable = response.write(block);
|
||||
if (!writable) await waitForResponseDrain(response, isActive);
|
||||
const active = isActive();
|
||||
if (active && name === "hwlab.event.v1") {
|
||||
emitLiveKafkaOtelSpan("hwlab.workbench.live_sse.business_event_write", payload, transport, {
|
||||
env: options.env,
|
||||
otelSpanEmitter: options.otelSpanEmitter ?? emitCodeAgentOtelSpan
|
||||
});
|
||||
}
|
||||
return active;
|
||||
};
|
||||
let writeChain = Promise.resolve(true);
|
||||
enqueueEvent = (name, payload, transport = null) => {
|
||||
writeChain = writeChain.then(() => writeEvent(name, payload, transport));
|
||||
return writeChain;
|
||||
};
|
||||
|
||||
const realtimeConnection = {
|
||||
close(fields = {}) {
|
||||
if (!isActive()) return false;
|
||||
realtimeCloseReason = textValue(fields.reason) || "server_shutdown";
|
||||
realtimeCloseSignal = textValue(fields.signal) || null;
|
||||
void enqueueEvent("workbench.server_draining", {
|
||||
type: "server.draining",
|
||||
status: "closing",
|
||||
reason: realtimeCloseReason,
|
||||
signal: realtimeCloseSignal,
|
||||
capabilities: realtimeCapabilities,
|
||||
lossPossible: true
|
||||
}).finally(() => {
|
||||
if (!response.writableEnded) response.end();
|
||||
});
|
||||
return true;
|
||||
},
|
||||
isActive
|
||||
};
|
||||
activeWorkbenchRealtimeConnections.add(realtimeConnection);
|
||||
cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection));
|
||||
|
||||
await enqueueEvent("workbench.connected", {
|
||||
type: "connected",
|
||||
status: "connected",
|
||||
capabilities: realtimeCapabilities,
|
||||
realtimeSource: "hwlab.event.v1",
|
||||
deliverySemantics: "live-only",
|
||||
liveOnly: true,
|
||||
replay: false,
|
||||
replaySupported: false,
|
||||
lossPossible: true,
|
||||
filters: { sessionId: authorizedSessionId, traceId: requestedTraceId }
|
||||
});
|
||||
liveDeliveryStarted = true;
|
||||
const readyBufferedEnvelopes = bufferedEnvelopes.splice(0);
|
||||
for (const { envelope, transport } of readyBufferedEnvelopes) void enqueueEvent("hwlab.event.v1", envelope, transport);
|
||||
await writeChain;
|
||||
if (!isActive()) {
|
||||
closeConnection();
|
||||
return;
|
||||
}
|
||||
const heartbeatTimer = setInterval(() => {
|
||||
void enqueueEvent("workbench.heartbeat", {
|
||||
type: "heartbeat",
|
||||
status: "connected",
|
||||
realtimeSource: "hwlab.event.v1",
|
||||
liveOnly: true,
|
||||
replay: false,
|
||||
lossPossible: true,
|
||||
serverSentAt: new Date().toISOString(),
|
||||
valuesPrinted: false
|
||||
});
|
||||
}, heartbeatMs);
|
||||
heartbeatTimer.unref?.();
|
||||
cleanup.push(() => clearInterval(heartbeatTimer));
|
||||
emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env);
|
||||
}
|
||||
|
||||
async function handleKafkaRefreshReplayWorkbenchRealtimeHttp(request, response, options, input) {
|
||||
const {
|
||||
bridge,
|
||||
heartbeatMs,
|
||||
realtimeCapabilities,
|
||||
requestedSessionId,
|
||||
requestedTraceId
|
||||
} = input;
|
||||
const refreshReplay = bridge?.refreshReplay;
|
||||
if (!refreshReplay || typeof bridge?.queryHwlabEventRetention !== "function") {
|
||||
sendJson(response, 503, workbenchError("workbench_kafka_refresh_unconfigured", "Workbench Kafka refresh replay is enabled without its retention query runtime."));
|
||||
return;
|
||||
}
|
||||
let closed = false;
|
||||
let realtimeCloseReason = "client_close";
|
||||
let realtimeCloseSignal = null;
|
||||
let failureClosing = false;
|
||||
const realtimeStartedAtMs = Date.now();
|
||||
const cleanup = [];
|
||||
const isActive = () => !closed && !response.destroyed && !response.writableEnded;
|
||||
const closeConnection = () => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
emitWorkbenchRealtimeClosedOtelSpan(request, options.env, {
|
||||
reason: realtimeCloseReason,
|
||||
signal: realtimeCloseSignal,
|
||||
startedAtMs: realtimeStartedAtMs,
|
||||
sessionId: requestedSessionId,
|
||||
threadId: null,
|
||||
traceId: requestedTraceId,
|
||||
activeConnectionCount: activeWorkbenchRealtimeConnections.size
|
||||
});
|
||||
for (const item of cleanup.splice(0)) item();
|
||||
};
|
||||
|
||||
response.once("close", closeConnection);
|
||||
request.once?.("aborted", closeConnection);
|
||||
request.socket?.once?.("close", closeConnection);
|
||||
cleanup.push(() => request.off?.("aborted", closeConnection));
|
||||
cleanup.push(() => request.socket?.off?.("close", closeConnection));
|
||||
|
||||
try {
|
||||
await (bridge.liveReady ?? bridge.ready);
|
||||
} catch (error) {
|
||||
for (const item of cleanup.splice(0)) item();
|
||||
throw error;
|
||||
}
|
||||
if (!isActive()) {
|
||||
closeConnection();
|
||||
return;
|
||||
}
|
||||
|
||||
response.writeHead(200, {
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"cache-control": "no-store, no-transform",
|
||||
connection: "keep-alive",
|
||||
"x-accel-buffering": "no",
|
||||
"x-content-type-options": "nosniff"
|
||||
});
|
||||
if (typeof response.flushHeaders === "function") response.flushHeaders();
|
||||
|
||||
const writeEvent = async (name, payload, transport = null) => {
|
||||
if (!isActive()) return false;
|
||||
const block = `event: ${name}\ndata: ${JSON.stringify(payload)}\n\n`;
|
||||
const writable = response.write(block);
|
||||
if (!writable) await waitForResponseDrain(response, isActive);
|
||||
const active = isActive();
|
||||
if (active && name === "hwlab.event.v1") {
|
||||
emitLiveKafkaOtelSpan("hwlab.workbench.live_sse.business_event_write", payload, transport, {
|
||||
env: options.env,
|
||||
otelSpanEmitter: options.otelSpanEmitter ?? emitCodeAgentOtelSpan
|
||||
});
|
||||
}
|
||||
return active;
|
||||
};
|
||||
let writeChain = Promise.resolve(true);
|
||||
const enqueueEvent = (name, payload, transport = null) => {
|
||||
writeChain = writeChain.then(() => writeEvent(name, payload, transport));
|
||||
return writeChain;
|
||||
};
|
||||
const closeWithRefreshFailure = async (error) => {
|
||||
if (failureClosing || !isActive()) return;
|
||||
failureClosing = true;
|
||||
await enqueueEvent("workbench.error", workbenchKafkaRefreshErrorPayload(error, {
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId
|
||||
}));
|
||||
await writeChain;
|
||||
if (!response.writableEnded) response.end();
|
||||
};
|
||||
|
||||
const realtimeConnection = {
|
||||
close(fields = {}) {
|
||||
if (!isActive()) return false;
|
||||
realtimeCloseReason = textValue(fields.reason) || "server_shutdown";
|
||||
realtimeCloseSignal = textValue(fields.signal) || null;
|
||||
void enqueueEvent("workbench.server_draining", {
|
||||
type: "server.draining",
|
||||
status: "closing",
|
||||
reason: realtimeCloseReason,
|
||||
signal: realtimeCloseSignal,
|
||||
capabilities: realtimeCapabilities,
|
||||
lossPossible: false
|
||||
}).finally(() => {
|
||||
if (!response.writableEnded) response.end();
|
||||
});
|
||||
return true;
|
||||
},
|
||||
isActive
|
||||
};
|
||||
activeWorkbenchRealtimeConnections.add(realtimeConnection);
|
||||
cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection));
|
||||
|
||||
const handoff = createWorkbenchKafkaRefreshHandoff({
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId,
|
||||
liveBufferLimit: refreshReplay.liveBufferLimit,
|
||||
subscribeLive: (listener) => bridge.subscribeLiveHwlabEvents(listener),
|
||||
queryRetention: ({ signal }) => bridge.queryHwlabEventRetention({
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId,
|
||||
limit: refreshReplay.matchedEventLimit,
|
||||
scanLimit: refreshReplay.scanLimit,
|
||||
timeoutMs: refreshReplay.timeoutMs,
|
||||
groupIdPrefix: refreshReplay.groupIdPrefix,
|
||||
fromBeginning: true,
|
||||
signal
|
||||
}),
|
||||
deliverEvent: (envelope, transport) => enqueueEvent("hwlab.event.v1", envelope, transport),
|
||||
deliverConnected: (summary) => enqueueEvent("workbench.connected", {
|
||||
type: "connected",
|
||||
status: "connected",
|
||||
capabilities: realtimeCapabilities,
|
||||
realtimeSource: "hwlab.event.v1",
|
||||
deliverySemantics: "kafka-retention-then-live",
|
||||
liveOnly: false,
|
||||
replay: true,
|
||||
replaySupported: true,
|
||||
lossPossible: false,
|
||||
filters: { sessionId: requestedSessionId, traceId: requestedTraceId },
|
||||
refreshReplay: summary
|
||||
}),
|
||||
onFailure: closeWithRefreshFailure
|
||||
});
|
||||
cleanup.push(() => handoff.stop("connection-closed"));
|
||||
|
||||
try {
|
||||
await handoff.start();
|
||||
} catch (error) {
|
||||
if (isActive()) await closeWithRefreshFailure(error);
|
||||
return;
|
||||
}
|
||||
if (!isActive()) return;
|
||||
|
||||
const heartbeatTimer = setInterval(() => {
|
||||
void enqueueEvent("workbench.heartbeat", {
|
||||
type: "heartbeat",
|
||||
status: "connected",
|
||||
realtimeSource: "hwlab.event.v1",
|
||||
deliverySemantics: "kafka-retention-then-live",
|
||||
liveOnly: false,
|
||||
replay: true,
|
||||
lossPossible: false,
|
||||
serverSentAt: new Date().toISOString(),
|
||||
valuesPrinted: false
|
||||
});
|
||||
}, heartbeatMs);
|
||||
heartbeatTimer.unref?.();
|
||||
cleanup.push(() => clearInterval(heartbeatTimer));
|
||||
emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env);
|
||||
}
|
||||
|
||||
async function authorizeLiveKafkaWorkbenchRealtimeScope(options, actor, { sessionId, traceId }) {
|
||||
const access = options.accessController;
|
||||
if ((sessionId && typeof access?.getAgentSession !== "function") || (traceId && typeof access?.getAgentSessionByTraceId !== "function")) {
|
||||
return {
|
||||
ok: false,
|
||||
status: 503,
|
||||
body: workbenchError("workbench_realtime_authorization_unconfigured", "Workbench live realtime ownership authorization is not configured.")
|
||||
};
|
||||
}
|
||||
|
||||
const [sessionById, sessionByTrace] = await Promise.all([
|
||||
sessionId ? access.getAgentSession(sessionId) : null,
|
||||
traceId ? access.getAgentSessionByTraceId(traceId) : null
|
||||
]);
|
||||
const sessionByIdId = sessionById ? safeSessionId(sessionById.id ?? sessionById.sessionId) : null;
|
||||
const sessionByTraceId = sessionByTrace ? safeSessionId(sessionByTrace.id ?? sessionByTrace.sessionId) : null;
|
||||
const scopeMissing = (sessionId && sessionByIdId !== sessionId)
|
||||
|| (traceId && !sessionByTraceId)
|
||||
|| (sessionId && traceId && sessionByIdId !== sessionByTraceId);
|
||||
const resolvedSessions = [sessionById, sessionByTrace].filter(Boolean);
|
||||
const ownedByActor = actor?.role === "admin" || (Boolean(actor?.id)
|
||||
&& resolvedSessions.length > 0
|
||||
&& resolvedSessions.every((session) => textValue(session.ownerUserId) === actor.id));
|
||||
if (scopeMissing || !ownedByActor) {
|
||||
return {
|
||||
ok: false,
|
||||
status: 404,
|
||||
body: workbenchError("workbench_realtime_scope_not_found", "Workbench realtime scope is not visible to the current actor.")
|
||||
};
|
||||
}
|
||||
return { ok: true, sessionId: sessionByIdId ?? sessionByTraceId };
|
||||
}
|
||||
|
||||
function liveKafkaEnvelopeMatches(envelope, sessionId, traceId) {
|
||||
if (!envelope || envelope.schema !== "hwlab.event.v1") return false;
|
||||
if (sessionId && safeSessionId(envelope.hwlabSessionId ?? envelope.sessionId) !== sessionId) return false;
|
||||
if (traceId && safeTraceId(envelope.traceId) !== traceId) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export function attachWorkbenchRealtimeOtelContext(request, fields = {}) {
|
||||
const context = request?.hwlabHttpRequestContext;
|
||||
if (!context) return;
|
||||
@@ -865,9 +455,9 @@ export function attachWorkbenchRealtimeOtelContext(request, fields = {}) {
|
||||
"workbench.thread_id": fields.threadId ?? null,
|
||||
"workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null,
|
||||
"workbench.sse.realtime_source": fields.realtimeSource ?? null,
|
||||
"workbench.sse.live_kafka_enabled": fields.realtimeCapabilities?.liveKafkaSse ?? null,
|
||||
"workbench.sse.projection_realtime_enabled": fields.realtimeCapabilities?.projectionRealtime ?? null,
|
||||
"workbench.sse.outbox_mode": fields.realtimeCapabilities?.liveKafkaSse === true ? false : true
|
||||
"workbench.sse.live_kafka_enabled": false,
|
||||
"workbench.sse.projection_realtime_enabled": true,
|
||||
"workbench.sse.outbox_mode": true
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -372,10 +372,7 @@ export async function buildHealthPayload(options = {}) {
|
||||
async function kafkaProjectorHealth(projector, { liveProbe = false } = {}) {
|
||||
if (!projector?.started) return { started: false, reason: projector?.reason ?? "disabled", valuesRedacted: true };
|
||||
const identity = {
|
||||
capabilities: projector.capabilities ?? null,
|
||||
directPublishGroupId: projector.directPublishGroupId ?? null,
|
||||
projectorGroupId: projector.projectorGroupId ?? null,
|
||||
hwlabEventGroupId: projector.hwlabEventGroupId ?? null,
|
||||
agentRunTopic: projector.agentRunTopic,
|
||||
hwlabTopic: projector.hwlabTopic
|
||||
};
|
||||
|
||||
@@ -144,6 +144,89 @@ test("authoritative assistant final seals terminal with a final response", () =>
|
||||
assert.equal(built.facts.checkpoints[0].projectionStatus, "caught_up");
|
||||
});
|
||||
|
||||
test("canonical event time owns running, terminal, replay, and duration", () => {
|
||||
const running = buildWorkbenchProjectionEventFacts({
|
||||
projectedSeq: 20,
|
||||
projectedAt: "2026-07-10T10:17:00.000Z",
|
||||
event: {
|
||||
traceId: "trc_canonical_duration",
|
||||
sessionId: "ses_canonical_duration",
|
||||
sourceEventId: "evt_canonical_running",
|
||||
sourceSeq: 41,
|
||||
type: "assistant",
|
||||
eventType: "assistant_progress",
|
||||
status: "running",
|
||||
createdAt: "2026-07-10T10:00:01.000Z",
|
||||
startedAt: "2026-07-10T09:43:00.000Z",
|
||||
durationMs: 1020000
|
||||
}
|
||||
});
|
||||
const runningCheckpoint = running.facts.checkpoints[0];
|
||||
assert.equal(runningCheckpoint.startedAt, "2026-07-10T10:00:01.000Z");
|
||||
assert.equal(runningCheckpoint.lastEventAt, "2026-07-10T10:00:01.000Z");
|
||||
assert.equal(runningCheckpoint.durationMs, null);
|
||||
|
||||
const terminal = buildWorkbenchProjectionEventFacts({
|
||||
projectedSeq: 21,
|
||||
projectedAt: "2026-07-10T10:18:00.000Z",
|
||||
previousCheckpoint: runningCheckpoint,
|
||||
event: {
|
||||
traceId: "trc_canonical_duration",
|
||||
sessionId: "ses_canonical_duration",
|
||||
sourceEventId: "evt_canonical_terminal",
|
||||
sourceSeq: 42,
|
||||
type: "assistant",
|
||||
eventType: "assistant",
|
||||
status: "completed",
|
||||
terminal: true,
|
||||
replyAuthority: true,
|
||||
text: "canonical final",
|
||||
createdAt: "2026-07-10T10:00:04.000Z",
|
||||
finishedAt: "2026-07-10T10:18:00.000Z",
|
||||
durationMs: 1020000
|
||||
}
|
||||
});
|
||||
const terminalCheckpoint = terminal.facts.checkpoints[0];
|
||||
assert.equal(terminalCheckpoint.startedAt, "2026-07-10T10:00:01.000Z");
|
||||
assert.equal(terminalCheckpoint.finishedAt, "2026-07-10T10:00:04.000Z");
|
||||
assert.equal(terminalCheckpoint.durationMs, 3000);
|
||||
|
||||
const [replayed] = projectionOutboxRealtimeEvents({
|
||||
events: [{
|
||||
outboxSeq: 72,
|
||||
entityFamily: "traceEvents",
|
||||
entityId: terminal.facts.traceEvents[0].id,
|
||||
projectedSeq: 21,
|
||||
projectionRevision: 1,
|
||||
traceId: "trc_canonical_duration",
|
||||
sessionId: "ses_canonical_duration",
|
||||
commitType: "event",
|
||||
terminal: true,
|
||||
sealed: true,
|
||||
payload: { family: "traceEvents", fact: terminal.facts.traceEvents[0] }
|
||||
}]
|
||||
});
|
||||
assert.deepEqual(replayed.payload.cursor, { outboxSeq: 72, traceSeq: 21 });
|
||||
assert.equal(replayed.payload.event.durationMs, 3000);
|
||||
const [refreshed] = projectionOutboxRealtimeEvents({
|
||||
events: [{
|
||||
outboxSeq: 72,
|
||||
entityFamily: "traceEvents",
|
||||
entityId: terminal.facts.traceEvents[0].id,
|
||||
projectedSeq: 21,
|
||||
projectionRevision: 1,
|
||||
traceId: "trc_canonical_duration",
|
||||
sessionId: "ses_canonical_duration",
|
||||
commitType: "event",
|
||||
terminal: true,
|
||||
sealed: true,
|
||||
payload: { family: "traceEvents", fact: terminal.facts.traceEvents[0] }
|
||||
}]
|
||||
});
|
||||
assert.deepEqual(refreshed.payload.cursor, replayed.payload.cursor);
|
||||
assert.equal(refreshed.payload.event.durationMs, replayed.payload.event.durationMs);
|
||||
});
|
||||
|
||||
test("completed terminal without final response remains visibly unsealed", () => {
|
||||
const built = buildWorkbenchProjectionEventFacts({
|
||||
projectedSeq: 1,
|
||||
|
||||
@@ -205,12 +205,9 @@ export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = {
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
const explicitEventTiming = normalizeTimingProjection(event.timing) ?? normalizeTimingProjection(event);
|
||||
const explicitDurationMs = durationValue(event.durationMs ?? explicitEventTiming?.durationMs);
|
||||
const explicitStartedAt = optionalTimestampValue(event.startedAt ?? event.traceStartedAt ?? event.runnerStartedAt ?? explicitEventTiming?.startedAt);
|
||||
const provisionalLastEventAt = latestTimestamp(previousTiming?.lastEventAt, explicitEventTiming?.lastEventAt, occurredAt);
|
||||
const provisionalFinishedAt = sourceTerminal ? latestTimestamp(explicitEventTiming?.finishedAt, optionalTimestampValue(event.finishedAt), provisionalLastEventAt, occurredAt) : null;
|
||||
const startedAt = previousTiming?.startedAt ?? explicitStartedAt ?? startedAtFromDuration(provisionalFinishedAt, explicitDurationMs) ?? (sourceSeq <= 1 ? occurredAt : null);
|
||||
const provisionalLastEventAt = latestTimestamp(previousTiming?.lastEventAt, occurredAt);
|
||||
const provisionalFinishedAt = sourceTerminal ? provisionalLastEventAt : null;
|
||||
const startedAt = previousTiming?.startedAt ?? occurredAt;
|
||||
const lastEventAt = provisionalLastEventAt;
|
||||
const finishedAt = sourceTerminal ? latestTimestamp(provisionalFinishedAt, lastEventAt) : null;
|
||||
const sourceEventId = workbenchSourceEventId(event, { traceId, sourceSeq, occurredAt });
|
||||
@@ -226,7 +223,7 @@ export function buildWorkbenchProjectionEventFacts({ event = {}, requestMeta = {
|
||||
const terminalSealBlocked = sourceTerminal && sourceStatus === "completed" && !messageFinalText;
|
||||
const effectiveTerminal = sourceTerminal && !terminalSealBlocked;
|
||||
const effectiveStatus = effectiveTerminal ? sourceStatus : "running";
|
||||
const timing = eventTimingProjection({ startedAt, lastEventAt, finishedAt, terminal: effectiveTerminal, durationMs: explicitDurationMs ?? previousTiming?.durationMs });
|
||||
const timing = eventTimingProjection({ startedAt, lastEventAt, finishedAt, terminal: effectiveTerminal });
|
||||
const timingAuthorityIssue = terminalTimingAuthorityIssue(timing, { terminal: effectiveTerminal, traceId, source: "kafka", status: effectiveStatus, label: event.label, sourceSeq });
|
||||
const projectionDiagnostic = { timingAuthorityIssue, terminalSealBlocked, sourceTerminal, sourceStatus, authority: "agentrun-kafka-projector", valuesRedacted: true };
|
||||
const userMessageFact = sessionId && previousCheckpoint?.userMessage ? normalizeMessageFact(previousCheckpoint.userMessage, 0, { traceId, sessionId, turnId, terminal: false, terminalStatus: "sent", finalText: null, timestamp: projectedAt, timing }) : null;
|
||||
@@ -499,13 +496,6 @@ function elapsedBetween(startedAt, endedAt) {
|
||||
return Math.trunc(end - start);
|
||||
}
|
||||
|
||||
function startedAtFromDuration(finishedAt, durationMs) {
|
||||
const end = Date.parse(String(finishedAt ?? ""));
|
||||
const duration = durationValue(durationMs);
|
||||
if (!Number.isFinite(end) || duration === null) return null;
|
||||
return new Date(Math.max(0, end - duration)).toISOString();
|
||||
}
|
||||
|
||||
function terminalTimingAuthorityIssue(timing, { terminal = false, traceId = null, source = null, status = null, label = null, sourceSeq = null } = {}) {
|
||||
if (!terminal) return null;
|
||||
const startedAt = optionalTimestampValue(timing?.startedAt);
|
||||
|
||||
@@ -1,35 +0,0 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench composable realtime capabilities.
|
||||
// Responsibility: validate YAML-owned capability switches without code defaults or implied mode selection.
|
||||
|
||||
export const WORKBENCH_REALTIME_CAPABILITY_ENVS = Object.freeze({
|
||||
directPublish: "HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
|
||||
liveKafkaSse: "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
|
||||
kafkaRefreshReplay: "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
|
||||
transactionalProjector: "HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
|
||||
projectionOutboxRelay: "HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
|
||||
projectionRealtime: "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
|
||||
});
|
||||
|
||||
export function workbenchRealtimeCapabilities(env = process.env) {
|
||||
return Object.fromEntries(Object.entries(WORKBENCH_REALTIME_CAPABILITY_ENVS).map(([key, name]) => [key, requiredBooleanEnv(env, name)]));
|
||||
}
|
||||
|
||||
export function requiredBooleanEnv(env, name) {
|
||||
const value = String(env?.[name] ?? "").trim().toLowerCase();
|
||||
if (value === "true" || value === "1") return true;
|
||||
if (value === "false" || value === "0") return false;
|
||||
throw capabilityError(
|
||||
"hwlab_workbench_realtime_capability_invalid",
|
||||
`${name} is required and must be explicitly true or false in the owning YAML.`,
|
||||
name
|
||||
);
|
||||
}
|
||||
|
||||
function capabilityError(code, message, envName) {
|
||||
return Object.assign(new Error(message), {
|
||||
code,
|
||||
envName,
|
||||
statusCode: 503,
|
||||
valuesRedacted: true
|
||||
});
|
||||
}
|
||||
@@ -400,11 +400,6 @@ function validateDisplayLocale(locale) {
|
||||
function workbenchRuntimeConfigFromEnv() {
|
||||
const traceTimeline = {};
|
||||
const result = {
|
||||
realtimeFeatures: {
|
||||
liveKafkaSse: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED, "HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED"),
|
||||
kafkaRefreshReplay: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED, "HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED"),
|
||||
projectionRealtime: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED, "HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED")
|
||||
},
|
||||
debugCapabilities: {
|
||||
isolatedKafka: requiredWorkbenchRealtimeFeature(process.env.HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED, "HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED"),
|
||||
rawHwlabEventWindow: {
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { mkdtemp, rm, writeFile } from "node:fs/promises";
|
||||
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import { createServer } from "node:http";
|
||||
import { connect } from "node:net";
|
||||
import os from "node:os";
|
||||
@@ -16,7 +16,25 @@ test("cloud web classifies Bun client disconnect ECONNRESET as non-fatal", () =>
|
||||
assert.equal(isClientDisconnectError(new Error("database migration failed")), false);
|
||||
});
|
||||
|
||||
test("serveCloudWeb rejects a missing refresh replay capability without opening health or SPA routes", async () => {
|
||||
test("repo feature schema owns one unique key per product feature", async () => {
|
||||
const schema = JSON.parse(await readFile(new URL("../../config/feature-config.schema.json", import.meta.url), "utf8"));
|
||||
assert.equal(schema.$schema, "https://json-schema.org/draft/2020-12/schema");
|
||||
assert.deepEqual(Object.keys(schema.properties), ["tracePanel", "rawEvents", "debugReplay", "views"]);
|
||||
const features = Object.values(schema.properties).map((property) => property["x-unidesk-feature"]);
|
||||
assert.equal(features.every((feature) => typeof feature === "string" && feature.length > 0), true);
|
||||
assert.equal(new Set(features).size, features.length);
|
||||
const encoded = JSON.stringify(schema);
|
||||
for (const architectureEnv of [
|
||||
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
|
||||
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
|
||||
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
|
||||
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
|
||||
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
|
||||
]) assert.equal(encoded.includes(architectureEnv), false);
|
||||
});
|
||||
|
||||
test("serveCloudWeb starts without obsolete architecture capabilities", async () => {
|
||||
const restoreEnv = withEnv({
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_ZONE: "Asia/Shanghai",
|
||||
HWLAB_CLOUD_WEB_DISPLAY_TIME_LOCALE: "zh-CN",
|
||||
@@ -29,19 +47,21 @@ test("serveCloudWeb rejects a missing refresh replay capability without opening
|
||||
HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_MAX_ENTRIES: "200",
|
||||
HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_MAX_RETAINED_BYTES: "1048576"
|
||||
});
|
||||
delete process.env.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED;
|
||||
const port = await unusedPort();
|
||||
let server = null;
|
||||
try {
|
||||
await assert.rejects(serveCloudWeb({
|
||||
server = await serveCloudWeb({
|
||||
port,
|
||||
serviceId: "hwlab-cloud-web",
|
||||
healthPayload: () => ({ status: "ok" }),
|
||||
sendJson() {
|
||||
assert.fail("missing runtime configuration must fail before serving any response");
|
||||
sendJson(response, statusCode, body) {
|
||||
response.writeHead(statusCode, { "content-type": "application/json" });
|
||||
response.end(JSON.stringify(body));
|
||||
}
|
||||
}), /HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED/u);
|
||||
assert.equal(await canConnect(port), false, "serveCloudWeb must not bind before runtime configuration is valid");
|
||||
});
|
||||
assert.equal(await canConnect(port), true);
|
||||
} finally {
|
||||
if (server) await close(server);
|
||||
restoreEnv();
|
||||
}
|
||||
});
|
||||
@@ -497,7 +517,7 @@ test("cloud web serves client deep links through the Vue shell", async () => {
|
||||
const html = await response.text();
|
||||
assert.match(html, /<div id="root"><\/div>/u, route);
|
||||
assert.match(html, /HWLAB_CLOUD_WEB_CONFIG/u, route);
|
||||
assert.match(html, /"realtimeFeatures":\{"liveKafkaSse":true,"kafkaRefreshReplay":true,"projectionRealtime":false\}/u, route);
|
||||
assert.doesNotMatch(html, /realtimeFeatures|liveKafkaSse|kafkaRefreshReplay|projectionRealtime/u, route);
|
||||
assert.match(html, /"debugCapabilities":\{"isolatedKafka":true,"rawHwlabEventWindow":\{"enabled":true,"maxEntries":200,"maxRetainedBytes":1048576\}\}/u, route);
|
||||
}
|
||||
|
||||
|
||||
@@ -687,11 +687,14 @@ test("v03 render keeps node identity as data instead of generated structure", as
|
||||
assert.equal(cloudApiEnv.get("HWLAB_CLOUD_DB_SSL_MODE"), "require");
|
||||
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_ENABLED"), "true");
|
||||
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED"), "true");
|
||||
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED"), "true");
|
||||
assert.equal(cloudApiEnv.get("HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED"), "true");
|
||||
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED"), "false");
|
||||
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED"), "false");
|
||||
assert.equal(cloudApiEnv.get("HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"), "false");
|
||||
for (const name of [
|
||||
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
|
||||
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
|
||||
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
|
||||
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
|
||||
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
|
||||
]) assert.equal(cloudApiEnv.has(name), false);
|
||||
assert.equal(cloudApiEnv.get("HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED"), "true");
|
||||
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_BOOTSTRAP_SERVERS"), "platform-infra-kafka-kafka-bootstrap.platform-infra.svc.cluster.local:9092");
|
||||
assert.equal(cloudApiEnv.get("HWLAB_KAFKA_STDIO_TOPIC"), "codex-stdio.raw.v1");
|
||||
@@ -709,8 +712,14 @@ test("v03 render keeps node identity as data instead of generated structure", as
|
||||
const cloudWeb = (workloadsJson.items ?? []).find((item) => item.kind === "Deployment" && item.metadata?.name === "hwlab-cloud-web");
|
||||
const cloudWebContainer = collectContainersFromItem(cloudWeb).find((container) => container.name === "hwlab-cloud-web");
|
||||
const cloudWebEnv = new Map((cloudWebContainer?.env ?? []).map((entry) => [entry.name, entry.value]));
|
||||
assert.equal(cloudWebEnv.get("HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED"), "true");
|
||||
assert.equal(cloudWebEnv.get("HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"), "false");
|
||||
for (const name of [
|
||||
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
|
||||
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
|
||||
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
|
||||
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
|
||||
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
|
||||
]) assert.equal(cloudWebEnv.has(name), false);
|
||||
assert.equal(cloudWebEnv.get("HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED"), "true");
|
||||
assert.equal(cloudWebEnv.get("HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_ENABLED"), "1");
|
||||
assert.equal(cloudWebEnv.get("HWLAB_WORKBENCH_RAW_HWLAB_EVENT_WINDOW_MAX_ENTRIES"), "200");
|
||||
|
||||
@@ -3,13 +3,14 @@ import test from "node:test";
|
||||
|
||||
import { readStructuredFile } from "./src/structured-config.mjs";
|
||||
|
||||
const PRODUCT_CAPABILITIES = Object.freeze({
|
||||
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
||||
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
||||
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
||||
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
||||
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false"
|
||||
});
|
||||
const OBSOLETE_ARCHITECTURE_ENV = Object.freeze([
|
||||
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
|
||||
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
|
||||
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
|
||||
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
|
||||
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
|
||||
]);
|
||||
|
||||
const LIVE_KAFKA_CONTRACT = Object.freeze({
|
||||
HWLAB_KAFKA_STDIO_TOPIC: "codex-stdio.raw.v1",
|
||||
@@ -21,7 +22,6 @@ const LIVE_KAFKA_CONTRACT = Object.freeze({
|
||||
});
|
||||
|
||||
const UNIDESK_OWNED_REFRESH_REPLAY_ENV = Object.freeze([
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_GROUP_PREFIX",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_TIMEOUT_MS",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_SCAN_LIMIT",
|
||||
@@ -37,15 +37,16 @@ const ISOLATED_DEBUG_CONTRACT = Object.freeze({
|
||||
HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_TIMEOUT_MS: "15000"
|
||||
});
|
||||
|
||||
test("v03 YAML keeps pure Kafka realtime capabilities independently composable", async () => {
|
||||
test("v03 YAML fixes one transactional projection architecture without architecture env", async () => {
|
||||
const deploy = await readStructuredFile(process.cwd(), "deploy/deploy.yaml");
|
||||
const cloudApi = deploy.lanes?.v03?.services?.find((service) => service.serviceId === "hwlab-cloud-api");
|
||||
const cloudWeb = deploy.lanes?.v03?.serviceDeclarations?.["hwlab-cloud-web"];
|
||||
assert.ok(cloudApi, "deploy.lanes.v03.services must declare hwlab-cloud-api");
|
||||
assert.ok(cloudWeb, "deploy.lanes.v03.serviceDeclarations must declare hwlab-cloud-web");
|
||||
|
||||
for (const [name, expected] of Object.entries(PRODUCT_CAPABILITIES)) {
|
||||
assert.equal(cloudApi.env?.[name], expected, `${name} must be explicitly YAML-owned`);
|
||||
for (const name of OBSOLETE_ARCHITECTURE_ENV) {
|
||||
assert.equal(cloudApi.env?.[name], undefined, `${name} must not be rendered into cloud-api`);
|
||||
assert.equal(cloudWeb.env?.[name], undefined, `${name} must not be rendered into cloud-web`);
|
||||
}
|
||||
for (const [name, expected] of Object.entries(LIVE_KAFKA_CONTRACT)) {
|
||||
assert.equal(cloudApi.env?.[name], expected, `${name} must preserve the pure Kafka live contract`);
|
||||
@@ -59,8 +60,5 @@ test("v03 YAML keeps pure Kafka realtime capabilities independently composable",
|
||||
|
||||
assert.equal(cloudApi.env?.HWLAB_SESSION_COOKIE_DOMAIN, undefined, "internal and public origins require host-scoped session cookies");
|
||||
assert.notEqual(cloudApi.env?.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID, "hwlab-v03-agentrun-event-bridge");
|
||||
assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED, "true");
|
||||
assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED, "true");
|
||||
assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED, "false");
|
||||
assert.equal(cloudWeb.env?.HWLAB_WORKBENCH_KAFKA_DEBUG_REPLAY_ENABLED, "true");
|
||||
});
|
||||
|
||||
@@ -199,7 +199,6 @@ assertIncludes(workbenchRealtimePlanSource, "afterOutboxSeq: finiteNumber(recove
|
||||
assert.doesNotMatch(workbenchRealtimePlanSource, /force:\s*true/u, "Realtime recovery planner must not turn transport recovery into force-refresh work");
|
||||
assertIncludes(workbenchColadaSource, "const state = await queryCache.refresh(entry);", "Workbench reads must preserve Colada staleTime/min-interval governance");
|
||||
assert.doesNotMatch(workbenchColadaSource, /queryCache\.fetch\(entry/u, "Workbench reads must not call queryCache.fetch(entry), which bypasses Colada freshness governance");
|
||||
assertIncludes(workbenchStoreSource, "runtimePolicy.sessionListRealtimeRefreshDelayMs", "Realtime recovery delay must come from runtime policy instead of store constants");
|
||||
assertIncludes(workbenchStoreSource, "workbenchColadaQueries.fetchSession", "Realtime session detail recovery must enter Colada query facade");
|
||||
assertIncludes(workbenchStoreSource, "runtimePolicy.workbenchSessionDetailMinRefreshMs", "Realtime session detail recovery budget must come from runtime policy");
|
||||
assert.doesNotMatch(workbenchStoreSource, /refreshRealtimeSessionMessages[\s\S]{0,900}refreshSessionMessageProjectionPage\(id, \{ force: true \}\)/u, "Realtime session message recovery must not force-bypass the message projection refresh budget");
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import test from "node:test";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
const repositoryRoot = fileURLToPath(new URL("../../..", import.meta.url));
|
||||
const productionFiles = [
|
||||
"internal/cloud/kafka-event-bridge.ts",
|
||||
"internal/cloud/server-workbench-realtime-http.ts",
|
||||
"internal/cloud/server-workbench-facts.ts",
|
||||
"internal/cloud/server-code-agent-admission-http.ts",
|
||||
"internal/cloud/server.ts",
|
||||
"web/hwlab-cloud-web/src/config/runtime.ts",
|
||||
"web/hwlab-cloud-web/src/config/workbench-runtime-policy.ts",
|
||||
"web/hwlab-cloud-web/src/types/global.d.ts",
|
||||
"web/hwlab-cloud-web/src/api/workbench-events.ts",
|
||||
"web/hwlab-cloud-web/src/utils/workbench-stream-transport.ts",
|
||||
"web/hwlab-cloud-web/src/utils/workbench-realtime-runtime.ts",
|
||||
"web/hwlab-cloud-web/src/stores/workbench.ts",
|
||||
"web/hwlab-cloud-web/src/stores/workbench-colada-queries.ts",
|
||||
"web/hwlab-cloud-web/src/stores/workbench-event-reducer.ts",
|
||||
"web/hwlab-cloud-web/src/stores/workbench-realtime-plan.ts",
|
||||
"web/hwlab-cloud-web/src/stores/workbench-session.ts",
|
||||
"web/hwlab-cloud-web/src/stores/workbench-session-messages-read-budget.ts"
|
||||
];
|
||||
const forbiddenArchitectureIdentifiers = [
|
||||
"directPublish",
|
||||
"liveKafkaSse",
|
||||
"kafkaRefreshReplay",
|
||||
"transactionalProjector",
|
||||
"projectionOutboxRelay",
|
||||
"projectionRealtime",
|
||||
"startLiveHwlabKafkaEventBridge",
|
||||
"handleLiveKafkaWorkbenchRealtimeHttp",
|
||||
"handleKafkaRefreshReplayWorkbenchRealtimeHttp",
|
||||
"subscribeLiveHwlabEvents",
|
||||
"queryHwlabEventRetention",
|
||||
"WorkbenchRealtimeCapabilities",
|
||||
"workbenchRealtimeCapabilities",
|
||||
"workbenchHistoryAuthorityPolicy",
|
||||
"workbenchRealtimeTransportEnabled",
|
||||
"workbenchRealtimeTraceIdForCapabilities",
|
||||
"workbenchProjectionEventStreamPath",
|
||||
"scheduleSessionListRefresh",
|
||||
"invalidateSessionList",
|
||||
"fetchWorkbenchTurnStatus",
|
||||
"refreshTurnStatusByTraceId",
|
||||
"refreshMessageProjectionForTrace",
|
||||
"selectActiveTurnStatusRefreshTraceIds",
|
||||
"sessionDetailAutoReadDecision",
|
||||
"projectionMergeCommitSummary",
|
||||
"forceRead",
|
||||
"sealRestoredActiveTurnMessages",
|
||||
"messageNeedsRestoredTurnSeal",
|
||||
"refreshSessionStatusAuthority",
|
||||
"readTerminalTraceDetailGaps",
|
||||
"workbenchTraceMessagesWindowLimit",
|
||||
"handleWorkbenchSyncHttp",
|
||||
"/v1/workbench/sync"
|
||||
];
|
||||
const obsoleteArchitectureEnv = [
|
||||
"HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED",
|
||||
"HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED",
|
||||
"HWLAB_WORKBENCH_KAFKA_REFRESH_REPLAY_ENABLED",
|
||||
"HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED",
|
||||
"HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED",
|
||||
"HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED"
|
||||
];
|
||||
|
||||
test("Workbench production source has one fixed transactional projection architecture", async () => {
|
||||
const violations: string[] = [];
|
||||
for (const relativePath of productionFiles) {
|
||||
const source = await readFile(`${repositoryRoot}/${relativePath}`, "utf8");
|
||||
for (const identifier of forbiddenArchitectureIdentifiers) {
|
||||
if (source.includes(identifier)) violations.push(`${relativePath}: ${identifier}`);
|
||||
}
|
||||
}
|
||||
assert.deepEqual(violations, []);
|
||||
});
|
||||
|
||||
test("production deploy does not render architecture selection env", async () => {
|
||||
const deploySource = await readFile(`${repositoryRoot}/deploy/deploy.yaml`, "utf8");
|
||||
const violations = obsoleteArchitectureEnv.filter((name) => deploySource.includes(name));
|
||||
assert.deepEqual(violations, []);
|
||||
});
|
||||
|
||||
test("raw hwlab Kafka fixtures remain isolated to the explicit debug path", async () => {
|
||||
const fixtureSource = await readFile(`${repositoryRoot}/web/hwlab-cloud-web/src/stores/workbench-live-kafka-event.ts`, "utf8");
|
||||
const debugSource = await readFile(`${repositoryRoot}/web/hwlab-cloud-web/src/stores/workbench-isolated-kafka-debug.ts`, "utf8");
|
||||
assert.match(fixtureSource, /historical hwlab\.event\.debug\.v1 fixtures/u);
|
||||
assert.match(fixtureSource, /never imported by the product Workbench SSE path/u);
|
||||
assert.match(debugSource, /workbench-live-kafka-event/u);
|
||||
});
|
||||
@@ -6,9 +6,9 @@ import test from "node:test";
|
||||
|
||||
import type { ChatMessage } from "../src/types/index.ts";
|
||||
import { workbenchSessionDetailPathForTest, workbenchSessionMessagesPathForTest } from "../src/api/workbench.ts";
|
||||
import { workbenchEventStreamPath, workbenchProjectionEventStreamPath } from "../src/api/workbench-events.ts";
|
||||
import { workbenchEventStreamPath } from "../src/api/workbench-events.ts";
|
||||
import type { WorkbenchStreamTransportRecovery } from "../src/utils/workbench-realtime-runtime.ts";
|
||||
import { workbenchRealtimeTraceIdForCapabilities, workbenchRealtimeTransportEnabled } from "../src/utils/workbench-stream-transport.ts";
|
||||
import { workbenchRealtimeTraceId } from "../src/utils/workbench-stream-transport.ts";
|
||||
import { workbenchRuntimePolicy } from "../src/config/workbench-runtime-policy.ts";
|
||||
import { AsyncQueue, work } from "../src/utils/scheduler/async-queue.ts";
|
||||
import { createCoalescedEventQueue } from "../src/utils/scheduler/coalesced-event-queue.ts";
|
||||
@@ -20,13 +20,11 @@ import { checkWorkbenchHealth, createWorkbenchHealthProbeCache } from "../src/ut
|
||||
import { messageDiagnosticView } from "../src/utils/workbench-error-runtime.ts";
|
||||
import { projectRejectedWorkbenchAdmission } from "../src/stores/workbench-admission-failure.ts";
|
||||
import { WORKBENCH_TIMELINE_OPENCODE_PARITY, buildWorkbenchTimelineRows, normalizeWorkbenchTimelineMessages, workbenchTimelineSignature } from "../src/stores/workbench-timeline-model.ts";
|
||||
import { reduceWorkbenchRealtimeEvent, workbenchRealtimeEventIsBusinessActivity } from "../src/stores/workbench-event-reducer.ts";
|
||||
import { reduceWorkbenchLiveKafkaMessageState, workbenchAgentMessageIdForTrace } from "../src/stores/workbench-live-kafka-event.ts";
|
||||
import { reduceWorkbenchRealtimeEvent } from "../src/stores/workbench-event-reducer.ts";
|
||||
import { planWorkbenchRealtimeApply, planWorkbenchRealtimeRecovery } from "../src/stores/workbench-realtime-plan.ts";
|
||||
import { WORKBENCH_REALTIME_AUTHORITY_VERSION, workbenchRealtimePrimaryAuthorityDecision } from "../src/stores/workbench-realtime-authority.ts";
|
||||
import { cleanupWorkbenchServerStateDroppedSessions, cleanupWorkbenchServerStateSessions, createWorkbenchServerState, reduceWorkbenchServerState } from "../src/stores/workbench-server-state.ts";
|
||||
import { cleanupDroppedWorkbenchSessionCaches, trimWorkbenchSessionCache } from "../src/stores/workbench-session-cache.ts";
|
||||
import { selectActiveTurnStatusRefreshTraceIds } from "../src/stores/workbench-session.ts";
|
||||
|
||||
test("Workbench scoped keys encode delimiter characters", () => {
|
||||
assert.deepEqual(splitWorkbenchScopedKey(workbenchRealtimeScopeKey("ses|one", "trc/two")), ["workbench.realtime", "ses|one", "trc/two"]);
|
||||
@@ -41,7 +39,6 @@ test("Workbench runtime policy reads injected config while preserving defaults",
|
||||
traceDetailMaxPages: 2,
|
||||
workbenchSessionDetailMinRefreshMs: 1234,
|
||||
workbenchSessionMessagesWindowLimit: 9,
|
||||
workbenchTraceMessagesWindowLimit: 4,
|
||||
workbenchRealtimeErrorSyncReplayMinMs: 0,
|
||||
workbenchRealtimeFlushMaxItemsPerChunk: 2,
|
||||
workbenchRealtimeFlushMaxChunkMs: 6,
|
||||
@@ -53,7 +50,6 @@ test("Workbench runtime policy reads injected config while preserving defaults",
|
||||
assert.equal(policy.traceDetailMaxPages, 2);
|
||||
assert.equal(policy.workbenchSessionDetailMinRefreshMs, 1234);
|
||||
assert.equal(policy.workbenchSessionMessagesWindowLimit, 9);
|
||||
assert.equal(policy.workbenchTraceMessagesWindowLimit, 4);
|
||||
assert.equal(policy.workbenchRealtimeErrorSyncReplayMinMs, 0);
|
||||
assert.equal(policy.workbenchRealtimeFlushMaxItemsPerChunk, 2);
|
||||
assert.equal(policy.workbenchRealtimeFlushMaxChunkMs, 6);
|
||||
@@ -83,24 +79,16 @@ test("Workbench API uses metadata-only session detail and bounded messages paths
|
||||
assert.equal(workbenchSessionMessagesPathForTest("ses_metadata", { limit: 9 }), "/v1/workbench/sessions/ses_metadata/messages?limit=9");
|
||||
});
|
||||
|
||||
test("only the projection authority sends the durable afterSeq cursor", () => {
|
||||
assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, kafkaRefreshReplay: false, projectionRealtime: false }, sessionId: "ses_live", traceId: "trc_live", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_live&traceId=trc_live");
|
||||
assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: true }, sessionId: "ses_projection", traceId: "trc_projection", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_projection&traceId=trc_projection&afterSeq=42");
|
||||
assert.equal(workbenchEventStreamPath({ realtimeCapabilities: { liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: true }, sessionId: "ses_both", traceId: "trc_both", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_both&traceId=trc_both&afterSeq=42");
|
||||
assert.equal(workbenchProjectionEventStreamPath({ sessionId: "ses_both", traceId: "trc_both", afterSeq: 42 }), "/v1/workbench/projection-events?sessionId=ses_both&traceId=trc_both&afterSeq=42");
|
||||
assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: false }), false);
|
||||
assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, kafkaRefreshReplay: false, projectionRealtime: false }), false);
|
||||
assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: true }), true);
|
||||
assert.equal(workbenchRealtimeTransportEnabled({ liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: true }), true);
|
||||
test("the product EventSource always sends the durable outbox cursor", () => {
|
||||
assert.equal(workbenchEventStreamPath({ sessionId: "ses_projection", traceId: "trc_projection", afterSeq: 42 }), "/v1/workbench/events?sessionId=ses_projection&traceId=trc_projection&afterSeq=42");
|
||||
});
|
||||
|
||||
test("obsolete live flags cannot erase the active projection trace scope", () => {
|
||||
const capabilities = { liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: false };
|
||||
const beforeSubmit = workbenchRealtimeScopeKey("ses_live", workbenchRealtimeTraceIdForCapabilities(capabilities, null, null));
|
||||
const afterSubmit = workbenchRealtimeScopeKey("ses_live", workbenchRealtimeTraceIdForCapabilities(capabilities, "trc_current_request", "trc_message"));
|
||||
test("the active projection trace scope follows the current request", () => {
|
||||
const beforeSubmit = workbenchRealtimeScopeKey("ses_projection", workbenchRealtimeTraceId(null, null));
|
||||
const afterSubmit = workbenchRealtimeScopeKey("ses_projection", workbenchRealtimeTraceId("trc_current_request", "trc_message"));
|
||||
assert.notEqual(afterSubmit, beforeSubmit);
|
||||
assert.equal(afterSubmit, workbenchRealtimeScopeKey("ses_live", "trc_current_request"));
|
||||
assert.equal(workbenchRealtimeTraceIdForCapabilities({ liveKafkaSse: false, kafkaRefreshReplay: false, projectionRealtime: true }, "trc_current_request", "trc_message"), "trc_current_request");
|
||||
assert.equal(afterSubmit, workbenchRealtimeScopeKey("ses_projection", "trc_current_request"));
|
||||
assert.equal(workbenchRealtimeTraceId(null, "trc_message"), "trc_message");
|
||||
});
|
||||
|
||||
test("Error runtime owns Workbench message diagnostic view model", () => {
|
||||
@@ -480,42 +468,6 @@ test("server-state cleanup removes dropped session trace and turn authority", ()
|
||||
assert.deepEqual(Object.keys(cleanedById.traceById), []);
|
||||
});
|
||||
|
||||
test("passive message projection selects active turn status refresh without local request", () => {
|
||||
const traceIds = selectActiveTurnStatusRefreshTraceIds({
|
||||
messages: [
|
||||
agentMessage({ id: "m_old", traceId: "trc_old", status: "completed", text: "done" }),
|
||||
agentMessage({ id: "m_new", traceId: "trc_new", status: "running", text: "thinking" })
|
||||
],
|
||||
currentRequestTraceId: null,
|
||||
turnStatusAuthority: {}
|
||||
});
|
||||
|
||||
assert.deepEqual(traceIds, ["trc_new"]);
|
||||
});
|
||||
|
||||
test("passive message projection skips sealed terminal turn status refresh", () => {
|
||||
const traceIds = selectActiveTurnStatusRefreshTraceIds({
|
||||
messages: [agentMessage({ id: "m_done", traceId: "trc_done", status: "completed", text: "done" })],
|
||||
turnStatusAuthority: { trc_done: { traceId: "trc_done", status: "completed", running: false, terminal: true, sessionId: "ses_1" } }
|
||||
});
|
||||
|
||||
assert.deepEqual(traceIds, []);
|
||||
});
|
||||
|
||||
test("turn status refresh keeps local request trace priority", () => {
|
||||
const traceIds = selectActiveTurnStatusRefreshTraceIds({
|
||||
messages: [
|
||||
agentMessage({ id: "m_request", traceId: "trc_request", status: "pending", text: "" }),
|
||||
agentMessage({ id: "m_observed", traceId: "trc_observed", status: "running", text: "thinking" })
|
||||
],
|
||||
currentRequestTraceId: "trc_request",
|
||||
turnStatusAuthority: {},
|
||||
limit: 2
|
||||
});
|
||||
|
||||
assert.deepEqual(traceIds, ["trc_request", "trc_observed"]);
|
||||
});
|
||||
|
||||
test("realtime event reducer classifies SSE payloads before store side effects", () => {
|
||||
const trace = reduceWorkbenchRealtimeEvent(realtimeEvent({ type: "trace.event", traceId: "trc_1", event: { traceId: "trc_1", label: "delta" }, snapshot: { traceId: "trc_1", status: "running" }, entity: { family: "traceEvents", id: "trc_1:1", version: 1, projectionRevision: "prj_1" } }), "workbench.trace.event");
|
||||
assert.equal(trace.activityLabel, "realtime:trace.event");
|
||||
@@ -533,53 +485,6 @@ test("realtime event reducer classifies SSE payloads before store side effects",
|
||||
assert.equal(error.diagnostic.traceId, "trc_2");
|
||||
});
|
||||
|
||||
test("live transport connected and heartbeat frames do not extend business activity timeouts", () => {
|
||||
assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "connected" }, "workbench.connected", true), false);
|
||||
assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "heartbeat" }, "workbench.heartbeat", true), false);
|
||||
assert.equal(workbenchRealtimeEventIsBusinessActivity({ schema: "hwlab.event.v1", event: { type: "assistant" } }, "hwlab.event.v1", true), true);
|
||||
assert.equal(workbenchRealtimeEventIsBusinessActivity({ type: "trace.event" }, "workbench.trace.event", false), true);
|
||||
});
|
||||
|
||||
test("live hwlab envelope projects assistant, tool/output, and terminal without replay or finalizer", () => {
|
||||
const envelope = (event: Record<string, unknown>) => ({
|
||||
schema: "hwlab.event.v1",
|
||||
eventType: "hwlab.trace.event.projected",
|
||||
eventId: `hwlab:${String(event.sourceEventId ?? event.type)}`,
|
||||
hwlabSessionId: "ses_live_web",
|
||||
sessionId: "ses_live_web",
|
||||
traceId: "trc_live_web",
|
||||
runId: "run_live_web",
|
||||
commandId: "cmd_live_web",
|
||||
context: { runId: "run_live_web", commandId: "cmd_live_web", valuesRedacted: true },
|
||||
event
|
||||
});
|
||||
const assistantEnvelope = envelope({ type: "assistant", eventType: "assistant", sourceEventId: "evt_assistant", traceId: "trc_live_web", sessionId: "ses_live_web", status: "running", assistantText: "running increment", terminal: false });
|
||||
const assistant = reduceWorkbenchRealtimeEvent(assistantEnvelope, "hwlab.event.v1");
|
||||
assert.equal(assistant.action.type, "trace.event");
|
||||
let visible = reduceWorkbenchLiveKafkaMessageState({ text: "", status: "running", terminal: false }, assistantEnvelope.event);
|
||||
assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false });
|
||||
|
||||
const toolEnvelope = envelope({ type: "tool", eventType: "tool", sourceEventId: "evt_tool", traceId: "trc_live_web", sessionId: "ses_live_web", status: "completed", toolName: "commandExecution", outputSummary: "ok", terminal: false });
|
||||
assert.equal(reduceWorkbenchRealtimeEvent(toolEnvelope, "hwlab.event.v1").action.type, "trace.event");
|
||||
visible = reduceWorkbenchLiveKafkaMessageState(visible, toolEnvelope.event);
|
||||
assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false });
|
||||
|
||||
const outputEnvelope = envelope({ type: "output", eventType: "status", sourceEventId: "evt_output", traceId: "trc_live_web", sessionId: "ses_live_web", status: "running", message: "stdout increment", terminal: false });
|
||||
visible = reduceWorkbenchLiveKafkaMessageState(visible, outputEnvelope.event);
|
||||
assert.deepEqual(visible, { text: "running increment", status: "running", terminal: false });
|
||||
|
||||
const terminalEnvelope = envelope({ type: "result", eventType: "terminal", sourceEventId: "evt_terminal", traceId: "trc_live_web", sessionId: "ses_live_web", status: "completed", terminal: true });
|
||||
visible = reduceWorkbenchLiveKafkaMessageState(visible, terminalEnvelope.event);
|
||||
assert.deepEqual(visible, { text: "running increment", status: "completed", terminal: true });
|
||||
assert.equal(workbenchAgentMessageIdForTrace("trc_live_web"), "msg_live_web_agent");
|
||||
|
||||
const failed = reduceWorkbenchLiveKafkaMessageState(
|
||||
{ text: "partial response", status: "running", terminal: false },
|
||||
{ type: "result", eventType: "terminal", status: "failed", message: "provider stream disconnected", terminal: true }
|
||||
);
|
||||
assert.deepEqual(failed, { text: "provider stream disconnected", status: "failed", terminal: true });
|
||||
});
|
||||
|
||||
test("realtime apply planner turns reducer actions into store steps", () => {
|
||||
const reduced = reduceWorkbenchRealtimeEvent(realtimeEvent({ type: "trace.event", traceId: "trc_1", event: { traceId: "trc_1", label: "delta" }, snapshot: { traceId: "trc_1", status: "running" }, entity: { family: "traceEvents", id: "trc_1:1", version: 1, projectionRevision: "prj_1" } }), "workbench.trace.event");
|
||||
const tracePlan = planWorkbenchRealtimeApply(reduced.action);
|
||||
|
||||
@@ -270,7 +270,7 @@ test("Workbench sealed completed final body survives stale running session messa
|
||||
assert.equal(selectActiveSession(state, "ses_final_seal")?.status, "completed");
|
||||
});
|
||||
|
||||
test("Workbench session messages repair terminal zero timing from running projection", () => {
|
||||
test("Workbench session messages preserve canonical terminal zero timing", () => {
|
||||
let state = createWorkbenchServerState();
|
||||
state = reduceWorkbenchServerState(state, {
|
||||
type: "session.messages",
|
||||
@@ -288,8 +288,8 @@ test("Workbench session messages repair terminal zero timing from running projec
|
||||
|
||||
const agent = selectActiveMessages(state, "ses_terminal_zero_repair").find((message) => message.role === "agent");
|
||||
assert.equal(agent?.status, "completed");
|
||||
assert.equal(agent?.durationMs, 6_000);
|
||||
assert.equal(agent?.timing?.durationMs, 6_000);
|
||||
assert.equal(agent?.durationMs, 0);
|
||||
assert.equal(agent?.timing?.durationMs, 0);
|
||||
});
|
||||
|
||||
test("Workbench session messages can replace sealed terminal zero with positive timing", () => {
|
||||
|
||||
@@ -2,7 +2,6 @@ import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { connectWorkbenchEvents, realtimeCoalesceKey, workbenchEventStreamPath, type WorkbenchRealtimeEvent, type WorkbenchSseIngressFrame } from "./workbench-events";
|
||||
import { createCoalescedEventQueue } from "../utils/scheduler/coalesced-event-queue";
|
||||
|
||||
test("turn snapshot coalescing keys by trace instead of per sequence", () => {
|
||||
const first: WorkbenchRealtimeEvent = { type: "turn.snapshot", turn: { sessionId: "ses_queue", traceId: "trc_queue", status: "running" }, cursor: { traceSeq: 10 } };
|
||||
@@ -21,7 +20,6 @@ test("trace events keep sequence-specific coalescing keys", () => {
|
||||
test("product EventSource resumes the transactional projection authority with one durable outbox cursor", () => {
|
||||
assert.equal(
|
||||
workbenchEventStreamPath({
|
||||
realtimeCapabilities: { liveKafkaSse: true, kafkaRefreshReplay: false, projectionRealtime: true },
|
||||
sessionId: "ses_projection_authority",
|
||||
traceId: "trc_projection_authority",
|
||||
afterSeq: 42
|
||||
@@ -30,29 +28,7 @@ test("product EventSource resumes the transactional projection authority with on
|
||||
);
|
||||
});
|
||||
|
||||
test("live Kafka envelopes from one trace retain every assistant, tool, output, and terminal event", () => {
|
||||
const delivered: WorkbenchRealtimeEvent[] = [];
|
||||
const queue = createCoalescedEventQueue<WorkbenchRealtimeEvent>({
|
||||
keyOf: (event) => realtimeCoalesceKey(event, "hwlab.event.v1"),
|
||||
onFlush: (events) => delivered.push(...events)
|
||||
});
|
||||
for (const sourceEventId of ["evt_assistant", "evt_tool", "evt_output", "evt_terminal"]) {
|
||||
queue.push({
|
||||
schema: "hwlab.event.v1",
|
||||
eventId: `hwlab:${sourceEventId}`,
|
||||
sourceEventId,
|
||||
sessionId: "ses_live_queue",
|
||||
traceId: "trc_live_queue",
|
||||
event: { sourceEventId, traceId: "trc_live_queue", sessionId: "ses_live_queue" }
|
||||
});
|
||||
}
|
||||
|
||||
queue.flush();
|
||||
|
||||
assert.deepEqual(delivered.map((event) => event.sourceEventId), ["evt_assistant", "evt_tool", "evt_output", "evt_terminal"]);
|
||||
});
|
||||
|
||||
test("product EventSource tees every raw HWLAB frame once before decode and reducer delivery", async () => {
|
||||
test("product EventSource tees projection frames once before reducer delivery", async () => {
|
||||
const originalEventSource = globalThis.EventSource;
|
||||
const ingress: WorkbenchSseIngressFrame[] = [];
|
||||
const delivered: WorkbenchRealtimeEvent[] = [];
|
||||
@@ -60,7 +36,6 @@ test("product EventSource tees every raw HWLAB frame once before decode and redu
|
||||
globalThis.EventSource = FakeProductEventSource as unknown as typeof EventSource;
|
||||
try {
|
||||
const stream = connectWorkbenchEvents({
|
||||
realtimeCapabilities: { liveKafkaSse: true, kafkaRefreshReplay: true, projectionRealtime: false },
|
||||
sessionId: "ses_raw_tee",
|
||||
flushYieldMs: 1,
|
||||
onIngress: (frame) => ingress.push(frame),
|
||||
@@ -69,17 +44,17 @@ test("product EventSource tees every raw HWLAB frame once before decode and redu
|
||||
assert.ok(stream);
|
||||
const source = FakeProductEventSource.instances[0];
|
||||
assert.ok(source);
|
||||
source.emitRaw("hwlab.event.v1", '{"schema":"hwlab.event.v1","eventId":"evt_raw_tee"}');
|
||||
source.emitRaw("hwlab.event.v1", "{invalid-json");
|
||||
source.emitRaw("workbench.trace.event", '{"type":"trace.event","traceId":"trc_raw_tee","event":{"traceId":"trc_raw_tee","projectedSeq":1}}');
|
||||
source.emitRaw("workbench.trace.event", "{invalid-json");
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
|
||||
assert.equal(FakeProductEventSource.instances.length, 1);
|
||||
assert.deepEqual(ingress.map((frame) => [frame.eventName, frame.decodeStatus]), [
|
||||
["hwlab.event.v1", "accepted"],
|
||||
["hwlab.event.v1", "invalid-json"]
|
||||
["workbench.trace.event", "accepted"],
|
||||
["workbench.trace.event", "invalid-json"]
|
||||
]);
|
||||
assert.equal(delivered.length, 1);
|
||||
assert.equal(delivered[0]?.eventId, "evt_raw_tee");
|
||||
assert.equal(delivered[0]?.traceId, "trc_raw_tee");
|
||||
stream.close();
|
||||
} finally {
|
||||
globalThis.EventSource = originalEventSource;
|
||||
|
||||
@@ -6,7 +6,6 @@ import type { ApiResult, ChatMessage, ProjectionDiagnostic, TraceEvent } from "@
|
||||
import { createCoalescedEventQueue } from "@/utils/scheduler/coalesced-event-queue";
|
||||
import { composeWorkbenchScopedKey, firstScopePart } from "@/utils/workbench-key";
|
||||
import { recordWorkbenchRuntimeDiagnostic, recordWorkbenchSseLifecycle } from "@/utils/workbench-performance";
|
||||
import type { WorkbenchRealtimeCapabilities } from "@/config/runtime";
|
||||
import { decodeWorkbenchRealtimeEventFrame } from "./workbench-realtime-codec";
|
||||
|
||||
export interface WorkbenchRealtimeTraceSnapshot {
|
||||
@@ -29,11 +28,6 @@ export interface WorkbenchRealtimeEvent {
|
||||
eventType?: string | null;
|
||||
eventId?: string | null;
|
||||
sourceEventId?: string | null;
|
||||
capabilities?: WorkbenchRealtimeCapabilities | null;
|
||||
liveOnly?: boolean | null;
|
||||
replay?: boolean | null;
|
||||
deliverySemantics?: string | null;
|
||||
lossPossible?: boolean | null;
|
||||
type?: string;
|
||||
contractVersion?: string | null;
|
||||
realtimeAuthority?: string | null;
|
||||
@@ -76,7 +70,6 @@ export interface WorkbenchRealtimeEvent {
|
||||
}
|
||||
|
||||
export interface WorkbenchEventStreamOptions {
|
||||
realtimeCapabilities: WorkbenchRealtimeCapabilities;
|
||||
sessionId?: string | null;
|
||||
traceId?: string | null;
|
||||
afterSeq?: number | null;
|
||||
@@ -105,7 +98,6 @@ interface QueuedRealtimeEvent {
|
||||
}
|
||||
|
||||
const WORKBENCH_EVENT_NAMES = [
|
||||
"hwlab.event.v1",
|
||||
"workbench.connected",
|
||||
"workbench.trace.snapshot",
|
||||
"workbench.trace.event",
|
||||
@@ -183,20 +175,12 @@ export function connectWorkbenchEvents(options: WorkbenchEventStreamOptions): Wo
|
||||
};
|
||||
}
|
||||
|
||||
export function workbenchEventStreamPath(options: Pick<WorkbenchEventStreamOptions, "realtimeCapabilities" | "sessionId" | "traceId" | "afterSeq">): string {
|
||||
const params = new URLSearchParams();
|
||||
appendParam(params, "sessionId", options.sessionId);
|
||||
appendParam(params, "traceId", options.traceId);
|
||||
if (options.realtimeCapabilities.projectionRealtime) appendNumberParam(params, "afterSeq", options.afterSeq);
|
||||
return `/v1/workbench/events?${params.toString()}`;
|
||||
}
|
||||
|
||||
export function workbenchProjectionEventStreamPath(options: Pick<WorkbenchEventStreamOptions, "sessionId" | "traceId" | "afterSeq">): string {
|
||||
export function workbenchEventStreamPath(options: Pick<WorkbenchEventStreamOptions, "sessionId" | "traceId" | "afterSeq">): string {
|
||||
const params = new URLSearchParams();
|
||||
appendParam(params, "sessionId", options.sessionId);
|
||||
appendParam(params, "traceId", options.traceId);
|
||||
appendNumberParam(params, "afterSeq", options.afterSeq);
|
||||
return `/v1/workbench/projection-events?${params.toString()}`;
|
||||
return `/v1/workbench/events?${params.toString()}`;
|
||||
}
|
||||
|
||||
function appendParam(params: URLSearchParams, key: string, value: string | null | undefined): void {
|
||||
@@ -228,10 +212,6 @@ function scheduleRealtimeFlushYield(flush: () => void, yieldMs: number | null |
|
||||
}
|
||||
|
||||
export function realtimeCoalesceKey(event: WorkbenchRealtimeEvent, eventName: string): string | null {
|
||||
if (eventName === "hwlab.event.v1" || event.schema === "hwlab.event.v1") {
|
||||
const eventId = firstScopePart(event.eventId, event.sourceEventId, event.event?.sourceEventId);
|
||||
return eventId ? composeWorkbenchScopedKey("workbench.sse.live-kafka-event", eventId) : null;
|
||||
}
|
||||
const entity = event.entity;
|
||||
const entityFamily = firstScopePart(entity?.family);
|
||||
const entityId = firstScopePart(entity?.id);
|
||||
|
||||
@@ -129,7 +129,6 @@ export const workbenchAPI = {
|
||||
return fetchJson(sessionDetailPath(sessionId, detailOptions), { timeoutMs: requestTimeoutMs(detailOptions.timeoutMs, SESSION_DETAIL_TIMEOUT_MS), timeoutName: "workbench session" });
|
||||
},
|
||||
sessionMessages: (sessionId: string, options: SessionMessageOptions = {}): Promise<ApiResult<WorkbenchMessagePageResponse>> => fetchJson(sessionMessagesPath(sessionId, options), { timeoutMs: requestTimeoutMs(options.timeoutMs, SESSION_DETAIL_TIMEOUT_MS), timeoutName: "workbench session messages" }),
|
||||
turn: async (traceId: string, timeoutMs = 8000, activityRef?: ApiRequestOptions["activityRef"]): Promise<ApiResult<AgentChatResultResponse>> => normalizeWorkbenchTurnResult(await fetchJson<Record<string, unknown>>(`/v1/workbench/turns/${encodeURIComponent(traceId)}`, { timeoutMs, timeoutName: "workbench turn", activityRef }), traceId),
|
||||
traceEvents: async (traceId: string, timeoutMs = 8000, activityRef?: ApiRequestOptions["activityRef"], options?: WorkbenchTraceRequestOptions): Promise<ApiResult<AgentChatResultResponse>> => normalizeWorkbenchTraceResult(await fetchJson<Record<string, unknown>>(workbenchTracePath(traceId, options), { timeoutMs, timeoutName: "workbench trace", activityRef }), traceId)
|
||||
};
|
||||
|
||||
@@ -186,42 +185,6 @@ function workbenchTracePath(traceId: string, options: WorkbenchTraceRequestOptio
|
||||
return `/v1/workbench/traces/${encodeURIComponent(traceId)}/events${query ? `?${query}` : ""}`;
|
||||
}
|
||||
|
||||
function normalizeWorkbenchTurnResult(result: ApiResult<Record<string, unknown>>, traceId: string): ApiResult<AgentChatResultResponse> {
|
||||
if (!result.ok || !result.data) return result as ApiResult<AgentChatResultResponse>;
|
||||
const turn = recordValue(result.data.turn) ?? result.data;
|
||||
const trace = recordValue(turn.trace);
|
||||
const status = normalizeStatus(textValue(turn.status)) ?? "unknown";
|
||||
const projection = normalizeProjectionDiagnostic(result.data.projection, turn.projection, result.data);
|
||||
const timing = normalizeTimingProjection(turn);
|
||||
return {
|
||||
...result,
|
||||
data: {
|
||||
...turn,
|
||||
projection,
|
||||
timing,
|
||||
startedAt: timing?.startedAt ?? undefined,
|
||||
lastEventAt: timing?.lastEventAt ?? undefined,
|
||||
finishedAt: timing?.finishedAt ?? undefined,
|
||||
durationMs: timing?.durationMs ?? undefined,
|
||||
projectionStatus: projection?.projectionStatus ?? textValue(result.data.projectionStatus) ?? undefined,
|
||||
projectionHealth: projection?.projectionHealth ?? textValue(result.data.projectionHealth) ?? undefined,
|
||||
lastProjectedSeq: Number.isFinite(Number(result.data.lastProjectedSeq)) ? Number(result.data.lastProjectedSeq) : undefined,
|
||||
sourceRunId: projection?.sourceRunId ?? textValue(result.data.sourceRunId) ?? undefined,
|
||||
sourceCommandId: projection?.sourceCommandId ?? textValue(result.data.sourceCommandId) ?? undefined,
|
||||
staleMs: projection?.staleMs ?? numberValue(result.data.staleMs) ?? undefined,
|
||||
blocker: projection?.blocker ?? recordValue(result.data.blocker) ?? undefined,
|
||||
traceId: textValue(turn.traceId) ?? traceId,
|
||||
status,
|
||||
running: typeof turn.running === "boolean" ? turn.running : undefined,
|
||||
terminal: typeof turn.terminal === "boolean" ? turn.terminal : undefined,
|
||||
sessionId: textValue(turn.sessionId) ?? undefined,
|
||||
threadId: textValue(turn.threadId) ?? undefined,
|
||||
agentRun: recordValue(turn.agentRun) ?? undefined,
|
||||
updatedAt: textValue(turn.updatedAt) ?? textValue(result.data.observedAt) ?? undefined
|
||||
} as AgentChatResultResponse
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeWorkbenchTraceResult(result: ApiResult<Record<string, unknown>>, traceId: string): ApiResult<AgentChatResultResponse> {
|
||||
if (!result.ok || !result.data) return result as ApiResult<AgentChatResultResponse>;
|
||||
const events = Array.isArray(result.data.events) ? result.data.events : [];
|
||||
@@ -308,13 +271,6 @@ function normalizeTimingProjection(value: unknown): WorkbenchTurnTimingProjectio
|
||||
} as WorkbenchTurnTimingProjection;
|
||||
}
|
||||
|
||||
function normalizeStatus(value: unknown): string | null {
|
||||
const text = textValue(value);
|
||||
if (!text) return null;
|
||||
const normalized = text.toLowerCase().replace(/_/gu, "-");
|
||||
return normalized === "cancelled" ? "canceled" : normalized;
|
||||
}
|
||||
|
||||
function recordValue(value: unknown): Record<string, unknown> | null {
|
||||
return value && typeof value === "object" ? value as Record<string, unknown> : null;
|
||||
}
|
||||
|
||||
@@ -10,12 +10,6 @@ export interface OpenCodeFrameConfig {
|
||||
url: string;
|
||||
}
|
||||
|
||||
export interface WorkbenchRealtimeCapabilities {
|
||||
liveKafkaSse: boolean;
|
||||
kafkaRefreshReplay: boolean;
|
||||
projectionRealtime: boolean;
|
||||
}
|
||||
|
||||
export interface WorkbenchDebugCapabilities {
|
||||
isolatedKafka: boolean;
|
||||
rawHwlabEventWindow: WorkbenchRawHwlabEventWindowCapability;
|
||||
@@ -97,14 +91,6 @@ export function workbenchTraceTimelinePolicy(): WorkbenchTraceTimelinePolicy {
|
||||
};
|
||||
}
|
||||
|
||||
export function workbenchRealtimeCapabilities(): WorkbenchRealtimeCapabilities {
|
||||
const features = window.HWLAB_CLOUD_WEB_CONFIG?.workbench?.realtimeFeatures;
|
||||
if (typeof features?.liveKafkaSse !== "boolean" || typeof features?.kafkaRefreshReplay !== "boolean" || typeof features?.projectionRealtime !== "boolean") {
|
||||
throw new Error("HWLAB Cloud Web workbench.realtimeFeatures are required");
|
||||
}
|
||||
return { liveKafkaSse: features.liveKafkaSse, kafkaRefreshReplay: features.kafkaRefreshReplay, projectionRealtime: features.projectionRealtime };
|
||||
}
|
||||
|
||||
export function workbenchDebugCapabilities(): WorkbenchDebugCapabilities {
|
||||
const features = window.HWLAB_CLOUD_WEB_CONFIG?.workbench?.debugCapabilities;
|
||||
const rawWindow = features?.rawHwlabEventWindow;
|
||||
|
||||
@@ -14,16 +14,12 @@ export interface WorkbenchRuntimePolicy {
|
||||
sessionListPageLimit: number;
|
||||
workbenchDetailReadConcurrency: number;
|
||||
workbenchReadFailureCooldownMs: number;
|
||||
workbenchTurnStatusMinRefreshMs: number;
|
||||
workbenchTraceEventsMinRefreshMs: number;
|
||||
workbenchSessionDetailMinRefreshMs: number;
|
||||
workbenchSessionMessagesMinRefreshMs: number;
|
||||
workbenchSessionMessagesWindowLimit: number;
|
||||
workbenchTraceMessagesWindowLimit: number;
|
||||
workbenchRealtimeSessionMessagesMinRefreshMs: number;
|
||||
workbenchTraceEventsTimeoutMs: number;
|
||||
sessionListRealtimeRefreshDelayMs: number;
|
||||
sessionListTerminalRefreshDelayMs: number;
|
||||
sessionListMinRefreshIntervalMs: number;
|
||||
workbenchRealtimeErrorSyncReplayMinMs: number;
|
||||
workbenchRealtimeFlushMaxItemsPerChunk: number;
|
||||
@@ -44,16 +40,12 @@ const DEFAULT_WORKBENCH_RUNTIME_POLICY: WorkbenchRuntimePolicy = Object.freeze({
|
||||
sessionListPageLimit: 20,
|
||||
workbenchDetailReadConcurrency: 3,
|
||||
workbenchReadFailureCooldownMs: 5_000,
|
||||
workbenchTurnStatusMinRefreshMs: 2_000,
|
||||
workbenchTraceEventsMinRefreshMs: 4_000,
|
||||
workbenchSessionDetailMinRefreshMs: 5_000,
|
||||
workbenchSessionMessagesMinRefreshMs: 5_000,
|
||||
workbenchSessionMessagesWindowLimit: 20,
|
||||
workbenchTraceMessagesWindowLimit: 8,
|
||||
workbenchRealtimeSessionMessagesMinRefreshMs: 1_000,
|
||||
workbenchTraceEventsTimeoutMs: 5_000,
|
||||
sessionListRealtimeRefreshDelayMs: 5_000,
|
||||
sessionListTerminalRefreshDelayMs: 1_500,
|
||||
sessionListMinRefreshIntervalMs: 15_000,
|
||||
workbenchRealtimeErrorSyncReplayMinMs: 2_000,
|
||||
workbenchRealtimeFlushMaxItemsPerChunk: 4,
|
||||
@@ -76,16 +68,12 @@ export function workbenchRuntimePolicy(input: unknown = runtimePolicyConfig()):
|
||||
sessionListPageLimit: positiveInteger(source.sessionListPageLimit, DEFAULT_WORKBENCH_RUNTIME_POLICY.sessionListPageLimit),
|
||||
workbenchDetailReadConcurrency: positiveInteger(source.workbenchDetailReadConcurrency ?? source.workbenchReadHydrationConcurrency, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchDetailReadConcurrency),
|
||||
workbenchReadFailureCooldownMs: positiveNumber(source.workbenchReadFailureCooldownMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchReadFailureCooldownMs),
|
||||
workbenchTurnStatusMinRefreshMs: nonNegativeNumber(source.workbenchTurnStatusMinRefreshMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchTurnStatusMinRefreshMs),
|
||||
workbenchTraceEventsMinRefreshMs: nonNegativeNumber(source.workbenchTraceEventsMinRefreshMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchTraceEventsMinRefreshMs),
|
||||
workbenchSessionDetailMinRefreshMs: nonNegativeNumber(source.workbenchSessionDetailMinRefreshMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchSessionDetailMinRefreshMs),
|
||||
workbenchSessionMessagesMinRefreshMs: nonNegativeNumber(source.workbenchSessionMessagesMinRefreshMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchSessionMessagesMinRefreshMs),
|
||||
workbenchSessionMessagesWindowLimit: positiveInteger(source.workbenchSessionMessagesWindowLimit, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchSessionMessagesWindowLimit),
|
||||
workbenchTraceMessagesWindowLimit: positiveInteger(source.workbenchTraceMessagesWindowLimit, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchTraceMessagesWindowLimit),
|
||||
workbenchRealtimeSessionMessagesMinRefreshMs: nonNegativeNumber(source.workbenchRealtimeSessionMessagesMinRefreshMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeSessionMessagesMinRefreshMs),
|
||||
workbenchTraceEventsTimeoutMs: positiveNumber(source.workbenchTraceEventsTimeoutMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchTraceEventsTimeoutMs),
|
||||
sessionListRealtimeRefreshDelayMs: nonNegativeNumber(source.sessionListRealtimeRefreshDelayMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.sessionListRealtimeRefreshDelayMs),
|
||||
sessionListTerminalRefreshDelayMs: nonNegativeNumber(source.sessionListTerminalRefreshDelayMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.sessionListTerminalRefreshDelayMs),
|
||||
sessionListMinRefreshIntervalMs: nonNegativeNumber(source.sessionListMinRefreshIntervalMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.sessionListMinRefreshIntervalMs),
|
||||
workbenchRealtimeErrorSyncReplayMinMs: nonNegativeNumber(source.workbenchRealtimeErrorSyncReplayMinMs ?? source.workbenchRealtimeErrorGapFillMinMs, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeErrorSyncReplayMinMs),
|
||||
workbenchRealtimeFlushMaxItemsPerChunk: positiveInteger(source.workbenchRealtimeFlushMaxItemsPerChunk, DEFAULT_WORKBENCH_RUNTIME_POLICY.workbenchRealtimeFlushMaxItemsPerChunk),
|
||||
|
||||
@@ -31,8 +31,6 @@ export const workbenchColadaKeys = {
|
||||
historySessionMessagesRoot: (): EntryKey => ["workbench", "history", "session-messages"],
|
||||
historySessionMessages: (sessionId: string, input: WorkbenchSessionMessagesKeyInput = {}): EntryKey => ["workbench", "history", "session-messages", sessionId, normalizeKeyObject({ cursor: input.cursor ?? null, limit: finiteNumber(input.limit) })],
|
||||
detailRoot: (): EntryKey => ["workbench", "detail"],
|
||||
detailTurnRoot: (): EntryKey => ["workbench", "detail", "turn"],
|
||||
detailTurn: (traceId: string): EntryKey => ["workbench", "detail", "turn", traceId],
|
||||
detailTraceEventsRoot: (): EntryKey => ["workbench", "detail", "trace-events"],
|
||||
detailTraceEvents: (traceId: string, input: WorkbenchTraceEventsKeyInput = {}): EntryKey => ["workbench", "detail", "trace-events", traceId, normalizeKeyObject({ afterProjectedSeq: finiteNumber(input.afterProjectedSeq), limit: finiteNumber(input.limit) })],
|
||||
sessionsRoot: (): EntryKey => workbenchColadaKeys.snapshotSessionsRoot(),
|
||||
@@ -41,8 +39,6 @@ export const workbenchColadaKeys = {
|
||||
session: (sessionId: string): EntryKey => workbenchColadaKeys.snapshotSession(sessionId),
|
||||
sessionMessagesRoot: (): EntryKey => workbenchColadaKeys.historySessionMessagesRoot(),
|
||||
sessionMessages: (sessionId: string, input: WorkbenchSessionMessagesKeyInput = {}): EntryKey => workbenchColadaKeys.historySessionMessages(sessionId, input),
|
||||
turnRoot: (): EntryKey => workbenchColadaKeys.detailTurnRoot(),
|
||||
turn: (traceId: string): EntryKey => workbenchColadaKeys.detailTurn(traceId),
|
||||
traceEventsRoot: (): EntryKey => workbenchColadaKeys.detailTraceEventsRoot(),
|
||||
traceEvents: (traceId: string, input: WorkbenchTraceEventsKeyInput = {}): EntryKey => workbenchColadaKeys.detailTraceEvents(traceId, input),
|
||||
providerProfiles: (): EntryKey => ["workbench", "provider-profiles"],
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
|
||||
import { useQueryCache, type EntryKey, type QueryCache } from "@pinia/colada";
|
||||
import { api } from "@/api";
|
||||
import type { ApiRequestOptions } from "@/api/client";
|
||||
import type { AgentChatResultResponse, ApiResult } from "@/types";
|
||||
import type { SessionListOptions, SessionMessageOptions, WorkbenchMessagePageResponse, WorkbenchSessionDetailResponse, WorkbenchSessionListResponse, WorkbenchTraceRequestOptions } from "@/api/workbench";
|
||||
import { workbenchColadaKeys } from "./workbench-colada-keys";
|
||||
@@ -13,14 +12,9 @@ interface QueryRunOptions {
|
||||
minIntervalMs?: number | null;
|
||||
}
|
||||
|
||||
export interface WorkbenchTurnQueryOptions extends QueryRunOptions {
|
||||
timeoutMs?: number;
|
||||
activityRef?: ApiRequestOptions["activityRef"];
|
||||
}
|
||||
|
||||
export interface WorkbenchTraceEventsQueryOptions extends QueryRunOptions, WorkbenchTraceRequestOptions {
|
||||
timeoutMs?: number;
|
||||
activityRef?: ApiRequestOptions["activityRef"];
|
||||
activityRef?: import("@/api/client").ApiRequestOptions["activityRef"];
|
||||
}
|
||||
|
||||
export interface WorkbenchSessionDetailQueryOptions extends QueryRunOptions {
|
||||
@@ -33,13 +27,10 @@ export interface WorkbenchColadaQueries {
|
||||
fetchSessions: (options?: SessionListOptions & QueryRunOptions) => Promise<ApiResult<WorkbenchSessionListResponse>>;
|
||||
fetchSession: (sessionId: string, options?: WorkbenchSessionDetailQueryOptions) => Promise<ApiResult<WorkbenchSessionDetailResponse>>;
|
||||
fetchSessionMessages: (sessionId: string, options?: SessionMessageOptions & QueryRunOptions) => Promise<ApiResult<WorkbenchMessagePageResponse>>;
|
||||
fetchTurn: (traceId: string, options?: WorkbenchTurnQueryOptions) => Promise<ApiResult<AgentChatResultResponse>>;
|
||||
fetchTraceEvents: (traceId: string, options?: WorkbenchTraceEventsQueryOptions) => Promise<ApiResult<AgentChatResultResponse>>;
|
||||
fetchProviderProfiles: (options?: QueryRunOptions) => Promise<ApiResult<unknown>>;
|
||||
invalidateSessionList: () => Promise<unknown>;
|
||||
invalidateSession: (sessionId: string | null | undefined) => Promise<unknown>;
|
||||
invalidateSessionMessages: (sessionId: string | null | undefined) => Promise<unknown>;
|
||||
invalidateTurn: (traceId: string | null | undefined) => Promise<unknown>;
|
||||
invalidateTraceEvents: (traceId: string | null | undefined) => Promise<unknown>;
|
||||
invalidateTraceScope: (input: { sessionId?: string | null; traceId?: string | null }) => Promise<unknown[]>;
|
||||
}
|
||||
@@ -52,13 +43,10 @@ export function useWorkbenchColadaQueries(): WorkbenchColadaQueries {
|
||||
fetchSessions: (options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.sessions(options), () => api.workbench.sessions(options), options),
|
||||
fetchSession: (sessionId, options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.session(sessionId), () => api.workbench.session(sessionId, { timeoutMs: options.timeoutMs ?? null, includeMessages: false }), options),
|
||||
fetchSessionMessages: (sessionId, options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.sessionMessages(sessionId, options), () => api.workbench.sessionMessages(sessionId, options), options),
|
||||
fetchTurn: (traceId, options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.turn(traceId), () => api.workbench.turn(traceId, options.timeoutMs ?? 8000, options.activityRef), options),
|
||||
fetchTraceEvents: (traceId, options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.traceEvents(traceId, options), () => api.workbench.traceEvents(traceId, options.timeoutMs ?? 8000, options.activityRef, { afterProjectedSeq: options.afterProjectedSeq, limit: options.limit }), options),
|
||||
fetchProviderProfiles: (options = {}) => runWorkbenchQuery(queryCache, workbenchColadaKeys.providerProfiles(), () => api.providerProfiles.catalog(), options),
|
||||
invalidateSessionList: () => queryCache.invalidateQueries({ key: workbenchColadaKeys.sessionsRoot() }),
|
||||
invalidateSession: (sessionId) => sessionId ? queryCache.invalidateQueries({ key: workbenchColadaKeys.session(sessionId) }) : Promise.resolve(),
|
||||
invalidateSessionMessages: (sessionId) => sessionId ? queryCache.invalidateQueries({ key: workbenchColadaKeys.sessionMessagesRoot().concat(sessionId) }) : Promise.resolve(),
|
||||
invalidateTurn: (traceId) => traceId ? queryCache.invalidateQueries({ key: workbenchColadaKeys.turn(traceId) }) : Promise.resolve(),
|
||||
invalidateTraceEvents: (traceId) => traceId ? queryCache.invalidateQueries({ key: workbenchColadaKeys.traceEventsRoot().concat(traceId) }) : Promise.resolve(),
|
||||
invalidateTraceScope: async (input) => Promise.all([
|
||||
queryCache.invalidateQueries({ key: workbenchColadaKeys.snapshotRoot() }),
|
||||
|
||||
@@ -48,15 +48,7 @@ export function reduceWorkbenchRealtimeEvent(event: WorkbenchRealtimeEvent, even
|
||||
};
|
||||
}
|
||||
|
||||
export function workbenchRealtimeEventIsBusinessActivity(event: WorkbenchRealtimeEvent, eventName: string, liveKafkaSse: boolean): boolean {
|
||||
if (!liveKafkaSse) return true;
|
||||
return eventName === "hwlab.event.v1" || event.schema === "hwlab.event.v1";
|
||||
}
|
||||
|
||||
function reduceRealtimeAction(event: WorkbenchRealtimeEvent, eventName: string): WorkbenchRealtimeAction {
|
||||
if (event.schema === "hwlab.event.v1" && event.event) {
|
||||
return { type: "trace.event", traceId: realtimeTraceId(event), event: event.event, snapshot: null, realtimeEvent: event };
|
||||
}
|
||||
const authority = primaryAuthority(event);
|
||||
if (authority) return authority;
|
||||
switch (event.type) {
|
||||
|
||||
@@ -5,9 +5,7 @@ import type { WorkbenchKafkaSseDebugEvent } from "@/api/workbench-debug";
|
||||
import type { WorkbenchRealtimeEvent } from "@/api/workbench-events";
|
||||
import type { ChatMessage } from "@/types";
|
||||
import { firstNonEmptyString } from "@/utils";
|
||||
import { reduceWorkbenchRealtimeEvent } from "./workbench-event-reducer";
|
||||
import { projectWorkbenchLiveKafkaMessage } from "./workbench-live-kafka-event";
|
||||
import { planWorkbenchRealtimeApply } from "./workbench-realtime-plan";
|
||||
|
||||
export interface WorkbenchIsolatedKafkaDebugLog {
|
||||
id: string;
|
||||
@@ -48,22 +46,17 @@ export function applyWorkbenchIsolatedKafkaDebugEvent(
|
||||
if (!traceId) return rejected(base, debugEvent, null, "debug-envelope-trace-missing");
|
||||
if (expected && traceId !== expected) return rejected(base, debugEvent, traceId, "debug-envelope-trace-mismatch");
|
||||
|
||||
const event = { ...debugEvent, schema: "hwlab.event.v1" } as WorkbenchRealtimeEvent;
|
||||
const reduced = reduceWorkbenchRealtimeEvent(event, "hwlab.event.v1");
|
||||
const plan = planWorkbenchRealtimeApply(reduced.action);
|
||||
const traceStep = plan.steps.find((step) => step.type === "apply-trace-event");
|
||||
if (!traceStep || traceStep.type !== "apply-trace-event" || !traceStep.event) {
|
||||
return withLog(base, event, traceId, reduced.action.type, plan.steps.map((step) => step.type), false, reduced.action.type === "ignore" ? reduced.action.reason : "debug-envelope-not-a-trace-event");
|
||||
}
|
||||
const traceEvent = debugEvent.event;
|
||||
if (!traceEvent) return withLog(base, debugEvent, traceId, "ignore", [], false, "debug-envelope-not-a-trace-event");
|
||||
const message = projectWorkbenchLiveKafkaMessage({
|
||||
previous: state.message,
|
||||
traceId,
|
||||
sessionId: firstNonEmptyString(event.hwlabSessionId, event.sessionId, traceStep.event.sessionId, state.message?.sessionId) ?? "ses_workbench_isolated_debug",
|
||||
event: traceStep.event,
|
||||
sessionId: firstNonEmptyString(debugEvent.hwlabSessionId, debugEvent.sessionId, traceEvent.sessionId, state.message?.sessionId) ?? "ses_workbench_isolated_debug",
|
||||
event: traceEvent,
|
||||
receivedAt: new Date().toISOString(),
|
||||
title: "Code Agent · 隔离调试"
|
||||
});
|
||||
return withLog({ ...base, message, appliedCount: state.appliedCount + 1, error: null }, event, traceId, reduced.action.type, plan.steps.map((step) => step.type), true, null);
|
||||
return withLog({ ...base, message, appliedCount: state.appliedCount + 1, error: null }, debugEvent, traceId, "debug.trace.event", ["debug-project-message"], true, null);
|
||||
}
|
||||
|
||||
export function workbenchCurrentDebugTraceId(messages: ChatMessage[], sessionLastTraceId?: string | null): string | null {
|
||||
|
||||
@@ -1,28 +0,0 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { workbenchHistoryAuthorityPolicy } from "./workbench-kafka-refresh-policy";
|
||||
|
||||
test("every capability combination keeps automatic HTTP terminal and history readers disabled", () => {
|
||||
for (const liveKafkaSse of [false, true]) {
|
||||
for (const kafkaRefreshReplay of [false, true]) {
|
||||
for (const projectionRealtime of [false, true]) {
|
||||
const policy = workbenchHistoryAuthorityPolicy({ liveKafkaSse, kafkaRefreshReplay, projectionRealtime });
|
||||
const calls = { fetchSessionMessagesPage: 0, fetchTurn: 0, fetchTraceEvents: 0 };
|
||||
if (policy.sessionMessagesHydrate) calls.fetchSessionMessagesPage += 1;
|
||||
if (policy.turnStatusHydrate) calls.fetchTurn += 1;
|
||||
if (policy.traceEventsHydrate) calls.fetchTraceEvents += 1;
|
||||
|
||||
assert.deepEqual(policy, {
|
||||
kafkaRetention: false,
|
||||
sessionMetadataRead: true,
|
||||
sessionMessagesHydrate: false,
|
||||
turnStatusHydrate: false,
|
||||
traceEventsHydrate: false,
|
||||
syncReplay: projectionRealtime
|
||||
});
|
||||
assert.deepEqual(calls, { fetchSessionMessagesPage: 0, fetchTurn: 0, fetchTraceEvents: 0 });
|
||||
}
|
||||
}
|
||||
}
|
||||
});
|
||||
@@ -1,21 +0,0 @@
|
||||
import type { WorkbenchRealtimeCapabilities } from "@/config/runtime";
|
||||
|
||||
export interface WorkbenchHistoryAuthorityPolicy {
|
||||
kafkaRetention: boolean;
|
||||
sessionMetadataRead: true;
|
||||
sessionMessagesHydrate: false;
|
||||
turnStatusHydrate: false;
|
||||
traceEventsHydrate: false;
|
||||
syncReplay: boolean;
|
||||
}
|
||||
|
||||
export function workbenchHistoryAuthorityPolicy(capabilities: WorkbenchRealtimeCapabilities): WorkbenchHistoryAuthorityPolicy {
|
||||
return {
|
||||
kafkaRetention: false,
|
||||
sessionMetadataRead: true,
|
||||
sessionMessagesHydrate: false,
|
||||
turnStatusHydrate: false,
|
||||
traceEventsHydrate: false,
|
||||
syncReplay: capabilities.projectionRealtime
|
||||
};
|
||||
}
|
||||
@@ -1,5 +1,5 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench live Kafka SSE.
|
||||
// Responsibility: project one transparent hwlab.event.v1 business event into visible turn state without replay/finalizer/polling.
|
||||
// SPEC: PJ2026-0104010803 isolated Kafka debug fixture.
|
||||
// Responsibility: project historical hwlab.event.debug.v1 fixtures for the admin-only isolated debugger; never imported by the product Workbench SSE path.
|
||||
|
||||
import { mergeRunnerTrace } from "../composables/workbench-trace-snapshot";
|
||||
import type { ChatMessage, TraceEvent, WorkbenchTurnTimingProjection } from "../types";
|
||||
|
||||
@@ -137,17 +137,17 @@ test("workbench active terminal paths seal final response from turn authority",
|
||||
const projectBlock = source.slice(source.indexOf("function projectTurnAuthorityToMessages"), source.indexOf("async function submitMessage"));
|
||||
const realtimeTurnBlock = source.slice(source.indexOf("function applyRealtimeTurnSnapshot"), source.indexOf("function scheduleRealtimeTurnProjection"));
|
||||
const realtimeTurnProjectionBlock = source.slice(source.indexOf("function flushRealtimeTurnProjection"), source.indexOf("function installRealtimeVisibilityHandler"));
|
||||
const completeBlock = source.slice(source.indexOf("function completeTrace"), source.indexOf("async function hydrateTerminalMessageDiagnostics"));
|
||||
const completeBlock = source.slice(source.indexOf("function completeTrace"), source.indexOf("function applyTerminalResultDiagnostics"));
|
||||
const terminalDiagnosticsBlock = source.slice(source.indexOf("function applyTerminalResultDiagnostics"), source.indexOf("function applyRealtimeProjectionError"));
|
||||
const crossTabSyncBlock = source.slice(source.indexOf("async function refreshCrossTabSessionFromSyncReplay"), source.indexOf("function completeTrace"));
|
||||
const sessionDetailReadBlock = source.slice(source.indexOf("function fetchSessionDetailPage"), source.indexOf("function sessionMessageProjectionWindowLimit"));
|
||||
const traceDetailReadBlock = source.slice(source.indexOf("async function readTraceEventsForExplicitDetailPages"), source.indexOf("async function fetchTraceDetailEventsPage"));
|
||||
const loadBlock = source.slice(source.indexOf("async function loadWorkbenchSession"), source.indexOf("async function sealRestoredActiveTurnMessages"));
|
||||
const restoreSealBlock = source.slice(source.indexOf("async function sealRestoredActiveTurnMessages"), source.indexOf("function reattachRestoredActiveTrace"));
|
||||
const loadBlock = source.slice(source.indexOf("async function loadWorkbenchSession"), source.indexOf("function reattachRestoredActiveTrace"));
|
||||
|
||||
assert.match(projectBlock, /terminalAuthorityMessageFromTurnResult\(result\)/u);
|
||||
assert.match(projectBlock, /type:\s*"message\.upsert"/u);
|
||||
assert.match(projectBlock, /\.\.\.terminalPatch/u);
|
||||
assert.match(source, /createKeyedSingleflight<ApiResult<AgentChatResultResponse>>/u);
|
||||
assert.match(source, /const traceDetailReadSingleflight = createKeyedSingleflight<void>\(\)/u);
|
||||
assert.match(source, /traceDetailReadSingleflight\.run/u);
|
||||
assert.doesNotMatch(source, new RegExp(["refresh", "RealtimeSessionMessages"].join(""), "u"));
|
||||
assert.match(realtimeTurnBlock, /rememberTurnStatus\(traceId, result\)[\s\S]*scheduleRealtimeTurnProjection\(\{ traceId, result, terminalTurn \}\)/u);
|
||||
@@ -156,7 +156,8 @@ test("workbench active terminal paths seal final response from turn authority",
|
||||
assert.match(realtimeTurnProjectionBlock, /workbench_realtime_turn_projection_budget/u);
|
||||
assert.doesNotMatch(realtimeTurnProjectionBlock, /refreshWorkbenchSyncReplay|refreshTerminalTraceFromSyncReplay/u);
|
||||
assert.match(completeBlock, /projectTurnAuthorityToMessages\(traceId, result, "complete-trace"\)/u);
|
||||
assert.match(completeBlock, /options\.forceRead[\s\S]*refreshMessageProjectionForTrace\(ownerSessionId, traceId, \{ force: true \}\)/u);
|
||||
assert.doesNotMatch(completeBlock, /forceRead|refreshMessageProjectionForTrace|fetchWorkbenchTurnStatus|refreshTurnStatusByTraceId/u);
|
||||
assert.doesNotMatch(terminalDiagnosticsBlock, /readTraceEventsForMessages|fetchSessionMessagesPage|fetchWorkbenchTurnStatus/u);
|
||||
assert.doesNotMatch(completeBlock, /scheduleActiveTraceSyncReplay|refreshWorkbenchSyncReplay/u);
|
||||
assert.match(source.slice(source.indexOf("function handleWorkbenchProjectionSignal"), source.indexOf("function applyTraceSnapshot")), /restartRealtime\("cross-tab-session-projection", true\)/u);
|
||||
assert.doesNotMatch(source, /scheduleActiveTraceSyncReplay|refreshActiveTraceFromSyncReplay/u);
|
||||
@@ -169,10 +170,10 @@ test("workbench active terminal paths seal final response from turn authority",
|
||||
assert.match(loadBlock, /fetchSessionDetailPage\(requestId, \{ reason: "load-session:detail", force: true \}\)/u);
|
||||
assert.match(loadBlock, /sessionFromWorkbenchSession\(detail\.data\?\.session, \{ includeMessages: false \}\)/u);
|
||||
assert.match(loadBlock, /const fallbackMessages = seed\?\.sessionId === id \? seed\.messages \?\? \[\] : \[\];/u);
|
||||
assert.match(loadBlock, /fetchSessionMessagesPage\(normalizedRequestId, \{ limit: messageLimit, reason: "load-session:messages", force: true \}\)/u);
|
||||
assert.doesNotMatch(loadBlock, /messages:\s*\[\]/u);
|
||||
assert.doesNotMatch(loadBlock, /limit:\s*100/u);
|
||||
assert.match(restoreSealBlock, /workbench_restored_turn_auto_read_disabled/u);
|
||||
assert.doesNotMatch(restoreSealBlock, /fetchWorkbenchTurnStatus|Promise\.all/u);
|
||||
assert.doesNotMatch(restoreSealBlock, /force:\s*true/u);
|
||||
assert.doesNotMatch(source, /sealRestoredActiveTurnMessages|messageNeedsRestoredTurnSeal|refreshSessionStatusAuthority|readTerminalTraceDetailGaps/u);
|
||||
});
|
||||
|
||||
test("workbench active SSE path only recovers by reconnecting the events cursor", () => {
|
||||
@@ -181,8 +182,7 @@ test("workbench active SSE path only recovers by reconnecting the events cursor"
|
||||
source.slice(source.indexOf("async function submitMessage"), source.indexOf("async function cancelAgentMessage")),
|
||||
source.slice(source.indexOf("function reattachTrace"), source.indexOf("function restartRealtime")),
|
||||
source.slice(source.indexOf("function flushRealtimeTurnProjection"), source.indexOf("function installRealtimeVisibilityHandler")),
|
||||
source.slice(source.indexOf("function completeTrace"), source.indexOf("async function hydrateTerminalMessageDiagnostics")),
|
||||
source.slice(source.indexOf("async function sealRestoredActiveTurnMessages"), source.indexOf("function reattachRestoredActiveTrace"))
|
||||
source.slice(source.indexOf("function completeTrace"), source.indexOf("function applyTerminalResultDiagnostics"))
|
||||
].join("\n");
|
||||
|
||||
assert.doesNotMatch(activeBlocks, /refreshWorkbenchSyncReplay|scheduleActiveTraceSyncReplay|setTimeout|setInterval/u);
|
||||
|
||||
@@ -1,13 +1,7 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import type { ChatMessage } from "@/types";
|
||||
import { projectionMergeCommitSummary, sessionDetailAutoReadDecision, traceEventsAutoReadDecision, workbenchSessionDetailReadKey, workbenchSessionMessagesReadKey, workbenchTraceEventsReadKey } from "./workbench-session-messages-read-budget";
|
||||
import { selectActiveTurnStatusRefreshTraceIds } from "./workbench-session";
|
||||
|
||||
function message(id: string): ChatMessage {
|
||||
return { id, role: "agent", text: "", status: "running", createdAt: "2026-07-02T00:00:00.000Z" } as ChatMessage;
|
||||
}
|
||||
import { traceEventsAutoReadDecision, workbenchSessionDetailReadKey, workbenchSessionMessagesReadKey, workbenchTraceEventsReadKey } from "./workbench-session-messages-read-budget";
|
||||
|
||||
test("session messages singleflight key separates limit and force class", () => {
|
||||
const limit8 = workbenchSessionMessagesReadKey({ sessionId: "ses_key", limit: 8, force: false });
|
||||
@@ -18,7 +12,6 @@ test("session messages singleflight key separates limit and force class", () =>
|
||||
assert.notEqual(limit20, force20);
|
||||
assert.equal(limit20, workbenchSessionMessagesReadKey({ sessionId: "ses_key", limit: 20, force: false }));
|
||||
});
|
||||
|
||||
test("session detail singleflight key separates force freshness class", () => {
|
||||
const fresh = workbenchSessionDetailReadKey({ sessionId: "ses_key", force: false });
|
||||
const forced = workbenchSessionDetailReadKey({ sessionId: "ses_key", force: true });
|
||||
@@ -26,7 +19,6 @@ test("session detail singleflight key separates force freshness class", () => {
|
||||
assert.notEqual(fresh, forced);
|
||||
assert.equal(fresh, workbenchSessionDetailReadKey({ sessionId: "ses_key" }));
|
||||
});
|
||||
|
||||
test("trace events auto read skips initial and completed equivalent ranges", () => {
|
||||
const key = workbenchTraceEventsReadKey({ traceId: "trc_range", afterProjectedSeq: 12, limit: 50 });
|
||||
|
||||
@@ -35,25 +27,3 @@ test("trace events auto read skips initial and completed equivalent ranges", ()
|
||||
assert.equal(traceEventsAutoReadDecision({ traceId: "trc_range", afterProjectedSeq: 12, limit: 50, cachedRange: { traceId: "trc_range", afterProjectedSeq: 12, limit: 50, nextProjectedSeq: 40, hasMore: true } }).read, true);
|
||||
assert.equal(key, workbenchTraceEventsReadKey({ traceId: "trc_range", afterProjectedSeq: 12, limit: 50 }));
|
||||
});
|
||||
|
||||
test("session detail auto read defers trace scoped realtime refresh to turn authority", () => {
|
||||
assert.deepEqual(sessionDetailAutoReadDecision({ traceId: "trc_turn", nowMs: 10_000, minIntervalMs: 5_000 }), { read: false, reason: "turn-authority-first" });
|
||||
assert.deepEqual(sessionDetailAutoReadDecision({ nowMs: 10_000, lastReadAtMs: 9_000, minIntervalMs: 5_000 }), { read: false, reason: "min-interval" });
|
||||
assert.deepEqual(sessionDetailAutoReadDecision({ nowMs: 10_000, lastReadAtMs: 1_000, minIntervalMs: 5_000 }), { read: true, reason: "metadata-only" });
|
||||
});
|
||||
|
||||
test("projection merge summary reports no write without blocking follow-up hydrate", () => {
|
||||
const existing = [{ ...message("msg_same"), traceId: "trc_current", sessionId: "ses_same" }];
|
||||
const summary = projectionMergeCommitSummary(existing, existing);
|
||||
const hydrateTraceIds = selectActiveTurnStatusRefreshTraceIds({ messages: existing, currentRequestTraceId: "trc_current", limit: 1 });
|
||||
|
||||
assert.deepEqual(summary, { changed: false, changedCount: 0 });
|
||||
assert.deepEqual(hydrateTraceIds, ["trc_current"]);
|
||||
});
|
||||
|
||||
test("projection merge summary reports changed message references", () => {
|
||||
const existing = [message("msg_old")];
|
||||
const merged = [message("msg_new")];
|
||||
|
||||
assert.deepEqual(projectionMergeCommitSummary(existing, merged), { changed: true, changedCount: 1 });
|
||||
});
|
||||
|
||||
@@ -1,7 +1,6 @@
|
||||
// SPEC: pikasTech/HWLAB#2356 Workbench bounded request storm follow-up.
|
||||
// Responsibility: Pure budgeting helpers for Workbench detail read de-duplication and projection merge writes.
|
||||
|
||||
import type { ChatMessage } from "@/types";
|
||||
import { firstNonEmptyString } from "@/utils";
|
||||
import { composeWorkbenchScopedKey } from "@/utils/workbench-key";
|
||||
|
||||
@@ -16,11 +15,6 @@ export interface SessionDetailReadKeyInput {
|
||||
force?: boolean | null;
|
||||
}
|
||||
|
||||
export interface ProjectionMergeCommitSummary {
|
||||
changed: boolean;
|
||||
changedCount: number;
|
||||
}
|
||||
|
||||
export interface TraceEventsReadRangeInput {
|
||||
traceId: string | null | undefined;
|
||||
afterProjectedSeq: number | null | undefined;
|
||||
@@ -38,15 +32,6 @@ export interface TraceEventsAutoReadDecisionInput extends TraceEventsReadRangeIn
|
||||
cachedRange?: TraceEventsReadRangeRecord | null;
|
||||
}
|
||||
|
||||
export interface SessionDetailAutoReadDecisionInput {
|
||||
force?: boolean | null;
|
||||
traceId?: string | null;
|
||||
terminalBodyVisible?: boolean | null;
|
||||
lastReadAtMs?: number | null;
|
||||
nowMs: number;
|
||||
minIntervalMs: number;
|
||||
}
|
||||
|
||||
export interface ReadDetailDecision {
|
||||
read: boolean;
|
||||
reason: string;
|
||||
@@ -89,25 +74,6 @@ export function traceEventsAutoReadDecision(input: TraceEventsAutoReadDecisionIn
|
||||
return { read: true, reason: "delta-range" };
|
||||
}
|
||||
|
||||
export function sessionDetailAutoReadDecision(input: SessionDetailAutoReadDecisionInput): ReadDetailDecision {
|
||||
if (input.force === true) return { read: true, reason: "force" };
|
||||
if (input.terminalBodyVisible === true) return { read: false, reason: "terminal-visible" };
|
||||
if (firstNonEmptyString(input.traceId)) return { read: false, reason: "turn-authority-first" };
|
||||
const lastReadAtMs = Number(input.lastReadAtMs);
|
||||
const minIntervalMs = Math.max(0, Math.trunc(Number(input.minIntervalMs)));
|
||||
if (Number.isFinite(lastReadAtMs) && input.nowMs - lastReadAtMs < minIntervalMs) return { read: false, reason: "min-interval" };
|
||||
return { read: true, reason: "metadata-only" };
|
||||
}
|
||||
|
||||
export function projectionMergeCommitSummary(existing: ChatMessage[], merged: ChatMessage[]): ProjectionMergeCommitSummary {
|
||||
let changedCount = Math.max(0, merged.length - existing.length);
|
||||
const sharedLength = Math.min(existing.length, merged.length);
|
||||
for (let index = 0; index < sharedLength; index += 1) {
|
||||
if (existing[index] !== merged[index]) changedCount += 1;
|
||||
}
|
||||
return { changed: changedCount > 0, changedCount };
|
||||
}
|
||||
|
||||
function sessionMessagesReadLimitPart(value: number | null | undefined): number {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) && number > 0 ? Math.trunc(number) : 1;
|
||||
|
||||
@@ -2,7 +2,7 @@ import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import type { ChatMessage, WorkbenchSessionRecord } from "../types";
|
||||
import { resolveCancelableAgentMessage, resolveComposerState, selectActiveTurnStatusRefreshTraceIds, sessionToSessionTab, shouldReadTerminalTraceDetail } from "./workbench-session";
|
||||
import { resolveCancelableAgentMessage, resolveComposerState, sessionToSessionTab, shouldReadTerminalTraceDetail } from "./workbench-session";
|
||||
import type { TurnStatusAuthority } from "./workbench-session";
|
||||
|
||||
function agentMessage(input: Partial<ChatMessage> & Pick<ChatMessage, "id" | "traceId" | "sessionId">): ChatMessage {
|
||||
@@ -96,22 +96,6 @@ test("cancel target remains available for failed message until terminal body exi
|
||||
assert.equal(resolveCancelableAgentMessage({ messages: [sealed], targetTraceId: traceId, targetSessionId: sessionId, turnStatusAuthority: { [traceId]: turnStatus({ traceId, sessionId, status: "running", running: true, terminal: false }) } }), null);
|
||||
});
|
||||
|
||||
test("turn status refresh selector keeps automatic hydrate bounded to the active trace", () => {
|
||||
const sessionId = "ses_turn_hydrate_budget";
|
||||
const older = agentMessage({ id: "msg_older", traceId: "trc_older", sessionId, status: "running" });
|
||||
const current = agentMessage({ id: "msg_current", traceId: "trc_current", sessionId, status: "running" });
|
||||
|
||||
assert.deepEqual(selectActiveTurnStatusRefreshTraceIds({ messages: [older, current], currentRequestTraceId: current.traceId, limit: 1 }), [current.traceId]);
|
||||
});
|
||||
|
||||
test("turn status refresh selector skips sealed terminal traces", () => {
|
||||
const sessionId = "ses_turn_detail_sealed";
|
||||
const traceId = "trc_sealed";
|
||||
const sealed = agentMessage({ id: "msg_sealed", traceId, sessionId, status: "completed", text: "final" });
|
||||
|
||||
assert.deepEqual(selectActiveTurnStatusRefreshTraceIds({ messages: [sealed], currentRequestTraceId: traceId, turnStatusAuthority: { [traceId]: turnStatus({ traceId, sessionId, status: "completed", running: false, terminal: true }) }, limit: 1 }), []);
|
||||
});
|
||||
|
||||
test("terminal detail read skips sealed terminal trace authority", () => {
|
||||
const sessionId = "ses_terminal_detail_sealed";
|
||||
const traceId = "trc_terminal_detail_sealed";
|
||||
|
||||
@@ -117,22 +117,6 @@ export function resolveCancelableAgentMessage(input: { messages: ChatMessage[];
|
||||
return null;
|
||||
}
|
||||
|
||||
export function selectActiveTurnStatusRefreshTraceIds(input: { messages: ChatMessage[]; currentRequestTraceId?: string | null; turnStatusAuthority?: TurnStatusAuthorityMap; limit?: number }): string[] {
|
||||
const traceIds = new Set<string>();
|
||||
const limit = Math.max(1, Math.trunc(input.limit ?? 3));
|
||||
const currentRequestTraceId = firstNonEmptyString(input.currentRequestTraceId);
|
||||
if (currentRequestTraceId && traceNeedsTurnStatusRefresh(currentRequestTraceId, input.messages, input.turnStatusAuthority)) traceIds.add(currentRequestTraceId);
|
||||
if (traceIds.size >= limit) return [...traceIds];
|
||||
for (const message of [...input.messages].reverse()) {
|
||||
const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId);
|
||||
if (!traceId || traceIds.has(traceId)) continue;
|
||||
if (!messageHasActiveTurnStatusEvidence(message, traceId, currentRequestTraceId)) continue;
|
||||
if (traceNeedsTurnStatusRefresh(traceId, input.messages, input.turnStatusAuthority)) traceIds.add(traceId);
|
||||
if (traceIds.size >= limit) break;
|
||||
}
|
||||
return [...traceIds];
|
||||
}
|
||||
|
||||
export function shouldReadTerminalTraceDetail(input: { traceId?: string | null; messages: ChatMessage[]; turnStatusAuthority?: TurnStatusAuthorityMap }): boolean {
|
||||
const traceId = firstNonEmptyString(input.traceId);
|
||||
if (!traceId) return false;
|
||||
@@ -317,21 +301,6 @@ function latestMessageForTrace(messages: ChatMessage[], traceId: string): ChatMe
|
||||
return [...messages].reverse().find((message) => firstNonEmptyString(message.traceId, message.runnerTrace?.traceId) === traceId) ?? null;
|
||||
}
|
||||
|
||||
function messageHasActiveTurnStatusEvidence(message: ChatMessage, traceId: string, currentRequestTraceId: string | null): boolean {
|
||||
if (currentRequestTraceId === traceId) return true;
|
||||
return isActiveStatus(message.status) || message.traceAutoLifecycle === "running";
|
||||
}
|
||||
|
||||
function traceNeedsTurnStatusRefresh(traceId: string | null | undefined, messages: ChatMessage[], turnStatusAuthority: TurnStatusAuthorityMap | undefined): boolean {
|
||||
const id = firstNonEmptyString(traceId);
|
||||
if (!id) return false;
|
||||
const turn = turnStatusAuthority?.[id];
|
||||
const message = latestMessageForTrace(messages, id);
|
||||
if (turnStatusAuthorityIsSealed(turn, message)) return false;
|
||||
if (messageIsSealedTerminal(message)) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function resolveSessionTabStatus(session: WorkbenchSessionRecord, authority: SessionStatusAuthority | null | undefined, latestAgent: ChatMessage | null = latestAgentMessage(session.messages)): string {
|
||||
const latestTraceId = firstNonEmptyString(latestAgent?.traceId, latestAgent?.runnerTrace?.traceId);
|
||||
const authorityTraceId = firstNonEmptyString(authority?.lastTraceId);
|
||||
|
||||
@@ -4,29 +4,27 @@
|
||||
import { computed, nextTick, ref } from "vue";
|
||||
import { defineStore } from "pinia";
|
||||
import { api } from "@/api";
|
||||
import { workbenchDebugCapabilities, workbenchRealtimeCapabilities } from "@/config/runtime";
|
||||
import { workbenchDebugCapabilities } from "@/config/runtime";
|
||||
import { workbenchRuntimePolicy } from "@/config/workbench-runtime-policy";
|
||||
import { createKeyedSingleflight } from "@/utils/scheduler/keyed-singleflight";
|
||||
import { createWorkbenchHealthProbeCache } from "@/utils/workbench-health";
|
||||
import { agentErrorFromProjection, normalizeApiErrorRecord, normalizeErrorDiagnostic, normalizeProjectionDiagnostic, projectionDiagnosticFromApiFailure, projectionDiagnosticFromFailure } from "@/utils/workbench-error-runtime";
|
||||
import { readWorkbenchJson, readWorkbenchNumber, readWorkbenchString, removeWorkbenchStorageKey, writeWorkbenchJson, writeWorkbenchString } from "@/utils/workbench-storage-runtime";
|
||||
import { createWorkbenchStreamTransportRuntime, workbenchRealtimeTraceIdForCapabilities, type WorkbenchRealtimeEvent, type WorkbenchStreamTransportRecovery } from "@/utils/workbench-realtime-runtime";
|
||||
import { createWorkbenchStreamTransportRuntime, workbenchRealtimeTraceId, type WorkbenchRealtimeEvent, type WorkbenchStreamTransportRecovery } from "@/utils/workbench-realtime-runtime";
|
||||
import { mergeRunnerTrace, snapshotToRunnerTrace, type TraceSnapshot } from "@/composables/workbench-trace-snapshot";
|
||||
import type { WorkbenchMessagePageResponse, WorkbenchSessionDetailResponse } from "@/api/workbench";
|
||||
import type { AgentChatResponse, AgentChatResultResponse, AgentRunProvenance, ApiError, ApiResult, ChatMessage, ErrorDiagnostic, LiveSurface, ProjectionBlocker, ProjectionDiagnostic, ProviderProfile, TraceEvent, WorkbenchSessionRecord, WorkbenchTurnTimingProjection } from "@/types";
|
||||
import { firstNonEmptyString, nextProtocolId, normalizeWorkbenchSessionId, normalizeWorkbenchSessionRouteId } from "@/utils";
|
||||
import { composeWorkbenchScopedKey, workbenchRealtimeScopeKey } from "@/utils/workbench-key";
|
||||
import { failWorkbenchSessionSwitch, failWorkbenchSubmitJourney, finishWorkbenchSessionSwitchFullLoad, markWorkbenchSubmitApiAccepted, markWorkbenchTraceEventsReceived, markWorkbenchTraceProjected, recordWorkbenchLoadingState, recordWorkbenchRuntimeDiagnostic, startWorkbenchSessionSwitch, startWorkbenchSubmitJourney } from "@/utils/workbench-performance";
|
||||
import { RECENT_DRAFTS_STORAGE_KEY, appendSessionPage, defaultProviderProfileOptions, isArchivedSession, mergeSessionIntoList, normalizeChatMessageStatus, normalizeRecentDrafts, normalizeWorkbenchMessageTitle, providerProfileOptionsFromPayload, recordRecentDraft, resolveCancelableAgentMessage, resolveComposerState, selectActiveTurnStatusRefreshTraceIds, shouldReadTerminalTraceDetail, shouldShowSessionListLoading, sortSessionTabs, stableSessionList, type DraftEntry, type ProviderProfileOption, type TurnStatusAuthority } from "./workbench-session";
|
||||
import { RECENT_DRAFTS_STORAGE_KEY, appendSessionPage, defaultProviderProfileOptions, isArchivedSession, mergeSessionIntoList, normalizeChatMessageStatus, normalizeRecentDrafts, normalizeWorkbenchMessageTitle, providerProfileOptionsFromPayload, recordRecentDraft, resolveCancelableAgentMessage, resolveComposerState, shouldReadTerminalTraceDetail, shouldShowSessionListLoading, sortSessionTabs, stableSessionList, type DraftEntry, type ProviderProfileOption, type TurnStatusAuthority } from "./workbench-session";
|
||||
import { initialWorkbenchSessionIdFromLocation } from "./workbench-projection";
|
||||
import { cleanupWorkbenchServerStateSessions, selectActiveMessages, selectActiveSession, selectSessionList, selectSessionStatusAuthority, selectTraceAuthorityById, selectTurnStatusAuthority, type WorkbenchServerAction } from "./workbench-server-state";
|
||||
import { cleanupDroppedWorkbenchSessionCaches, trimWorkbenchSessionCache } from "./workbench-session-cache";
|
||||
import { reduceWorkbenchRealtimeEvent, workbenchRealtimeEventIsBusinessActivity, type WorkbenchRealtimeAction } from "./workbench-event-reducer";
|
||||
import { projectWorkbenchLiveKafkaMessage, projectWorkbenchLiveKafkaUserMessage, workbenchAgentMessageIdForTrace, workbenchLiveKafkaEnvelope, workbenchLiveKafkaProjectionTarget, workbenchUserMessageIdForTrace } from "./workbench-live-kafka-event";
|
||||
import { reduceWorkbenchRealtimeEvent, type WorkbenchRealtimeAction } from "./workbench-event-reducer";
|
||||
import { projectRejectedWorkbenchAdmission } from "./workbench-admission-failure";
|
||||
import { workbenchHistoryAuthorityPolicy } from "./workbench-kafka-refresh-policy";
|
||||
import { messageHasSealedTerminalResult, messageIsSealedTerminal, traceAuthorityIsSealed } from "./workbench-terminal-authority";
|
||||
import { boundedProjectionMessageLimit, mergeBoundedProjectionMessages, selectProjectionMessageWindow, traceProjectionIsTerminalSealed } from "./workbench-message-projection-budget";
|
||||
import { boundedProjectionMessageLimit } from "./workbench-message-projection-budget";
|
||||
import {
|
||||
agentErrorDisplayText,
|
||||
agentErrorFromApiFailure,
|
||||
@@ -39,7 +37,6 @@ import {
|
||||
isTerminalMessageStatus,
|
||||
isTraceActiveStatus,
|
||||
messageHasTerminalResponse,
|
||||
messageNeedsTerminalDiagnostics,
|
||||
messageNeedsTraceDetailRead,
|
||||
messageStatusPatchForTerminalMerge,
|
||||
terminalAuthorityMessageFromTurnResult,
|
||||
@@ -48,7 +45,6 @@ import {
|
||||
messageTimingPatch,
|
||||
messageTimingPatchForMerge,
|
||||
messageTimingPatchFromProjection,
|
||||
messageTimingSealPatchForProjectionMerge,
|
||||
mergeTerminalResultTrace,
|
||||
nonBlockingProjection,
|
||||
normalizeAgentError,
|
||||
@@ -65,7 +61,6 @@ import {
|
||||
terminalMessageTimingPatchForNormalize,
|
||||
turnResultIsTerminalForMerge,
|
||||
turnResultStatusForMerge,
|
||||
traceHasTerminalResponse,
|
||||
traceHasEvents,
|
||||
traceResultHasTerminalEvidence,
|
||||
} from "./workbench-message-projection-runtime";
|
||||
@@ -73,7 +68,7 @@ import { planWorkbenchRealtimeApply, planWorkbenchRealtimeRecovery, type Workben
|
||||
import { useWorkbenchColadaMutations } from "./workbench-colada-mutations";
|
||||
import { useWorkbenchColadaQueries } from "./workbench-colada-queries";
|
||||
import { useWorkbenchColadaReducer } from "./workbench-colada-reducer";
|
||||
import { projectionMergeCommitSummary, traceEventsAutoReadDecision, workbenchSessionDetailReadKey, workbenchSessionMessagesReadKey, workbenchTraceEventsReadKey, type TraceEventsReadRangeRecord } from "./workbench-session-messages-read-budget";
|
||||
import { traceEventsAutoReadDecision, workbenchSessionDetailReadKey, workbenchSessionMessagesReadKey, workbenchTraceEventsReadKey, type TraceEventsReadRangeRecord } from "./workbench-session-messages-read-budget";
|
||||
import { realtimeSnapshotToTraceSnapshot, terminalSealResultWithoutTraceEvents, traceDetailProjectedSeq, traceNextProjectedSeq } from "./workbench-trace-detail";
|
||||
import { appendRawHwlabIngressFrame, createRawHwlabIngressState } from "./workbench-raw-hwlab-ingress";
|
||||
|
||||
@@ -108,15 +103,21 @@ interface RealtimeTurnProjectionItem {
|
||||
terminalTurn: boolean;
|
||||
}
|
||||
|
||||
function workbenchMessageIdForTrace(traceId: string, role: "user" | "agent"): string {
|
||||
const suffix = firstNonEmptyString(traceId)
|
||||
?.replace(/^trc_/u, "")
|
||||
.replace(/[^A-Za-z0-9_.:-]/gu, "_")
|
||||
.slice(0, 48);
|
||||
if (!suffix) throw new Error("traceId must produce a stable Workbench message identity");
|
||||
return `msg_${suffix}_${role}`;
|
||||
}
|
||||
|
||||
export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
const runtimePolicy = workbenchRuntimePolicy();
|
||||
const realtimeCapabilities = workbenchRealtimeCapabilities();
|
||||
const historyAuthorityPolicy = workbenchHistoryAuthorityPolicy(realtimeCapabilities);
|
||||
const debugCapabilities = workbenchDebugCapabilities();
|
||||
const workbenchColadaReducer = useWorkbenchColadaReducer();
|
||||
const workbenchColadaQueries = useWorkbenchColadaQueries();
|
||||
const workbenchColadaMutations = useWorkbenchColadaMutations();
|
||||
const turnStatusReadSingleflight = createKeyedSingleflight<ApiResult<AgentChatResultResponse>>();
|
||||
const traceDetailReadSingleflight = createKeyedSingleflight<void>();
|
||||
const sessionMessagesReadSingleflight = createKeyedSingleflight<ApiResult<WorkbenchMessagePageResponse>>();
|
||||
const sessionDetailReadSingleflight = createKeyedSingleflight<ApiResult<WorkbenchSessionDetailResponse>>();
|
||||
@@ -138,7 +139,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
const sessionListLoadingMore = ref(false);
|
||||
const sessionListLoadMoreError = ref<string | null>(null);
|
||||
const chatPending = ref(false);
|
||||
const liveRealtimeReadySessionId = ref<string | null>(null);
|
||||
const rawHwlabIngress = ref(createRawHwlabIngressState());
|
||||
const error = ref<string | null>(null);
|
||||
const activityRef = ref({ lastActivityAt: Date.now(), lastActivityIso: new Date().toISOString(), waitingFor: "idle", lastEventLabel: null as string | null });
|
||||
@@ -169,7 +169,7 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
const sessionListLoadedCount = computed(() => sessionTabs.value.length);
|
||||
const sessionListLoading = computed(() => shouldShowSessionListLoading({ loading: loading.value, sessionsReady: sessionsReady.value }));
|
||||
const sessionDetailLoading = computed(() => Boolean(sessionDetailLoadingId.value || switchingSessionId.value || (loading.value && messages.value.length === 0)));
|
||||
const composer = computed(() => resolveComposerState({ messages: messages.value, sessions: sessions.value, activeSessionId: activeSessionId.value, chatPending: chatPending.value, realtimeReady: !realtimeCapabilities.liveKafkaSse || liveRealtimeReadySessionId.value === activeSessionId.value, currentRequest: currentRequest.value, turnStatusAuthority: turnStatusAuthority.value }));
|
||||
const composer = computed(() => resolveComposerState({ messages: messages.value, sessions: sessions.value, activeSessionId: activeSessionId.value, chatPending: chatPending.value, realtimeReady: true, currentRequest: currentRequest.value, turnStatusAuthority: turnStatusAuthority.value }));
|
||||
|
||||
function recordActivity(label = "user-activity"): void {
|
||||
const now = Date.now();
|
||||
@@ -228,7 +228,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
const nextSessions = stableSessionList(sessions.value, listedSessions, stableSelected?.sessionId ?? selected?.sessionId ?? routeSessionId ?? includeSessionId, stableSelected);
|
||||
rememberSessionList(nextSessions);
|
||||
sessionsReady.value = sessionsResult.ok || nextSessions.length > 0;
|
||||
void refreshSessionStatusAuthority(nextSessions);
|
||||
clearSessionDetailLoading(targetSessionId);
|
||||
loading.value = false;
|
||||
recordWorkbenchLoadingState({ scope: "session_detail", active: false, reason: "hydrate", sessionId: targetSessionId });
|
||||
@@ -376,23 +375,12 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
rememberSessionList(next);
|
||||
applySessionPagination(response.data);
|
||||
sessionsReady.value = true;
|
||||
if (historyAuthorityPolicy.turnStatusHydrate) await refreshSessionStatusAuthority(next);
|
||||
return;
|
||||
}
|
||||
sessionsReady.value = sessions.value.length > 0;
|
||||
if (sessions.value.length === 0) error.value = response.error ?? "session list unavailable";
|
||||
}
|
||||
|
||||
function scheduleSessionListRefresh(includeSessionId: string | null | undefined = activeSessionId.value, delayMs = runtimePolicy.sessionListRealtimeRefreshDelayMs): void {
|
||||
if (realtimeCapabilities.liveKafkaSse) return;
|
||||
void includeSessionId;
|
||||
if (typeof window === "undefined") {
|
||||
void workbenchColadaQueries.invalidateSessionList();
|
||||
return;
|
||||
}
|
||||
window.setTimeout(() => { void workbenchColadaQueries.invalidateSessionList(); }, delayMs);
|
||||
}
|
||||
|
||||
async function loadMoreSessions(): Promise<void> {
|
||||
if (sessionListLoadingMore.value || !sessionListHasMore.value) return;
|
||||
const cursor = sessionListNextCursor.value;
|
||||
@@ -412,7 +400,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
rememberSessionList(next);
|
||||
applySessionPagination(response.data);
|
||||
sessionsReady.value = true;
|
||||
if (historyAuthorityPolicy.turnStatusHydrate) await refreshSessionStatusAuthority(next);
|
||||
}
|
||||
|
||||
function currentSessionListLimit(): number {
|
||||
@@ -511,15 +498,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
return projectedActiveSession.value ?? null;
|
||||
}
|
||||
|
||||
async function refreshSessionStatusAuthority(source: WorkbenchSessionRecord[] = sessions.value): Promise<void> {
|
||||
void source;
|
||||
}
|
||||
|
||||
function fetchWorkbenchTurnStatus(traceId: string, useActivityTimeout = shouldUseActivityTimeoutForTrace(traceId), options: { force?: boolean } = {}): Promise<ApiResult<AgentChatResultResponse>> {
|
||||
const activitySource = useActivityTimeout ? () => activityRef.value : null;
|
||||
return turnStatusReadSingleflight.run(traceId, () => workbenchColadaQueries.fetchTurn(traceId, { timeoutMs: 8000, activityRef: activitySource, minIntervalMs: runtimePolicy.workbenchTurnStatusMinRefreshMs, force: options.force }), { reason: options.force ? "force-turn-status" : "turn-status" });
|
||||
}
|
||||
|
||||
function fetchWorkbenchTraceEvents(traceId: string, afterProjectedSeq: number, useActivityTimeout = shouldUseActivityTimeoutForTrace(traceId), options: { force?: boolean } = {}): Promise<ApiResult<AgentChatResultResponse>> {
|
||||
const activitySource = useActivityTimeout ? () => activityRef.value : null;
|
||||
return workbenchColadaQueries.fetchTraceEvents(traceId, { timeoutMs: runtimePolicy.workbenchTraceEventsTimeoutMs, activityRef: activitySource, afterProjectedSeq, limit: runtimePolicy.traceDetailPageLimit, minIntervalMs: runtimePolicy.workbenchTraceEventsMinRefreshMs, force: options.force });
|
||||
@@ -536,25 +514,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
return isTraceActiveStatus(turn?.status) || isTraceActiveStatus(message?.status);
|
||||
}
|
||||
|
||||
async function refreshMessageProjectionForTrace(sessionId: string | null | undefined, traceId: string, options: { force?: boolean } = {}): Promise<void> {
|
||||
const id = normalizeWorkbenchSessionId(sessionId);
|
||||
if (!id) return;
|
||||
const existing = serverState.value.messagesBySessionId[id] ?? [];
|
||||
if (traceProjectionIsTerminalSealed(traceId, existing)) return;
|
||||
const response = await fetchSessionMessagesPage(id, { limit: traceMessageProjectionWindowLimit(), reason: `trace-message-page:${traceId}`, force: options.force });
|
||||
if (!response.ok || !response.data) {
|
||||
if (shouldSuppressTransientWorkbenchReadFailure(response)) return;
|
||||
if (traceHasTerminalResponse(traceId, messages.value)) return;
|
||||
applyProjectionDiagnostic(traceId, projectionDiagnosticFromApiFailure(response, { code: "message_projection_refresh_failed", message: response.error ?? "消息投影刷新失败,主消息正文保持上一份 canonical projection。", health: response.status === 0 ? "unavailable" : "degraded" }));
|
||||
return;
|
||||
}
|
||||
const pageMessages = Array.isArray(response.data.messages) ? response.data.messages.map((message) => normalizeChatMessage(message as ChatMessage)) : [];
|
||||
const merged = mergeMessageProjectionPage(id, pageMessages, { traceId, limit: traceMessageProjectionWindowLimit() });
|
||||
commitMergedSessionMessages(id, pageMessages, merged, { traceId, limit: traceMessageProjectionWindowLimit(), reason: "trace-message-page" });
|
||||
await hydrateTurnStatusAuthority(merged, { traceId, limit: 1, reason: "trace-message-page" });
|
||||
if (!traceProjectionIsTerminalSealed(traceId, merged)) readTerminalTraceDetailGaps(merged, `trace-message-page:${traceId}`);
|
||||
}
|
||||
|
||||
function fetchSessionMessagesPage(sessionId: string, options: SessionMessagesReadOptions): Promise<ApiResult<WorkbenchMessagePageResponse>> {
|
||||
const key = workbenchSessionMessagesReadKey({ sessionId, limit: options.limit, force: options.force });
|
||||
return sessionMessagesReadSingleflight.run(key, () => workbenchColadaQueries.fetchSessionMessages(sessionId, { limit: options.limit, minIntervalMs: runtimePolicy.workbenchSessionMessagesMinRefreshMs, force: options.force }), { reason: options.reason });
|
||||
@@ -569,121 +528,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
return boundedProjectionMessageLimit(runtimePolicy.workbenchSessionMessagesWindowLimit, runtimePolicy.sessionListPageLimit);
|
||||
}
|
||||
|
||||
function traceMessageProjectionWindowLimit(): number {
|
||||
return boundedProjectionMessageLimit(runtimePolicy.workbenchTraceMessagesWindowLimit, runtimePolicy.traceDetailAutoQueueLimit);
|
||||
}
|
||||
|
||||
function mergeMessageProjectionPage(sessionId: string, pageMessages: ChatMessage[], options: { traceId?: string | null; limit: number }): ChatMessage[] {
|
||||
const existing = serverState.value.messagesBySessionId[sessionId] ?? [];
|
||||
const windowMessages = selectProjectionMessageWindow(pageMessages, options);
|
||||
const incoming = windowMessages.map((message) => mergeMessageProjectionMessage(message, existing));
|
||||
return mergeBoundedProjectionMessages(existing, incoming);
|
||||
}
|
||||
|
||||
function commitMergedSessionMessages(sessionId: string, incoming: ChatMessage[], merged: ChatMessage[], options: { traceId?: string | null; limit: number; reason: string }): void {
|
||||
const existing = serverState.value.messagesBySessionId[sessionId] ?? [];
|
||||
const summary = projectionMergeCommitSummary(existing, merged);
|
||||
recordWorkbenchRuntimeDiagnostic({
|
||||
module: "workbench-message-projection",
|
||||
sessionId,
|
||||
traceId: firstNonEmptyString(options.traceId) ?? null,
|
||||
outcome: "ok",
|
||||
diagnostic: {
|
||||
code: "workbench_message_projection_merge",
|
||||
reason: options.reason,
|
||||
source: "session-messages-page",
|
||||
inputCount: incoming.length,
|
||||
existingCount: existing.length,
|
||||
mergedCount: merged.length,
|
||||
changedCount: summary.changedCount,
|
||||
limit: options.limit,
|
||||
valuesRedacted: true
|
||||
}
|
||||
});
|
||||
if (summary.changed) rememberSessionMessages(sessionId, merged);
|
||||
}
|
||||
|
||||
function mergeMessageProjectionMessage(message: ChatMessage, existing: ChatMessage[]): ChatMessage {
|
||||
const previous = findExistingProjectionMessage(message, existing);
|
||||
const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId, previous?.traceId, previous?.runnerTrace?.traceId);
|
||||
const authority = traceId ? traceAuthorityById.value[traceId] ?? null : null;
|
||||
const preservedTrace = authority ?? previous?.runnerTrace ?? null;
|
||||
const timingSealPatch = messageTimingSealPatchForProjectionMerge(message, previous);
|
||||
if (!preservedTrace) return { ...message, ...timingSealPatch };
|
||||
const runnerTrace = message.runnerTrace ? mergeRunnerTrace(message.runnerTrace, preservedTrace) : preservedTrace;
|
||||
return { ...message, ...timingSealPatch, runnerTrace };
|
||||
}
|
||||
|
||||
function findExistingProjectionMessage(message: ChatMessage, existing: ChatMessage[]): ChatMessage | null {
|
||||
const messageId = firstNonEmptyString(message.messageId, message.id);
|
||||
if (messageId) {
|
||||
const byId = existing.find((item) => firstNonEmptyString(item.messageId, item.id) === messageId);
|
||||
if (byId) return byId;
|
||||
}
|
||||
const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId);
|
||||
if (!traceId) return null;
|
||||
return existing.find((item) => item.role === message.role && firstNonEmptyString(item.traceId, item.runnerTrace?.traceId) === traceId) ?? null;
|
||||
}
|
||||
|
||||
async function hydrateTurnStatusAuthority(source: ChatMessage[] = messages.value, options: { traceId?: string | null; limit?: number; reason?: string; force?: boolean } = {}): Promise<void> {
|
||||
const traceIds = activeTurnStatusRefreshTraceIds(source, options);
|
||||
if (options.force !== true) {
|
||||
if (traceIds.length > 0) {
|
||||
recordWorkbenchRuntimeDiagnostic({
|
||||
module: "workbench-turn-status",
|
||||
traceId: options.traceId ?? traceIds[0] ?? null,
|
||||
outcome: "ok",
|
||||
diagnostic: { code: "workbench_turn_status_auto_read_disabled", reason: options.reason ?? "auto-turn-status-hydrate", source: "projection-sse-authority", requestedCount: traceIds.length, valuesRedacted: true }
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
for (const traceId of traceIds) {
|
||||
await refreshTurnStatusByTraceId(traceId);
|
||||
}
|
||||
}
|
||||
|
||||
function activeTurnStatusRefreshTraceIds(source: ChatMessage[] = messages.value, options: { traceId?: string | null; limit?: number; reason?: string; force?: boolean } = {}): string[] {
|
||||
const requestedTraceId = firstNonEmptyString(options.traceId, currentRequest.value?.traceId);
|
||||
const traceIds = selectActiveTurnStatusRefreshTraceIds({ messages: source, currentRequestTraceId: requestedTraceId, turnStatusAuthority: turnStatusAuthority.value, limit: options.limit ?? 1 });
|
||||
recordWorkbenchRuntimeDiagnostic({
|
||||
module: "workbench-turn-status",
|
||||
traceId: requestedTraceId ?? traceIds[0] ?? null,
|
||||
outcome: "ok",
|
||||
diagnostic: {
|
||||
code: "workbench_turn_status_hydrate_budget",
|
||||
reason: options.reason ?? "auto-turn-status-hydrate",
|
||||
source: "session-message-projection",
|
||||
requestedCount: traceIds.length,
|
||||
limit: options.limit ?? 1,
|
||||
valuesRedacted: true
|
||||
}
|
||||
});
|
||||
return traceIds;
|
||||
}
|
||||
|
||||
async function refreshTurnStatusByTraceId(traceId: string | null | undefined, options: { force?: boolean } = {}): Promise<void> {
|
||||
const id = firstNonEmptyString(traceId);
|
||||
if (!id) return;
|
||||
if (options.force !== true) {
|
||||
recordWorkbenchRuntimeDiagnostic({ module: "workbench-turn-status", traceId: id, outcome: "ok", diagnostic: { code: "workbench_turn_status_auto_read_disabled", source: "projection-sse-authority", valuesRedacted: true } });
|
||||
return;
|
||||
}
|
||||
if (traceTerminalBodyIsVisible(id)) {
|
||||
recordWorkbenchRuntimeDiagnostic({ module: "workbench-terminal-priority", traceId: id, outcome: "ok", diagnostic: { code: "terminal_low_priority_turn_skip", source: "turn-status", valuesRedacted: true } });
|
||||
return;
|
||||
}
|
||||
const response = await fetchWorkbenchTurnStatus(id, shouldUseActivityTimeoutForTrace(id), { force: options.force });
|
||||
if (response.ok && response.data) {
|
||||
applyTurnStatusSnapshot(id, response.data);
|
||||
if (turnResultIsTerminalForMerge(response.data as AgentChatResultResponse)) completeTrace(id, response.data, { forceRead: options.force });
|
||||
return;
|
||||
}
|
||||
if (shouldSuppressTransientWorkbenchReadFailure(response)) return;
|
||||
if (traceHasTerminalResponse(id, messages.value)) return;
|
||||
applyProjectionDiagnostic(id, projectionDiagnosticFromApiFailure(response, { code: "turn_status_poll_failed", message: response.error ?? "状态更新失败,当前 turn 状态暂不可见。", health: response.status === 0 ? "unavailable" : "degraded" }));
|
||||
}
|
||||
|
||||
function applyTurnStatusSnapshot(traceId: string, result: AgentChatResultResponse | TraceSnapshot): void {
|
||||
const ownerSessionId = traceOwnerSessionId(traceId, traceResultSessionId(result));
|
||||
if (!ownerSessionId) return;
|
||||
@@ -763,14 +607,9 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
const submitEntry = steerMode ? "steer" : "existing";
|
||||
const traceId = steerMode && composer.value.targetTraceId ? composer.value.targetTraceId : nextProtocolId("trc");
|
||||
const steerTraceId = steerMode ? nextProtocolId("trc_steer") : null;
|
||||
const userMessageId = workbenchUserMessageIdForTrace(steerTraceId ?? traceId);
|
||||
const agentMessageId = workbenchAgentMessageIdForTrace(traceId);
|
||||
const userMessageId = workbenchMessageIdForTrace(steerTraceId ?? traceId, "user");
|
||||
const agentMessageId = workbenchMessageIdForTrace(traceId, "agent");
|
||||
const sessionId = composer.value.sessionId;
|
||||
if (realtimeCapabilities.liveKafkaSse && liveRealtimeReadySessionId.value !== sessionId) {
|
||||
error.value = "realtime_connecting";
|
||||
restartRealtime("submit-readiness");
|
||||
return false;
|
||||
}
|
||||
const threadId = composer.value.threadId;
|
||||
const providerThreadId = providerThreadIdForRequest(threadId);
|
||||
const submittedAt = new Date().toISOString();
|
||||
@@ -820,7 +659,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
status: "running"
|
||||
};
|
||||
void readTraceEventsForMessages(messages.value);
|
||||
scheduleSessionListRefresh(sessionId, runtimePolicy.sessionListTerminalRefreshDelayMs);
|
||||
restartRealtime("steer");
|
||||
return true;
|
||||
}
|
||||
@@ -842,7 +680,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
};
|
||||
applyTurnStatusSnapshot(canonicalTraceId, response.data);
|
||||
publishWorkbenchProjectionSignal(sessionId, canonicalTraceId, "submit-admitted");
|
||||
scheduleSessionListRefresh(sessionId, runtimePolicy.sessionListTerminalRefreshDelayMs);
|
||||
if (turnResultIsTerminalForMerge(response.data as AgentChatResultResponse)) {
|
||||
completeTrace(canonicalTraceId, response.data as AgentChatResultResponse);
|
||||
return true;
|
||||
@@ -864,7 +701,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
if (message.status === "running") chatPending.value = false;
|
||||
currentRequest.value = null;
|
||||
void clearActiveTrace(traceId, "cancel-agent-message");
|
||||
scheduleSessionListRefresh(sessionId, runtimePolicy.sessionListTerminalRefreshDelayMs);
|
||||
}
|
||||
|
||||
async function cancelRunningTrace(): Promise<void> {
|
||||
@@ -966,13 +802,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
return fetchWorkbenchTraceEvents(traceId, afterProjectedSeq, shouldUseActivityTimeoutForTrace(traceId), { force: options.force });
|
||||
}
|
||||
|
||||
function readTerminalTraceDetailGaps(source: ChatMessage[], reason: string): void {
|
||||
// Terminal trace rows are loaded on explicit force/detail paths. Auto-filling
|
||||
// historical terminal traces makes multi-turn Workbench sessions main-thread bound.
|
||||
void source;
|
||||
void reason;
|
||||
}
|
||||
|
||||
async function readTraceEventsForMessages(source: ChatMessage[] = messages.value, options: { force?: boolean } = {}): Promise<void> {
|
||||
const candidates = traceDetailReadCandidates(source);
|
||||
if (options.force !== true) {
|
||||
@@ -1128,10 +957,8 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
void reason;
|
||||
const realtimeScopeKey = workbenchRealtimeScopeKey(sessionId, traceId);
|
||||
const scopeChanged = realtimeTransport.currentKey() !== realtimeScopeKey;
|
||||
if (realtimeCapabilities.liveKafkaSse && scopeChanged) liveRealtimeReadySessionId.value = null;
|
||||
if (debugCapabilities.rawHwlabEventWindow.enabled && scopeChanged) rawHwlabIngress.value = createRawHwlabIngressState(realtimeScopeKey);
|
||||
realtimeTransport.restart({
|
||||
realtimeCapabilities,
|
||||
sessionId,
|
||||
traceId,
|
||||
forceReconnect,
|
||||
@@ -1142,31 +969,8 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
onIngress: debugCapabilities.rawHwlabEventWindow.enabled
|
||||
? (frame) => { rawHwlabIngress.value = appendRawHwlabIngressFrame(rawHwlabIngress.value, frame, debugCapabilities.rawHwlabEventWindow); }
|
||||
: undefined,
|
||||
onOpen: () => undefined,
|
||||
onError: () => {
|
||||
if (realtimeCapabilities.liveKafkaSse) liveRealtimeReadySessionId.value = null;
|
||||
},
|
||||
onState: (state) => {
|
||||
if (realtimeCapabilities.liveKafkaSse && ["connecting", "error", "closed", "blocked"].includes(state.phase)) liveRealtimeReadySessionId.value = null;
|
||||
},
|
||||
onRecovery: (recovery) => handleRealtimeRecovery(recovery),
|
||||
onEvent: (event, eventName) => {
|
||||
if (realtimeCapabilities.liveKafkaSse && eventName === "workbench.connected") {
|
||||
const filters = recordValue(event.filters);
|
||||
const connectedSessionId = normalizeWorkbenchSessionId(filters?.sessionId);
|
||||
const refreshReplay = realtimeCapabilities.kafkaRefreshReplay;
|
||||
const deliveryValid = refreshReplay
|
||||
? event.deliverySemantics === "kafka-retention-then-live" && event.liveOnly === false && event.replay === true && event.lossPossible === false
|
||||
: event.deliverySemantics === "live-only" && event.liveOnly === true && event.replay === false;
|
||||
const valid = deliveryValid
|
||||
&& event.capabilities?.liveKafkaSse === true
|
||||
&& event.capabilities?.kafkaRefreshReplay === refreshReplay
|
||||
&& connectedSessionId === sessionId;
|
||||
liveRealtimeReadySessionId.value = valid ? connectedSessionId : null;
|
||||
if (!valid) error.value = "workbench_live_realtime_contract_invalid";
|
||||
}
|
||||
applyRealtimeEvent(event, eventName);
|
||||
}
|
||||
onEvent: (event, eventName) => applyRealtimeEvent(event, eventName)
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1193,13 +997,11 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
}
|
||||
|
||||
function stopRealtime(): void {
|
||||
liveRealtimeReadySessionId.value = null;
|
||||
realtimeTransport.stop();
|
||||
}
|
||||
|
||||
function realtimeTraceId(): string | null {
|
||||
return workbenchRealtimeTraceIdForCapabilities(
|
||||
realtimeCapabilities,
|
||||
return workbenchRealtimeTraceId(
|
||||
currentRequest.value?.traceId,
|
||||
activeTraceIdFromMessages(messages.value, turnStatusAuthority.value)
|
||||
);
|
||||
@@ -1217,7 +1019,7 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
const snapshot = recordValue(event.snapshot);
|
||||
const traceEvent = recordValue(event.event);
|
||||
const sessionId = firstNonEmptyString(event.hwlabSessionId, event.sessionId, turn?.sessionId, snapshot?.sessionId, traceEvent?.sessionId);
|
||||
return event.schema === "hwlab.event.v1" ? normalizeWorkbenchSessionId(sessionId) : normalizeTraceAuthoritySessionId(sessionId);
|
||||
return normalizeTraceAuthoritySessionId(sessionId);
|
||||
}
|
||||
|
||||
function traceResultSessionId(result: AgentChatResultResponse | TraceSnapshot | Record<string, unknown> | null | undefined): string | null {
|
||||
@@ -1284,7 +1086,7 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
|
||||
function applyRealtimeEvent(event: WorkbenchRealtimeEvent, eventName: string): void {
|
||||
const reduced = reduceWorkbenchRealtimeEvent(event, eventName);
|
||||
if (workbenchRealtimeEventIsBusinessActivity(event, eventName, realtimeCapabilities.liveKafkaSse)) recordActivity(reduced.activityLabel);
|
||||
recordActivity(reduced.activityLabel);
|
||||
applyWorkbenchRealtimeAction(reduced.action);
|
||||
}
|
||||
|
||||
@@ -1340,77 +1142,18 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
function applyRealtimeTraceEvent(traceId: string | null | undefined, event: WorkbenchRealtimeEvent["event"], snapshot: WorkbenchRealtimeEvent["snapshot"], realtimeEvent?: WorkbenchRealtimeEvent | null): void {
|
||||
const id = firstNonEmptyString(traceId, event?.traceId, snapshot?.traceId);
|
||||
if (!id) return;
|
||||
const liveKafkaEnvelope = workbenchLiveKafkaEnvelope(realtimeEvent?.schema);
|
||||
const sessionId = realtimeEvent ? realtimeEventSessionId(realtimeEvent) : traceResultSessionId(snapshot ?? event ?? null);
|
||||
if (!shouldApplyActiveTraceAuthority(id, sessionId, liveKafkaEnvelope)) return;
|
||||
if (traceTerminalBodyIsVisible(id, sessionId) && !liveKafkaEnvelope) {
|
||||
if (!shouldApplyActiveTraceAuthority(id, sessionId)) return;
|
||||
if (traceTerminalBodyIsVisible(id, sessionId)) {
|
||||
recordWorkbenchRuntimeDiagnostic({ module: "workbench-terminal-priority", sessionId, traceId: id, outcome: "ok", diagnostic: { code: "terminal_low_priority_sse_trace_skip", source: "realtime-trace-event", valuesRedacted: true } });
|
||||
return;
|
||||
}
|
||||
const events = event ? [event] : Array.isArray(snapshot?.events) ? snapshot.events : [];
|
||||
markWorkbenchTraceEventsReceived({ traceId: id, events, transport: "sse", serverSentAt: realtimeEvent?.serverSentAt, eventCreatedAt: realtimeEvent?.eventCreatedAt, traceSeq: realtimeEvent?.traceSeq ?? realtimeEvent?.cursor?.traceSeq });
|
||||
if (liveKafkaEnvelope && event) {
|
||||
applyLiveKafkaBusinessEvent(id, sessionId, event, new Date().toISOString());
|
||||
return;
|
||||
}
|
||||
const eventSnapshot = snapshot ?? { traceId: id, status: event?.status, events };
|
||||
applyTraceSnapshot(id, realtimeSnapshotToTraceSnapshot(id, eventSnapshot, events));
|
||||
}
|
||||
|
||||
function applyLiveKafkaBusinessEvent(traceId: string, authoritySessionId: string | null, event: TraceEvent, receivedAt: string): void {
|
||||
const eventSessionId = firstNonEmptyString(event.sessionId);
|
||||
const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId ?? eventSessionId, true);
|
||||
if (!ownerSessionId) return;
|
||||
if (workbenchLiveKafkaProjectionTarget(event) === "user") {
|
||||
const messageId = firstNonEmptyString(event.userMessageId, event.messageId);
|
||||
const previousUserMessage = messageId
|
||||
? (serverState.value.messagesBySessionId[ownerSessionId] ?? []).find((message) => (message.messageId ?? message.id) === messageId) ?? null
|
||||
: null;
|
||||
const userMessage = projectWorkbenchLiveKafkaUserMessage({ previous: previousUserMessage, traceId, sessionId: ownerSessionId, event, receivedAt });
|
||||
if (userMessage) reduceServerState({ type: "message.upsert", sessionId: ownerSessionId, message: userMessage });
|
||||
else error.value = "workbench_live_user_message_invalid";
|
||||
return;
|
||||
}
|
||||
const previousMessage = (serverState.value.messagesBySessionId[ownerSessionId] ?? []).find((message) => messageMatchesTraceAuthority(message, traceId, authoritySessionId ?? eventSessionId, ownerSessionId, true)) ?? null;
|
||||
const message = projectWorkbenchLiveKafkaMessage({
|
||||
previous: previousMessage,
|
||||
traceId,
|
||||
sessionId: ownerSessionId,
|
||||
event,
|
||||
receivedAt
|
||||
});
|
||||
reduceServerState({
|
||||
type: "message.upsert",
|
||||
sessionId: ownerSessionId,
|
||||
message
|
||||
});
|
||||
if (message.runnerTrace) rememberTraceAuthority(message.runnerTrace);
|
||||
markWorkbenchTraceProjected(traceId);
|
||||
const terminal = message.traceAutoLifecycle === "terminal";
|
||||
const status = firstNonEmptyString(message.status) ?? "running";
|
||||
const finalResponse = terminal && message.text ? { text: message.text, status, traceId } : undefined;
|
||||
rememberTurnStatus(traceId, {
|
||||
traceId,
|
||||
sessionId: ownerSessionId,
|
||||
status,
|
||||
running: !terminal,
|
||||
terminal,
|
||||
finalResponse,
|
||||
timing: message.timing,
|
||||
startedAt: message.timing?.startedAt,
|
||||
lastEventAt: message.timing?.lastEventAt,
|
||||
finishedAt: message.timing?.finishedAt,
|
||||
durationMs: message.timing?.durationMs,
|
||||
updatedAt: receivedAt
|
||||
} as AgentChatResultResponse);
|
||||
const existing = sessions.value.find((session) => session.sessionId === ownerSessionId) ?? null;
|
||||
if (existing) rememberSessionList(mergeSessionIntoList(sessions.value, { ...existing, status, lastTraceId: traceId, updatedAt: receivedAt }));
|
||||
if (terminal && currentRequest.value?.traceId === traceId) {
|
||||
chatPending.value = false;
|
||||
currentRequest.value = null;
|
||||
}
|
||||
}
|
||||
|
||||
function applyRealtimeTurnSnapshot(turn: Record<string, unknown>): void {
|
||||
const traceId = firstNonEmptyString(turn.traceId);
|
||||
if (!traceId) return;
|
||||
@@ -1513,7 +1256,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
}
|
||||
|
||||
function publishWorkbenchProjectionSignal(sessionId: string | null | undefined, traceId: string | null | undefined, reason: string): void {
|
||||
if (!realtimeCapabilities.projectionRealtime) return;
|
||||
if (typeof window === "undefined") return;
|
||||
const id = normalizeWorkbenchSessionId(sessionId);
|
||||
if (!id) return;
|
||||
@@ -1523,7 +1265,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
}
|
||||
|
||||
function handleWorkbenchProjectionSignal(value: unknown): void {
|
||||
if (!realtimeCapabilities.projectionRealtime) return;
|
||||
const record = recordValue(value);
|
||||
if (!record || record.sourceId === workbenchProjectionSignalSourceId) return;
|
||||
if (firstNonEmptyString(record.type) !== "session-projection") return;
|
||||
@@ -1557,10 +1298,9 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
return { ...message, ...messageTimingPatchForMerge(message, trace), ...messageStatusPatchForTerminalMerge(message, traceStatus, terminal), runnerTrace, error: clearCompletedDiagnostics ? null : error ?? message.error ?? null, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, updatedAt: new Date().toISOString() };
|
||||
}));
|
||||
markWorkbenchTraceProjected(traceId);
|
||||
if (!traceProjectionIsTerminalSealed(traceId, serverState.value.messagesBySessionId[ownerSessionId] ?? [])) scheduleSessionListRefresh(ownerSessionId, runtimePolicy.sessionListRealtimeRefreshDelayMs);
|
||||
}
|
||||
|
||||
function completeTrace(traceId: string, result: AgentChatResultResponse, options: { forceRead?: boolean } = {}): void {
|
||||
function completeTrace(traceId: string, result: AgentChatResultResponse): void {
|
||||
const authoritySessionId = traceResultSessionId(result);
|
||||
const ownerSessionId = traceOwnerSessionId(traceId, authoritySessionId);
|
||||
if (!ownerSessionId) return;
|
||||
@@ -1568,25 +1308,12 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
projectTurnAuthorityToMessages(traceId, result, "complete-trace");
|
||||
rememberTurnStatus(traceId, result);
|
||||
markWorkbenchTraceProjected(traceId);
|
||||
const ownerMessages = serverState.value.messagesBySessionId[ownerSessionId] ?? [];
|
||||
const terminalMessage = latestMessageForTrace(traceId, ownerMessages);
|
||||
if (!traceProjectionIsTerminalSealed(traceId, ownerMessages)) {
|
||||
if (options.forceRead) void refreshMessageProjectionForTrace(ownerSessionId, traceId, { force: true });
|
||||
}
|
||||
if (options.forceRead && terminalMessage && !traceProjectionIsTerminalSealed(traceId, ownerMessages)) void readTraceEventsForMessage(terminalMessage, { force: true });
|
||||
if (ownerSessionId === activeSessionId.value) {
|
||||
chatPending.value = false;
|
||||
currentRequest.value = null;
|
||||
void clearActiveTrace(traceId, "trace-terminal");
|
||||
restartRealtime("trace-terminal");
|
||||
}
|
||||
if (!traceProjectionIsTerminalSealed(traceId, serverState.value.messagesBySessionId[ownerSessionId] ?? [])) scheduleSessionListRefresh(ownerSessionId, runtimePolicy.sessionListTerminalRefreshDelayMs);
|
||||
}
|
||||
|
||||
async function hydrateTerminalMessageDiagnostics(): Promise<void> {
|
||||
const targets = messages.value.filter(messageNeedsTerminalDiagnostics).slice(-6);
|
||||
void targets;
|
||||
void readTraceEventsForMessages(messages.value);
|
||||
}
|
||||
|
||||
function applyTerminalResultDiagnostics(traceId: string, result: AgentChatResultResponse): void {
|
||||
@@ -1608,8 +1335,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
const projection = clearCompletedDiagnostics ? nonBlockingProjection(resultProjection) : resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null);
|
||||
return { ...message, ...messageTimingPatchForMerge(message, result), ...messageStatusPatchForTerminalMerge(message, resultStatus, terminal), title: normalizeWorkbenchMessageTitle(message.role, message.title), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() };
|
||||
}));
|
||||
void readTraceEventsForMessages(serverState.value.messagesBySessionId[ownerSessionId] ?? []);
|
||||
scheduleSessionListRefresh(ownerSessionId, runtimePolicy.sessionListTerminalRefreshDelayMs);
|
||||
}
|
||||
|
||||
function applyRealtimeProjectionError(event: WorkbenchRealtimeEvent): void {
|
||||
@@ -1619,7 +1344,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
const code = firstNonEmptyString(errorRecord.code, "workbench_realtime_error") ?? "workbench_realtime_error";
|
||||
const phase = firstNonEmptyString(event.phase, errorRecord.phase, "realtime") ?? "realtime";
|
||||
const message = firstNonEmptyString(errorRecord.message, errorRecord.summary, "Workbench 实时状态更新异常,最新运行记录暂不可见。") ?? "Workbench 实时状态更新异常,最新运行记录暂不可见。";
|
||||
liveRealtimeReadySessionId.value = null;
|
||||
error.value = `${phase} · ${code} · ${message}`;
|
||||
return;
|
||||
}
|
||||
@@ -1651,7 +1375,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
chatPending.value = false;
|
||||
currentRequest.value = null;
|
||||
void clearActiveTrace(traceId, "trace-infrastructure-error");
|
||||
scheduleSessionListRefresh(selectedSessionId.value, runtimePolicy.sessionListTerminalRefreshDelayMs);
|
||||
}
|
||||
|
||||
function projectRejectedAdmissionFailure(input: { traceId: string; message: string; error: ChatMessage["error"]; projection: ProjectionDiagnostic; submittedAt: string }): void {
|
||||
@@ -1793,12 +1516,6 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
rememberSessionDetail({ ...session, messages: projectedMessages, messageCount: session.messageCount ?? projectedMessages.length });
|
||||
sessionsReady.value = true;
|
||||
currentRequest.value = null;
|
||||
if (historyAuthorityPolicy.kafkaRetention) {
|
||||
restartRealtime("apply-selected-session:kafka-retention");
|
||||
return;
|
||||
}
|
||||
void hydrateTurnStatusAuthority(messages.value);
|
||||
void hydrateTerminalMessageDiagnostics();
|
||||
void readTraceEventsForMessages(messages.value);
|
||||
reattachRestoredActiveTrace();
|
||||
restartRealtime("apply-selected-session");
|
||||
@@ -1825,8 +1542,10 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
if (!requestId) return null;
|
||||
const normalizedRequestId = normalizeWorkbenchSessionId(requestId);
|
||||
const messageLimit = sessionMessageProjectionWindowLimit();
|
||||
const eagerMessages = historyAuthorityPolicy.sessionMessagesHydrate && normalizedRequestId ? fetchSessionMessagesPage(normalizedRequestId, { limit: messageLimit, reason: "load-session:eager", force: true }) : null;
|
||||
const detail = await fetchSessionDetailPage(requestId, { reason: "load-session:detail", force: true });
|
||||
const [detail, messagePage] = await Promise.all([
|
||||
fetchSessionDetailPage(requestId, { reason: "load-session:detail", force: true }),
|
||||
normalizedRequestId ? fetchSessionMessagesPage(normalizedRequestId, { limit: messageLimit, reason: "load-session:messages", force: true }) : Promise.resolve(null)
|
||||
]);
|
||||
if (!detail.ok) return null;
|
||||
const detailSession = sessionFromWorkbenchSession(detail.data?.session, { includeMessages: false });
|
||||
const id = detailSession?.sessionId ?? normalizedRequestId ?? seed?.sessionId;
|
||||
@@ -1834,32 +1553,13 @@ export const useWorkbenchStore = defineStore("workbench", () => {
|
||||
const fallbackMessages = seed?.sessionId === id ? seed.messages ?? [] : [];
|
||||
const base = detailSession ? { ...detailSession, messages: fallbackMessages } : seed;
|
||||
if (!base) return null;
|
||||
if (!historyAuthorityPolicy.sessionMessagesHydrate) return { ...base, sessionId: id, messages: [], messageCount: base.messageCount ?? 0 };
|
||||
const messages = eagerMessages && id === normalizedRequestId ? await eagerMessages : await fetchSessionMessagesPage(id, { limit: messageLimit, reason: "load-session", force: true });
|
||||
const page = messages.ok ? messages.data : null;
|
||||
const pageMessages = Array.isArray(page?.messages) ? await sealRestoredActiveTurnMessages(page.messages.map((message) => normalizeChatMessage(message as ChatMessage))) : fallbackMessages;
|
||||
return { ...base, sessionId: id, messages: pageMessages, messageCount: page?.total ?? pageMessages?.length ?? base.messageCount };
|
||||
}
|
||||
|
||||
async function sealRestoredActiveTurnMessages(source: ChatMessage[]): Promise<ChatMessage[]> {
|
||||
const targets = source.filter(messageNeedsRestoredTurnSeal).slice(-1);
|
||||
if (targets.length === 0) return source;
|
||||
for (const message of targets) {
|
||||
const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId);
|
||||
if (!traceId || traceProjectionIsTerminalSealed(traceId, source)) continue;
|
||||
recordWorkbenchRuntimeDiagnostic({
|
||||
module: "workbench-turn-status",
|
||||
sessionId: message.sessionId ?? message.runnerTrace?.sessionId ?? null,
|
||||
traceId,
|
||||
outcome: "ok",
|
||||
diagnostic: { code: "workbench_restored_turn_auto_read_disabled", reason: "load-session:restored-turn", source: "projection-sse-authority", valuesRedacted: true }
|
||||
});
|
||||
}
|
||||
return source;
|
||||
const hydratedMessages = messagePage?.ok && Array.isArray(messagePage.data?.messages)
|
||||
? messagePage.data.messages.map((message) => normalizeChatMessage(message as ChatMessage))
|
||||
: fallbackMessages;
|
||||
return { ...base, sessionId: id, messages: hydratedMessages, messageCount: base.messageCount ?? hydratedMessages.length };
|
||||
}
|
||||
|
||||
function reattachRestoredActiveTrace(): void {
|
||||
void readTraceEventsForMessages(messages.value);
|
||||
const traceId = activeTraceIdFromMessages(messages.value, turnStatusAuthority.value);
|
||||
if (traceId) reattachTrace(traceId);
|
||||
}
|
||||
@@ -1936,14 +1636,6 @@ function routeSelectedSessionRecord(sessionId: string | null | undefined, loadin
|
||||
} as WorkbenchSessionRecord;
|
||||
}
|
||||
|
||||
function messageNeedsRestoredTurnSeal(message: ChatMessage): boolean {
|
||||
if (message.role !== "agent") return false;
|
||||
if (messageHasTerminalResponse(message)) return false;
|
||||
const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId);
|
||||
if (!traceId) return false;
|
||||
return isTraceActiveStatus(message.status) || isTerminalMessageStatus(message.status) || message.traceAutoLifecycle === "running";
|
||||
}
|
||||
|
||||
function providerThreadIdForRequest(threadId: string | null | undefined): string | null {
|
||||
const value = firstNonEmptyString(threadId) ?? null;
|
||||
if (!value) return null;
|
||||
|
||||
-9
@@ -8,11 +8,6 @@ declare global {
|
||||
label?: string;
|
||||
};
|
||||
workbench?: {
|
||||
realtimeFeatures?: {
|
||||
liveKafkaSse?: boolean;
|
||||
kafkaRefreshReplay?: boolean;
|
||||
projectionRealtime?: boolean;
|
||||
};
|
||||
debugCapabilities?: {
|
||||
isolatedKafka?: boolean;
|
||||
rawHwlabEventWindow?: {
|
||||
@@ -55,16 +50,12 @@ declare global {
|
||||
/** @deprecated Use workbenchDetailReadConcurrency. */
|
||||
workbenchReadHydrationConcurrency?: number;
|
||||
workbenchReadFailureCooldownMs?: number;
|
||||
workbenchTurnStatusMinRefreshMs?: number;
|
||||
workbenchTraceEventsMinRefreshMs?: number;
|
||||
workbenchSessionDetailMinRefreshMs?: number;
|
||||
workbenchSessionMessagesMinRefreshMs?: number;
|
||||
workbenchSessionMessagesWindowLimit?: number;
|
||||
workbenchTraceMessagesWindowLimit?: number;
|
||||
workbenchRealtimeSessionMessagesMinRefreshMs?: number;
|
||||
workbenchTraceEventsTimeoutMs?: number;
|
||||
sessionListRealtimeRefreshDelayMs?: number;
|
||||
sessionListTerminalRefreshDelayMs?: number;
|
||||
sessionListMinRefreshIntervalMs?: number;
|
||||
workbenchRealtimeErrorSyncReplayMinMs?: number;
|
||||
/** @deprecated Use workbenchRealtimeErrorSyncReplayMinMs. */
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
// SPEC: PJ2026-0106050514 Workbench实时运行面 draft-2026-06-30-p0-1297-spec-first.
|
||||
// Responsibility: Production Workbench realtime runtime entry point for SSE transport, event coalescing, and scoped cursor ownership.
|
||||
|
||||
export { createWorkbenchStreamTransportRuntime, WorkbenchStreamTransportRuntime, workbenchRealtimeTraceIdForCapabilities, type WorkbenchStreamTransportRecovery, type WorkbenchStreamTransportRestartInput, type WorkbenchStreamTransportRestartResult, type WorkbenchStreamTransportState, type WorkbenchRealtimeEvent } from "@/utils/workbench-stream-transport";
|
||||
export { createWorkbenchStreamTransportRuntime, WorkbenchStreamTransportRuntime, workbenchRealtimeTraceId, type WorkbenchStreamTransportRecovery, type WorkbenchStreamTransportRestartInput, type WorkbenchStreamTransportRestartResult, type WorkbenchStreamTransportState, type WorkbenchRealtimeEvent } from "@/utils/workbench-stream-transport";
|
||||
|
||||
@@ -8,13 +8,11 @@
|
||||
// - OpenCode stream.transport.ts:504-530 reduce output/trace.
|
||||
|
||||
import { connectWorkbenchEvents, type WorkbenchEventStream, type WorkbenchRealtimeEvent, type WorkbenchSseIngressFrame } from "@/api/workbench-events";
|
||||
import type { WorkbenchRealtimeCapabilities } from "@/config/runtime";
|
||||
import { workbenchRealtimeScopeKey } from "@/utils/workbench-key";
|
||||
|
||||
export type { WorkbenchRealtimeEvent };
|
||||
|
||||
export interface WorkbenchStreamTransportRestartInput {
|
||||
realtimeCapabilities: WorkbenchRealtimeCapabilities;
|
||||
sessionId?: string | null;
|
||||
traceId?: string | null;
|
||||
afterSeq?: number | null;
|
||||
@@ -78,8 +76,7 @@ interface WorkbenchTransportCursor {
|
||||
traceSeq: number | null;
|
||||
}
|
||||
|
||||
export function workbenchRealtimeTraceIdForCapabilities(
|
||||
_capabilities: WorkbenchRealtimeCapabilities,
|
||||
export function workbenchRealtimeTraceId(
|
||||
...candidates: Array<string | null | undefined>
|
||||
): string | null {
|
||||
for (const candidate of candidates) {
|
||||
@@ -89,10 +86,6 @@ export function workbenchRealtimeTraceIdForCapabilities(
|
||||
return null;
|
||||
}
|
||||
|
||||
export function workbenchRealtimeTransportEnabled(capabilities: WorkbenchRealtimeCapabilities): boolean {
|
||||
return capabilities.projectionRealtime;
|
||||
}
|
||||
|
||||
export class WorkbenchStreamTransportRuntime {
|
||||
private stream: WorkbenchEventStream | null = null;
|
||||
private key = "";
|
||||
@@ -107,11 +100,9 @@ export class WorkbenchStreamTransportRuntime {
|
||||
this.stop();
|
||||
this.key = key;
|
||||
this.armWait(key);
|
||||
if (!workbenchRealtimeTransportEnabled(input.realtimeCapabilities)) return { key, changed: true, streamStarted: false };
|
||||
if (!input.sessionId && !input.traceId) return { key, changed: true, streamStarted: false };
|
||||
this.emitState(input, "connecting", null);
|
||||
this.stream = connectWorkbenchEvents({
|
||||
realtimeCapabilities: input.realtimeCapabilities,
|
||||
sessionId: input.sessionId ?? null,
|
||||
traceId: input.traceId ?? null,
|
||||
afterSeq: input.afterSeq ?? this.cursorByKey.get(key)?.outboxSeq ?? null,
|
||||
@@ -202,7 +193,7 @@ export class WorkbenchStreamTransportRuntime {
|
||||
const sessionId = input.sessionId ?? null;
|
||||
const traceId = input.traceId ?? null;
|
||||
const cursor = this.currentCursor(key);
|
||||
const actions: WorkbenchStreamTransportRecoveryAction[] = input.realtimeCapabilities.projectionRealtime && (sessionId || traceId) ? ["events-reconnect"] : [];
|
||||
const actions: WorkbenchStreamTransportRecoveryAction[] = sessionId || traceId ? ["events-reconnect"] : [];
|
||||
const diagnostic = this.diagnosticEnvelope("workbench_sse_recovery", reason, key, actions);
|
||||
input.onRecovery?.({ key, sessionId, traceId, tick: this.wait?.tick ?? this.tick, reason, actions, outboxSeq: cursor.outboxSeq, traceSeq: cursor.traceSeq, diagnostic });
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user