root
664 lines
29 KiB
TypeScript
664 lines
29 KiB
TypeScript
// SPEC: PJ2026-0104010803 Workbench事件流可见性 draft-2026-07-09-p0-kafka-authority.
|
|
// Responsibility: lock AgentRun Kafka event -> HWLAB Kafka event projection contract without requiring a live Kafka broker.
|
|
|
|
import assert from "node:assert/strict";
|
|
import { test } from "bun:test";
|
|
|
|
import { createCloudApiBunServer } from "./bun-server.ts";
|
|
import { buildCloudApiReadiness } from "./health-contract.ts";
|
|
import { decodeCanonicalAgentRunKafkaMessage, kafkaEventBridgeConfig, liveKafkaOtelSpanAttributes, projectAgentRunKafkaEventToHwlabEvent, publishAgentRunKafkaMessageLive, relayHwlabKafkaOutboxOnce, shouldEmitLiveKafkaOtelSpan, startHwlabKafkaEventBridge } from "./kafka-event-bridge.ts";
|
|
|
|
const PROJECTOR_ENV = Object.freeze({
|
|
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false",
|
|
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false",
|
|
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true",
|
|
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "true",
|
|
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true",
|
|
HWLAB_KAFKA_ENABLED: "true",
|
|
HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true",
|
|
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092",
|
|
HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC: "agentrun.event.v1",
|
|
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1",
|
|
HWLAB_KAFKA_CLIENT_ID: "hwlab-projector-test",
|
|
HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-projector-test-v1",
|
|
HWLAB_KAFKA_PROJECTOR_GROUP_ID: "hwlab-projector-test-v1",
|
|
HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "10",
|
|
HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS: "60000",
|
|
HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE: "1",
|
|
HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS: "30000",
|
|
HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS: "10000",
|
|
HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS: "1000"
|
|
});
|
|
|
|
const LIVE_ENV = Object.freeze({
|
|
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "true",
|
|
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "true",
|
|
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
|
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
|
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "false",
|
|
HWLAB_KAFKA_ENABLED: "true",
|
|
HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED: "true",
|
|
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092",
|
|
HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC: "agentrun.event.v1",
|
|
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1",
|
|
HWLAB_KAFKA_CLIENT_ID: "hwlab-live-test",
|
|
HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID: "hwlab-live-bridge-fixed",
|
|
HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID: "hwlab-live-fanout-fixed"
|
|
});
|
|
|
|
test("projects AgentRun assistant_message Kafka event into HWLAB trace event", () => {
|
|
const projected = projectAgentRunKafkaEventToHwlabEvent({
|
|
schema: "agentrun.event.v1",
|
|
eventType: "agentrun.run.event",
|
|
source: "agentrun-manager",
|
|
producedAt: "2026-07-09T10:00:00.000Z",
|
|
run: {
|
|
runId: "run_kafka_projection",
|
|
sessionId: "ses_kafka_projection",
|
|
conversationId: "cnv_kafka_projection",
|
|
threadId: "thread-kafka-projection",
|
|
projectId: "prj_kafka_projection",
|
|
backendProfile: "deepseek"
|
|
},
|
|
command: { commandId: "cmd_kafka_projection" },
|
|
event: {
|
|
seq: 7,
|
|
type: "assistant_message",
|
|
createdAt: "2026-07-09T10:00:01.000Z",
|
|
payload: {
|
|
traceId: "trc_kafka_projection",
|
|
commandId: "cmd_kafka_projection",
|
|
text: "partial answer",
|
|
itemId: "item_assistant_1",
|
|
final: false
|
|
}
|
|
}
|
|
}, { source: "hwlab-test", producedAt: "2026-07-09T10:00:02.000Z", sourceTopic: "agentrun.event.v1", sourceOffset: "42" });
|
|
|
|
assert.equal(projected.schema, "hwlab.event.v1");
|
|
assert.equal(projected.eventType, "hwlab.trace.event.projected");
|
|
assert.equal(projected.source, "hwlab-test");
|
|
assert.equal(projected.traceId, "trc_kafka_projection");
|
|
assert.equal(projected.hwlabSessionId, "ses_kafka_projection");
|
|
assert.equal(projected.sessionId, "ses_kafka_projection");
|
|
assert.equal(projected.runId, "run_kafka_projection");
|
|
assert.equal(projected.commandId, "cmd_kafka_projection");
|
|
assert.equal(projected.context.runId, "run_kafka_projection");
|
|
assert.equal(projected.context.commandId, "cmd_kafka_projection");
|
|
assert.equal(projected.context.sourceSeq, 7);
|
|
assert.equal(projected.context.agentRunEventType, "assistant_message");
|
|
assert.equal(projected.event.type, "assistant");
|
|
assert.equal(projected.event.label, "agentrun:assistant:message");
|
|
assert.equal(projected.event.status, "running");
|
|
assert.equal(projected.event.text, "partial answer");
|
|
assert.equal(projected.event.assistantText, "partial answer");
|
|
assert.equal(projected.event.finalResponse, null);
|
|
assert.equal(projected.event.terminal, false);
|
|
assert.equal(projected.sourceEvent.topic, "agentrun.event.v1");
|
|
assert.equal(projected.sourceEvent.offset, "42");
|
|
assert.equal(projected.valuesPrinted, false);
|
|
});
|
|
|
|
test("projects final AgentRun assistant text without sealing before terminal_status", () => {
|
|
const projected = projectAgentRunKafkaEventToHwlabEvent({
|
|
schema: "agentrun.event.v1",
|
|
eventType: "agentrun.run.event",
|
|
run: { runId: "run_final", sessionId: "ses_final" },
|
|
command: { commandId: "cmd_final" },
|
|
event: { seq: 9, type: "assistant_message", payload: { traceId: "trc_final", text: "durable final answer", final: true, replyAuthority: true } }
|
|
}, { source: "hwlab-test" });
|
|
|
|
assert.equal(projected.event.assistantText, "durable final answer");
|
|
assert.equal(projected.event.status, "running");
|
|
assert.equal(projected.event.terminal, false);
|
|
assert.equal(projected.event.finalResponse.text, "durable final answer");
|
|
assert.equal(projected.event.finalResponse.traceId, "trc_final");
|
|
});
|
|
|
|
test("projects AgentRun terminal_status Kafka event into terminal HWLAB event", () => {
|
|
const projected = projectAgentRunKafkaEventToHwlabEvent({
|
|
schema: "agentrun.event.v1",
|
|
eventType: "agentrun.run.event",
|
|
run: { runId: "run_terminal", sessionId: "ses_terminal" },
|
|
command: { commandId: "cmd_terminal" },
|
|
event: {
|
|
seq: 12,
|
|
type: "terminal_status",
|
|
payload: {
|
|
traceId: "trc_terminal",
|
|
terminalStatus: "completed",
|
|
message: "AgentRun completed"
|
|
}
|
|
}
|
|
}, { source: "hwlab-test" });
|
|
|
|
assert.equal(projected.traceId, "trc_terminal");
|
|
assert.equal(projected.event.type, "result");
|
|
assert.equal(projected.event.eventType, "terminal");
|
|
assert.equal(projected.event.status, "completed");
|
|
assert.equal(projected.event.label, "agentrun:terminal:completed");
|
|
assert.equal(projected.event.terminal, true);
|
|
assert.equal(projected.event.message, "AgentRun completed");
|
|
});
|
|
|
|
test("live Kafka mode publishes canonical AgentRun input directly and fans one broker event to all SSE subscribers", async () => {
|
|
const consumerConfigs = [];
|
|
const subscriptions = [];
|
|
const runOptions = [];
|
|
const runOrder = [];
|
|
const produced = [];
|
|
const otelSpans = [];
|
|
const consumers = [0, 1].map((index) => ({
|
|
async connect() {},
|
|
async subscribe(input) { subscriptions[index] = input; },
|
|
async run(input) { runOptions[index] = input; runOrder.push(index); },
|
|
async stop() {},
|
|
async disconnect() {}
|
|
}));
|
|
const producer = {
|
|
async connect() {},
|
|
async send(input) { produced.push(input); return [{ topicName: input.topic, partition: 4, baseOffset: "91" }]; },
|
|
async disconnect() {}
|
|
};
|
|
let consumerIndex = 0;
|
|
const kafkaFactory = () => ({
|
|
consumer(config) {
|
|
consumerConfigs.push(config);
|
|
return consumers[consumerIndex++];
|
|
},
|
|
producer() { return producer; }
|
|
});
|
|
const forbiddenRuntimeStore = new Proxy({}, {
|
|
get() { throw new Error("live mode must not touch the runtime store"); }
|
|
});
|
|
const bridge = startHwlabKafkaEventBridge({
|
|
env: LIVE_ENV,
|
|
runtimeStore: forbiddenRuntimeStore,
|
|
kafkaFactory,
|
|
logger: null,
|
|
otelSpanEmitter(name, traceId, env, options) { otelSpans.push({ name, traceId, env, options }); }
|
|
});
|
|
await bridge.ready;
|
|
|
|
assert.deepEqual(consumerConfigs.map((item) => item.groupId), ["hwlab-live-bridge-fixed", "hwlab-live-fanout-fixed"]);
|
|
assert.deepEqual(runOrder, [1, 0], "fanout group must be ready before direct publish can create a new HWLAB event");
|
|
assert.deepEqual(subscriptions, [
|
|
{ topic: "agentrun.event.v1", fromBeginning: false },
|
|
{ topic: "hwlab.event.v1", fromBeginning: false }
|
|
]);
|
|
const message = canonicalMessage({ offset: "70" });
|
|
await runOptions[0].eachMessage({ topic: "agentrun.event.v1", partition: 0, message });
|
|
assert.equal(produced.length, 1);
|
|
assert.equal(produced[0].topic, "hwlab.event.v1");
|
|
const envelope = JSON.parse(produced[0].messages[0].value);
|
|
assert.equal(envelope.schema, "hwlab.event.v1");
|
|
assert.equal(envelope.traceId, "trc_projector_test");
|
|
assert.equal(envelope.hwlabSessionId, "ses_projector_test");
|
|
assert.equal(envelope.runId, "run_projector_test");
|
|
assert.equal(envelope.commandId, "cmd_projector_test");
|
|
assert.equal(envelope.event.assistantText, "projected answer");
|
|
assert.equal(otelSpans[0].name, "hwlab.kafka.live.direct_publish");
|
|
assert.equal(otelSpans[0].traceId, "trc_projector_test");
|
|
assert.deepEqual(otelSpans[0].options.attributes, {
|
|
businessTraceId: "trc_projector_test",
|
|
hwlabSessionId: "ses_projector_test",
|
|
runId: "run_projector_test",
|
|
commandId: "cmd_projector_test",
|
|
topic: "hwlab.event.v1",
|
|
partition: 4,
|
|
offset: "91",
|
|
sourceTopic: "agentrun.event.v1",
|
|
sourcePartition: 0,
|
|
sourceOffset: "70",
|
|
eventType: "assistant_message",
|
|
terminal: false,
|
|
valuesRedacted: true
|
|
});
|
|
|
|
const first = [];
|
|
const second = [];
|
|
const firstTransports = [];
|
|
bridge.subscribeLiveHwlabEvents((value, transport) => { first.push(value); firstTransports.push(transport); });
|
|
bridge.subscribeLiveHwlabEvents((value) => second.push(value));
|
|
await runOptions[1].eachBatch({
|
|
batch: { topic: "hwlab.event.v1", partition: 0, highWatermark: "1", messages: [{ offset: "0", value: Buffer.from(JSON.stringify(envelope)) }] },
|
|
resolveOffset() {},
|
|
async heartbeat() {},
|
|
isRunning: () => true,
|
|
isStale: () => false
|
|
});
|
|
assert.equal(bridge.liveSubscriberCount(), 2);
|
|
assert.deepEqual(first, [envelope]);
|
|
assert.deepEqual(second, [envelope]);
|
|
assert.deepEqual(firstTransports, [{ topic: "hwlab.event.v1", partition: 0, offset: "0" }]);
|
|
assert.equal(otelSpans[1].name, "hwlab.kafka.live.fanout_receive");
|
|
assert.equal(otelSpans[1].options.attributes.topic, "hwlab.event.v1");
|
|
assert.equal(otelSpans[1].options.attributes.partition, 0);
|
|
assert.equal(otelSpans[1].options.attributes.offset, "0");
|
|
assert.equal(consumerConfigs.length, 2, "browser subscribers must not allocate Kafka consumer groups");
|
|
assert.equal((await bridge.status()).consumerLag.total, 0);
|
|
await bridge.stop();
|
|
});
|
|
|
|
test("live Kafka publisher has no PG inbox, facts, checkpoint, or outbox dependency", async () => {
|
|
const sent = [];
|
|
const runtimeStore = new Proxy({}, { get() { throw new Error("unexpected DB access"); } });
|
|
void runtimeStore;
|
|
const result = await publishAgentRunKafkaMessageLive({ topic: "agentrun.event.v1", partition: 3, message: canonicalMessage({ offset: "71" }) }, {
|
|
producer: { async send(input) { sent.push(input); } },
|
|
config: { clientId: "hwlab-live-test", hwlabTopic: "hwlab.event.v1" },
|
|
logger: null
|
|
});
|
|
assert.equal(result.published, true);
|
|
assert.equal(sent.length, 1);
|
|
assert.equal(JSON.parse(sent[0].messages[0].value).sourceEvent.offset, "71");
|
|
});
|
|
|
|
test("live Kafka OTel sampling is deterministic and bounded without payload attributes", () => {
|
|
const envelope = (agentRunEventType, sourceSeq, event = {}) => ({
|
|
schema: "hwlab.event.v1",
|
|
eventId: `hwlab:evt_otel_${sourceSeq}`,
|
|
sourceEventId: `evt_otel_${sourceSeq}`,
|
|
traceId: "trc_live_otel",
|
|
hwlabSessionId: "ses_live_otel",
|
|
runId: "run_live_otel",
|
|
commandId: "cmd_live_otel",
|
|
context: { agentRunEventType, sourceSeq, valuesRedacted: true },
|
|
event: { type: "backend", eventType: "backend", terminal: false, message: "must-not-leak", payload: { password: "must-not-leak" }, ...event }
|
|
});
|
|
|
|
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("assistant_message", 1, { type: "assistant" })), true);
|
|
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("tool_call", 2, { type: "tool" })), true);
|
|
assert.equal(shouldEmitLiveKafkaOtelSpan(envelope("terminal_status", 3, { type: "result", terminal: true })), true);
|
|
const decisions = Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" })));
|
|
assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => (index + 1) % 32 === 0));
|
|
assert.deepEqual(decisions, Array.from({ length: 64 }, (_, index) => shouldEmitLiveKafkaOtelSpan(envelope("command_output", index + 1, { type: "output" }))));
|
|
|
|
const attributes = liveKafkaOtelSpanAttributes(envelope("assistant_message", 1, { type: "assistant" }), {
|
|
topic: "hwlab.event.v1",
|
|
partition: 2,
|
|
offset: "14"
|
|
});
|
|
assert.deepEqual(attributes, {
|
|
businessTraceId: "trc_live_otel",
|
|
hwlabSessionId: "ses_live_otel",
|
|
runId: "run_live_otel",
|
|
commandId: "cmd_live_otel",
|
|
topic: "hwlab.event.v1",
|
|
partition: 2,
|
|
offset: "14",
|
|
sourceTopic: null,
|
|
sourcePartition: null,
|
|
sourceOffset: null,
|
|
eventType: "assistant_message",
|
|
terminal: false,
|
|
valuesRedacted: true
|
|
});
|
|
assert.equal(JSON.stringify(attributes).includes("must-not-leak"), false);
|
|
});
|
|
|
|
test("composable Kafka capabilities reject consumer group collisions", () => {
|
|
assert.throws(
|
|
() => kafkaEventBridgeConfig({
|
|
...LIVE_ENV,
|
|
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "true",
|
|
HWLAB_KAFKA_PROJECTOR_GROUP_ID: LIVE_ENV.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID,
|
|
HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS: "1000"
|
|
}),
|
|
(error) => error?.code === "hwlab_kafka_consumer_groups_not_distinct"
|
|
);
|
|
});
|
|
|
|
test("projection realtime subscribes to projection commits without enabling Kafka ingest or relay", async () => {
|
|
let subscribeCount = 0;
|
|
let unsubscribeCount = 0;
|
|
const bridge = startHwlabKafkaEventBridge({
|
|
env: {
|
|
HWLAB_KAFKA_DIRECT_PUBLISH_ENABLED: "false",
|
|
HWLAB_WORKBENCH_LIVE_KAFKA_SSE_ENABLED: "false",
|
|
HWLAB_KAFKA_TRANSACTIONAL_PROJECTOR_ENABLED: "false",
|
|
HWLAB_KAFKA_PROJECTION_OUTBOX_RELAY_ENABLED: "false",
|
|
HWLAB_WORKBENCH_PROJECTION_REALTIME_ENABLED: "true"
|
|
},
|
|
runtimeStore: {
|
|
async assertWorkbenchTransactionalRealtimeReady() { return { ready: true }; },
|
|
async subscribeWorkbenchProjectionCommits() {
|
|
subscribeCount += 1;
|
|
return () => { unsubscribeCount += 1; };
|
|
}
|
|
},
|
|
kafkaFactory() { throw new Error("projection realtime must not create a Kafka client"); },
|
|
logger: null
|
|
});
|
|
|
|
await bridge.ready;
|
|
assert.equal(subscribeCount, 1);
|
|
await bridge.stop();
|
|
assert.equal(unsubscribeCount, 1);
|
|
});
|
|
|
|
test("projects AgentRun run-created envelope even when no source event is present", () => {
|
|
const projected = projectAgentRunKafkaEventToHwlabEvent({
|
|
schema: "agentrun.event.v1",
|
|
eventType: "agentrun.run.created",
|
|
run: {
|
|
runId: "run_created",
|
|
sessionId: "ses_created",
|
|
traceId: "trc_created",
|
|
backendProfile: "codex"
|
|
}
|
|
}, { source: "hwlab-test" });
|
|
|
|
assert.equal(projected.traceId, "trc_created");
|
|
assert.equal(projected.sessionId, "ses_created");
|
|
assert.equal(projected.context.runId, "run_created");
|
|
assert.equal(projected.context.sourceEventType, "agentrun.run.created");
|
|
assert.equal(projected.event.label, "agentrun:event:agentrun.run.created");
|
|
assert.equal(projected.event.status, "running");
|
|
});
|
|
|
|
test("accepts canonical runner-job-created payload with redacted tool credential references", () => {
|
|
const message = canonicalMessage({ offset: "60" });
|
|
const envelope = JSON.parse(message.value.toString("utf8"));
|
|
envelope.event.type = "backend_status";
|
|
envelope.event.payload = {
|
|
traceId: envelope.traceId,
|
|
hwlabSessionId: envelope.hwlabSessionId,
|
|
commandId: envelope.commandId,
|
|
phase: "runner-job-created",
|
|
toolCredentials: [{
|
|
tool: "unidesk-ssh",
|
|
purpose: "ssh-passthrough",
|
|
name: "agentrun-v02-tool-unidesk-ssh",
|
|
namespace: "agentrun-v02",
|
|
keys: ["UNIDESK_SSH_CLIENT_TOKEN"],
|
|
projection: { kind: "env", envName: "UNIDESK_SSH_CLIENT_TOKEN" },
|
|
valuesPrinted: false
|
|
}],
|
|
valuesPrinted: false
|
|
};
|
|
message.value = Buffer.from(JSON.stringify(envelope));
|
|
|
|
const decoded = decodeCanonicalAgentRunKafkaMessage({ topic: "agentrun.event.v1", partition: 0, message });
|
|
assert.equal(decoded.ok, true);
|
|
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, { source: "hwlab-test" });
|
|
assert.equal(projected.event.label, "agentrun:backend:runner-job-created");
|
|
});
|
|
|
|
test("Kafka projector commits next offset only after durable projection", async () => {
|
|
const calls = [];
|
|
let commitInput = null;
|
|
const harness = kafkaProjectorHarness({
|
|
async commitAgentRunKafkaProjection(input) {
|
|
commitInput = input;
|
|
calls.push("db");
|
|
return { projectedSeq: 1 };
|
|
}
|
|
}, calls);
|
|
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
|
await bridge.ready;
|
|
await harness.runBatch(canonicalMessage({ offset: "41" }));
|
|
|
|
assert.ok(calls.indexOf("heartbeat") < calls.indexOf("db"));
|
|
assert.ok(calls.indexOf("db") < calls.indexOf("resolve:41"));
|
|
assert.equal(commitInput.partitionKey, "ses_projector_test");
|
|
assert.deepEqual(calls.slice(-3), ["resolve:41", "commit:42", "heartbeat"]);
|
|
await bridge.stop();
|
|
});
|
|
|
|
test("Kafka projector leaves offset uncommitted when durable projection throws", async () => {
|
|
const calls = [];
|
|
const expected = Object.assign(new Error("database unavailable"), { code: "db_unavailable" });
|
|
const harness = kafkaProjectorHarness({
|
|
async commitAgentRunKafkaProjection() {
|
|
calls.push("db");
|
|
throw expected;
|
|
}
|
|
}, calls);
|
|
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
|
await bridge.ready;
|
|
|
|
await assert.rejects(harness.runBatch(canonicalMessage({ offset: "42" })), expected);
|
|
assert.equal(calls.some((entry) => entry.startsWith("commit:")), false);
|
|
assert.equal(calls.some((entry) => entry.startsWith("resolve:")), false);
|
|
await bridge.stop();
|
|
});
|
|
|
|
test("Kafka projector signals committed projection but does not commit a stale generation offset", async () => {
|
|
const calls = [];
|
|
let signal = null;
|
|
let harness;
|
|
harness = kafkaProjectorHarness({
|
|
async commitAgentRunKafkaProjection() {
|
|
calls.push("db");
|
|
harness.setStale(true);
|
|
return { projectedSeq: 1 };
|
|
}
|
|
}, calls);
|
|
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
|
bridge.subscribeProjectionCommits((value) => { signal = value; });
|
|
await bridge.ready;
|
|
|
|
await harness.runBatch(canonicalMessage({ offset: "43" }));
|
|
|
|
assert.equal(calls.includes("resolve:43"), false);
|
|
assert.equal(calls.includes("commit:44"), false);
|
|
assert.equal(signal.sessionId, "ses_projector_test");
|
|
assert.equal(signal.traceId, "trc_projector_test");
|
|
await bridge.stop();
|
|
});
|
|
|
|
test("Kafka projector commits a suppressed post-seal offset without publishing a projection signal", async () => {
|
|
const calls = [];
|
|
let signal = null;
|
|
const harness = kafkaProjectorHarness({
|
|
async commitAgentRunKafkaProjection() {
|
|
calls.push("db");
|
|
return { projectedSeq: 9, projectionWritten: false, suppressedAfterSeal: true };
|
|
}
|
|
}, calls);
|
|
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
|
bridge.subscribeProjectionCommits((value) => { signal = value; });
|
|
await bridge.ready;
|
|
|
|
await harness.runBatch(canonicalMessage({ offset: "44" }));
|
|
|
|
assert.equal(signal, null);
|
|
assert.deepEqual(calls.slice(-3), ["resolve:44", "commit:45", "heartbeat"]);
|
|
await bridge.stop();
|
|
});
|
|
|
|
test("Kafka projector persists malformed and non-HWLAB events before committing offsets", async () => {
|
|
const calls = [];
|
|
const harness = kafkaProjectorHarness({}, calls);
|
|
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
|
await bridge.ready;
|
|
await harness.runBatch({ offset: "50", value: Buffer.from("{}") });
|
|
await harness.runBatch(canonicalMessage({ offset: "51", traceId: null, hwlabSessionId: null }));
|
|
|
|
assert.deepEqual(calls.filter((entry) => ["dlq", "ignored", "commit:51", "commit:52"].includes(entry)), ["dlq", "commit:51", "ignored", "commit:52"]);
|
|
await bridge.stop();
|
|
});
|
|
|
|
test("Kafka projector exposes startup failure as rejected readiness and blocked status", async () => {
|
|
const expected = Object.assign(new Error("broker unavailable"), { code: "kafka_unavailable" });
|
|
const harness = kafkaProjectorHarness({}, []);
|
|
harness.kafkaFactory = () => ({
|
|
consumer: () => harness.consumer,
|
|
producer: () => ({ ...harness.producer, connect: async () => { throw expected; } })
|
|
});
|
|
const bridge = startHwlabKafkaEventBridge({ env: PROJECTOR_ENV, runtimeStore: harness.runtimeStore, kafkaFactory: harness.kafkaFactory, logger: null });
|
|
|
|
await assert.rejects(bridge.ready, expected);
|
|
assert.equal((await bridge.status()).status, "blocked");
|
|
assert.equal((await bridge.status()).errorCode, "kafka_unavailable");
|
|
await bridge.stop();
|
|
});
|
|
|
|
test("cloud API Bun entry does not listen when Kafka projector startup fails", async () => {
|
|
const expected = Object.assign(new Error("broker unavailable before listen"), { code: "kafka_startup_blocked" });
|
|
const harness = kafkaProjectorHarness({}, []);
|
|
let bunServeCalls = 0;
|
|
const kafkaFactory = () => ({
|
|
consumer: () => harness.consumer,
|
|
producer: () => ({ ...harness.producer, connect: async () => { throw expected; } })
|
|
});
|
|
|
|
await assert.rejects(createCloudApiBunServer({
|
|
env: { ...PROJECTOR_ENV },
|
|
runtimeStore: harness.runtimeStore,
|
|
kafkaFactory,
|
|
accessController: { async ensureBootstrap() {}, async authenticate() { return { ok: false }; } },
|
|
bunServe() { bunServeCalls += 1; throw new Error("Bun.serve must not run"); },
|
|
installSignalHandlers: false,
|
|
logger: { log() {}, info() {}, warn() {}, error() {} }
|
|
}), expected);
|
|
|
|
assert.equal(bunServeCalls, 0);
|
|
});
|
|
|
|
test("cloud readiness is blocked while the Kafka projector is blocked", () => {
|
|
const readiness = buildCloudApiReadiness({
|
|
db: {},
|
|
codeAgent: {},
|
|
runtime: {},
|
|
kafkaProjector: { started: true, status: "blocked", errorCode: "kafka_startup_blocked", message: "broker unavailable" }
|
|
});
|
|
|
|
assert.equal(readiness.components.kafkaProjector, "blocked");
|
|
assert.equal(readiness.blockerCodes.includes("kafka_startup_blocked"), true);
|
|
assert.equal(readiness.ready, false);
|
|
});
|
|
|
|
test("HWLAB Kafka relay claims just in time and forwards fencing token", async () => {
|
|
const calls = [];
|
|
const pending = [
|
|
{ outboxSeq: 1, eventId: "evt-relay-1", topic: "hwlab.event.v1", partitionKey: "trc-relay", payload: { eventId: "evt-relay-1" }, headers: {}, attemptCount: 2, leaseOwner: "relay-owner", leaseExpiresAt: "2026-07-10T10:02:30.000Z" },
|
|
{ outboxSeq: 2, eventId: "evt-relay-2", topic: "hwlab.event.v1", partitionKey: "trc-relay", payload: { eventId: "evt-relay-2" }, headers: {}, attemptCount: 1, leaseOwner: "relay-owner", leaseExpiresAt: "2026-07-10T10:02:30.000Z" }
|
|
];
|
|
const runtimeStore = {
|
|
async claimHwlabKafkaOutbox(params) { calls.push(`claim:${params.limit}`); return pending.length ? [pending.shift()] : []; },
|
|
async completeHwlabKafkaOutbox(item) { calls.push(`complete:${item.outboxSeq}:${item.attemptCount}:${item.leaseOwner}`); }
|
|
};
|
|
const producer = { async send(record) { calls.push(`send:${record.messages[0].value.includes("evt-relay-1") ? 1 : 2}`); } };
|
|
const config = { relay: { batchSize: 2, leaseMs: 30000, retryBackoffMs: 1000 } };
|
|
|
|
const result = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-owner", logger: null });
|
|
|
|
assert.equal(result.deliveredCount, 2);
|
|
assert.deepEqual(calls, ["claim:1", "send:1", "complete:1:2:relay-owner", "claim:1", "send:2", "complete:2:1:relay-owner"]);
|
|
});
|
|
|
|
test("HWLAB Kafka relay retries a failed send and clears backlog on the next cycle", async () => {
|
|
const calls = [];
|
|
let pending = true;
|
|
let attemptCount = 0;
|
|
const runtimeStore = {
|
|
async claimHwlabKafkaOutbox() {
|
|
if (!pending) return [];
|
|
attemptCount += 1;
|
|
return [{ outboxSeq: 7, eventId: "evt-relay-retry", topic: "hwlab.event.v1", partitionKey: "trc-relay-retry", payload: { eventId: "evt-relay-retry" }, headers: {}, attemptCount, leaseOwner: "relay-retry-owner", leaseExpiresAt: `2026-07-10T10:0${attemptCount}:30.000Z` }];
|
|
},
|
|
async completeHwlabKafkaOutbox(item) {
|
|
calls.push(`complete:${item.attemptCount}`);
|
|
pending = false;
|
|
},
|
|
async retryHwlabKafkaOutbox(item, error, retryAt) {
|
|
calls.push(`retry:${item.attemptCount}:${error.message}:${retryAt}`);
|
|
},
|
|
async hwlabKafkaProjectorStatus() {
|
|
return { backlogCount: pending ? 1 : 0, retryCount: pending && attemptCount > 0 ? 1 : 0, failedInboxCount: 0, dlqCount: 0 };
|
|
}
|
|
};
|
|
let sendCount = 0;
|
|
const producer = {
|
|
async send() {
|
|
sendCount += 1;
|
|
calls.push(`send:${sendCount}`);
|
|
if (sendCount === 1) throw new Error("broker unavailable");
|
|
}
|
|
};
|
|
const config = { relay: { batchSize: 1, leaseMs: 30000, sendTimeoutMs: 10000, retryBackoffMs: 1000 } };
|
|
|
|
const failedCycle = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-retry-owner", now: () => "2026-07-10T10:00:00.000Z", logger: null });
|
|
assert.deepEqual(failedCycle, { claimedCount: 1, deliveredCount: 0, retryCount: 1, valuesPrinted: false });
|
|
assert.deepEqual(await runtimeStore.hwlabKafkaProjectorStatus(), { backlogCount: 1, retryCount: 1, failedInboxCount: 0, dlqCount: 0 });
|
|
|
|
const recoveredCycle = await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: "relay-retry-owner", now: () => "2026-07-10T10:00:02.000Z", logger: null });
|
|
assert.deepEqual(recoveredCycle, { claimedCount: 1, deliveredCount: 1, retryCount: 0, valuesPrinted: false });
|
|
assert.deepEqual(await runtimeStore.hwlabKafkaProjectorStatus(), { backlogCount: 0, retryCount: 0, failedInboxCount: 0, dlqCount: 0 });
|
|
assert.deepEqual(calls, [
|
|
"send:1",
|
|
"retry:1:broker unavailable:2026-07-10T10:00:01.000Z",
|
|
"send:2",
|
|
"complete:2"
|
|
]);
|
|
});
|
|
|
|
function kafkaProjectorHarness(overrides = {}, calls = []) {
|
|
let runOptions = null;
|
|
let stale = false;
|
|
const consumer = {
|
|
async connect() {},
|
|
async subscribe() {},
|
|
async run(options) { runOptions = options; },
|
|
async commitOffsets(offsets) { calls.push(`commit:${offsets[0].offset}`); },
|
|
async stop() {},
|
|
async disconnect() {}
|
|
};
|
|
const producer = { async connect() {}, async send() {}, async disconnect() {} };
|
|
const runtimeStore = {
|
|
async assertWorkbenchTransactionalRealtimeReady() { return { ready: true }; },
|
|
async subscribeWorkbenchProjectionCommits() { return () => {}; },
|
|
async commitAgentRunKafkaProjection() { return { projectedSeq: 1 }; },
|
|
async recordFailedAgentRunKafkaMessage() { calls.push("dlq"); },
|
|
async recordIgnoredAgentRunKafkaMessage() { calls.push("ignored"); return { ignored: true }; },
|
|
async claimHwlabKafkaOutbox() { return []; },
|
|
async completeHwlabKafkaOutbox() {},
|
|
async retryHwlabKafkaOutbox() {},
|
|
async hwlabKafkaProjectorStatus() { return { backlogCount: 0 }; },
|
|
...overrides
|
|
};
|
|
return {
|
|
consumer,
|
|
producer,
|
|
runtimeStore,
|
|
kafkaFactory: () => ({ consumer: () => consumer, producer: () => producer }),
|
|
setStale(value) { stale = value === true; },
|
|
async runBatch(message) {
|
|
assert.ok(runOptions?.eachBatch);
|
|
await runOptions.eachBatch({
|
|
batch: { topic: "agentrun.event.v1", partition: 2, messages: [message] },
|
|
resolveOffset(offset) { calls.push(`resolve:${offset}`); },
|
|
async heartbeat() { calls.push("heartbeat"); },
|
|
isRunning: () => true,
|
|
isStale: () => stale
|
|
});
|
|
}
|
|
};
|
|
}
|
|
|
|
function canonicalMessage({ offset, traceId = "trc_projector_test", hwlabSessionId = "ses_projector_test" }) {
|
|
const eventId = `evt_projector_${offset}`;
|
|
const runId = "run_projector_test";
|
|
return {
|
|
offset,
|
|
key: Buffer.from(runId),
|
|
value: Buffer.from(JSON.stringify({
|
|
schema: "agentrun.event.v1",
|
|
eventType: "agentrun.event.committed",
|
|
source: "agentrun-v02",
|
|
eventId,
|
|
sourceSeq: Number(offset) + 1,
|
|
traceId,
|
|
hwlabSessionId,
|
|
runId,
|
|
commandId: "cmd_projector_test",
|
|
run: { runId, status: "running", valuesPrinted: false },
|
|
command: { commandId: "cmd_projector_test", runId, seq: 1, type: "prompt", state: "running", valuesPrinted: false },
|
|
event: { id: eventId, runId, seq: Number(offset) + 1, type: "assistant_message", payload: { traceId, hwlabSessionId, commandId: "cmd_projector_test", text: "projected answer" }, createdAt: "2026-07-10T10:00:00.000Z" },
|
|
valuesPrinted: false
|
|
}))
|
|
};
|
|
}
|