root
331 lines
14 KiB
TypeScript
331 lines
14 KiB
TypeScript
import assert from "node:assert/strict";
|
|
import { mkdtemp, readFile, writeFile } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
|
|
import { test } from "bun:test";
|
|
|
|
import { queryKafkaEventStream } from "../../internal/cloud/kafka-event-bridge.ts";
|
|
import { mapAgentRunRecordsToHwlabDebugEvents, runKafkaCli } from "../src/hwlab-cli/kafka-regenerate.ts";
|
|
|
|
const SESSION_ID = "ses_agentrun_5ec4e141_6abc_466d_9afe_049f7c0ac105";
|
|
const HWLAB_SESSION_ID = "ses_5ec4e141-6abc-466d-9afe-049f7c0ac105";
|
|
const TRACE_ID = "trc_mretx18t3jl4tg";
|
|
const RUN_ID = "run_a78cbcb05f2c4afaa748a3158db44998";
|
|
|
|
test("offline JSONL maps 35 stdio reconstructions to 35 ordered HWLAB debug events without runtime dependencies", async () => {
|
|
const cwd = await mkdtemp(path.join(os.tmpdir(), "hwlab-kafka-debug-jsonl-"));
|
|
const inputFile = path.join(cwd, "agentrun-events.jsonl");
|
|
const outputFile = path.join(cwd, "hwlab-events.jsonl");
|
|
const records = Array.from({ length: 35 }, (_, index) => reconstructionRecord(index + 1));
|
|
await writeFile(inputFile, records.map((record) => JSON.stringify(record)).join("\n") + "\n", "utf8");
|
|
|
|
const result = await runKafkaCli([
|
|
"regenerate", "hwlab",
|
|
"--from", "jsonl",
|
|
"--session-id", SESSION_ID,
|
|
"--trace-id", TRACE_ID,
|
|
"--jsonl-file", inputFile,
|
|
"--output-jsonl", outputFile,
|
|
"--expect-count", "35",
|
|
"--no-publish",
|
|
"--json"
|
|
], {
|
|
cwd,
|
|
env: {},
|
|
now: () => "2026-07-10T12:00:00.000Z",
|
|
async createProducer() { throw new Error("offline JSONL must not create a producer"); }
|
|
});
|
|
|
|
assert.equal(result.exitCode, 0);
|
|
assert.equal(result.payload.input.readCount, 35);
|
|
assert.equal(result.payload.output.count, 35);
|
|
assert.equal(result.payload.output.publishedCount, 0);
|
|
assert.deepEqual(result.payload.validation, {
|
|
expectedCount: 35,
|
|
oneToOne: true,
|
|
orderPreserved: true,
|
|
lineagePreserved: true,
|
|
firstSourceSeq: 1,
|
|
lastSourceSeq: 35,
|
|
inputKinds: ["stdio-derived partial reconstruction"]
|
|
});
|
|
assert.deepEqual(result.payload.runtimeDependencies, {
|
|
cloudApi: false,
|
|
database: false,
|
|
transactionalProjector: false,
|
|
kafka: false
|
|
});
|
|
|
|
const output = (await readFile(outputFile, "utf8")).trim().split("\n").map(JSON.parse);
|
|
assert.equal(output.length, 35);
|
|
assert.deepEqual(output.map((row) => row.value.debugLineage.sourceSeq), Array.from({ length: 35 }, (_, index) => index + 1));
|
|
assert.ok(output.every((row) => row.topic === "hwlab.event.debug.v1"));
|
|
assert.ok(output.every((row) => row.value.schema === "hwlab.event.debug.v1"));
|
|
assert.ok(output.every((row) => row.value.eventId === null && row.value.sourceEventId === null && row.value.outboxSeq === null));
|
|
assert.ok(output.every((row) => row.value.debugLineage.inputEventId === null && row.value.debugLineage.inputOutboxSeq === null));
|
|
assert.ok(output.every((row) => row.value.sourceEvent.partition === null && row.value.sourceEvent.offset === null));
|
|
assert.ok(output.every((row) => row.value.traceId === TRACE_ID && row.value.sessionId === HWLAB_SESSION_ID));
|
|
assert.ok(output.every((row) => row.value.debugLineage.inputSessionId === SESSION_ID));
|
|
assert.ok(output.every((row) => row.value.debugLineage.inputHwlabSessionId === HWLAB_SESSION_ID));
|
|
assert.ok(output.every((row) => row.value.reconstruction.kind === "stdio-derived partial reconstruction"));
|
|
assert.ok(output.every((row) => row.value.reconstruction.originalEventId === null && row.value.reconstruction.originalOutboxSeq === null));
|
|
assert.ok(output.every((row) => row.value.reconstruction.identityDerivation.kind === "agentrun-session-uuid-to-hwlab-session"));
|
|
assert.ok(output.every((row) => row.value.reconstruction.identityDerivation.reversible === true));
|
|
});
|
|
|
|
test("Kafka reader and producer are injected while canonical AgentRun events use strict decode and isolated output", async () => {
|
|
const cwd = await mkdtemp(path.join(os.tmpdir(), "hwlab-kafka-debug-canonical-"));
|
|
const records = Array.from({ length: 35 }, (_, index) => canonicalRecord(index + 1));
|
|
const readerCalls: any[] = [];
|
|
const producerCalls: any[] = [];
|
|
let connected = 0;
|
|
let disconnected = 0;
|
|
const result = await runKafkaCli([
|
|
"regenerate", "hwlab",
|
|
"--from", "kafka",
|
|
"--session-id", SESSION_ID,
|
|
"--trace-id", TRACE_ID,
|
|
"--input-topic", "agentrun.event.v1",
|
|
"--expect-count", "35",
|
|
"--output-jsonl", path.join(cwd, "canonical-output.jsonl"),
|
|
"--json"
|
|
], {
|
|
cwd,
|
|
env: {
|
|
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092",
|
|
HWLAB_KAFKA_HWLAB_DEBUG_EVENT_TOPIC: "hwlab.event.debug.v1",
|
|
HWLAB_KAFKA_HWLAB_DEBUG_GROUP_PREFIX: "hwlab-v03-workbench-isolated-debug"
|
|
},
|
|
now: () => "2026-07-10T12:00:00.000Z",
|
|
async readKafka(input) {
|
|
readerCalls.push(input);
|
|
return { ok: true, groupId: "hwlab-v03-agentrun-event-debug-regenerate-test", events: records };
|
|
},
|
|
async createProducer() {
|
|
return {
|
|
async connect() { connected += 1; },
|
|
async send(input: any) { producerCalls.push(input); return [{ topicName: input.topic, partition: 0, baseOffset: "100" }]; },
|
|
async disconnect() { disconnected += 1; }
|
|
};
|
|
}
|
|
});
|
|
|
|
assert.equal(result.exitCode, 0);
|
|
assert.equal(readerCalls.length, 1);
|
|
assert.equal(readerCalls[0].groupIdPrefix, "hwlab-v03-workbench-isolated-debug");
|
|
assert.equal(readerCalls[0].sessionId, SESSION_ID);
|
|
assert.equal(readerCalls[0].limit, 36, "exact count verification must leave room to detect one extra event");
|
|
assert.equal(connected, 1);
|
|
assert.equal(disconnected, 1);
|
|
assert.equal(producerCalls.length, 1);
|
|
assert.equal(producerCalls[0].topic, "hwlab.event.debug.v1");
|
|
assert.equal(producerCalls[0].messages.length, 35);
|
|
const envelopes = producerCalls[0].messages.map((message: any) => JSON.parse(message.value));
|
|
assert.deepEqual(envelopes.map((event: any) => event.debugLineage.sourceSeq), Array.from({ length: 35 }, (_, index) => index + 1));
|
|
assert.deepEqual(envelopes.map((event: any) => event.eventId), Array.from({ length: 35 }, (_, index) => `hwlab:evt_${index + 1}`));
|
|
assert.deepEqual(envelopes.map((event: any) => event.debugLineage.inputOutboxSeq), Array.from({ length: 35 }, (_, index) => index + 1));
|
|
assert.ok(envelopes.every((event: any) => event.diagnostics.inputKind === "canonical-durable"));
|
|
assert.ok(envelopes.every((event: any) => event.schema === "hwlab.event.debug.v1"));
|
|
assert.equal(result.payload.output.publishedCount, 35);
|
|
assert.deepEqual(result.payload.config.outputTopic, { value: "hwlab.event.debug.v1", source: "env:HWLAB_KAFKA_HWLAB_DEBUG_EVENT_TOPIC" });
|
|
assert.deepEqual(result.payload.config.groupPrefix, { value: "hwlab-v03-workbench-isolated-debug", source: "env:HWLAB_KAFKA_HWLAB_DEBUG_GROUP_PREFIX" });
|
|
assert.equal(result.payload.validation.orderPreserved, true);
|
|
assert.equal(result.payload.validation.lineagePreserved, true);
|
|
});
|
|
|
|
test("reconstruction identity fabrication and product output topic fail closed", async () => {
|
|
const cwd = await mkdtemp(path.join(os.tmpdir(), "hwlab-kafka-debug-invalid-"));
|
|
const forged = reconstructionRecord(1);
|
|
forged.value.eventId = "evt_forged";
|
|
const inputFile = path.join(cwd, "forged.jsonl");
|
|
await writeFile(inputFile, `${JSON.stringify(forged)}\n`, "utf8");
|
|
|
|
const forgedResult = await runKafkaCli([
|
|
"regenerate", "hwlab", "--from", "jsonl", "--session-id", SESSION_ID,
|
|
"--jsonl-file", inputFile, "--no-publish"
|
|
], { cwd, env: {}, now: () => "2026-07-10T12:00:00.000Z" });
|
|
assert.equal(forgedResult.exitCode, 1);
|
|
assert.equal(forgedResult.payload.error.code, "hwlab_kafka_debug_identity_invalid");
|
|
|
|
const productResult = await runKafkaCli([
|
|
"regenerate", "hwlab", "--from", "jsonl", "--session-id", SESSION_ID,
|
|
"--jsonl-file", inputFile, "--output-topic", "hwlab.event.v1", "--no-publish"
|
|
], { cwd, env: {}, now: () => "2026-07-10T12:00:00.000Z" });
|
|
assert.equal(productResult.exitCode, 1);
|
|
assert.equal(productResult.payload.error.code, "product_topic_forbidden");
|
|
});
|
|
|
|
test("Kafka query exposes the isolated generated group and original value hash", async () => {
|
|
const valueText = JSON.stringify(canonicalRecord(1).value);
|
|
const consumerConfigs: any[] = [];
|
|
const consumer = {
|
|
async connect() {},
|
|
async subscribe() {},
|
|
async run({ eachMessage }: any) {
|
|
await eachMessage({
|
|
topic: "agentrun.event.v1",
|
|
partition: 0,
|
|
message: { offset: "1601", key: Buffer.from(SESSION_ID), value: Buffer.from(valueText), timestamp: "1783681200000" }
|
|
});
|
|
},
|
|
async stop() {},
|
|
async disconnect() {}
|
|
};
|
|
const queried = await queryKafkaEventStream({
|
|
env: { HWLAB_KAFKA_BOOTSTRAP_SERVERS: "kafka.test:9092", HWLAB_KAFKA_CLIENT_ID: "hwlab-test" },
|
|
topic: "agentrun.event.v1",
|
|
sessionId: SESSION_ID,
|
|
limit: 1,
|
|
groupIdPrefix: "hwlab-v03-workbench-isolated-debug",
|
|
kafkaFactory() {
|
|
return { consumer(config: any) { consumerConfigs.push(config); return consumer; } };
|
|
}
|
|
});
|
|
assert.match(queried.groupId, /^hwlab-v03-workbench-isolated-debug-\d+-[a-f0-9]{8}$/u);
|
|
assert.equal(consumerConfigs[0].groupId, queried.groupId);
|
|
assert.equal(queried.events[0].valueSha256, createSha256(valueText));
|
|
});
|
|
|
|
test("non-reversible AgentRun session identity stays only in debug lineage", () => {
|
|
const rawSessionId = "ses_agentrun_not_uuid";
|
|
const record = reconstructionRecord(1);
|
|
record.key = rawSessionId;
|
|
record.value.sessionId = rawSessionId;
|
|
record.value.hwlabSessionId = null;
|
|
record.value.run.sessionId = rawSessionId;
|
|
record.value.run.hwlabSessionId = null;
|
|
record.value.event.payload.sessionId = rawSessionId;
|
|
record.value.reconstruction.identityDerivation = {
|
|
kind: "none",
|
|
sourceField: "sessionId",
|
|
targetField: "hwlabSessionId",
|
|
reversible: false,
|
|
reason: "source sessionId is not ses_agentrun_<uuid-with-underscores>",
|
|
valuesPrinted: false
|
|
};
|
|
const mapped = mapAgentRunRecordsToHwlabDebugEvents([record], {
|
|
sessionId: rawSessionId,
|
|
traceId: TRACE_ID,
|
|
expectedCount: 1,
|
|
producedAt: "2026-07-10T12:00:00.000Z"
|
|
});
|
|
assert.equal(mapped.events[0].sessionId, null);
|
|
assert.equal(mapped.events[0].hwlabSessionId, null);
|
|
assert.equal(mapped.events[0].event.sessionId, null);
|
|
assert.equal(mapped.events[0].debugLineage.inputSessionId, rawSessionId);
|
|
assert.equal(mapped.events[0].reconstruction.identityDerivation.kind, "none");
|
|
});
|
|
|
|
function reconstructionRecord(seq: number): any {
|
|
const value = {
|
|
schema: "agentrun.event.reconstruction.v1",
|
|
eventType: "agentrun.event.reconstructed",
|
|
source: "agentrun-cli-stdio-reconstruction",
|
|
eventId: null,
|
|
outboxSeq: null,
|
|
sourceSeq: seq,
|
|
observedAt: `2026-07-10T11:00:${String(seq).padStart(2, "0")}.000Z`,
|
|
traceId: TRACE_ID,
|
|
sessionId: SESSION_ID,
|
|
hwlabSessionId: HWLAB_SESSION_ID,
|
|
runId: RUN_ID,
|
|
commandId: "cmd_9fff03fc124b4203b065d04bbb7c4f1e",
|
|
run: { runId: RUN_ID, sessionId: SESSION_ID, hwlabSessionId: HWLAB_SESSION_ID, status: "running", valuesPrinted: false },
|
|
command: { commandId: "cmd_9fff03fc124b4203b065d04bbb7c4f1e", runId: RUN_ID, seq: 1, type: "run", state: "running", valuesPrinted: false },
|
|
event: {
|
|
id: null,
|
|
runId: RUN_ID,
|
|
seq,
|
|
type: "backend_status",
|
|
payload: {
|
|
traceId: TRACE_ID,
|
|
sessionId: SESSION_ID,
|
|
phase: `reconstructed-${seq}`,
|
|
reconstructed: true,
|
|
reconstructionKind: "stdio-derived partial reconstruction"
|
|
},
|
|
createdAt: `2026-07-10T11:00:${String(seq).padStart(2, "0")}.000Z`
|
|
},
|
|
reconstruction: {
|
|
kind: "stdio-derived partial reconstruction",
|
|
completeLifecycle: false,
|
|
originalEventId: null,
|
|
originalOutboxSeq: null,
|
|
sourceTopic: "codex-stdio.raw.v1",
|
|
sourcePartition: 0,
|
|
sourceOffset: String(3500 + seq),
|
|
sourceFrameSeq: seq,
|
|
sourceValueSha256: `sha256:${String(seq).padStart(64, "0")}`,
|
|
identityDerivation: {
|
|
kind: "agentrun-session-uuid-to-hwlab-session",
|
|
sourceField: "sessionId",
|
|
targetField: "hwlabSessionId",
|
|
reversible: true,
|
|
sourcePrefix: "ses_agentrun_",
|
|
targetPrefix: "ses_",
|
|
separatorTransform: "underscore-to-hyphen",
|
|
valuesPrinted: false
|
|
}
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
return { topic: "agentrun.event.debug.v1", partition: null, offset: null, key: SESSION_ID, value, headers: { "x-trace-id": TRACE_ID } };
|
|
}
|
|
|
|
function canonicalRecord(seq: number): any {
|
|
const eventId = `evt_${seq}`;
|
|
const value = {
|
|
schema: "agentrun.event.v1",
|
|
eventType: "agentrun.event.committed",
|
|
source: "agentrun-manager",
|
|
eventId,
|
|
outboxSeq: seq,
|
|
sourceSeq: seq,
|
|
committedAt: `2026-07-10T11:00:${String(seq).padStart(2, "0")}.000Z`,
|
|
traceId: TRACE_ID,
|
|
sessionId: SESSION_ID,
|
|
hwlabSessionId: HWLAB_SESSION_ID,
|
|
runId: RUN_ID,
|
|
commandId: "cmd_9fff03fc124b4203b065d04bbb7c4f1e",
|
|
run: {
|
|
runId: RUN_ID,
|
|
status: "running",
|
|
terminalStatus: null,
|
|
failureKind: null,
|
|
tenantId: "tenant-test",
|
|
projectId: "project-test",
|
|
providerId: "gpt.pika",
|
|
backendProfile: "gpt.pika",
|
|
sessionId: SESSION_ID,
|
|
hwlabSessionId: HWLAB_SESSION_ID,
|
|
conversationId: null,
|
|
threadId: null,
|
|
valuesPrinted: false
|
|
},
|
|
command: {
|
|
commandId: "cmd_9fff03fc124b4203b065d04bbb7c4f1e",
|
|
runId: RUN_ID,
|
|
seq: 1,
|
|
type: "run",
|
|
state: "running",
|
|
payloadHash: "hash",
|
|
valuesPrinted: false
|
|
},
|
|
event: {
|
|
id: eventId,
|
|
runId: RUN_ID,
|
|
seq,
|
|
type: "backend_status",
|
|
payload: { traceId: TRACE_ID, sessionId: SESSION_ID, phase: `canonical-${seq}` },
|
|
createdAt: `2026-07-10T11:00:${String(seq).padStart(2, "0")}.000Z`
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
return { topic: "agentrun.event.v1", partition: 0, offset: String(1600 + seq), key: SESSION_ID, value };
|
|
}
|
|
|
|
function createSha256(value: string) {
|
|
return new Bun.CryptoHasher("sha256").update(value).digest("hex");
|
|
}
|