root
1644 lines
79 KiB
TypeScript
1644 lines
79 KiB
TypeScript
// SPEC: PJ2026-0104010803 Workbench事件流可见性 draft-2026-07-09-p0-kafka-authority.
|
|
// Responsibility: subscribe AgentRun Kafka events, project them into HWLAB Kafka events, and provide bounded Kafka stream queries for diagnostics.
|
|
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
|
|
import { Kafka, logLevel } from "kafkajs";
|
|
import { emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
|
import { buildWorkbenchProjectionEventFacts } from "./workbench-projection-writer.ts";
|
|
import { workbenchRealtimeCapabilities } from "./workbench-realtime-capabilities.ts";
|
|
|
|
const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
|
|
export const DEFAULT_AGENTRUN_EVENT_TOPIC = "agentrun.event.v1";
|
|
export const DEFAULT_HWLAB_EVENT_TOPIC = "hwlab.event.v1";
|
|
const DEFAULT_STDIO_TOPIC = "codex-stdio.raw.v1";
|
|
export const DEFAULT_AGENTRUN_DEBUG_EVENT_TOPIC = "agentrun.event.debug.v1";
|
|
export const DEFAULT_HWLAB_DEBUG_EVENT_TOPIC = "hwlab.event.debug.v1";
|
|
export const AGENTRUN_STDIO_RECONSTRUCTION_SCHEMA = "agentrun.event.reconstruction.v1";
|
|
export const AGENTRUN_STDIO_RECONSTRUCTION_KIND = "stdio-derived partial reconstruction";
|
|
const DEFAULT_CLIENT_ID = "hwlab-v03-cloud-api";
|
|
const DEFAULT_QUERY_TIMEOUT_MS = 5000;
|
|
const DEFAULT_QUERY_LIMIT = 50;
|
|
const LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS = 32;
|
|
const REQUIRED_KAFKA_ENV = Object.freeze([
|
|
"HWLAB_KAFKA_BOOTSTRAP_SERVERS",
|
|
"HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC",
|
|
"HWLAB_KAFKA_EVENT_TOPIC",
|
|
"HWLAB_KAFKA_CLIENT_ID"
|
|
]);
|
|
|
|
export function kafkaEventBridgeConfig(env = process.env) {
|
|
const legacyKafkaEnabled = truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED);
|
|
const kafkaConfigured = REQUIRED_KAFKA_ENV.every((name) => stringValue(env[name]));
|
|
if (!legacyKafkaEnabled && !kafkaConfigured) return null;
|
|
const capabilities = workbenchRealtimeCapabilities();
|
|
const required = [
|
|
...REQUIRED_KAFKA_ENV,
|
|
"HWLAB_KAFKA_PROJECTOR_GROUP_ID",
|
|
"HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS",
|
|
"HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS",
|
|
"HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE",
|
|
"HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS",
|
|
"HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS",
|
|
"HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS"
|
|
];
|
|
const missing = required.filter((name) => !stringValue(env[name]));
|
|
if (missing.length > 0) {
|
|
const error = new Error(`HWLAB Kafka capability configuration is incomplete: ${missing.join(", ")}`);
|
|
error.code = "hwlab_kafka_realtime_config_incomplete";
|
|
error.missing = missing;
|
|
throw error;
|
|
}
|
|
const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS);
|
|
const agentRunTopic = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC);
|
|
const hwlabTopic = stringValue(env.HWLAB_KAFKA_EVENT_TOPIC);
|
|
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID);
|
|
const relay = {
|
|
intervalMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS, "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS"),
|
|
batchSize: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE, "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE"),
|
|
leaseMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS, "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS"),
|
|
sendTimeoutMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS, "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS"),
|
|
retryBackoffMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS, "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS")
|
|
};
|
|
if (relay && relay.sendTimeoutMs + 5000 >= relay.leaseMs) {
|
|
const error = new Error("HWLAB Kafka relay lease must exceed producer send timeout by at least 5000ms.");
|
|
error.code = "hwlab_kafka_relay_lease_invalid";
|
|
throw error;
|
|
}
|
|
return {
|
|
capabilities,
|
|
brokers,
|
|
agentRunTopic,
|
|
hwlabTopic,
|
|
clientId,
|
|
directPublishGroupId: null,
|
|
projectorGroupId: stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID),
|
|
hwlabEventGroupId: null,
|
|
refreshReplay: null,
|
|
projectorHeartbeatIntervalMs: strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS"),
|
|
relay
|
|
};
|
|
}
|
|
|
|
export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString(), otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
|
const config = kafkaEventBridgeConfig(env);
|
|
if (!config) return { started: false, reason: "disabled_or_unconfigured", stop() {}, subscribeProjectionCommits() { return () => {}; }, valuesPrinted: false };
|
|
return startTransactionalHwlabKafkaEventBridge({ config, logger, kafkaFactory, runtimeStore, now });
|
|
}
|
|
|
|
function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString() } = {}) {
|
|
requireKafkaProjectorStore(runtimeStore);
|
|
|
|
let stopped = false;
|
|
let consumer = null;
|
|
let producer = null;
|
|
let relayTimer = null;
|
|
let relayRunning = false;
|
|
let stopProjectionNotifications = null;
|
|
const projectionCommitListeners = new Set();
|
|
const publishProjectionCommit = (signal = {}) => {
|
|
for (const listener of [...projectionCommitListeners]) {
|
|
try { listener({ sessionId: signal.sessionId ?? null, traceId: signal.traceId ?? null, outboxSeq: signal.outboxSeq ?? null, recovery: signal.recovery === true, valuesRedacted: true }); } catch {}
|
|
}
|
|
};
|
|
const relayOwner = `${config.clientId}:outbox-relay:${randomUUID()}`;
|
|
let startupError = null;
|
|
let startupReady = false;
|
|
const ready = (async () => {
|
|
await runtimeStore.assertWorkbenchTransactionalRealtimeReady();
|
|
const kafka = kafkaFactory(config);
|
|
consumer = kafka.consumer({ groupId: config.projectorGroupId, allowAutoTopicCreation: false });
|
|
producer = kafka.producer({ allowAutoTopicCreation: false });
|
|
await producer.connect();
|
|
await consumer.connect();
|
|
stopProjectionNotifications = await runtimeStore.subscribeWorkbenchProjectionCommits(publishProjectionCommit);
|
|
await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
|
|
await consumer.run({
|
|
autoCommit: false,
|
|
eachBatchAutoResolve: false,
|
|
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => {
|
|
for (const message of batch.messages) {
|
|
if (stopped || !isRunning() || isStale()) break;
|
|
await heartbeat();
|
|
const heartbeatLoop = startKafkaHeartbeatLoop(heartbeat, config.projectorHeartbeatIntervalMs);
|
|
let projection;
|
|
try {
|
|
projection = await projectAgentRunKafkaMessage({ topic: batch.topic, partition: batch.partition, message }, { runtimeStore, config, logger });
|
|
} finally {
|
|
await heartbeatLoop.stop();
|
|
}
|
|
if (projection?.projected) publishProjectionCommit(projection);
|
|
if (heartbeatLoop.error) throw heartbeatLoop.error;
|
|
if (stopped || !isRunning() || isStale()) {
|
|
logWarn(logger, "hwlab-kafka-offset-commit-skipped", { topic: batch.topic, partition: batch.partition, offset: message.offset, reason: isStale() ? "stale_generation" : "consumer_stopped", valuesPrinted: false });
|
|
break;
|
|
}
|
|
resolveOffset(message.offset);
|
|
await consumer.commitOffsets([{ topic: batch.topic, partition: batch.partition, offset: nextKafkaOffset(message.offset) }]);
|
|
await heartbeat();
|
|
}
|
|
}
|
|
});
|
|
const runRelay = async () => {
|
|
if (stopped || relayRunning) return;
|
|
relayRunning = true;
|
|
try {
|
|
await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: relayOwner, now, logger });
|
|
} finally {
|
|
relayRunning = false;
|
|
}
|
|
};
|
|
relayTimer = setInterval(() => { void runRelay().catch((error) => logWarn(logger, "hwlab-kafka-outbox-relay-failed", { message: errorMessage(error), valuesPrinted: false })); }, config.relay.intervalMs);
|
|
relayTimer.unref?.();
|
|
await runRelay();
|
|
startupReady = true;
|
|
logInfo(logger, "hwlab-kafka-event-bridge-started", {
|
|
agentRunTopic: config.agentRunTopic,
|
|
hwlabTopic: config.hwlabTopic,
|
|
clientId: config.clientId,
|
|
projectorGroupId: config.projectorGroupId,
|
|
capabilities: config.capabilities,
|
|
valuesPrinted: false
|
|
});
|
|
})();
|
|
void ready.catch((error) => {
|
|
startupError = error;
|
|
logWarn(logger, "hwlab-kafka-event-bridge-start-failed", {
|
|
message: errorMessage(error),
|
|
agentRunTopic: config.agentRunTopic,
|
|
hwlabTopic: config.hwlabTopic,
|
|
valuesPrinted: false
|
|
});
|
|
});
|
|
|
|
return {
|
|
started: true,
|
|
...config,
|
|
ready,
|
|
async stop() {
|
|
stopped = true;
|
|
if (relayTimer) clearInterval(relayTimer);
|
|
if (typeof stopProjectionNotifications === "function") {
|
|
try { await stopProjectionNotifications(); } catch {}
|
|
}
|
|
if (consumer?.stop) await consumer.stop().catch(() => undefined);
|
|
await Promise.allSettled([consumer?.disconnect?.(), producer?.disconnect?.()]);
|
|
},
|
|
async status() {
|
|
if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
|
if (!startupReady) return { status: "initializing", valuesRedacted: true };
|
|
const transactionalReadiness = typeof runtimeStore.workbenchTransactionalRealtimeReadiness === "function"
|
|
? await runtimeStore.workbenchTransactionalRealtimeReadiness()
|
|
: { ready: true, valuesRedacted: true };
|
|
const projectorStatus = await runtimeStore.hwlabKafkaProjectorStatus();
|
|
return { status: transactionalReadiness.ready === false ? "blocked" : "ready", capabilities: config.capabilities, transactionalReadiness, ...projectorStatus, valuesRedacted: true };
|
|
},
|
|
startupStatus() {
|
|
if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
|
return { status: startupReady ? "ready" : "initializing", valuesRedacted: true };
|
|
},
|
|
subscribeProjectionCommits(listener) {
|
|
if (typeof listener !== "function") return () => {};
|
|
projectionCommitListeners.add(listener);
|
|
return () => projectionCommitListeners.delete(listener);
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function startLiveHwlabKafkaEventBridge({ config, env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
|
let stopped = false;
|
|
let bridgeConsumer = null;
|
|
let fanoutConsumer = null;
|
|
let producer = null;
|
|
let startupError = null;
|
|
let startupReady = false;
|
|
const subscribers = new Set();
|
|
const lagByPartition = new Map();
|
|
const publishLiveEnvelope = (envelope, transport) => {
|
|
for (const listener of [...subscribers]) {
|
|
try {
|
|
listener(envelope, transport);
|
|
} catch (error) {
|
|
logWarn(logger, "hwlab-live-kafka-subscriber-failed", { message: errorMessage(error), valuesPrinted: false });
|
|
}
|
|
}
|
|
};
|
|
const ready = (async () => {
|
|
const kafka = kafkaFactory(config);
|
|
if (config.capabilities.directPublish) {
|
|
bridgeConsumer = kafka.consumer({ groupId: config.directPublishGroupId, allowAutoTopicCreation: false });
|
|
producer = kafka.producer({ allowAutoTopicCreation: false });
|
|
}
|
|
if (config.capabilities.liveKafkaSse) fanoutConsumer = kafka.consumer({ groupId: config.hwlabEventGroupId, allowAutoTopicCreation: false });
|
|
if (producer) await producer.connect();
|
|
if (bridgeConsumer) await bridgeConsumer.connect();
|
|
if (fanoutConsumer) await fanoutConsumer.connect();
|
|
if (bridgeConsumer) await bridgeConsumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
|
|
if (fanoutConsumer) await fanoutConsumer.subscribe({ topic: config.hwlabTopic, fromBeginning: false });
|
|
if (fanoutConsumer) await fanoutConsumer.run({
|
|
eachBatchAutoResolve: false,
|
|
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => {
|
|
let lastOffset = null;
|
|
for (const message of batch.messages) {
|
|
if (stopped || !isRunning() || isStale()) break;
|
|
const envelope = parseJson(message?.value ? Buffer.from(message.value).toString("utf8") : "");
|
|
if (!envelope || envelope.schema !== "hwlab.event.v1") {
|
|
logWarn(logger, "hwlab-live-kafka-envelope-ignored", { reason: "schema-invalid", valuesPrinted: false });
|
|
} else {
|
|
const transport = { topic: batch.topic, partition: batch.partition, offset: message.offset };
|
|
emitLiveKafkaOtelSpan("hwlab.kafka.live.fanout_receive", envelope, transport, { env, otelSpanEmitter });
|
|
publishLiveEnvelope(envelope, transport);
|
|
}
|
|
resolveOffset(message.offset);
|
|
lastOffset = message.offset;
|
|
await heartbeat();
|
|
}
|
|
if (lastOffset !== null) lagByPartition.set(batch.partition, kafkaPartitionLag(batch.highWatermark, lastOffset));
|
|
}
|
|
});
|
|
if (bridgeConsumer) await bridgeConsumer.run({
|
|
eachMessage: async ({ topic, partition, message }) => {
|
|
if (stopped) return;
|
|
await publishAgentRunKafkaMessageLive({ topic, partition, message }, { producer, config, env, logger, otelSpanEmitter });
|
|
}
|
|
});
|
|
startupReady = true;
|
|
logInfo(logger, "hwlab-live-kafka-event-bridge-started", {
|
|
capabilities: config.capabilities,
|
|
agentRunTopic: config.agentRunTopic,
|
|
hwlabTopic: config.hwlabTopic,
|
|
clientId: config.clientId,
|
|
directPublishGroupId: config.directPublishGroupId,
|
|
hwlabEventGroupId: config.hwlabEventGroupId,
|
|
valuesPrinted: false
|
|
});
|
|
})();
|
|
void ready.catch((error) => {
|
|
startupError = error;
|
|
logWarn(logger, "hwlab-live-kafka-event-bridge-start-failed", {
|
|
message: errorMessage(error),
|
|
agentRunTopic: config.agentRunTopic,
|
|
hwlabTopic: config.hwlabTopic,
|
|
valuesPrinted: false
|
|
});
|
|
});
|
|
return {
|
|
started: true,
|
|
...config,
|
|
ready,
|
|
liveReady: ready,
|
|
async stop() {
|
|
stopped = true;
|
|
subscribers.clear();
|
|
await Promise.allSettled([bridgeConsumer?.stop?.(), fanoutConsumer?.stop?.()]);
|
|
await Promise.allSettled([bridgeConsumer?.disconnect?.(), fanoutConsumer?.disconnect?.(), producer?.disconnect?.()]);
|
|
},
|
|
async status() {
|
|
if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
|
return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, subscriberCount: subscribers.size, consumerLag: liveKafkaLagSummary(lagByPartition), lossPossible: true, valuesRedacted: true };
|
|
},
|
|
startupStatus() {
|
|
if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
|
return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, valuesRedacted: true };
|
|
},
|
|
subscribeProjectionCommits() { return () => {}; },
|
|
subscribeLiveHwlabEvents(listener) {
|
|
if (typeof listener !== "function") return () => {};
|
|
subscribers.add(listener);
|
|
return () => subscribers.delete(listener);
|
|
},
|
|
queryHwlabEventRetention(params = {}) {
|
|
return queryKafkaEventStream({ ...params, env, stream: "hwlab", topic: config.hwlabTopic, kafkaFactory });
|
|
},
|
|
liveSubscriberCount() { return subscribers.size; },
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function kafkaPartitionLag(highWatermark, lastOffset) {
|
|
try {
|
|
const lag = BigInt(String(highWatermark ?? "0")) - (BigInt(String(lastOffset ?? "0")) + 1n);
|
|
return Number(lag > 0n ? lag : 0n);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function liveKafkaLagSummary(lagByPartition) {
|
|
const partitions = [...lagByPartition.entries()].map(([partition, lag]) => ({ partition, lag })).sort((left, right) => left.partition - right.partition);
|
|
const known = partitions.map((item) => item.lag).filter((lag) => Number.isFinite(lag));
|
|
return {
|
|
known: known.length > 0,
|
|
total: known.length > 0 ? known.reduce((sum, lag) => sum + lag, 0) : null,
|
|
partitions,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function combineKafkaEventBridgeComponents(config, components) {
|
|
const ready = Promise.all(components.map((component) => component.ready));
|
|
const liveOwner = components.find((component) => typeof component.subscribeLiveHwlabEvents === "function");
|
|
const componentStartup = () => components.map((component) => component.startupStatus?.() ?? { status: "initializing" });
|
|
return {
|
|
started: true,
|
|
...config,
|
|
ready,
|
|
liveReady: liveOwner?.liveReady ?? liveOwner?.ready ?? ready,
|
|
async stop() { await Promise.allSettled(components.map((component) => component.stop())); },
|
|
async status() {
|
|
const statuses = await Promise.all(components.map((component) => component.status()));
|
|
const blocked = statuses.find((status) => status.status === "blocked");
|
|
const initializing = statuses.some((status) => status.status !== "ready");
|
|
return {
|
|
status: blocked ? "blocked" : initializing ? "initializing" : "ready",
|
|
capabilities: config.capabilities,
|
|
components: statuses,
|
|
errorCode: blocked?.errorCode ?? null,
|
|
message: blocked?.message ?? null,
|
|
valuesRedacted: true
|
|
};
|
|
},
|
|
startupStatus() {
|
|
const statuses = componentStartup();
|
|
const blocked = statuses.find((status) => status.status === "blocked");
|
|
return {
|
|
status: blocked ? "blocked" : statuses.every((status) => status.status === "ready") ? "ready" : "initializing",
|
|
capabilities: config.capabilities,
|
|
components: statuses,
|
|
errorCode: blocked?.errorCode ?? null,
|
|
message: blocked?.message ?? null,
|
|
valuesRedacted: true
|
|
};
|
|
},
|
|
subscribeProjectionCommits(listener) {
|
|
const stops = components.map((component) => component.subscribeProjectionCommits?.(listener)).filter((stop) => typeof stop === "function");
|
|
return () => stops.forEach((stop) => stop());
|
|
},
|
|
subscribeLiveHwlabEvents(listener) {
|
|
return liveOwner?.subscribeLiveHwlabEvents(listener) ?? (() => {});
|
|
},
|
|
queryHwlabEventRetention(params = {}) {
|
|
if (typeof liveOwner?.queryHwlabEventRetention !== "function") throw contractError("hwlab_kafka_refresh_query_unconfigured", "Kafka refresh retention query is not configured.");
|
|
return liveOwner.queryHwlabEventRetention(params);
|
|
},
|
|
liveSubscriberCount() {
|
|
const owner = components.find((component) => typeof component.liveSubscriberCount === "function");
|
|
return owner?.liveSubscriberCount() ?? 0;
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
export async function publishAgentRunKafkaMessageLive(kafkaMessage = {}, { producer, config, env = process.env, logger = console, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
|
const transport = kafkaTransport(kafkaMessage);
|
|
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
|
|
if (!decoded.ok) {
|
|
logWarn(logger, "hwlab-live-kafka-message-invalid", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false });
|
|
return { handled: true, invalid: true, published: false, valuesPrinted: false };
|
|
}
|
|
if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) {
|
|
return { handled: true, ignored: true, published: false, valuesPrinted: false };
|
|
}
|
|
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, {
|
|
source: config.clientId,
|
|
sourceTopic: transport.sourceTopic,
|
|
sourcePartition: transport.sourcePartition,
|
|
sourceOffset: transport.sourceOffset,
|
|
sourceKey: transport.sourceKey,
|
|
inputSha256: transport.inputSha256
|
|
});
|
|
const publishMetadata = await producer.send({ topic: config.hwlabTopic, messages: [buildHwlabKafkaProducerMessage(projected)] });
|
|
const publishedRecord = Array.isArray(publishMetadata) ? publishMetadata[0] : null;
|
|
emitLiveKafkaOtelSpan("hwlab.kafka.live.direct_publish", projected, {
|
|
topic: config.hwlabTopic,
|
|
partition: publishedRecord?.partition,
|
|
offset: publishedRecord?.baseOffset ?? publishedRecord?.offset,
|
|
sourceTopic: transport.sourceTopic,
|
|
sourcePartition: transport.sourcePartition,
|
|
sourceOffset: transport.sourceOffset
|
|
}, { env, otelSpanEmitter });
|
|
return { handled: true, published: true, envelope: projected, sessionId: projected.sessionId, traceId: projected.traceId, valuesPrinted: false };
|
|
}
|
|
|
|
export function shouldEmitLiveKafkaOtelSpan(envelope = {}) {
|
|
const event = objectValue(envelope.event);
|
|
const context = objectValue(envelope.context);
|
|
const eventType = firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType)?.toLowerCase();
|
|
if (event.terminal === true || eventType === "terminal" || eventType === "terminal_status") return true;
|
|
if (["assistant", "assistant_progress", "assistant_message", "tool", "tool_call"].includes(eventType)) return true;
|
|
if (eventType !== "command_output" && event.type !== "output") return true;
|
|
const sourceSeq = integerValue(context.sourceSeq ?? event.sourceSeq);
|
|
if (Number.isInteger(sourceSeq) && sourceSeq > 0) return sourceSeq % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0;
|
|
const identity = firstText(envelope.sourceEventId, envelope.eventId, envelope.traceId, envelope.hwlabSessionId, envelope.sessionId);
|
|
if (!identity) return false;
|
|
return createHash("sha256").update(identity).digest().readUInt32BE(0) % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0;
|
|
}
|
|
|
|
export function liveKafkaOtelSpanAttributes(envelope = {}, transport = {}) {
|
|
const event = objectValue(envelope.event);
|
|
const context = objectValue(envelope.context);
|
|
const sourceEvent = objectValue(envelope.sourceEvent);
|
|
return {
|
|
businessTraceId: firstText(envelope.traceId, event.traceId),
|
|
hwlabSessionId: firstText(envelope.hwlabSessionId, envelope.sessionId, event.sessionId),
|
|
runId: firstText(envelope.runId, context.runId, event.runId),
|
|
commandId: firstText(envelope.commandId, context.commandId, event.commandId),
|
|
topic: firstText(transport.topic, sourceEvent.topic),
|
|
partition: integerValue(transport.partition ?? sourceEvent.partition),
|
|
offset: firstText(transport.offset, sourceEvent.offset),
|
|
sourceTopic: firstText(transport.sourceTopic),
|
|
sourcePartition: integerValue(transport.sourcePartition),
|
|
sourceOffset: firstText(transport.sourceOffset),
|
|
eventType: firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType),
|
|
terminal: event.terminal === true,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
export function emitLiveKafkaOtelSpan(name, envelope = {}, transport = {}, { env = process.env, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
|
if (typeof otelSpanEmitter !== "function" || !shouldEmitLiveKafkaOtelSpan(envelope)) return { emitted: false, valuesRedacted: true };
|
|
const attributes = liveKafkaOtelSpanAttributes(envelope, transport);
|
|
if (!attributes.businessTraceId) return { emitted: false, valuesRedacted: true };
|
|
try {
|
|
const pending = otelSpanEmitter(name, attributes.businessTraceId, env, { attributes });
|
|
void Promise.resolve(pending).catch(() => undefined);
|
|
} catch {
|
|
// OTel is best effort and must never change the live Kafka delivery path.
|
|
}
|
|
return { emitted: true, valuesRedacted: true };
|
|
}
|
|
|
|
export async function projectAgentRunKafkaMessage(kafkaMessage = {}, { runtimeStore, config, logger = console } = {}) {
|
|
const transport = kafkaTransport(kafkaMessage);
|
|
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
|
|
if (!decoded.ok) {
|
|
await runtimeStore.recordFailedAgentRunKafkaMessage({ transport, sourceEventId: decoded.sourceEventId, envelope: decoded.envelope, error: decoded.error });
|
|
logWarn(logger, "hwlab-kafka-message-dlq", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false });
|
|
return { handled: true, invalid: true, valuesPrinted: false };
|
|
}
|
|
if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) {
|
|
const ignored = await runtimeStore.recordIgnoredAgentRunKafkaMessage({ transport, envelope: decoded.event, reason: "hwlab-correlation-absent" });
|
|
return { handled: true, ignored: true, duplicate: ignored?.duplicate === true, valuesPrinted: false };
|
|
}
|
|
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, {
|
|
source: config.clientId,
|
|
sourceTopic: transport.sourceTopic,
|
|
sourcePartition: transport.sourcePartition,
|
|
sourceOffset: transport.sourceOffset,
|
|
sourceKey: transport.sourceKey,
|
|
inputSha256: transport.inputSha256
|
|
});
|
|
const result = await runtimeStore.commitAgentRunKafkaProjection({
|
|
transport,
|
|
canonicalEvent: decoded.event,
|
|
projectedEvent: projected.event,
|
|
requestMeta: { traceId: projected.traceId, sessionId: projected.sessionId, agentSessionId: projected.sessionId, valuesPrinted: false },
|
|
factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({ event: projected.event, requestMeta: { traceId: projected.traceId, sessionId: projected.sessionId }, previousCheckpoint, projectedSeq, projectedAt }),
|
|
hwlabEvent: projected,
|
|
hwlabTopic: config.hwlabTopic,
|
|
partitionKey: kafkaMessageKey(projected),
|
|
headers: kafkaHeaders(projected)
|
|
});
|
|
if (result.conflict) logWarn(logger, "hwlab-kafka-message-conflict", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: result.diagnostic?.code, valuesPrinted: false });
|
|
return {
|
|
handled: true,
|
|
projected: !result.duplicate && !result.conflict && result.projectionWritten !== false && result.suppressedAfterSeal !== true,
|
|
duplicate: result.duplicate === true,
|
|
conflict: result.conflict === true,
|
|
suppressedAfterSeal: result.suppressedAfterSeal === true,
|
|
projectedSeq: result.projectedSeq,
|
|
sessionId: projected.sessionId,
|
|
traceId: projected.traceId,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
export async function relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner, now = () => new Date().toISOString(), logger = console } = {}) {
|
|
let claimedCount = 0;
|
|
let deliveredCount = 0;
|
|
let retryCount = 0;
|
|
for (let index = 0; index < config.relay.batchSize; index += 1) {
|
|
const item = (await runtimeStore.claimHwlabKafkaOutbox({ owner, leaseMs: config.relay.leaseMs, limit: 1 }))[0];
|
|
if (!item) break;
|
|
claimedCount += 1;
|
|
try {
|
|
await producer.send({ topic: item.topic, timeout: config.relay.sendTimeoutMs, messages: [{ key: item.partitionKey, value: JSON.stringify(item.payload), headers: item.headers }] });
|
|
await runtimeStore.completeHwlabKafkaOutbox(item);
|
|
deliveredCount += 1;
|
|
} catch (error) {
|
|
const retryAt = new Date(Date.parse(now()) + config.relay.retryBackoffMs).toISOString();
|
|
await runtimeStore.retryHwlabKafkaOutbox(item, error, retryAt);
|
|
retryCount += 1;
|
|
logWarn(logger, "hwlab-kafka-outbox-publish-retry", { outboxSeq: item.outboxSeq, eventId: item.eventId, retryAt, message: errorMessage(error), valuesPrinted: false });
|
|
break;
|
|
}
|
|
}
|
|
return { claimedCount, deliveredCount, retryCount, valuesPrinted: false };
|
|
}
|
|
|
|
export function startKafkaHeartbeatLoop(heartbeat, intervalMs) {
|
|
let stopped = false;
|
|
let running = null;
|
|
let heartbeatError = null;
|
|
const pulse = () => {
|
|
if (stopped || running || typeof heartbeat !== "function") return;
|
|
running = Promise.resolve()
|
|
.then(() => heartbeat())
|
|
.catch((error) => { heartbeatError ??= error; })
|
|
.finally(() => { running = null; });
|
|
};
|
|
pulse();
|
|
const timer = setInterval(pulse, intervalMs);
|
|
timer.unref?.();
|
|
return {
|
|
get error() { return heartbeatError; },
|
|
async stop() {
|
|
stopped = true;
|
|
clearInterval(timer);
|
|
if (running) await running;
|
|
}
|
|
};
|
|
}
|
|
|
|
export function projectAgentRunKafkaMessageToHwlabEvent(kafkaMessage = {}, options = {}) {
|
|
const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : "";
|
|
const input = parseJson(valueText);
|
|
if (!input) return null;
|
|
return projectAgentRunKafkaEventToHwlabEvent(input, {
|
|
...options,
|
|
sourceTopic: kafkaMessage.topic,
|
|
sourcePartition: kafkaMessage.partition,
|
|
sourceOffset: kafkaMessage.message?.offset,
|
|
sourceKey: kafkaMessage.message?.key ? Buffer.from(kafkaMessage.message.key).toString("utf8") : null,
|
|
inputSha256: sha256(valueText)
|
|
});
|
|
}
|
|
|
|
export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {}) {
|
|
if (!input || typeof input !== "object" || Array.isArray(input)) return null;
|
|
const run = objectValue(input.run ?? input.agentRun?.run);
|
|
const command = objectValue(input.command ?? input.agentRun?.command);
|
|
const sourceEvent = objectValue(input.event ?? input.payload?.event ?? input.agentRunEvent);
|
|
const payload = objectValue(sourceEvent.payload ?? input.payload);
|
|
const hwlab = objectValue(input.hwlab ?? payload.hwlab);
|
|
const context = objectValue(input.context ?? payload.context);
|
|
const sourceSeq = integerValue(sourceEvent.seq ?? input.seq ?? input.sourceSeq);
|
|
const traceId = firstText(input.traceId, hwlab.traceId, context.traceId, sourceEvent.traceId, payload.traceId, payload.hwlabTraceId, payload.businessTraceId, payload.metadata?.traceId, command.traceId, run.traceId);
|
|
const sourceEventId = firstText(input.eventId, sourceEvent.id);
|
|
const sessionId = firstText(input.hwlabSessionId, payload.hwlabSessionId, hwlab.sessionId, context.hwlabSessionId, input.sessionId, context.sessionId, run.hwlabSessionId, run.sessionId, payload.sessionId);
|
|
const runId = firstText(run.runId, sourceEvent.runId, payload.runId, input.runId);
|
|
const commandId = firstText(command.commandId, sourceEvent.commandId, payload.commandId, input.commandId);
|
|
const event = mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, command, traceId, sessionId, sourceSeq, runId, commandId });
|
|
const producedAt = timestampValue(options.producedAt) || new Date().toISOString();
|
|
return {
|
|
schema: "hwlab.event.v1",
|
|
eventType: "hwlab.trace.event.projected",
|
|
eventId: sourceEventId ? `hwlab:${sourceEventId}` : null,
|
|
sourceEventId,
|
|
source: stringValue(options.source) || DEFAULT_CLIENT_ID,
|
|
producedAt,
|
|
traceId,
|
|
hwlabSessionId: sessionId,
|
|
sessionId,
|
|
runId,
|
|
commandId,
|
|
context: {
|
|
conversationId: firstText(input.conversationId, hwlab.conversationId, context.conversationId, run.conversationId, payload.conversationId),
|
|
threadId: firstText(input.threadId, hwlab.threadId, context.threadId, run.threadId, payload.threadId),
|
|
projectId: firstText(input.projectId, hwlab.projectId, context.projectId, run.projectId, payload.projectId),
|
|
runId,
|
|
commandId,
|
|
sourceSeq,
|
|
sourceEventType: firstText(input.eventType, sourceEvent.eventType),
|
|
agentRunEventType: firstText(sourceEvent.type, payload.type),
|
|
valuesRedacted: true
|
|
},
|
|
event: { ...event, sourceEventId },
|
|
sourceEvent: {
|
|
schema: firstText(input.schema),
|
|
topic: stringValue(options.sourceTopic),
|
|
partition: integerValue(options.sourcePartition),
|
|
offset: stringValue(options.sourceOffset),
|
|
key: stringValue(options.sourceKey),
|
|
sha256: stringValue(options.inputSha256) || sha256(JSON.stringify(input)),
|
|
valuesRedacted: true
|
|
},
|
|
diagnostics: {
|
|
projectedBy: "hwlab-kafka-event-bridge",
|
|
valuesPrinted: false
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
export function decodeCanonicalAgentRunKafkaMessage(kafkaMessage = {}) {
|
|
const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : "";
|
|
const envelope = parseJson(valueText);
|
|
const sourceEventId = firstText(envelope?.eventId, envelope?.event?.id);
|
|
try {
|
|
assertCanonicalAgentRunEvent(envelope);
|
|
return { ok: true, event: envelope, sourceEventId, inputSha256: sha256(valueText), valuesPrinted: false };
|
|
} catch (error) {
|
|
return { ok: false, envelope, sourceEventId, inputSha256: sha256(valueText), error: { code: error?.code ?? "workbench_kafka_message_invalid", message: errorMessage(error), valuesRedacted: true }, valuesPrinted: false };
|
|
}
|
|
}
|
|
|
|
export function decodeAgentRunDebugProjectionMessage(kafkaMessage = {}) {
|
|
const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : "";
|
|
const envelope = parseJson(valueText);
|
|
if (envelope?.schema === "agentrun.event.v1") {
|
|
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
|
|
return decoded.ok
|
|
? { ...decoded, inputKind: "canonical-durable", reconstruction: null }
|
|
: { ...decoded, inputKind: "canonical-invalid", reconstruction: null };
|
|
}
|
|
try {
|
|
assertAgentRunStdioReconstruction(envelope);
|
|
return {
|
|
ok: true,
|
|
event: envelope,
|
|
sourceEventId: null,
|
|
inputSha256: sha256(valueText),
|
|
inputKind: AGENTRUN_STDIO_RECONSTRUCTION_KIND,
|
|
reconstruction: envelope.reconstruction,
|
|
valuesPrinted: false
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
envelope,
|
|
sourceEventId: null,
|
|
inputSha256: sha256(valueText),
|
|
inputKind: "reconstruction-invalid",
|
|
reconstruction: envelope?.reconstruction ?? null,
|
|
error: {
|
|
code: error?.code ?? "workbench_kafka_debug_message_invalid",
|
|
message: errorMessage(error),
|
|
valuesRedacted: true
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
}
|
|
|
|
export function projectAgentRunKafkaMessageToHwlabDebugEvent(kafkaMessage = {}, options = {}) {
|
|
const decoded = decodeAgentRunDebugProjectionMessage(kafkaMessage);
|
|
if (!decoded.ok) return decoded;
|
|
const transport = kafkaTransport(kafkaMessage);
|
|
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, {
|
|
source: stringValue(options.source) || "hwlab-cli-kafka-debug",
|
|
producedAt: options.producedAt,
|
|
sourceTopic: transport.sourceTopic,
|
|
sourcePartition: transport.sourcePartition,
|
|
sourceOffset: transport.sourceOffset,
|
|
sourceKey: transport.sourceKey,
|
|
inputSha256: transport.inputSha256
|
|
});
|
|
if (!stringValue(projected?.traceId) || (!decoded.reconstruction && !stringValue(projected?.sessionId))) {
|
|
return {
|
|
ok: false,
|
|
inputKind: decoded.inputKind,
|
|
error: {
|
|
code: "hwlab_kafka_debug_correlation_missing",
|
|
message: decoded.reconstruction
|
|
? "AgentRun reconstruction debug projection requires traceId lineage."
|
|
: "Canonical AgentRun debug projection requires traceId and sessionId lineage.",
|
|
valuesRedacted: true
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
const debugSessionId = decoded.reconstruction ? stringValue(decoded.event.hwlabSessionId) : stringValue(projected.sessionId);
|
|
const replayId = stringValue(options.replayId);
|
|
const replayIndex = integerValue(options.replayIndex);
|
|
const replayCount = integerValue(options.replayCount);
|
|
const debugEnvelope = {
|
|
...projected,
|
|
schema: DEFAULT_HWLAB_DEBUG_EVENT_TOPIC,
|
|
eventType: "hwlab.trace.event.debug-projected",
|
|
sessionId: debugSessionId,
|
|
hwlabSessionId: debugSessionId,
|
|
event: decoded.reconstruction ? { ...projected.event, sessionId: debugSessionId } : projected.event,
|
|
outboxSeq: null,
|
|
...(decoded.reconstruction ? { eventId: null, sourceEventId: null } : {}),
|
|
...(replayId ? {
|
|
replayId,
|
|
debugReplay: {
|
|
replayId,
|
|
index: replayIndex,
|
|
count: replayCount,
|
|
valuesPrinted: false
|
|
}
|
|
} : {}),
|
|
diagnostics: {
|
|
projectedBy: "hwlab-kafka-event-bridge",
|
|
projectionMode: "source-direct-debug",
|
|
inputKind: decoded.inputKind,
|
|
valuesPrinted: false
|
|
},
|
|
debugLineage: {
|
|
inputSchema: firstText(decoded.event.schema),
|
|
inputEventType: firstText(decoded.event.eventType),
|
|
inputEventId: firstText(decoded.event.eventId),
|
|
inputOutboxSeq: integerValue(decoded.event.outboxSeq),
|
|
inputTraceId: firstText(decoded.event.traceId, decoded.event.event?.payload?.traceId),
|
|
inputSessionId: firstText(decoded.event.sessionId, decoded.event.run?.sessionId),
|
|
inputHwlabSessionId: firstText(decoded.event.hwlabSessionId, decoded.event.run?.hwlabSessionId),
|
|
inputRunId: firstText(decoded.event.runId, decoded.event.run?.runId),
|
|
inputCommandId: firstText(decoded.event.commandId, decoded.event.command?.commandId),
|
|
sourceSeq: integerValue(decoded.event.sourceSeq ?? decoded.event.event?.seq),
|
|
replayId,
|
|
valuesPrinted: false
|
|
},
|
|
...(decoded.reconstruction ? {
|
|
reconstruction: {
|
|
...decoded.reconstruction,
|
|
observedAt: timestampValue(decoded.event.observedAt),
|
|
inputSchema: decoded.event.schema,
|
|
inputEventType: decoded.event.eventType,
|
|
valuesPrinted: false
|
|
}
|
|
} : {}),
|
|
valuesPrinted: false
|
|
};
|
|
return {
|
|
ok: true,
|
|
inputKind: decoded.inputKind,
|
|
envelope: debugEnvelope,
|
|
sourceSeq: integerValue(decoded.event.sourceSeq ?? decoded.event.event?.seq),
|
|
traceId: debugEnvelope.traceId,
|
|
sessionId: debugEnvelope.sessionId,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function assertAgentRunStdioReconstruction(value) {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("hwlab_kafka_debug_json_invalid", "AgentRun reconstruction value must be a JSON object.");
|
|
if (value.schema !== AGENTRUN_STDIO_RECONSTRUCTION_SCHEMA || value.eventType !== "agentrun.event.reconstructed") {
|
|
throw contractError("hwlab_kafka_debug_schema_invalid", "AgentRun debug input must be a canonical committed event or a stdio reconstruction envelope.");
|
|
}
|
|
if (value.eventId !== null || value.outboxSeq !== null) {
|
|
throw contractError("hwlab_kafka_debug_identity_invalid", "Stdio reconstruction must keep eventId and outboxSeq null.");
|
|
}
|
|
for (const [name, field] of [["traceId", value.traceId], ["sessionId", value.sessionId], ["runId", value.runId]]) {
|
|
if (!stringValue(field)) throw contractError("hwlab_kafka_debug_lineage_invalid", `Stdio reconstruction ${name} is required.`);
|
|
}
|
|
if (!Number.isInteger(Number(value.sourceSeq)) || Number(value.sourceSeq) <= 0) {
|
|
throw contractError("hwlab_kafka_debug_sequence_invalid", "Stdio reconstruction sourceSeq must be a positive integer.");
|
|
}
|
|
if (!value.event || typeof value.event !== "object" || Array.isArray(value.event) || value.event.id !== null || Number(value.event.seq) !== Number(value.sourceSeq)) {
|
|
throw contractError("hwlab_kafka_debug_identity_invalid", "Stdio reconstruction event identity must remain null and its seq must match sourceSeq.");
|
|
}
|
|
if (!value.event.payload || typeof value.event.payload !== "object" || Array.isArray(value.event.payload)) {
|
|
throw contractError("hwlab_kafka_debug_schema_invalid", "Stdio reconstruction event payload must be an object.");
|
|
}
|
|
if (value.event.payload.reconstructed !== true || value.event.payload.reconstructionKind !== AGENTRUN_STDIO_RECONSTRUCTION_KIND) {
|
|
throw contractError("hwlab_kafka_debug_lineage_invalid", "Stdio reconstruction payload must disclose its partial reconstruction kind.");
|
|
}
|
|
const reconstruction = value.reconstruction;
|
|
if (!reconstruction || typeof reconstruction !== "object" || Array.isArray(reconstruction)
|
|
|| reconstruction.kind !== AGENTRUN_STDIO_RECONSTRUCTION_KIND
|
|
|| reconstruction.completeLifecycle !== false
|
|
|| reconstruction.originalEventId !== null
|
|
|| reconstruction.originalOutboxSeq !== null) {
|
|
throw contractError("hwlab_kafka_debug_lineage_invalid", "Stdio reconstruction lineage is incomplete or claims durable identity.");
|
|
}
|
|
assertReconstructionIdentityDerivation(value, reconstruction.identityDerivation);
|
|
if (value.valuesPrinted !== false || containsSecretLikeKey(value)) {
|
|
throw contractError("hwlab_kafka_debug_redaction_invalid", "Stdio reconstruction must be redacted and declare valuesPrinted=false.");
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function assertReconstructionIdentityDerivation(value, derivation) {
|
|
if (!derivation || typeof derivation !== "object" || Array.isArray(derivation) || derivation.valuesPrinted !== false
|
|
|| derivation.sourceField !== "sessionId" || derivation.targetField !== "hwlabSessionId") {
|
|
throw contractError("hwlab_kafka_debug_identity_derivation_invalid", "Stdio reconstruction must disclose the session identity derivation contract.");
|
|
}
|
|
const rawSessionId = stringValue(value.sessionId);
|
|
const match = rawSessionId?.match(/^ses_agentrun_([0-9a-fA-F]{8})_([0-9a-fA-F]{4})_([0-9a-fA-F]{4})_([0-9a-fA-F]{4})_([0-9a-fA-F]{12})$/u);
|
|
if (match) {
|
|
const expectedHwlabSessionId = `ses_${match.slice(1).join("-").toLowerCase()}`;
|
|
if (value.hwlabSessionId !== expectedHwlabSessionId || value.run?.hwlabSessionId !== expectedHwlabSessionId
|
|
|| derivation.kind !== "agentrun-session-uuid-to-hwlab-session" || derivation.reversible !== true
|
|
|| derivation.sourcePrefix !== "ses_agentrun_" || derivation.targetPrefix !== "ses_"
|
|
|| derivation.separatorTransform !== "underscore-to-hyphen") {
|
|
throw contractError("hwlab_kafka_debug_identity_derivation_invalid", "Derived hwlabSessionId does not match the disclosed reversible AgentRun session identity transform.");
|
|
}
|
|
return;
|
|
}
|
|
if (value.hwlabSessionId !== null || value.run?.hwlabSessionId !== null
|
|
|| derivation.kind !== "none" || derivation.reversible !== false
|
|
|| derivation.reason !== "source sessionId is not ses_agentrun_<uuid-with-underscores>") {
|
|
throw contractError("hwlab_kafka_debug_identity_derivation_invalid", "Non-reversible AgentRun session identity must keep hwlabSessionId null and disclose why.");
|
|
}
|
|
}
|
|
|
|
function assertCanonicalAgentRunEvent(value) {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("workbench_kafka_json_invalid", "Kafka value must be a JSON object.");
|
|
assertClosedObject(value, ["schema", "eventType", "source", "eventId", "outboxSeq", "sourceSeq", "committedAt", "traceId", "sessionId", "hwlabSessionId", "runId", "commandId", "run", "command", "event", "valuesPrinted"], "AgentRun envelope");
|
|
if (value.schema !== "agentrun.event.v1" || value.eventType !== "agentrun.event.committed") throw contractError("workbench_kafka_schema_invalid", "Kafka value is not a canonical AgentRun committed event.");
|
|
for (const [name, field] of [["eventId", value.eventId], ["runId", value.runId]]) {
|
|
if (!stringValue(field)) throw contractError("workbench_kafka_schema_invalid", `Canonical AgentRun ${name} is required.`);
|
|
}
|
|
for (const [name, field] of [["traceId", value.traceId], ["hwlabSessionId", value.hwlabSessionId]]) {
|
|
if (field !== null && field !== undefined && !stringValue(field)) throw contractError("workbench_kafka_schema_invalid", `Canonical AgentRun ${name} must be a string or null.`);
|
|
}
|
|
if (!Number.isInteger(Number(value.sourceSeq)) || Number(value.sourceSeq) <= 0) throw contractError("workbench_kafka_schema_invalid", "Canonical AgentRun sourceSeq must be a positive integer.");
|
|
if (value.valuesPrinted !== false) throw contractError("workbench_kafka_redaction_invalid", "Canonical AgentRun event must declare valuesPrinted=false.");
|
|
assertClosedObject(value.run, ["runId", "status", "terminalStatus", "failureKind", "tenantId", "projectId", "providerId", "backendProfile", "sessionId", "hwlabSessionId", "conversationId", "threadId", "valuesPrinted"], "AgentRun run");
|
|
if (value.command !== null && value.command !== undefined) assertClosedObject(value.command, ["commandId", "runId", "seq", "type", "state", "payloadHash", "valuesPrinted"], "AgentRun command");
|
|
assertClosedObject(value.event, ["id", "runId", "seq", "type", "payload", "createdAt"], "AgentRun event");
|
|
if (value.event.id !== value.eventId || Number(value.event.seq) !== Number(value.sourceSeq)) throw contractError("workbench_kafka_identity_invalid", "Canonical AgentRun event identity does not match the envelope.");
|
|
if (!value.event.payload || typeof value.event.payload !== "object" || Array.isArray(value.event.payload)) throw contractError("workbench_kafka_schema_invalid", "Canonical AgentRun event payload must be an object.");
|
|
if (containsSecretLikeKey(value)) throw contractError("workbench_kafka_redaction_invalid", "Canonical AgentRun event contains a secret-like field.");
|
|
return value;
|
|
}
|
|
|
|
function assertClosedObject(value, allowed, label) {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("workbench_kafka_schema_invalid", `${label} must be an object.`);
|
|
const unknown = Object.keys(value).filter((key) => !allowed.includes(key));
|
|
if (unknown.length > 0) throw contractError("workbench_kafka_schema_invalid", `${label} contains unsupported fields: ${unknown.join(", ")}`);
|
|
}
|
|
|
|
function containsSecretLikeKey(value, depth = 0) {
|
|
if (depth > 12 || value === null || value === undefined) return false;
|
|
if (Array.isArray(value)) return value.some((item) => containsSecretLikeKey(item, depth + 1));
|
|
if (typeof value !== "object") return false;
|
|
return Object.entries(value).some(([key, entry]) => {
|
|
const normalized = key.replace(/[^a-z0-9]/giu, "").toLowerCase();
|
|
if (/^(apikey|accesstoken|refreshtoken|password|authorization|credentialvalue|secretvalue|privatekey|clientsecret|bearertoken)$/u.test(normalized)) return !redactedSecretMarker(entry);
|
|
if (typeof entry === "string" && /^(?:https?|postgres(?:ql)?):\/\/[^/@\s]+:[^/@\s]+@/iu.test(entry)) return true;
|
|
return containsSecretLikeKey(entry, depth + 1);
|
|
});
|
|
}
|
|
|
|
function redactedSecretMarker(value) {
|
|
if (value === null || value === undefined) return true;
|
|
const normalized = stringValue(value)?.toLowerCase();
|
|
return normalized === "redacted" || normalized === "[redacted]" || normalized === "present" || normalized === "missing";
|
|
}
|
|
|
|
function kafkaTransport(kafkaMessage = {}) {
|
|
const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : "";
|
|
return {
|
|
sourceTopic: stringValue(kafkaMessage.topic),
|
|
sourcePartition: integerValue(kafkaMessage.partition),
|
|
sourceOffset: stringValue(kafkaMessage.message?.offset),
|
|
sourceKey: kafkaMessage.message?.key ? Buffer.from(kafkaMessage.message.key).toString("utf8") : null,
|
|
inputSha256: stringValue(kafkaMessage.inputSha256) || sha256(valueText),
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function requireKafkaProjectorStore(runtimeStore) {
|
|
const required = [
|
|
"commitAgentRunKafkaProjection",
|
|
"recordFailedAgentRunKafkaMessage",
|
|
"recordIgnoredAgentRunKafkaMessage",
|
|
"claimHwlabKafkaOutbox",
|
|
"completeHwlabKafkaOutbox",
|
|
"retryHwlabKafkaOutbox",
|
|
"subscribeWorkbenchProjectionCommits",
|
|
"hwlabKafkaProjectorStatus",
|
|
"assertWorkbenchTransactionalRealtimeReady"
|
|
];
|
|
const missing = required.filter((name) => typeof runtimeStore?.[name] !== "function");
|
|
if (missing.length > 0) throw contractError("hwlab_kafka_projector_store_invalid", `Kafka durable capabilities require a runtime store: ${missing.join(", ")}`);
|
|
}
|
|
|
|
export async function queryKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, traceId = null, sessionId = null, runId = null, commandId = null, limit = DEFAULT_QUERY_LIMIT, scanLimit = null, timeoutMs = DEFAULT_QUERY_TIMEOUT_MS, fromBeginning = true, groupIdPrefix = null, signal = null, kafkaFactory = defaultKafkaFactory } = {}) {
|
|
const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS);
|
|
if (brokers.length === 0) throw new Error("HWLAB_KAFKA_BOOTSTRAP_SERVERS is required for Kafka event queries.");
|
|
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID;
|
|
const resolvedTopic = stringValue(topic) || kafkaTopicForStream(stream, env);
|
|
const maxEvents = Math.max(1, integerValue(limit) || DEFAULT_QUERY_LIMIT);
|
|
const maxScannedRecords = scanLimit === null || scanLimit === undefined
|
|
? null
|
|
: strictPositiveInteger(scanLimit, "scanLimit");
|
|
const budgetMs = Math.max(250, integerValue(timeoutMs) || DEFAULT_QUERY_TIMEOUT_MS);
|
|
const deadlineMs = Date.now() + budgetMs;
|
|
const kafka = kafkaFactory({ brokers, clientId: `${clientId}-query` });
|
|
const endOffsetSnapshot = await kafkaTopicEndOffsetSnapshot(kafka, resolvedTopic, deadlineMs, signal);
|
|
const endOffsetByPartition = new Map(endOffsetSnapshot.offsets.map((entry) => [entry.partition, entry.endOffset]));
|
|
const startOffsetByPartition = new Map(endOffsetSnapshot.offsets.map((entry) => [entry.partition, entry.startOffset]));
|
|
const reachedEndPartitions = new Set(endOffsetSnapshot.offsets.filter(kafkaOffsetPartitionIsEmpty).map((entry) => entry.partition));
|
|
const verifiedStartPartitions = new Set(reachedEndPartitions);
|
|
const resolvedGroupIdPrefix = stringValue(groupIdPrefix) || `${clientId}-query`;
|
|
const groupId = `${resolvedGroupIdPrefix}-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
|
const consumer = kafka.consumer({ groupId, allowAutoTopicCreation: false });
|
|
const events = [];
|
|
const firstScannedOffsetByPartition = new Map();
|
|
const lastScannedOffsetByPartition = new Map();
|
|
let scannedCount = 0;
|
|
let parsedCount = 0;
|
|
let invalidJsonCount = 0;
|
|
let filterRejectedCount = 0;
|
|
let excludedPostBarrierCount = 0;
|
|
let completionReason = "timeout";
|
|
let timer = null;
|
|
let running = false;
|
|
let removeAbortListener = null;
|
|
try {
|
|
if (signal?.aborted === true) {
|
|
completionReason = "aborted";
|
|
return kafkaQueryResult();
|
|
}
|
|
if (remainingKafkaQueryBudget(deadlineMs) <= 0) return kafkaQueryResult();
|
|
if (fromBeginning !== false && endOffsetSnapshot.available && endOffsetSnapshot.offsets.some((entry) => entry.startOffset === null)) {
|
|
completionReason = "retention-start-unavailable";
|
|
return kafkaQueryResult();
|
|
}
|
|
await withinKafkaQueryBudget(consumer.connect(), deadlineMs, "consumer-connect", signal);
|
|
await withinKafkaQueryBudget(consumer.subscribe({ topic: resolvedTopic, fromBeginning: fromBeginning !== false }), deadlineMs, "consumer-subscribe", signal);
|
|
const alreadyAtBoundedEnd = fromBeginning !== false && endOffsetSnapshot.available && reachedEndPartitions.size === endOffsetByPartition.size;
|
|
if (alreadyAtBoundedEnd) completionReason = "end-offset";
|
|
if (alreadyAtBoundedEnd) return kafkaQueryResult();
|
|
await new Promise((resolve, reject) => {
|
|
let finished = false;
|
|
const finish = (reason = "timeout") => {
|
|
if (finished) return;
|
|
finished = true;
|
|
completionReason = reason;
|
|
resolve(reason);
|
|
};
|
|
timer = setTimeout(() => finish("timeout"), Math.max(1, remainingKafkaQueryBudget(deadlineMs)));
|
|
timer.unref?.();
|
|
if (typeof signal?.addEventListener === "function") {
|
|
const onAbort = () => finish("aborted");
|
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
removeAbortListener = () => signal.removeEventListener?.("abort", onAbort);
|
|
}
|
|
running = true;
|
|
consumer.run({
|
|
eachMessage: async ({ topic: messageTopic, partition, message }) => {
|
|
if (finished) return;
|
|
const endOffset = endOffsetByPartition.get(partition);
|
|
if (fromBeginning !== false && endOffsetSnapshot.available && !firstScannedOffsetByPartition.has(partition)) {
|
|
const actualStartOffset = stringValue(message.offset);
|
|
const expectedStartOffset = startOffsetByPartition.get(partition);
|
|
firstScannedOffsetByPartition.set(partition, actualStartOffset);
|
|
if (!endOffsetByPartition.has(partition) || !expectedStartOffset || actualStartOffset !== expectedStartOffset) {
|
|
finish("retention-start-moved");
|
|
return;
|
|
}
|
|
verifiedStartPartitions.add(partition);
|
|
}
|
|
if (fromBeginning !== false && endOffset && kafkaOffsetAtOrBeyond(message.offset, endOffset)) {
|
|
excludedPostBarrierCount += 1;
|
|
reachedEndPartitions.add(partition);
|
|
if (endOffsetSnapshot.available && reachedEndPartitions.size === endOffsetByPartition.size) {
|
|
finish(verifiedStartPartitions.size === endOffsetByPartition.size ? "end-offset" : "retention-start-unverified");
|
|
}
|
|
return;
|
|
}
|
|
scannedCount += 1;
|
|
lastScannedOffsetByPartition.set(partition, stringValue(message.offset));
|
|
const valueText = message.value ? Buffer.from(message.value).toString("utf8") : "";
|
|
const value = parseJson(valueText);
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) invalidJsonCount += 1;
|
|
else {
|
|
parsedCount += 1;
|
|
if (!eventMatchesFilters(value, { traceId, sessionId, runId, commandId })) filterRejectedCount += 1;
|
|
else {
|
|
events.push({
|
|
topic: messageTopic,
|
|
partition,
|
|
offset: message.offset,
|
|
key: message.key ? Buffer.from(message.key).toString("utf8") : null,
|
|
timestamp: message.timestamp ?? null,
|
|
valueSha256: sha256(valueText),
|
|
value
|
|
});
|
|
}
|
|
}
|
|
if (fromBeginning !== false && endOffset && kafkaOffsetReached(message.offset, endOffset)) reachedEndPartitions.add(partition);
|
|
if (endOffsetSnapshot.available && reachedEndPartitions.size === endOffsetByPartition.size) {
|
|
return finish(verifiedStartPartitions.size === endOffsetByPartition.size ? "end-offset" : "retention-start-unverified");
|
|
}
|
|
if (events.length >= maxEvents) finish("limit");
|
|
else if (maxScannedRecords !== null && scannedCount >= maxScannedRecords) finish("scan-limit");
|
|
}
|
|
}).catch(reject);
|
|
});
|
|
} catch (error) {
|
|
if (error?.code === "kafka_query_aborted") completionReason = "aborted";
|
|
else throw error;
|
|
} finally {
|
|
if (timer) clearTimeout(timer);
|
|
removeAbortListener?.();
|
|
if (running) await boundedKafkaPromise(consumer.stop(), remainingKafkaQueryCleanupBudget(deadlineMs)).catch(() => undefined);
|
|
await boundedDisconnect(consumer, remainingKafkaQueryCleanupBudget(deadlineMs));
|
|
}
|
|
return kafkaQueryResult();
|
|
|
|
function kafkaQueryResult() {
|
|
const retentionStartVerified = endOffsetSnapshot.available && verifiedStartPartitions.size === endOffsetByPartition.size;
|
|
const complete = completionReason === "end-offset" && retentionStartVerified;
|
|
return {
|
|
ok: true,
|
|
stream,
|
|
topic: resolvedTopic,
|
|
groupId,
|
|
count: events.length,
|
|
scannedCount,
|
|
parsedCount,
|
|
matchedCount: events.length,
|
|
invalidJsonCount,
|
|
filterRejectedCount,
|
|
excludedPostBarrierCount,
|
|
completionReason,
|
|
completion: {
|
|
reason: completionReason,
|
|
complete,
|
|
barrierReached: complete,
|
|
retentionStartVerified,
|
|
retentionStartMoved: completionReason === "retention-start-moved",
|
|
timeout: completionReason === "timeout",
|
|
matchedEventLimitReached: completionReason === "limit",
|
|
scanLimitReached: completionReason === "scan-limit",
|
|
aborted: completionReason === "aborted",
|
|
valuesRedacted: true
|
|
},
|
|
reachedEndOffsets: complete,
|
|
endOffsets: endOffsetSnapshot.offsets,
|
|
endOffsetsAvailable: endOffsetSnapshot.available,
|
|
endOffsetsError: endOffsetSnapshot.error,
|
|
firstScannedOffsets: [...firstScannedOffsetByPartition.entries()].map(([partition, offset]) => ({ partition, offset })).sort((left, right) => left.partition - right.partition),
|
|
lastScannedOffsets: [...lastScannedOffsetByPartition.entries()].map(([partition, offset]) => ({ partition, offset })).sort((left, right) => left.partition - right.partition),
|
|
limit: maxEvents,
|
|
scanLimit: maxScannedRecords,
|
|
timeoutMs: budgetMs,
|
|
filters: compactObject({ traceId, sessionId, runId, commandId }),
|
|
events,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
}
|
|
|
|
async function kafkaTopicEndOffsetSnapshot(kafka, topic, deadlineMs, signal = null) {
|
|
if (typeof kafka?.admin !== "function") return { available: false, offsets: [], error: { code: "kafka_admin_unavailable", valuesRedacted: true } };
|
|
const admin = kafka.admin();
|
|
try {
|
|
await withinKafkaQueryBudget(admin.connect(), deadlineMs, "admin-connect", signal);
|
|
const fetched = await withinKafkaQueryBudget(admin.fetchTopicOffsets(topic), deadlineMs, "fetch-topic-offsets", signal);
|
|
const offsets = Array.isArray(fetched)
|
|
? fetched.map((entry) => ({
|
|
partition: integerValue(entry.partition),
|
|
startOffset: kafkaOffsetValue(entry.low),
|
|
endOffset: kafkaOffsetValue(entry.high ?? entry.offset)
|
|
})).filter((entry) => entry.partition !== null && entry.endOffset !== null)
|
|
: [];
|
|
if (offsets.length === 0) return { available: false, offsets: [], error: { code: "kafka_end_offsets_empty", valuesRedacted: true } };
|
|
return {
|
|
available: true,
|
|
offsets,
|
|
error: null
|
|
};
|
|
} catch (error) {
|
|
return { available: false, offsets: [], error: { code: error?.code ?? "kafka_end_offsets_failed", message: errorMessage(error), valuesRedacted: true } };
|
|
} finally {
|
|
await boundedDisconnect(admin, remainingKafkaQueryCleanupBudget(deadlineMs));
|
|
}
|
|
}
|
|
|
|
function kafkaOffsetPartitionIsEmpty(entry) {
|
|
if (entry.startOffset !== null) return entry.startOffset === entry.endOffset;
|
|
return entry.endOffset === "0";
|
|
}
|
|
|
|
function remainingKafkaQueryBudget(deadlineMs) {
|
|
return Math.max(0, deadlineMs - Date.now());
|
|
}
|
|
|
|
function remainingKafkaQueryCleanupBudget(deadlineMs) {
|
|
return Math.max(1, remainingKafkaQueryBudget(deadlineMs));
|
|
}
|
|
|
|
async function withinKafkaQueryBudget(promise, deadlineMs, stage, signal = null) {
|
|
const remainingMs = remainingKafkaQueryBudget(deadlineMs);
|
|
if (remainingMs <= 0) throw contractError("kafka_query_timeout", `Kafka query timed out before ${stage}.`);
|
|
if (signal?.aborted === true) throw contractError("kafka_query_aborted", `Kafka query was aborted before ${stage}.`);
|
|
let timer = null;
|
|
let removeAbortListener = null;
|
|
try {
|
|
const contenders = [
|
|
promise,
|
|
new Promise((_, reject) => {
|
|
timer = setTimeout(() => reject(contractError("kafka_query_timeout", `Kafka query timed out during ${stage}.`)), remainingMs);
|
|
timer.unref?.();
|
|
})
|
|
];
|
|
if (typeof signal?.addEventListener === "function") {
|
|
contenders.push(new Promise((_, reject) => {
|
|
const onAbort = () => reject(contractError("kafka_query_aborted", `Kafka query was aborted during ${stage}.`));
|
|
signal.addEventListener("abort", onAbort, { once: true });
|
|
removeAbortListener = () => signal.removeEventListener?.("abort", onAbort);
|
|
}));
|
|
}
|
|
return await Promise.race(contenders);
|
|
} finally {
|
|
if (timer) clearTimeout(timer);
|
|
removeAbortListener?.();
|
|
}
|
|
}
|
|
|
|
function kafkaOffsetReached(offset, endOffset) {
|
|
try {
|
|
return BigInt(String(offset)) + 1n >= BigInt(String(endOffset));
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function kafkaOffsetAtOrBeyond(offset, endOffset) {
|
|
try {
|
|
return BigInt(String(offset)) >= BigInt(String(endOffset));
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
export async function openKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, groupIdPrefix = null, traceId = null, sessionId = null, runId = null, commandId = null, replayId = null, offsetRange = null, fromBeginning = false, onEvent, onRecord = null, onBarrierReady = null, onBarrierComplete = null, onError = null, kafkaFactory = defaultKafkaFactory } = {}) {
|
|
if (typeof onEvent !== "function") throw new Error("onEvent callback is required for Kafka event streaming.");
|
|
const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS);
|
|
if (brokers.length === 0) throw new Error("HWLAB_KAFKA_BOOTSTRAP_SERVERS is required for Kafka event streams.");
|
|
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID;
|
|
const resolvedTopic = stringValue(topic) || kafkaTopicForStream(stream, env);
|
|
const filters = compactObject({ traceId, sessionId, runId, commandId, replayId });
|
|
const resolvedOffsetRange = normalizeKafkaOffsetRange(offsetRange);
|
|
const kafka = kafkaFactory({ brokers, clientId: `${clientId}-debug-sse` });
|
|
const resolvedGroupIdPrefix = stringValue(groupIdPrefix) || `${clientId}-debug-sse`;
|
|
const groupId = `${resolvedGroupIdPrefix}-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
|
const consumer = kafka.consumer({ groupId, allowAutoTopicCreation: false });
|
|
let running = false;
|
|
let stopped = false;
|
|
let seekApplied = false;
|
|
let barrierCompleted = false;
|
|
const seenRecordOffsets = new Set();
|
|
await consumer.connect();
|
|
await consumer.subscribe({ topic: resolvedTopic, fromBeginning: fromBeginning === true });
|
|
running = true;
|
|
try {
|
|
await consumer.run({
|
|
eachMessage: async ({ topic: messageTopic, partition, message }) => {
|
|
if (stopped) return;
|
|
if (resolvedOffsetRange && !seekApplied) return;
|
|
const valueText = message.value ? Buffer.from(message.value).toString("utf8") : "";
|
|
const value = parseJson(valueText);
|
|
const parsed = Boolean(value && typeof value === "object" && !Array.isArray(value));
|
|
const inOffsetRange = kafkaRecordInOffsetRange(partition, message.offset, resolvedOffsetRange);
|
|
const filterMatches = parsed ? eventFilterResults(value, filters) : {};
|
|
const offsetKey = `${messageTopic}:${partition}:${message.offset}`;
|
|
const duplicate = Boolean(resolvedOffsetRange && seenRecordOffsets.has(offsetKey));
|
|
if (resolvedOffsetRange) seenRecordOffsets.add(offsetKey);
|
|
const matched = !duplicate && parsed && inOffsetRange && Object.values(filterMatches).every(Boolean);
|
|
if (typeof onRecord === "function") {
|
|
await onRecord({
|
|
topic: messageTopic,
|
|
partition,
|
|
offset: message.offset,
|
|
parsed,
|
|
duplicate,
|
|
inOffsetRange,
|
|
filterMatches,
|
|
matched,
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
if (matched) {
|
|
await onEvent({
|
|
topic: messageTopic,
|
|
partition,
|
|
offset: message.offset,
|
|
key: message.key ? Buffer.from(message.key).toString("utf8") : null,
|
|
timestamp: message.timestamp ?? null,
|
|
value
|
|
});
|
|
}
|
|
if (!barrierCompleted && kafkaRecordCompletesOffsetRange(partition, message.offset, resolvedOffsetRange)) {
|
|
barrierCompleted = true;
|
|
if (typeof onBarrierComplete === "function") {
|
|
await onBarrierComplete({
|
|
topic: messageTopic,
|
|
partition,
|
|
offset: message.offset,
|
|
offsetRange: resolvedOffsetRange,
|
|
valuesPrinted: false
|
|
});
|
|
}
|
|
}
|
|
}
|
|
});
|
|
if (resolvedOffsetRange) {
|
|
if (typeof consumer.seek !== "function") throw new Error("Kafka consumer seek is required for correlated replay offset barriers.");
|
|
await consumer.seek({ topic: resolvedTopic, partition: resolvedOffsetRange.partition, offset: resolvedOffsetRange.firstOffset });
|
|
seekApplied = true;
|
|
if (typeof onBarrierReady === "function") {
|
|
await onBarrierReady({ topic: resolvedTopic, offsetRange: resolvedOffsetRange, seekApplied: true, valuesPrinted: false });
|
|
}
|
|
}
|
|
} catch (error) {
|
|
if (typeof onError === "function") onError(error);
|
|
stopped = true;
|
|
if (running) await consumer.stop().catch(() => undefined);
|
|
await boundedDisconnect(consumer);
|
|
throw error;
|
|
}
|
|
return {
|
|
ok: true,
|
|
stream,
|
|
topic: resolvedTopic,
|
|
groupId,
|
|
filters,
|
|
offsetRange: resolvedOffsetRange,
|
|
seekApplied,
|
|
async stop() {
|
|
stopped = true;
|
|
if (running) await consumer.stop().catch(() => undefined);
|
|
await boundedDisconnect(consumer);
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, command, traceId, sessionId, sourceSeq, runId, commandId }) {
|
|
const type = firstText(sourceEvent.type, payload.type, input.eventType) || "event";
|
|
const base = {
|
|
traceId,
|
|
sessionId,
|
|
source: "agentrun.kafka",
|
|
sourceSeq,
|
|
runId,
|
|
commandId,
|
|
attemptId: firstText(command.attemptId, sourceEvent.attemptId, payload.attemptId),
|
|
runnerId: firstText(command.runnerId, sourceEvent.runnerId, payload.runnerId),
|
|
backend: firstText(run.backendProfile, payload.backendProfile, payload.providerProfile),
|
|
agentRunEventType: type,
|
|
createdAt: timestampValue(sourceEvent.createdAt ?? input.producedAt),
|
|
valuesPrinted: false
|
|
};
|
|
if (type === "user_message") {
|
|
const userMessageId = firstText(payload.userMessageId);
|
|
const userText = textPayload(payload, "user message");
|
|
return {
|
|
...base,
|
|
type: "user",
|
|
eventType: "user",
|
|
status: "sent",
|
|
label: "agentrun:user:message",
|
|
message: userText,
|
|
text: userText,
|
|
role: "user",
|
|
messageId: userMessageId,
|
|
userMessageId,
|
|
targetTraceId: firstText(payload.targetTraceId),
|
|
userSource: firstText(payload.source),
|
|
terminal: false
|
|
};
|
|
}
|
|
if (type === "assistant_message") {
|
|
const authoritative = payload.replyAuthority === true || payload.final === true;
|
|
const assistantText = textPayload(payload, "assistant message");
|
|
return {
|
|
...base,
|
|
type: "assistant",
|
|
eventType: "assistant",
|
|
status: "running",
|
|
label: "agentrun:assistant:message",
|
|
message: assistantText,
|
|
text: assistantText,
|
|
assistantText,
|
|
finalResponse: authoritative ? { text: assistantText, status: "completed", traceId, source: "agentrun.kafka", valuesPrinted: false } : null,
|
|
itemId: firstText(payload.itemId),
|
|
assistantSource: firstText(payload.source),
|
|
messageIndex: integerValue(payload.messageIndex),
|
|
messageCount: integerValue(payload.messageCount),
|
|
progress: payload.progress === true,
|
|
progressFlush: payload.progressFlush === true,
|
|
durableText: payload.durableText === true,
|
|
replyAuthority: payload.replyAuthority === true,
|
|
final: payload.final === true,
|
|
textBytes: integerValue(payload.textBytes),
|
|
textTruncated: payload.textTruncated === true,
|
|
outputBytes: integerValue(payload.outputBytes),
|
|
outputTruncated: payload.outputTruncated === true,
|
|
terminal: false
|
|
};
|
|
}
|
|
if (type === "assistant_progress") {
|
|
const assistantText = textPayload(payload, "assistant progress");
|
|
return {
|
|
...base,
|
|
type: "assistant",
|
|
eventType: "assistant_progress",
|
|
status: "running",
|
|
label: "agentrun:assistant:progress",
|
|
message: assistantText,
|
|
text: assistantText,
|
|
assistantText,
|
|
finalResponse: null,
|
|
itemId: firstText(payload.itemId),
|
|
assistantSource: firstText(payload.source),
|
|
messageIndex: integerValue(payload.messageIndex),
|
|
messageCount: integerValue(payload.messageCount),
|
|
progress: payload.progress === true,
|
|
progressFlush: payload.progressFlush === true,
|
|
durableText: false,
|
|
replyAuthority: false,
|
|
final: false,
|
|
textBytes: integerValue(payload.textBytes),
|
|
textTruncated: payload.textTruncated === true,
|
|
outputBytes: integerValue(payload.outputBytes),
|
|
outputTruncated: payload.outputTruncated === true,
|
|
terminal: false
|
|
};
|
|
}
|
|
if (type === "backend_status") {
|
|
const phase = firstText(payload.phase) || "status";
|
|
return {
|
|
...base,
|
|
type: "backend",
|
|
eventType: "backend",
|
|
status: "running",
|
|
label: `agentrun:backend:${phase}`,
|
|
message: textPayload(payload, phase),
|
|
itemId: firstText(payload.itemId),
|
|
finalSeal: payload.finalSeal === true,
|
|
replyRef: payload.replyRef && typeof payload.replyRef === "object" && !Array.isArray(payload.replyRef) ? payload.replyRef : null,
|
|
terminal: false
|
|
};
|
|
}
|
|
if (type === "terminal_status") {
|
|
const terminalStatus = firstText(payload.terminalStatus, sourceEvent.terminalStatus, payload.status, sourceEvent.status) || "failed";
|
|
const normalized = terminalStatus === "cancelled" ? "canceled" : terminalStatus;
|
|
return { ...base, type: "result", eventType: "terminal", status: normalized, label: `agentrun:terminal:${terminalStatus}`, errorCode: firstText(payload.failureKind, payload.errorCode, sourceEvent.failureKind, sourceEvent.errorCode), failureKind: firstText(payload.failureKind, sourceEvent.failureKind), message: textPayload({ ...sourceEvent, ...payload }, `AgentRun terminal status ${terminalStatus}`), terminal: true };
|
|
}
|
|
if (type === "tool_call") {
|
|
const toolName = firstText(payload.toolName, payload.name, payload.method) || "tool";
|
|
return {
|
|
...base,
|
|
type: "tool",
|
|
eventType: "tool",
|
|
status: firstText(payload.status) || "running",
|
|
label: `agentrun:tool:${toolName}`,
|
|
message: textPayload(payload, `AgentRun tool ${toolName}`),
|
|
toolName,
|
|
toolType: firstText(payload.type),
|
|
method: firstText(payload.method),
|
|
itemId: firstText(payload.itemId),
|
|
command: firstText(payload.command, payload.commandLine),
|
|
cwd: firstText(payload.cwd),
|
|
processId: firstText(payload.processId),
|
|
exitCode: integerValue(payload.exitCode),
|
|
durationMs: integerValue(payload.durationMs),
|
|
outputSummary: firstText(payload.outputSummary, payload.summary?.text),
|
|
stdoutSummary: firstText(payload.stdoutSummary),
|
|
stderrSummary: firstText(payload.stderrSummary),
|
|
commandTruncated: payload.commandTruncated === true,
|
|
outputBytes: integerValue(payload.outputBytes ?? payload.summary?.outputBytes),
|
|
outputTruncated: payload.outputTruncated === true || payload.summary?.outputTruncated === true,
|
|
...(Array.isArray(payload.changes) ? { changes: payload.changes, fileCount: integerValue(payload.fileCount) } : {}),
|
|
terminal: false
|
|
};
|
|
}
|
|
if (type === "diff") {
|
|
return {
|
|
...base,
|
|
type: "diff",
|
|
eventType: "diff",
|
|
status: firstText(payload.status) || "updated",
|
|
label: "agentrun:diff",
|
|
message: firstText(payload.diff) || "",
|
|
diff: firstText(payload.diff) || "",
|
|
threadId: firstText(payload.threadId),
|
|
turnId: firstText(payload.turnId),
|
|
terminal: false
|
|
};
|
|
}
|
|
if (type === "command_output") {
|
|
const stream = payload.stream === "stderr" ? "stderr" : "stdout";
|
|
return { ...base, type: "output", eventType: "status", status: "running", label: `agentrun:output:${stream}`, stream, message: textPayload(payload, ""), outputTruncated: payload.outputTruncated === true };
|
|
}
|
|
if (type === "error") {
|
|
const failureKind = firstText(payload.failureKind, payload.errorCode, sourceEvent.failureKind, sourceEvent.errorCode) || "backend";
|
|
return { ...base, type: "error", eventType: "error", status: "failed", label: `agentrun:error:${failureKind}`, errorCode: failureKind, failureKind, message: textPayload(payload, "AgentRun error"), terminal: false };
|
|
}
|
|
return { ...base, type: "backend", eventType: "backend", status: "running", label: `agentrun:event:${type}`, message: textPayload(payload, firstText(input.eventType, type) || "AgentRun event") };
|
|
}
|
|
|
|
function eventMatchesFilters(value, filters) {
|
|
return Object.values(eventFilterResults(value, filters)).every(Boolean);
|
|
}
|
|
|
|
function eventFilterResults(value, filters) {
|
|
const results = {};
|
|
for (const [name, expected] of Object.entries(filters)) {
|
|
const needle = stringValue(expected);
|
|
if (!needle) continue;
|
|
results[name] = eventFieldCandidates(value, name).includes(needle);
|
|
}
|
|
return results;
|
|
}
|
|
|
|
function eventFieldCandidates(value, name) {
|
|
const event = objectValue(value.event);
|
|
const context = objectValue(value.context);
|
|
const run = objectValue(value.run);
|
|
const command = objectValue(value.command);
|
|
const payload = objectValue(value.payload);
|
|
const metadata = objectValue(value.metadata ?? value.meta);
|
|
const trace = objectValue(value.trace);
|
|
const ids = objectValue(value.ids);
|
|
const stdio = objectValue(value.stdio ?? value.frame ?? value.codexStdio);
|
|
const sourceEvent = objectValue(value.sourceEvent);
|
|
const debugReplay = objectValue(value.debugReplay);
|
|
const debugLineage = objectValue(value.debugLineage);
|
|
const nestedEvent = objectValue(value.event?.sourceEvent ?? value.agentRunEvent ?? payload.event);
|
|
const nestedPayload = objectValue(nestedEvent.payload);
|
|
if (name === "traceId") return textCandidates(value.traceId, value.trace_id, event.traceId, event.trace_id, context.traceId, context.trace_id, sourceEvent.traceId, sourceEvent.trace_id, nestedEvent.traceId, nestedEvent.trace_id, nestedPayload.traceId, nestedPayload.trace_id, payload.traceId, payload.trace_id, metadata.traceId, metadata.trace_id, trace.traceId, trace.trace_id, ids.traceId, ids.trace_id, stdio.traceId, stdio.trace_id);
|
|
if (name === "sessionId") return textCandidates(value.sessionId, value.session_id, value.hwlabSessionId, value.hwlab_session_id, event.sessionId, event.session_id, event.hwlabSessionId, event.hwlab_session_id, context.sessionId, context.session_id, context.hwlabSessionId, context.hwlab_session_id, run.sessionId, run.session_id, run.hwlabSessionId, run.hwlab_session_id, nestedPayload.sessionId, nestedPayload.session_id, nestedPayload.hwlabSessionId, nestedPayload.hwlab_session_id, payload.sessionId, payload.session_id, payload.hwlabSessionId, payload.hwlab_session_id, metadata.sessionId, metadata.session_id, metadata.hwlabSessionId, metadata.hwlab_session_id, ids.sessionId, ids.session_id, ids.hwlabSessionId, ids.hwlab_session_id, stdio.sessionId, stdio.session_id, stdio.hwlabSessionId, stdio.hwlab_session_id);
|
|
if (name === "runId") return textCandidates(value.runId, value.run_id, event.runId, event.run_id, context.runId, context.run_id, run.runId, run.run_id, nestedEvent.runId, nestedEvent.run_id, nestedPayload.runId, nestedPayload.run_id, payload.runId, payload.run_id, metadata.runId, metadata.run_id, ids.runId, ids.run_id, stdio.runId, stdio.run_id);
|
|
if (name === "commandId") return textCandidates(value.commandId, value.command_id, event.commandId, event.command_id, context.commandId, context.command_id, command.commandId, command.command_id, nestedEvent.commandId, nestedEvent.command_id, nestedPayload.commandId, nestedPayload.command_id, payload.commandId, payload.command_id, metadata.commandId, metadata.command_id, ids.commandId, ids.command_id, stdio.commandId, stdio.command_id);
|
|
if (name === "replayId") return textCandidates(value.replayId, value.replay_id, debugReplay.replayId, debugReplay.replay_id, debugLineage.replayId, debugLineage.replay_id);
|
|
return [];
|
|
}
|
|
|
|
function normalizeKafkaOffsetRange(value) {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) return null;
|
|
const partition = integerValue(value.partition);
|
|
const firstOffset = kafkaOffsetValue(value.firstOffset ?? value.startOffset);
|
|
const lastOffset = kafkaOffsetValue(value.lastOffset ?? value.endOffset);
|
|
if (partition === null || firstOffset === null || lastOffset === null || BigInt(firstOffset) > BigInt(lastOffset)) return null;
|
|
return { partition, firstOffset, lastOffset };
|
|
}
|
|
|
|
function kafkaRecordInOffsetRange(partitionValue, offsetValue, range) {
|
|
if (!range) return true;
|
|
const partition = integerValue(partitionValue);
|
|
const offset = kafkaOffsetValue(offsetValue);
|
|
return partition === range.partition && offset !== null && BigInt(offset) >= BigInt(range.firstOffset) && BigInt(offset) <= BigInt(range.lastOffset);
|
|
}
|
|
|
|
function kafkaRecordCompletesOffsetRange(partitionValue, offsetValue, range) {
|
|
if (!range) return false;
|
|
const partition = integerValue(partitionValue);
|
|
const offset = kafkaOffsetValue(offsetValue);
|
|
return partition === range.partition && offset !== null && BigInt(offset) === BigInt(range.lastOffset);
|
|
}
|
|
|
|
function kafkaOffsetValue(value) {
|
|
const text = stringValue(value);
|
|
if (!text || !/^\d+$/u.test(text)) return null;
|
|
try { return BigInt(text).toString(); } catch { return null; }
|
|
}
|
|
|
|
function kafkaTopicForStream(stream, env) {
|
|
const normalized = stringValue(stream) || "hwlab";
|
|
if (normalized === "stdio" || normalized === "codex-stdio") return stringValue(env.HWLAB_KAFKA_STDIO_TOPIC ?? env.AGENTRUN_KAFKA_STDIO_TOPIC) || DEFAULT_STDIO_TOPIC;
|
|
if (normalized === "agentrun") return stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC) || DEFAULT_AGENTRUN_EVENT_TOPIC;
|
|
if (normalized === "hwlab") return stringValue(env.HWLAB_KAFKA_EVENT_TOPIC) || DEFAULT_HWLAB_EVENT_TOPIC;
|
|
return normalized;
|
|
}
|
|
|
|
function defaultKafkaFactory(config) {
|
|
return new Kafka({ clientId: config.clientId, brokers: config.brokers, logLevel: logLevel.NOTHING });
|
|
}
|
|
|
|
async function boundedDisconnect(consumer, timeoutMs = 2000) {
|
|
return boundedKafkaPromise(consumer.disconnect(), timeoutMs);
|
|
}
|
|
|
|
async function boundedKafkaPromise(promise, timeoutMs) {
|
|
let timer = null;
|
|
try {
|
|
await Promise.race([
|
|
promise,
|
|
new Promise((resolve) => {
|
|
timer = setTimeout(resolve, Math.max(1, timeoutMs));
|
|
timer.unref?.();
|
|
})
|
|
]);
|
|
} finally {
|
|
if (timer) clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
export function buildHwlabKafkaProducerMessage(projected) {
|
|
return {
|
|
key: kafkaMessageKey(projected),
|
|
value: JSON.stringify(projected),
|
|
headers: kafkaHeaders(projected)
|
|
};
|
|
}
|
|
|
|
function kafkaMessageKey(projected) {
|
|
return firstText(projected.sessionId, projected.traceId, projected.context?.commandId, projected.context?.runId) || "hwlab-event";
|
|
}
|
|
|
|
function kafkaHeaders(projected) {
|
|
return {
|
|
"x-trace-id": projected.traceId ?? "",
|
|
"x-session-id": projected.sessionId ?? "",
|
|
"x-run-id": projected.context?.runId ?? "",
|
|
"x-command-id": projected.context?.commandId ?? "",
|
|
"x-values-printed": "false",
|
|
...(projected.replayId ? { "x-debug-replay-id": projected.replayId } : {})
|
|
};
|
|
}
|
|
|
|
function textPayload(payload, fallback) {
|
|
return firstText(payload.text, payload.message, payload.summary, payload.output, payload.errorMessage, payload.reason) || fallback;
|
|
}
|
|
|
|
function truthy(value) {
|
|
return TRUE_VALUES.has(String(value ?? "").trim().toLowerCase());
|
|
}
|
|
|
|
function csv(value) {
|
|
return String(value ?? "").split(",").map((entry) => entry.trim()).filter(Boolean);
|
|
}
|
|
|
|
function parseJson(text) {
|
|
try {
|
|
return JSON.parse(text);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function objectValue(value) {
|
|
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
}
|
|
|
|
function firstText(...values) {
|
|
for (const value of values) {
|
|
const text = stringValue(value);
|
|
if (text) return text;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function textCandidates(...values) {
|
|
return values.map(stringValue).filter(Boolean);
|
|
}
|
|
|
|
function stringValue(value) {
|
|
const text = typeof value === "string" ? value.trim() : value === null || value === undefined ? "" : String(value).trim();
|
|
return text.length > 0 ? text : null;
|
|
}
|
|
|
|
function integerValue(value) {
|
|
if (value === null || value === undefined || (typeof value === "string" && value.trim() === "")) return null;
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? Math.floor(parsed) : null;
|
|
}
|
|
|
|
function nextKafkaOffset(value) {
|
|
try {
|
|
return (BigInt(String(value)) + 1n).toString();
|
|
} catch {
|
|
throw contractError("workbench_kafka_offset_invalid", "Kafka source offset must be an integer string.");
|
|
}
|
|
}
|
|
|
|
function strictPositiveInteger(value, name) {
|
|
const parsed = Number(value);
|
|
if (!Number.isInteger(parsed) || parsed <= 0) throw contractError("hwlab_kafka_projector_config_invalid", `${name} must be a positive integer.`);
|
|
return parsed;
|
|
}
|
|
|
|
function contractError(code, message) {
|
|
const error = new Error(message);
|
|
error.code = code;
|
|
return error;
|
|
}
|
|
|
|
function timestampValue(value) {
|
|
const text = stringValue(value);
|
|
if (!text) return null;
|
|
const ms = Date.parse(text);
|
|
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
|
|
}
|
|
|
|
function sha256(text) {
|
|
return createHash("sha256").update(String(text ?? "")).digest("hex");
|
|
}
|
|
|
|
function compactObject(value) {
|
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => stringValue(entry)));
|
|
}
|
|
|
|
function errorMessage(error) {
|
|
return error instanceof Error ? error.message : String(error);
|
|
}
|
|
|
|
function logInfo(logger, code, details) {
|
|
logger?.log?.(JSON.stringify({ code, component: "hwlab-kafka-event-bridge", ...details, valuesPrinted: false }));
|
|
}
|
|
|
|
function logWarn(logger, code, details) {
|
|
logger?.warn?.(JSON.stringify({ code, component: "hwlab-kafka-event-bridge", ...details, valuesPrinted: false }));
|
|
}
|