root
1277 lines
64 KiB
TypeScript
1277 lines
64 KiB
TypeScript
// SPEC: PJ2026-0104010803 Workbench事件流可见性 draft-2026-07-09-p0-kafka-authority.
|
|
// Responsibility: subscribe AgentRun Kafka events, project them into HWLAB Kafka events, and provide bounded Kafka stream queries for diagnostics.
|
|
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
|
|
import { Kafka, logLevel } from "kafkajs";
|
|
import { emitCodeAgentOtelSpan } from "./otel-trace.ts";
|
|
import { buildWorkbenchProjectionEventFacts } from "./workbench-projection-writer.ts";
|
|
import {
|
|
WORKBENCH_REALTIME_CAPABILITY_ENVS,
|
|
workbenchRealtimeCapabilities
|
|
} from "./workbench-realtime-capabilities.ts";
|
|
|
|
const TRUE_VALUES = new Set(["1", "true", "yes", "on"]);
|
|
const DEFAULT_AGENTRUN_EVENT_TOPIC = "agentrun.event.v1";
|
|
const DEFAULT_HWLAB_EVENT_TOPIC = "hwlab.event.v1";
|
|
const DEFAULT_STDIO_TOPIC = "codex-stdio.raw.v1";
|
|
export const DEFAULT_AGENTRUN_DEBUG_EVENT_TOPIC = "agentrun.event.debug.v1";
|
|
export const DEFAULT_HWLAB_DEBUG_EVENT_TOPIC = "hwlab.event.debug.v1";
|
|
export const AGENTRUN_STDIO_RECONSTRUCTION_SCHEMA = "agentrun.event.reconstruction.v1";
|
|
export const AGENTRUN_STDIO_RECONSTRUCTION_KIND = "stdio-derived partial reconstruction";
|
|
const DEFAULT_CLIENT_ID = "hwlab-v03-cloud-api";
|
|
const DEFAULT_QUERY_TIMEOUT_MS = 5000;
|
|
const DEFAULT_QUERY_LIMIT = 50;
|
|
const LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS = 32;
|
|
const REQUIRED_KAFKA_ENV = Object.freeze([
|
|
"HWLAB_KAFKA_BOOTSTRAP_SERVERS",
|
|
"HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC",
|
|
"HWLAB_KAFKA_EVENT_TOPIC",
|
|
"HWLAB_KAFKA_CLIENT_ID"
|
|
]);
|
|
|
|
export function kafkaEventBridgeConfig(env = process.env) {
|
|
const featureEnvPresent = Object.values(WORKBENCH_REALTIME_CAPABILITY_ENVS).some((name) => stringValue(env[name]));
|
|
const legacyKafkaEnabled = truthy(env.HWLAB_KAFKA_AGENTRUN_EVENT_CONSUME_ENABLED ?? env.HWLAB_KAFKA_AGENTRUN_CONSUME_ENABLED ?? env.HWLAB_KAFKA_ENABLED);
|
|
if (!featureEnvPresent && !legacyKafkaEnabled) return null;
|
|
const capabilities = workbenchRealtimeCapabilities(env);
|
|
const kafkaEnabled = capabilities.directPublish || capabilities.liveKafkaSse || capabilities.transactionalProjector || capabilities.projectionOutboxRelay;
|
|
const transactionalEnabled = capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime;
|
|
if (!kafkaEnabled && !transactionalEnabled) return null;
|
|
const required = [...REQUIRED_KAFKA_ENV];
|
|
if (!kafkaEnabled) required.length = 0;
|
|
if (capabilities.directPublish) required.push("HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID");
|
|
if (capabilities.liveKafkaSse) required.push("HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID");
|
|
if (capabilities.transactionalProjector) required.push("HWLAB_KAFKA_PROJECTOR_GROUP_ID", "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS");
|
|
if (capabilities.projectionOutboxRelay) required.push(
|
|
"HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS",
|
|
"HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE",
|
|
"HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS",
|
|
"HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS",
|
|
"HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS"
|
|
);
|
|
const missing = required.filter((name) => !stringValue(env[name]));
|
|
if (missing.length > 0) {
|
|
const error = new Error(`HWLAB Kafka capability configuration is incomplete: ${missing.join(", ")}`);
|
|
error.code = "hwlab_kafka_realtime_config_incomplete";
|
|
error.missing = missing;
|
|
throw error;
|
|
}
|
|
const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS);
|
|
const agentRunTopic = stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC);
|
|
const hwlabTopic = stringValue(env.HWLAB_KAFKA_EVENT_TOPIC);
|
|
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID);
|
|
const relay = capabilities.projectionOutboxRelay ? {
|
|
intervalMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS, "HWLAB_KAFKA_OUTBOX_RELAY_INTERVAL_MS"),
|
|
batchSize: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE, "HWLAB_KAFKA_OUTBOX_RELAY_BATCH_SIZE"),
|
|
leaseMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS, "HWLAB_KAFKA_OUTBOX_RELAY_LEASE_MS"),
|
|
sendTimeoutMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS, "HWLAB_KAFKA_OUTBOX_RELAY_SEND_TIMEOUT_MS"),
|
|
retryBackoffMs: strictPositiveInteger(env.HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS, "HWLAB_KAFKA_OUTBOX_RELAY_RETRY_BACKOFF_MS")
|
|
} : null;
|
|
if (relay && relay.sendTimeoutMs + 5000 >= relay.leaseMs) {
|
|
const error = new Error("HWLAB Kafka relay lease must exceed producer send timeout by at least 5000ms.");
|
|
error.code = "hwlab_kafka_relay_lease_invalid";
|
|
throw error;
|
|
}
|
|
const enabledConsumerGroups = [
|
|
capabilities.directPublish ? stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID) : null,
|
|
capabilities.transactionalProjector ? stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID) : null,
|
|
capabilities.liveKafkaSse ? stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID) : null
|
|
].filter(Boolean);
|
|
if (new Set(enabledConsumerGroups).size !== enabledConsumerGroups.length) {
|
|
const error = new Error("HWLAB Kafka direct publisher, projector, and live SSE consumer groups must be distinct when enabled together.");
|
|
error.code = "hwlab_kafka_consumer_groups_not_distinct";
|
|
error.groupIds = enabledConsumerGroups;
|
|
throw error;
|
|
}
|
|
return {
|
|
capabilities,
|
|
brokers,
|
|
agentRunTopic,
|
|
hwlabTopic,
|
|
clientId,
|
|
directPublishGroupId: stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_GROUP_ID),
|
|
projectorGroupId: stringValue(env.HWLAB_KAFKA_PROJECTOR_GROUP_ID),
|
|
hwlabEventGroupId: stringValue(env.HWLAB_KAFKA_HWLAB_EVENT_GROUP_ID),
|
|
projectorHeartbeatIntervalMs: capabilities.transactionalProjector ? strictPositiveInteger(env.HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS, "HWLAB_KAFKA_PROJECTOR_HEARTBEAT_INTERVAL_MS") : null,
|
|
relay
|
|
};
|
|
}
|
|
|
|
export function startHwlabKafkaEventBridge({ env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString(), otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
|
const config = kafkaEventBridgeConfig(env);
|
|
if (!config) return { started: false, reason: "disabled_or_unconfigured", stop() {}, subscribeProjectionCommits() { return () => {}; }, valuesPrinted: false };
|
|
const components = [];
|
|
if (config.capabilities.directPublish || config.capabilities.liveKafkaSse) {
|
|
components.push(startLiveHwlabKafkaEventBridge({ config, env, logger, kafkaFactory, otelSpanEmitter }));
|
|
}
|
|
if (config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay || config.capabilities.projectionRealtime) {
|
|
components.push(startTransactionalHwlabKafkaEventBridge({ config, logger, kafkaFactory, runtimeStore, now }));
|
|
}
|
|
return components.length === 1 ? components[0] : combineKafkaEventBridgeComponents(config, components);
|
|
}
|
|
|
|
function startTransactionalHwlabKafkaEventBridge({ config, logger = console, kafkaFactory = defaultKafkaFactory, runtimeStore = null, now = () => new Date().toISOString() } = {}) {
|
|
requireKafkaProjectorStore(runtimeStore, config.capabilities);
|
|
|
|
let stopped = false;
|
|
let consumer = null;
|
|
let producer = null;
|
|
let relayTimer = null;
|
|
let relayRunning = false;
|
|
let stopProjectionNotifications = null;
|
|
const projectionCommitListeners = new Set();
|
|
const publishProjectionCommit = (signal = {}) => {
|
|
for (const listener of [...projectionCommitListeners]) {
|
|
try { listener({ sessionId: signal.sessionId ?? null, traceId: signal.traceId ?? null, outboxSeq: signal.outboxSeq ?? null, recovery: signal.recovery === true, valuesRedacted: true }); } catch {}
|
|
}
|
|
};
|
|
const relayOwner = `${config.clientId}:outbox-relay:${randomUUID()}`;
|
|
let startupError = null;
|
|
let startupReady = false;
|
|
const ready = (async () => {
|
|
await runtimeStore.assertWorkbenchTransactionalRealtimeReady();
|
|
const kafka = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay ? kafkaFactory(config) : null;
|
|
if (config.capabilities.transactionalProjector) consumer = kafka.consumer({ groupId: config.projectorGroupId, allowAutoTopicCreation: false });
|
|
if (config.capabilities.projectionOutboxRelay) producer = kafka.producer({ allowAutoTopicCreation: false });
|
|
if (producer) await producer.connect();
|
|
if (consumer) await consumer.connect();
|
|
if (config.capabilities.projectionRealtime && typeof runtimeStore.subscribeWorkbenchProjectionCommits === "function") {
|
|
stopProjectionNotifications = await runtimeStore.subscribeWorkbenchProjectionCommits(publishProjectionCommit);
|
|
}
|
|
if (consumer) await consumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
|
|
if (consumer) await consumer.run({
|
|
autoCommit: false,
|
|
eachBatchAutoResolve: false,
|
|
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => {
|
|
for (const message of batch.messages) {
|
|
if (stopped || !isRunning() || isStale()) break;
|
|
await heartbeat();
|
|
const heartbeatLoop = startKafkaHeartbeatLoop(heartbeat, config.projectorHeartbeatIntervalMs);
|
|
let projection;
|
|
try {
|
|
projection = await projectAgentRunKafkaMessage({ topic: batch.topic, partition: batch.partition, message }, { runtimeStore, config, logger });
|
|
} finally {
|
|
await heartbeatLoop.stop();
|
|
}
|
|
if (projection?.projected) publishProjectionCommit(projection);
|
|
if (heartbeatLoop.error) throw heartbeatLoop.error;
|
|
if (stopped || !isRunning() || isStale()) {
|
|
logWarn(logger, "hwlab-kafka-offset-commit-skipped", { topic: batch.topic, partition: batch.partition, offset: message.offset, reason: isStale() ? "stale_generation" : "consumer_stopped", valuesPrinted: false });
|
|
break;
|
|
}
|
|
resolveOffset(message.offset);
|
|
await consumer.commitOffsets([{ topic: batch.topic, partition: batch.partition, offset: nextKafkaOffset(message.offset) }]);
|
|
await heartbeat();
|
|
}
|
|
}
|
|
});
|
|
const runRelay = async () => {
|
|
if (!config.capabilities.projectionOutboxRelay) return;
|
|
if (stopped || relayRunning) return;
|
|
relayRunning = true;
|
|
try {
|
|
await relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner: relayOwner, now, logger });
|
|
} finally {
|
|
relayRunning = false;
|
|
}
|
|
};
|
|
if (config.capabilities.projectionOutboxRelay) {
|
|
relayTimer = setInterval(() => { void runRelay().catch((error) => logWarn(logger, "hwlab-kafka-outbox-relay-failed", { message: errorMessage(error), valuesPrinted: false })); }, config.relay.intervalMs);
|
|
relayTimer.unref?.();
|
|
await runRelay();
|
|
}
|
|
startupReady = true;
|
|
logInfo(logger, "hwlab-kafka-event-bridge-started", {
|
|
agentRunTopic: config.agentRunTopic,
|
|
hwlabTopic: config.hwlabTopic,
|
|
clientId: config.clientId,
|
|
projectorGroupId: config.projectorGroupId,
|
|
capabilities: config.capabilities,
|
|
valuesPrinted: false
|
|
});
|
|
})();
|
|
void ready.catch((error) => {
|
|
startupError = error;
|
|
logWarn(logger, "hwlab-kafka-event-bridge-start-failed", {
|
|
message: errorMessage(error),
|
|
agentRunTopic: config.agentRunTopic,
|
|
hwlabTopic: config.hwlabTopic,
|
|
valuesPrinted: false
|
|
});
|
|
});
|
|
|
|
return {
|
|
started: true,
|
|
...config,
|
|
ready,
|
|
async stop() {
|
|
stopped = true;
|
|
if (relayTimer) clearInterval(relayTimer);
|
|
if (typeof stopProjectionNotifications === "function") {
|
|
try { await stopProjectionNotifications(); } catch {}
|
|
}
|
|
if (consumer?.stop) await consumer.stop().catch(() => undefined);
|
|
await Promise.allSettled([consumer?.disconnect?.(), producer?.disconnect?.()]);
|
|
},
|
|
async status() {
|
|
if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
|
if (!startupReady) return { status: "initializing", valuesRedacted: true };
|
|
const transactionalReadiness = typeof runtimeStore.workbenchTransactionalRealtimeReadiness === "function"
|
|
? await runtimeStore.workbenchTransactionalRealtimeReadiness()
|
|
: { ready: true, valuesRedacted: true };
|
|
const projectorStatus = config.capabilities.transactionalProjector || config.capabilities.projectionOutboxRelay
|
|
? await runtimeStore.hwlabKafkaProjectorStatus()
|
|
: {};
|
|
return { status: transactionalReadiness.ready === false ? "blocked" : "ready", capabilities: config.capabilities, transactionalReadiness, ...projectorStatus, valuesRedacted: true };
|
|
},
|
|
startupStatus() {
|
|
if (startupError) return { status: "blocked", errorCode: startupError.code ?? "hwlab_kafka_projector_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
|
return { status: startupReady ? "ready" : "initializing", valuesRedacted: true };
|
|
},
|
|
subscribeProjectionCommits(listener) {
|
|
if (typeof listener !== "function") return () => {};
|
|
projectionCommitListeners.add(listener);
|
|
return () => projectionCommitListeners.delete(listener);
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function startLiveHwlabKafkaEventBridge({ config, env = process.env, logger = console, kafkaFactory = defaultKafkaFactory, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
|
let stopped = false;
|
|
let bridgeConsumer = null;
|
|
let fanoutConsumer = null;
|
|
let producer = null;
|
|
let startupError = null;
|
|
let startupReady = false;
|
|
const subscribers = new Set();
|
|
const lagByPartition = new Map();
|
|
const publishLiveEnvelope = (envelope, transport) => {
|
|
for (const listener of [...subscribers]) {
|
|
try {
|
|
listener(envelope, transport);
|
|
} catch (error) {
|
|
logWarn(logger, "hwlab-live-kafka-subscriber-failed", { message: errorMessage(error), valuesPrinted: false });
|
|
}
|
|
}
|
|
};
|
|
const ready = (async () => {
|
|
const kafka = kafkaFactory(config);
|
|
if (config.capabilities.directPublish) {
|
|
bridgeConsumer = kafka.consumer({ groupId: config.directPublishGroupId, allowAutoTopicCreation: false });
|
|
producer = kafka.producer({ allowAutoTopicCreation: false });
|
|
}
|
|
if (config.capabilities.liveKafkaSse) fanoutConsumer = kafka.consumer({ groupId: config.hwlabEventGroupId, allowAutoTopicCreation: false });
|
|
if (producer) await producer.connect();
|
|
if (bridgeConsumer) await bridgeConsumer.connect();
|
|
if (fanoutConsumer) await fanoutConsumer.connect();
|
|
if (bridgeConsumer) await bridgeConsumer.subscribe({ topic: config.agentRunTopic, fromBeginning: false });
|
|
if (fanoutConsumer) await fanoutConsumer.subscribe({ topic: config.hwlabTopic, fromBeginning: false });
|
|
if (fanoutConsumer) await fanoutConsumer.run({
|
|
eachBatchAutoResolve: false,
|
|
eachBatch: async ({ batch, resolveOffset, heartbeat, isRunning, isStale }) => {
|
|
let lastOffset = null;
|
|
for (const message of batch.messages) {
|
|
if (stopped || !isRunning() || isStale()) break;
|
|
const envelope = parseJson(message?.value ? Buffer.from(message.value).toString("utf8") : "");
|
|
if (!envelope || envelope.schema !== "hwlab.event.v1") {
|
|
logWarn(logger, "hwlab-live-kafka-envelope-ignored", { reason: "schema-invalid", valuesPrinted: false });
|
|
} else {
|
|
const transport = { topic: batch.topic, partition: batch.partition, offset: message.offset };
|
|
emitLiveKafkaOtelSpan("hwlab.kafka.live.fanout_receive", envelope, transport, { env, otelSpanEmitter });
|
|
publishLiveEnvelope(envelope, transport);
|
|
}
|
|
resolveOffset(message.offset);
|
|
lastOffset = message.offset;
|
|
await heartbeat();
|
|
}
|
|
if (lastOffset !== null) lagByPartition.set(batch.partition, kafkaPartitionLag(batch.highWatermark, lastOffset));
|
|
}
|
|
});
|
|
if (bridgeConsumer) await bridgeConsumer.run({
|
|
eachMessage: async ({ topic, partition, message }) => {
|
|
if (stopped) return;
|
|
await publishAgentRunKafkaMessageLive({ topic, partition, message }, { producer, config, env, logger, otelSpanEmitter });
|
|
}
|
|
});
|
|
startupReady = true;
|
|
logInfo(logger, "hwlab-live-kafka-event-bridge-started", {
|
|
capabilities: config.capabilities,
|
|
agentRunTopic: config.agentRunTopic,
|
|
hwlabTopic: config.hwlabTopic,
|
|
clientId: config.clientId,
|
|
directPublishGroupId: config.directPublishGroupId,
|
|
hwlabEventGroupId: config.hwlabEventGroupId,
|
|
valuesPrinted: false
|
|
});
|
|
})();
|
|
void ready.catch((error) => {
|
|
startupError = error;
|
|
logWarn(logger, "hwlab-live-kafka-event-bridge-start-failed", {
|
|
message: errorMessage(error),
|
|
agentRunTopic: config.agentRunTopic,
|
|
hwlabTopic: config.hwlabTopic,
|
|
valuesPrinted: false
|
|
});
|
|
});
|
|
return {
|
|
started: true,
|
|
...config,
|
|
ready,
|
|
async stop() {
|
|
stopped = true;
|
|
subscribers.clear();
|
|
await Promise.allSettled([bridgeConsumer?.stop?.(), fanoutConsumer?.stop?.()]);
|
|
await Promise.allSettled([bridgeConsumer?.disconnect?.(), fanoutConsumer?.disconnect?.(), producer?.disconnect?.()]);
|
|
},
|
|
async status() {
|
|
if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
|
return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, subscriberCount: subscribers.size, consumerLag: liveKafkaLagSummary(lagByPartition), lossPossible: true, valuesRedacted: true };
|
|
},
|
|
startupStatus() {
|
|
if (startupError) return { status: "blocked", capabilities: config.capabilities, errorCode: startupError.code ?? "hwlab_live_kafka_start_failed", message: errorMessage(startupError), valuesRedacted: true };
|
|
return { status: startupReady ? "ready" : "initializing", capabilities: config.capabilities, valuesRedacted: true };
|
|
},
|
|
subscribeProjectionCommits() { return () => {}; },
|
|
subscribeLiveHwlabEvents(listener) {
|
|
if (typeof listener !== "function") return () => {};
|
|
subscribers.add(listener);
|
|
return () => subscribers.delete(listener);
|
|
},
|
|
liveSubscriberCount() { return subscribers.size; },
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function kafkaPartitionLag(highWatermark, lastOffset) {
|
|
try {
|
|
const lag = BigInt(String(highWatermark ?? "0")) - (BigInt(String(lastOffset ?? "0")) + 1n);
|
|
return Number(lag > 0n ? lag : 0n);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function liveKafkaLagSummary(lagByPartition) {
|
|
const partitions = [...lagByPartition.entries()].map(([partition, lag]) => ({ partition, lag })).sort((left, right) => left.partition - right.partition);
|
|
const known = partitions.map((item) => item.lag).filter((lag) => Number.isFinite(lag));
|
|
return {
|
|
known: known.length > 0,
|
|
total: known.length > 0 ? known.reduce((sum, lag) => sum + lag, 0) : null,
|
|
partitions,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
function combineKafkaEventBridgeComponents(config, components) {
|
|
const ready = Promise.all(components.map((component) => component.ready));
|
|
const componentStartup = () => components.map((component) => component.startupStatus?.() ?? { status: "initializing" });
|
|
return {
|
|
started: true,
|
|
...config,
|
|
ready,
|
|
async stop() { await Promise.allSettled(components.map((component) => component.stop())); },
|
|
async status() {
|
|
const statuses = await Promise.all(components.map((component) => component.status()));
|
|
const blocked = statuses.find((status) => status.status === "blocked");
|
|
const initializing = statuses.some((status) => status.status !== "ready");
|
|
return {
|
|
status: blocked ? "blocked" : initializing ? "initializing" : "ready",
|
|
capabilities: config.capabilities,
|
|
components: statuses,
|
|
errorCode: blocked?.errorCode ?? null,
|
|
message: blocked?.message ?? null,
|
|
valuesRedacted: true
|
|
};
|
|
},
|
|
startupStatus() {
|
|
const statuses = componentStartup();
|
|
const blocked = statuses.find((status) => status.status === "blocked");
|
|
return {
|
|
status: blocked ? "blocked" : statuses.every((status) => status.status === "ready") ? "ready" : "initializing",
|
|
capabilities: config.capabilities,
|
|
components: statuses,
|
|
errorCode: blocked?.errorCode ?? null,
|
|
message: blocked?.message ?? null,
|
|
valuesRedacted: true
|
|
};
|
|
},
|
|
subscribeProjectionCommits(listener) {
|
|
const stops = components.map((component) => component.subscribeProjectionCommits?.(listener)).filter((stop) => typeof stop === "function");
|
|
return () => stops.forEach((stop) => stop());
|
|
},
|
|
subscribeLiveHwlabEvents(listener) {
|
|
const owner = components.find((component) => typeof component.subscribeLiveHwlabEvents === "function");
|
|
return owner?.subscribeLiveHwlabEvents(listener) ?? (() => {});
|
|
},
|
|
liveSubscriberCount() {
|
|
const owner = components.find((component) => typeof component.liveSubscriberCount === "function");
|
|
return owner?.liveSubscriberCount() ?? 0;
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
export async function publishAgentRunKafkaMessageLive(kafkaMessage = {}, { producer, config, env = process.env, logger = console, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
|
const transport = kafkaTransport(kafkaMessage);
|
|
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
|
|
if (!decoded.ok) {
|
|
logWarn(logger, "hwlab-live-kafka-message-invalid", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false });
|
|
return { handled: true, invalid: true, published: false, valuesPrinted: false };
|
|
}
|
|
if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) {
|
|
return { handled: true, ignored: true, published: false, valuesPrinted: false };
|
|
}
|
|
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, {
|
|
source: config.clientId,
|
|
sourceTopic: transport.sourceTopic,
|
|
sourcePartition: transport.sourcePartition,
|
|
sourceOffset: transport.sourceOffset,
|
|
sourceKey: transport.sourceKey,
|
|
inputSha256: transport.inputSha256
|
|
});
|
|
const publishMetadata = await producer.send({ topic: config.hwlabTopic, messages: [buildHwlabKafkaProducerMessage(projected)] });
|
|
const publishedRecord = Array.isArray(publishMetadata) ? publishMetadata[0] : null;
|
|
emitLiveKafkaOtelSpan("hwlab.kafka.live.direct_publish", projected, {
|
|
topic: config.hwlabTopic,
|
|
partition: publishedRecord?.partition,
|
|
offset: publishedRecord?.baseOffset ?? publishedRecord?.offset,
|
|
sourceTopic: transport.sourceTopic,
|
|
sourcePartition: transport.sourcePartition,
|
|
sourceOffset: transport.sourceOffset
|
|
}, { env, otelSpanEmitter });
|
|
return { handled: true, published: true, envelope: projected, sessionId: projected.sessionId, traceId: projected.traceId, valuesPrinted: false };
|
|
}
|
|
|
|
export function shouldEmitLiveKafkaOtelSpan(envelope = {}) {
|
|
const event = objectValue(envelope.event);
|
|
const context = objectValue(envelope.context);
|
|
const eventType = firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType)?.toLowerCase();
|
|
if (event.terminal === true || eventType === "terminal" || eventType === "terminal_status") return true;
|
|
if (["assistant", "assistant_message", "tool", "tool_call"].includes(eventType)) return true;
|
|
if (eventType !== "command_output" && event.type !== "output") return true;
|
|
const sourceSeq = integerValue(context.sourceSeq ?? event.sourceSeq);
|
|
if (Number.isInteger(sourceSeq) && sourceSeq > 0) return sourceSeq % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0;
|
|
const identity = firstText(envelope.sourceEventId, envelope.eventId, envelope.traceId, envelope.hwlabSessionId, envelope.sessionId);
|
|
if (!identity) return false;
|
|
return createHash("sha256").update(identity).digest().readUInt32BE(0) % LIVE_KAFKA_COMMAND_OUTPUT_OTEL_SAMPLE_MODULUS === 0;
|
|
}
|
|
|
|
export function liveKafkaOtelSpanAttributes(envelope = {}, transport = {}) {
|
|
const event = objectValue(envelope.event);
|
|
const context = objectValue(envelope.context);
|
|
const sourceEvent = objectValue(envelope.sourceEvent);
|
|
return {
|
|
businessTraceId: firstText(envelope.traceId, event.traceId),
|
|
hwlabSessionId: firstText(envelope.hwlabSessionId, envelope.sessionId, event.sessionId),
|
|
runId: firstText(envelope.runId, context.runId, event.runId),
|
|
commandId: firstText(envelope.commandId, context.commandId, event.commandId),
|
|
topic: firstText(transport.topic, sourceEvent.topic),
|
|
partition: integerValue(transport.partition ?? sourceEvent.partition),
|
|
offset: firstText(transport.offset, sourceEvent.offset),
|
|
sourceTopic: firstText(transport.sourceTopic),
|
|
sourcePartition: integerValue(transport.sourcePartition),
|
|
sourceOffset: firstText(transport.sourceOffset),
|
|
eventType: firstText(context.agentRunEventType, event.eventType, event.type, envelope.eventType),
|
|
terminal: event.terminal === true,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
export function emitLiveKafkaOtelSpan(name, envelope = {}, transport = {}, { env = process.env, otelSpanEmitter = emitCodeAgentOtelSpan } = {}) {
|
|
if (typeof otelSpanEmitter !== "function" || !shouldEmitLiveKafkaOtelSpan(envelope)) return { emitted: false, valuesRedacted: true };
|
|
const attributes = liveKafkaOtelSpanAttributes(envelope, transport);
|
|
if (!attributes.businessTraceId) return { emitted: false, valuesRedacted: true };
|
|
try {
|
|
const pending = otelSpanEmitter(name, attributes.businessTraceId, env, { attributes });
|
|
void Promise.resolve(pending).catch(() => undefined);
|
|
} catch {
|
|
// OTel is best effort and must never change the live Kafka delivery path.
|
|
}
|
|
return { emitted: true, valuesRedacted: true };
|
|
}
|
|
|
|
export async function projectAgentRunKafkaMessage(kafkaMessage = {}, { runtimeStore, config, logger = console } = {}) {
|
|
const transport = kafkaTransport(kafkaMessage);
|
|
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
|
|
if (!decoded.ok) {
|
|
await runtimeStore.recordFailedAgentRunKafkaMessage({ transport, sourceEventId: decoded.sourceEventId, envelope: decoded.envelope, error: decoded.error });
|
|
logWarn(logger, "hwlab-kafka-message-dlq", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: decoded.error.code, valuesPrinted: false });
|
|
return { handled: true, invalid: true, valuesPrinted: false };
|
|
}
|
|
if (!stringValue(decoded.event.traceId) || !stringValue(decoded.event.hwlabSessionId)) {
|
|
const ignored = await runtimeStore.recordIgnoredAgentRunKafkaMessage({ transport, envelope: decoded.event, reason: "hwlab-correlation-absent" });
|
|
return { handled: true, ignored: true, duplicate: ignored?.duplicate === true, valuesPrinted: false };
|
|
}
|
|
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, {
|
|
source: config.clientId,
|
|
sourceTopic: transport.sourceTopic,
|
|
sourcePartition: transport.sourcePartition,
|
|
sourceOffset: transport.sourceOffset,
|
|
sourceKey: transport.sourceKey,
|
|
inputSha256: transport.inputSha256
|
|
});
|
|
const result = await runtimeStore.commitAgentRunKafkaProjection({
|
|
transport,
|
|
canonicalEvent: decoded.event,
|
|
projectedEvent: projected.event,
|
|
requestMeta: { traceId: projected.traceId, sessionId: projected.sessionId, agentSessionId: projected.sessionId, valuesPrinted: false },
|
|
factsFactory: ({ projectedSeq, previousCheckpoint, projectedAt }) => buildWorkbenchProjectionEventFacts({ event: projected.event, requestMeta: { traceId: projected.traceId, sessionId: projected.sessionId }, previousCheckpoint, projectedSeq, projectedAt }),
|
|
hwlabEvent: projected,
|
|
hwlabTopic: config.hwlabTopic,
|
|
partitionKey: kafkaMessageKey(projected),
|
|
headers: kafkaHeaders(projected)
|
|
});
|
|
if (result.conflict) logWarn(logger, "hwlab-kafka-message-conflict", { topic: transport.sourceTopic, partition: transport.sourcePartition, offset: transport.sourceOffset, errorCode: result.diagnostic?.code, valuesPrinted: false });
|
|
return {
|
|
handled: true,
|
|
projected: !result.duplicate && !result.conflict && result.projectionWritten !== false && result.suppressedAfterSeal !== true,
|
|
duplicate: result.duplicate === true,
|
|
conflict: result.conflict === true,
|
|
suppressedAfterSeal: result.suppressedAfterSeal === true,
|
|
projectedSeq: result.projectedSeq,
|
|
sessionId: projected.sessionId,
|
|
traceId: projected.traceId,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
export async function relayHwlabKafkaOutboxOnce({ runtimeStore, producer, config, owner, now = () => new Date().toISOString(), logger = console } = {}) {
|
|
let claimedCount = 0;
|
|
let deliveredCount = 0;
|
|
let retryCount = 0;
|
|
for (let index = 0; index < config.relay.batchSize; index += 1) {
|
|
const item = (await runtimeStore.claimHwlabKafkaOutbox({ owner, leaseMs: config.relay.leaseMs, limit: 1 }))[0];
|
|
if (!item) break;
|
|
claimedCount += 1;
|
|
try {
|
|
await producer.send({ topic: item.topic, timeout: config.relay.sendTimeoutMs, messages: [{ key: item.partitionKey, value: JSON.stringify(item.payload), headers: item.headers }] });
|
|
await runtimeStore.completeHwlabKafkaOutbox(item);
|
|
deliveredCount += 1;
|
|
} catch (error) {
|
|
const retryAt = new Date(Date.parse(now()) + config.relay.retryBackoffMs).toISOString();
|
|
await runtimeStore.retryHwlabKafkaOutbox(item, error, retryAt);
|
|
retryCount += 1;
|
|
logWarn(logger, "hwlab-kafka-outbox-publish-retry", { outboxSeq: item.outboxSeq, eventId: item.eventId, retryAt, message: errorMessage(error), valuesPrinted: false });
|
|
break;
|
|
}
|
|
}
|
|
return { claimedCount, deliveredCount, retryCount, valuesPrinted: false };
|
|
}
|
|
|
|
export function startKafkaHeartbeatLoop(heartbeat, intervalMs) {
|
|
let stopped = false;
|
|
let running = null;
|
|
let heartbeatError = null;
|
|
const pulse = () => {
|
|
if (stopped || running || typeof heartbeat !== "function") return;
|
|
running = Promise.resolve()
|
|
.then(() => heartbeat())
|
|
.catch((error) => { heartbeatError ??= error; })
|
|
.finally(() => { running = null; });
|
|
};
|
|
pulse();
|
|
const timer = setInterval(pulse, intervalMs);
|
|
timer.unref?.();
|
|
return {
|
|
get error() { return heartbeatError; },
|
|
async stop() {
|
|
stopped = true;
|
|
clearInterval(timer);
|
|
if (running) await running;
|
|
}
|
|
};
|
|
}
|
|
|
|
export function projectAgentRunKafkaMessageToHwlabEvent(kafkaMessage = {}, options = {}) {
|
|
const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : "";
|
|
const input = parseJson(valueText);
|
|
if (!input) return null;
|
|
return projectAgentRunKafkaEventToHwlabEvent(input, {
|
|
...options,
|
|
sourceTopic: kafkaMessage.topic,
|
|
sourcePartition: kafkaMessage.partition,
|
|
sourceOffset: kafkaMessage.message?.offset,
|
|
sourceKey: kafkaMessage.message?.key ? Buffer.from(kafkaMessage.message.key).toString("utf8") : null,
|
|
inputSha256: sha256(valueText)
|
|
});
|
|
}
|
|
|
|
export function projectAgentRunKafkaEventToHwlabEvent(input = {}, options = {}) {
|
|
if (!input || typeof input !== "object" || Array.isArray(input)) return null;
|
|
const run = objectValue(input.run ?? input.agentRun?.run);
|
|
const command = objectValue(input.command ?? input.agentRun?.command);
|
|
const sourceEvent = objectValue(input.event ?? input.payload?.event ?? input.agentRunEvent);
|
|
const payload = objectValue(sourceEvent.payload ?? input.payload);
|
|
const hwlab = objectValue(input.hwlab ?? payload.hwlab);
|
|
const context = objectValue(input.context ?? payload.context);
|
|
const sourceSeq = integerValue(sourceEvent.seq ?? input.seq ?? input.sourceSeq);
|
|
const traceId = firstText(input.traceId, hwlab.traceId, context.traceId, sourceEvent.traceId, payload.traceId, payload.hwlabTraceId, payload.businessTraceId, payload.metadata?.traceId, command.traceId, run.traceId);
|
|
const sourceEventId = firstText(input.eventId, sourceEvent.id);
|
|
const sessionId = firstText(input.hwlabSessionId, payload.hwlabSessionId, hwlab.sessionId, context.hwlabSessionId, input.sessionId, context.sessionId, run.hwlabSessionId, run.sessionId, payload.sessionId);
|
|
const runId = firstText(run.runId, sourceEvent.runId, payload.runId, input.runId);
|
|
const commandId = firstText(command.commandId, sourceEvent.commandId, payload.commandId, input.commandId);
|
|
const event = mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, command, traceId, sessionId, sourceSeq, runId, commandId });
|
|
const producedAt = timestampValue(options.producedAt) || new Date().toISOString();
|
|
return {
|
|
schema: "hwlab.event.v1",
|
|
eventType: "hwlab.trace.event.projected",
|
|
eventId: sourceEventId ? `hwlab:${sourceEventId}` : null,
|
|
sourceEventId,
|
|
source: stringValue(options.source) || DEFAULT_CLIENT_ID,
|
|
producedAt,
|
|
traceId,
|
|
hwlabSessionId: sessionId,
|
|
sessionId,
|
|
runId,
|
|
commandId,
|
|
context: {
|
|
conversationId: firstText(input.conversationId, hwlab.conversationId, context.conversationId, run.conversationId, payload.conversationId),
|
|
threadId: firstText(input.threadId, hwlab.threadId, context.threadId, run.threadId, payload.threadId),
|
|
projectId: firstText(input.projectId, hwlab.projectId, context.projectId, run.projectId, payload.projectId),
|
|
runId,
|
|
commandId,
|
|
sourceSeq,
|
|
sourceEventType: firstText(input.eventType, sourceEvent.eventType),
|
|
agentRunEventType: firstText(sourceEvent.type, payload.type),
|
|
valuesRedacted: true
|
|
},
|
|
event: { ...event, sourceEventId },
|
|
sourceEvent: {
|
|
schema: firstText(input.schema),
|
|
topic: stringValue(options.sourceTopic),
|
|
partition: integerValue(options.sourcePartition),
|
|
offset: stringValue(options.sourceOffset),
|
|
key: stringValue(options.sourceKey),
|
|
sha256: stringValue(options.inputSha256) || sha256(JSON.stringify(input)),
|
|
valuesRedacted: true
|
|
},
|
|
diagnostics: {
|
|
projectedBy: "hwlab-kafka-event-bridge",
|
|
valuesPrinted: false
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
export function decodeCanonicalAgentRunKafkaMessage(kafkaMessage = {}) {
|
|
const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : "";
|
|
const envelope = parseJson(valueText);
|
|
const sourceEventId = firstText(envelope?.eventId, envelope?.event?.id);
|
|
try {
|
|
assertCanonicalAgentRunEvent(envelope);
|
|
return { ok: true, event: envelope, sourceEventId, inputSha256: sha256(valueText), valuesPrinted: false };
|
|
} catch (error) {
|
|
return { ok: false, envelope, sourceEventId, inputSha256: sha256(valueText), error: { code: error?.code ?? "workbench_kafka_message_invalid", message: errorMessage(error), valuesRedacted: true }, valuesPrinted: false };
|
|
}
|
|
}
|
|
|
|
export function decodeAgentRunDebugProjectionMessage(kafkaMessage = {}) {
|
|
const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : "";
|
|
const envelope = parseJson(valueText);
|
|
if (envelope?.schema === "agentrun.event.v1") {
|
|
const decoded = decodeCanonicalAgentRunKafkaMessage(kafkaMessage);
|
|
return decoded.ok
|
|
? { ...decoded, inputKind: "canonical-durable", reconstruction: null }
|
|
: { ...decoded, inputKind: "canonical-invalid", reconstruction: null };
|
|
}
|
|
try {
|
|
assertAgentRunStdioReconstruction(envelope);
|
|
return {
|
|
ok: true,
|
|
event: envelope,
|
|
sourceEventId: null,
|
|
inputSha256: sha256(valueText),
|
|
inputKind: AGENTRUN_STDIO_RECONSTRUCTION_KIND,
|
|
reconstruction: envelope.reconstruction,
|
|
valuesPrinted: false
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
envelope,
|
|
sourceEventId: null,
|
|
inputSha256: sha256(valueText),
|
|
inputKind: "reconstruction-invalid",
|
|
reconstruction: envelope?.reconstruction ?? null,
|
|
error: {
|
|
code: error?.code ?? "workbench_kafka_debug_message_invalid",
|
|
message: errorMessage(error),
|
|
valuesRedacted: true
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
}
|
|
|
|
export function projectAgentRunKafkaMessageToHwlabDebugEvent(kafkaMessage = {}, options = {}) {
|
|
const decoded = decodeAgentRunDebugProjectionMessage(kafkaMessage);
|
|
if (!decoded.ok) return decoded;
|
|
const transport = kafkaTransport(kafkaMessage);
|
|
const projected = projectAgentRunKafkaEventToHwlabEvent(decoded.event, {
|
|
source: stringValue(options.source) || "hwlab-cli-kafka-debug",
|
|
producedAt: options.producedAt,
|
|
sourceTopic: transport.sourceTopic,
|
|
sourcePartition: transport.sourcePartition,
|
|
sourceOffset: transport.sourceOffset,
|
|
sourceKey: transport.sourceKey,
|
|
inputSha256: transport.inputSha256
|
|
});
|
|
if (!stringValue(projected?.traceId) || (!decoded.reconstruction && !stringValue(projected?.sessionId))) {
|
|
return {
|
|
ok: false,
|
|
inputKind: decoded.inputKind,
|
|
error: {
|
|
code: "hwlab_kafka_debug_correlation_missing",
|
|
message: decoded.reconstruction
|
|
? "AgentRun reconstruction debug projection requires traceId lineage."
|
|
: "Canonical AgentRun debug projection requires traceId and sessionId lineage.",
|
|
valuesRedacted: true
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
const debugSessionId = decoded.reconstruction ? stringValue(decoded.event.hwlabSessionId) : stringValue(projected.sessionId);
|
|
const debugEnvelope = {
|
|
...projected,
|
|
schema: DEFAULT_HWLAB_DEBUG_EVENT_TOPIC,
|
|
eventType: "hwlab.trace.event.debug-projected",
|
|
sessionId: debugSessionId,
|
|
hwlabSessionId: debugSessionId,
|
|
event: decoded.reconstruction ? { ...projected.event, sessionId: debugSessionId } : projected.event,
|
|
outboxSeq: null,
|
|
...(decoded.reconstruction ? { eventId: null, sourceEventId: null } : {}),
|
|
diagnostics: {
|
|
projectedBy: "hwlab-kafka-event-bridge",
|
|
projectionMode: "source-direct-debug",
|
|
inputKind: decoded.inputKind,
|
|
valuesPrinted: false
|
|
},
|
|
debugLineage: {
|
|
inputSchema: firstText(decoded.event.schema),
|
|
inputEventType: firstText(decoded.event.eventType),
|
|
inputEventId: firstText(decoded.event.eventId),
|
|
inputOutboxSeq: integerValue(decoded.event.outboxSeq),
|
|
inputTraceId: firstText(decoded.event.traceId, decoded.event.event?.payload?.traceId),
|
|
inputSessionId: firstText(decoded.event.sessionId, decoded.event.run?.sessionId),
|
|
inputHwlabSessionId: firstText(decoded.event.hwlabSessionId, decoded.event.run?.hwlabSessionId),
|
|
inputRunId: firstText(decoded.event.runId, decoded.event.run?.runId),
|
|
inputCommandId: firstText(decoded.event.commandId, decoded.event.command?.commandId),
|
|
sourceSeq: integerValue(decoded.event.sourceSeq ?? decoded.event.event?.seq),
|
|
valuesPrinted: false
|
|
},
|
|
...(decoded.reconstruction ? {
|
|
reconstruction: {
|
|
...decoded.reconstruction,
|
|
observedAt: timestampValue(decoded.event.observedAt),
|
|
inputSchema: decoded.event.schema,
|
|
inputEventType: decoded.event.eventType,
|
|
valuesPrinted: false
|
|
}
|
|
} : {}),
|
|
valuesPrinted: false
|
|
};
|
|
return {
|
|
ok: true,
|
|
inputKind: decoded.inputKind,
|
|
envelope: debugEnvelope,
|
|
sourceSeq: integerValue(decoded.event.sourceSeq ?? decoded.event.event?.seq),
|
|
traceId: debugEnvelope.traceId,
|
|
sessionId: debugEnvelope.sessionId,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function assertAgentRunStdioReconstruction(value) {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("hwlab_kafka_debug_json_invalid", "AgentRun reconstruction value must be a JSON object.");
|
|
if (value.schema !== AGENTRUN_STDIO_RECONSTRUCTION_SCHEMA || value.eventType !== "agentrun.event.reconstructed") {
|
|
throw contractError("hwlab_kafka_debug_schema_invalid", "AgentRun debug input must be a canonical committed event or a stdio reconstruction envelope.");
|
|
}
|
|
if (value.eventId !== null || value.outboxSeq !== null) {
|
|
throw contractError("hwlab_kafka_debug_identity_invalid", "Stdio reconstruction must keep eventId and outboxSeq null.");
|
|
}
|
|
for (const [name, field] of [["traceId", value.traceId], ["sessionId", value.sessionId], ["runId", value.runId]]) {
|
|
if (!stringValue(field)) throw contractError("hwlab_kafka_debug_lineage_invalid", `Stdio reconstruction ${name} is required.`);
|
|
}
|
|
if (!Number.isInteger(Number(value.sourceSeq)) || Number(value.sourceSeq) <= 0) {
|
|
throw contractError("hwlab_kafka_debug_sequence_invalid", "Stdio reconstruction sourceSeq must be a positive integer.");
|
|
}
|
|
if (!value.event || typeof value.event !== "object" || Array.isArray(value.event) || value.event.id !== null || Number(value.event.seq) !== Number(value.sourceSeq)) {
|
|
throw contractError("hwlab_kafka_debug_identity_invalid", "Stdio reconstruction event identity must remain null and its seq must match sourceSeq.");
|
|
}
|
|
if (!value.event.payload || typeof value.event.payload !== "object" || Array.isArray(value.event.payload)) {
|
|
throw contractError("hwlab_kafka_debug_schema_invalid", "Stdio reconstruction event payload must be an object.");
|
|
}
|
|
if (value.event.payload.reconstructed !== true || value.event.payload.reconstructionKind !== AGENTRUN_STDIO_RECONSTRUCTION_KIND) {
|
|
throw contractError("hwlab_kafka_debug_lineage_invalid", "Stdio reconstruction payload must disclose its partial reconstruction kind.");
|
|
}
|
|
const reconstruction = value.reconstruction;
|
|
if (!reconstruction || typeof reconstruction !== "object" || Array.isArray(reconstruction)
|
|
|| reconstruction.kind !== AGENTRUN_STDIO_RECONSTRUCTION_KIND
|
|
|| reconstruction.completeLifecycle !== false
|
|
|| reconstruction.originalEventId !== null
|
|
|| reconstruction.originalOutboxSeq !== null) {
|
|
throw contractError("hwlab_kafka_debug_lineage_invalid", "Stdio reconstruction lineage is incomplete or claims durable identity.");
|
|
}
|
|
assertReconstructionIdentityDerivation(value, reconstruction.identityDerivation);
|
|
if (value.valuesPrinted !== false || containsSecretLikeKey(value)) {
|
|
throw contractError("hwlab_kafka_debug_redaction_invalid", "Stdio reconstruction must be redacted and declare valuesPrinted=false.");
|
|
}
|
|
return value;
|
|
}
|
|
|
|
function assertReconstructionIdentityDerivation(value, derivation) {
|
|
if (!derivation || typeof derivation !== "object" || Array.isArray(derivation) || derivation.valuesPrinted !== false
|
|
|| derivation.sourceField !== "sessionId" || derivation.targetField !== "hwlabSessionId") {
|
|
throw contractError("hwlab_kafka_debug_identity_derivation_invalid", "Stdio reconstruction must disclose the session identity derivation contract.");
|
|
}
|
|
const rawSessionId = stringValue(value.sessionId);
|
|
const match = rawSessionId?.match(/^ses_agentrun_([0-9a-fA-F]{8})_([0-9a-fA-F]{4})_([0-9a-fA-F]{4})_([0-9a-fA-F]{4})_([0-9a-fA-F]{12})$/u);
|
|
if (match) {
|
|
const expectedHwlabSessionId = `ses_${match.slice(1).join("-").toLowerCase()}`;
|
|
if (value.hwlabSessionId !== expectedHwlabSessionId || value.run?.hwlabSessionId !== expectedHwlabSessionId
|
|
|| derivation.kind !== "agentrun-session-uuid-to-hwlab-session" || derivation.reversible !== true
|
|
|| derivation.sourcePrefix !== "ses_agentrun_" || derivation.targetPrefix !== "ses_"
|
|
|| derivation.separatorTransform !== "underscore-to-hyphen") {
|
|
throw contractError("hwlab_kafka_debug_identity_derivation_invalid", "Derived hwlabSessionId does not match the disclosed reversible AgentRun session identity transform.");
|
|
}
|
|
return;
|
|
}
|
|
if (value.hwlabSessionId !== null || value.run?.hwlabSessionId !== null
|
|
|| derivation.kind !== "none" || derivation.reversible !== false
|
|
|| derivation.reason !== "source sessionId is not ses_agentrun_<uuid-with-underscores>") {
|
|
throw contractError("hwlab_kafka_debug_identity_derivation_invalid", "Non-reversible AgentRun session identity must keep hwlabSessionId null and disclose why.");
|
|
}
|
|
}
|
|
|
|
function assertCanonicalAgentRunEvent(value) {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("workbench_kafka_json_invalid", "Kafka value must be a JSON object.");
|
|
assertClosedObject(value, ["schema", "eventType", "source", "eventId", "outboxSeq", "sourceSeq", "committedAt", "traceId", "sessionId", "hwlabSessionId", "runId", "commandId", "run", "command", "event", "valuesPrinted"], "AgentRun envelope");
|
|
if (value.schema !== "agentrun.event.v1" || value.eventType !== "agentrun.event.committed") throw contractError("workbench_kafka_schema_invalid", "Kafka value is not a canonical AgentRun committed event.");
|
|
for (const [name, field] of [["eventId", value.eventId], ["runId", value.runId]]) {
|
|
if (!stringValue(field)) throw contractError("workbench_kafka_schema_invalid", `Canonical AgentRun ${name} is required.`);
|
|
}
|
|
for (const [name, field] of [["traceId", value.traceId], ["hwlabSessionId", value.hwlabSessionId]]) {
|
|
if (field !== null && field !== undefined && !stringValue(field)) throw contractError("workbench_kafka_schema_invalid", `Canonical AgentRun ${name} must be a string or null.`);
|
|
}
|
|
if (!Number.isInteger(Number(value.sourceSeq)) || Number(value.sourceSeq) <= 0) throw contractError("workbench_kafka_schema_invalid", "Canonical AgentRun sourceSeq must be a positive integer.");
|
|
if (value.valuesPrinted !== false) throw contractError("workbench_kafka_redaction_invalid", "Canonical AgentRun event must declare valuesPrinted=false.");
|
|
assertClosedObject(value.run, ["runId", "status", "terminalStatus", "failureKind", "tenantId", "projectId", "providerId", "backendProfile", "sessionId", "hwlabSessionId", "conversationId", "threadId", "valuesPrinted"], "AgentRun run");
|
|
if (value.command !== null && value.command !== undefined) assertClosedObject(value.command, ["commandId", "runId", "seq", "type", "state", "payloadHash", "valuesPrinted"], "AgentRun command");
|
|
assertClosedObject(value.event, ["id", "runId", "seq", "type", "payload", "createdAt"], "AgentRun event");
|
|
if (value.event.id !== value.eventId || Number(value.event.seq) !== Number(value.sourceSeq)) throw contractError("workbench_kafka_identity_invalid", "Canonical AgentRun event identity does not match the envelope.");
|
|
if (!value.event.payload || typeof value.event.payload !== "object" || Array.isArray(value.event.payload)) throw contractError("workbench_kafka_schema_invalid", "Canonical AgentRun event payload must be an object.");
|
|
if (containsSecretLikeKey(value)) throw contractError("workbench_kafka_redaction_invalid", "Canonical AgentRun event contains a secret-like field.");
|
|
return value;
|
|
}
|
|
|
|
function assertClosedObject(value, allowed, label) {
|
|
if (!value || typeof value !== "object" || Array.isArray(value)) throw contractError("workbench_kafka_schema_invalid", `${label} must be an object.`);
|
|
const unknown = Object.keys(value).filter((key) => !allowed.includes(key));
|
|
if (unknown.length > 0) throw contractError("workbench_kafka_schema_invalid", `${label} contains unsupported fields: ${unknown.join(", ")}`);
|
|
}
|
|
|
|
function containsSecretLikeKey(value, depth = 0) {
|
|
if (depth > 12 || value === null || value === undefined) return false;
|
|
if (Array.isArray(value)) return value.some((item) => containsSecretLikeKey(item, depth + 1));
|
|
if (typeof value !== "object") return false;
|
|
return Object.entries(value).some(([key, entry]) => {
|
|
const normalized = key.replace(/[^a-z0-9]/giu, "").toLowerCase();
|
|
if (/^(apikey|accesstoken|refreshtoken|password|authorization|credentialvalue|secretvalue|privatekey|clientsecret|bearertoken)$/u.test(normalized)) return !redactedSecretMarker(entry);
|
|
if (typeof entry === "string" && /^(?:https?|postgres(?:ql)?):\/\/[^/@\s]+:[^/@\s]+@/iu.test(entry)) return true;
|
|
return containsSecretLikeKey(entry, depth + 1);
|
|
});
|
|
}
|
|
|
|
function redactedSecretMarker(value) {
|
|
if (value === null || value === undefined) return true;
|
|
const normalized = stringValue(value)?.toLowerCase();
|
|
return normalized === "redacted" || normalized === "[redacted]" || normalized === "present" || normalized === "missing";
|
|
}
|
|
|
|
function kafkaTransport(kafkaMessage = {}) {
|
|
const valueText = kafkaMessage?.message?.value ? Buffer.from(kafkaMessage.message.value).toString("utf8") : "";
|
|
return {
|
|
sourceTopic: stringValue(kafkaMessage.topic),
|
|
sourcePartition: integerValue(kafkaMessage.partition),
|
|
sourceOffset: stringValue(kafkaMessage.message?.offset),
|
|
sourceKey: kafkaMessage.message?.key ? Buffer.from(kafkaMessage.message.key).toString("utf8") : null,
|
|
inputSha256: stringValue(kafkaMessage.inputSha256) || sha256(valueText),
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function requireKafkaProjectorStore(runtimeStore, capabilities = {}) {
|
|
const required = [];
|
|
if (capabilities.transactionalProjector) required.push("commitAgentRunKafkaProjection", "recordFailedAgentRunKafkaMessage", "recordIgnoredAgentRunKafkaMessage");
|
|
if (capabilities.projectionOutboxRelay) required.push("claimHwlabKafkaOutbox", "completeHwlabKafkaOutbox", "retryHwlabKafkaOutbox");
|
|
if (capabilities.projectionRealtime) required.push("subscribeWorkbenchProjectionCommits");
|
|
if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay) required.push("hwlabKafkaProjectorStatus");
|
|
if (capabilities.transactionalProjector || capabilities.projectionOutboxRelay || capabilities.projectionRealtime) required.push("assertWorkbenchTransactionalRealtimeReady");
|
|
const missing = required.filter((name) => typeof runtimeStore?.[name] !== "function");
|
|
if (missing.length > 0) throw contractError("hwlab_kafka_projector_store_invalid", `Kafka durable capabilities require a runtime store: ${missing.join(", ")}`);
|
|
}
|
|
|
|
export async function queryKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, traceId = null, sessionId = null, runId = null, commandId = null, limit = DEFAULT_QUERY_LIMIT, timeoutMs = DEFAULT_QUERY_TIMEOUT_MS, fromBeginning = true, groupIdPrefix = null, kafkaFactory = defaultKafkaFactory } = {}) {
|
|
const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS);
|
|
if (brokers.length === 0) throw new Error("HWLAB_KAFKA_BOOTSTRAP_SERVERS is required for Kafka event queries.");
|
|
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID;
|
|
const resolvedTopic = stringValue(topic) || kafkaTopicForStream(stream, env);
|
|
const maxEvents = Math.max(1, integerValue(limit) || DEFAULT_QUERY_LIMIT);
|
|
const budgetMs = Math.max(250, integerValue(timeoutMs) || DEFAULT_QUERY_TIMEOUT_MS);
|
|
const kafka = kafkaFactory({ brokers, clientId: `${clientId}-query` });
|
|
const resolvedGroupIdPrefix = stringValue(groupIdPrefix) || `${clientId}-query`;
|
|
const groupId = `${resolvedGroupIdPrefix}-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
|
const consumer = kafka.consumer({ groupId, allowAutoTopicCreation: false });
|
|
const events = [];
|
|
let timer = null;
|
|
let running = false;
|
|
await consumer.connect();
|
|
await consumer.subscribe({ topic: resolvedTopic, fromBeginning: fromBeginning !== false });
|
|
try {
|
|
await new Promise((resolve, reject) => {
|
|
let finished = false;
|
|
const finish = (value = null) => {
|
|
if (finished) return;
|
|
finished = true;
|
|
resolve(value);
|
|
};
|
|
timer = setTimeout(finish, budgetMs);
|
|
timer.unref?.();
|
|
running = true;
|
|
consumer.run({
|
|
eachMessage: async ({ topic: messageTopic, partition, message }) => {
|
|
const valueText = message.value ? Buffer.from(message.value).toString("utf8") : "";
|
|
const value = parseJson(valueText);
|
|
if (!value || !eventMatchesFilters(value, { traceId, sessionId, runId, commandId })) return;
|
|
events.push({
|
|
topic: messageTopic,
|
|
partition,
|
|
offset: message.offset,
|
|
key: message.key ? Buffer.from(message.key).toString("utf8") : null,
|
|
timestamp: message.timestamp ?? null,
|
|
valueSha256: sha256(valueText),
|
|
value
|
|
});
|
|
if (events.length >= maxEvents) finish();
|
|
}
|
|
}).catch(reject);
|
|
});
|
|
} finally {
|
|
if (timer) clearTimeout(timer);
|
|
if (running) await consumer.stop().catch(() => undefined);
|
|
await boundedDisconnect(consumer);
|
|
}
|
|
return {
|
|
ok: true,
|
|
stream,
|
|
topic: resolvedTopic,
|
|
groupId,
|
|
count: events.length,
|
|
limit: maxEvents,
|
|
timeoutMs: budgetMs,
|
|
filters: compactObject({ traceId, sessionId, runId, commandId }),
|
|
events,
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
export async function openKafkaEventStream({ env = process.env, stream = "hwlab", topic = null, traceId = null, sessionId = null, runId = null, commandId = null, fromBeginning = false, onEvent, onError = null, kafkaFactory = defaultKafkaFactory } = {}) {
|
|
if (typeof onEvent !== "function") throw new Error("onEvent callback is required for Kafka event streaming.");
|
|
const brokers = csv(env.HWLAB_KAFKA_BOOTSTRAP_SERVERS);
|
|
if (brokers.length === 0) throw new Error("HWLAB_KAFKA_BOOTSTRAP_SERVERS is required for Kafka event streams.");
|
|
const clientId = stringValue(env.HWLAB_KAFKA_CLIENT_ID) || DEFAULT_CLIENT_ID;
|
|
const resolvedTopic = stringValue(topic) || kafkaTopicForStream(stream, env);
|
|
const filters = compactObject({ traceId, sessionId, runId, commandId });
|
|
const kafka = kafkaFactory({ brokers, clientId: `${clientId}-debug-sse` });
|
|
const groupId = `${clientId}-debug-sse-${Date.now()}-${randomUUID().slice(0, 8)}`;
|
|
const consumer = kafka.consumer({ groupId, allowAutoTopicCreation: false });
|
|
let running = false;
|
|
let stopped = false;
|
|
await consumer.connect();
|
|
await consumer.subscribe({ topic: resolvedTopic, fromBeginning: fromBeginning === true });
|
|
running = true;
|
|
try {
|
|
await consumer.run({
|
|
eachMessage: async ({ topic: messageTopic, partition, message }) => {
|
|
if (stopped) return;
|
|
const valueText = message.value ? Buffer.from(message.value).toString("utf8") : "";
|
|
const value = parseJson(valueText);
|
|
if (!value || !eventMatchesFilters(value, filters)) return;
|
|
await onEvent({
|
|
topic: messageTopic,
|
|
partition,
|
|
offset: message.offset,
|
|
key: message.key ? Buffer.from(message.key).toString("utf8") : null,
|
|
timestamp: message.timestamp ?? null,
|
|
value
|
|
});
|
|
}
|
|
});
|
|
} catch (error) {
|
|
if (typeof onError === "function") onError(error);
|
|
stopped = true;
|
|
if (running) await consumer.stop().catch(() => undefined);
|
|
await boundedDisconnect(consumer);
|
|
throw error;
|
|
}
|
|
return {
|
|
ok: true,
|
|
stream,
|
|
topic: resolvedTopic,
|
|
groupId,
|
|
filters,
|
|
async stop() {
|
|
stopped = true;
|
|
if (running) await consumer.stop().catch(() => undefined);
|
|
await boundedDisconnect(consumer);
|
|
},
|
|
valuesPrinted: false
|
|
};
|
|
}
|
|
|
|
function mapAgentRunSourceEventToHwlabEvent({ input, sourceEvent, payload, run, command, traceId, sessionId, sourceSeq, runId, commandId }) {
|
|
const base = {
|
|
traceId,
|
|
sessionId,
|
|
source: "agentrun.kafka",
|
|
sourceSeq,
|
|
runId,
|
|
commandId,
|
|
attemptId: firstText(command.attemptId, sourceEvent.attemptId, payload.attemptId),
|
|
runnerId: firstText(command.runnerId, sourceEvent.runnerId, payload.runnerId),
|
|
backend: firstText(run.backendProfile, payload.backendProfile, payload.providerProfile),
|
|
createdAt: timestampValue(sourceEvent.createdAt ?? input.producedAt),
|
|
valuesPrinted: false
|
|
};
|
|
const type = firstText(sourceEvent.type, payload.type, input.eventType) || "event";
|
|
if (type === "assistant_message") {
|
|
const authoritative = payload.replyAuthority === true || payload.final === true;
|
|
const assistantText = textPayload(payload, "assistant message");
|
|
return {
|
|
...base,
|
|
type: "assistant",
|
|
eventType: "assistant",
|
|
status: "running",
|
|
label: "agentrun:assistant:message",
|
|
message: assistantText,
|
|
text: assistantText,
|
|
assistantText,
|
|
finalResponse: authoritative ? { text: assistantText, status: "completed", traceId, source: "agentrun.kafka", valuesPrinted: false } : null,
|
|
itemId: firstText(payload.itemId),
|
|
replyAuthority: payload.replyAuthority === true,
|
|
final: payload.final === true,
|
|
terminal: false
|
|
};
|
|
}
|
|
if (type === "backend_status") {
|
|
const phase = firstText(payload.phase) || "status";
|
|
return { ...base, type: "backend", eventType: "backend", status: "running", label: `agentrun:backend:${phase}`, message: textPayload(payload, phase) };
|
|
}
|
|
if (type === "terminal_status") {
|
|
const terminalStatus = firstText(payload.terminalStatus, sourceEvent.terminalStatus, payload.status, sourceEvent.status) || "failed";
|
|
const normalized = terminalStatus === "cancelled" ? "canceled" : terminalStatus;
|
|
return { ...base, type: "result", eventType: "terminal", status: normalized, label: `agentrun:terminal:${terminalStatus}`, errorCode: firstText(payload.failureKind, payload.errorCode, sourceEvent.failureKind, sourceEvent.errorCode), failureKind: firstText(payload.failureKind, sourceEvent.failureKind), message: textPayload({ ...sourceEvent, ...payload }, `AgentRun terminal status ${terminalStatus}`), terminal: true };
|
|
}
|
|
if (type === "tool_call") {
|
|
const toolName = firstText(payload.toolName, payload.name, payload.method) || "tool";
|
|
return {
|
|
...base,
|
|
type: "tool",
|
|
eventType: "tool",
|
|
status: firstText(payload.status) || "running",
|
|
label: `agentrun:tool:${toolName}`,
|
|
message: textPayload(payload, `AgentRun tool ${toolName}`),
|
|
toolName,
|
|
method: firstText(payload.method),
|
|
itemId: firstText(payload.itemId),
|
|
exitCode: integerValue(payload.exitCode),
|
|
durationMs: integerValue(payload.durationMs),
|
|
outputSummary: firstText(payload.outputSummary, payload.summary?.text),
|
|
terminal: false
|
|
};
|
|
}
|
|
if (type === "command_output") {
|
|
const stream = payload.stream === "stderr" ? "stderr" : "stdout";
|
|
return { ...base, type: "output", eventType: "status", status: "running", label: `agentrun:output:${stream}`, stream, message: textPayload(payload, ""), outputTruncated: payload.outputTruncated === true };
|
|
}
|
|
if (type === "error") {
|
|
const failureKind = firstText(payload.failureKind, payload.errorCode, sourceEvent.failureKind, sourceEvent.errorCode) || "backend";
|
|
return { ...base, type: "error", eventType: "error", status: "failed", label: `agentrun:error:${failureKind}`, errorCode: failureKind, failureKind, message: textPayload(payload, "AgentRun error"), terminal: false };
|
|
}
|
|
return { ...base, type: "backend", eventType: "backend", status: "running", label: `agentrun:event:${type}`, message: textPayload(payload, firstText(input.eventType, type) || "AgentRun event") };
|
|
}
|
|
|
|
function eventMatchesFilters(value, filters) {
|
|
for (const [name, expected] of Object.entries(filters)) {
|
|
const needle = stringValue(expected);
|
|
if (!needle) continue;
|
|
if (!eventFieldCandidates(value, name).includes(needle)) return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
function eventFieldCandidates(value, name) {
|
|
const event = objectValue(value.event);
|
|
const context = objectValue(value.context);
|
|
const run = objectValue(value.run);
|
|
const command = objectValue(value.command);
|
|
const payload = objectValue(value.payload);
|
|
const metadata = objectValue(value.metadata ?? value.meta);
|
|
const trace = objectValue(value.trace);
|
|
const ids = objectValue(value.ids);
|
|
const stdio = objectValue(value.stdio ?? value.frame ?? value.codexStdio);
|
|
const sourceEvent = objectValue(value.sourceEvent);
|
|
const nestedEvent = objectValue(value.event?.sourceEvent ?? value.agentRunEvent ?? payload.event);
|
|
const nestedPayload = objectValue(nestedEvent.payload);
|
|
if (name === "traceId") return textCandidates(value.traceId, value.trace_id, event.traceId, event.trace_id, context.traceId, context.trace_id, sourceEvent.traceId, sourceEvent.trace_id, nestedEvent.traceId, nestedEvent.trace_id, nestedPayload.traceId, nestedPayload.trace_id, payload.traceId, payload.trace_id, metadata.traceId, metadata.trace_id, trace.traceId, trace.trace_id, ids.traceId, ids.trace_id, stdio.traceId, stdio.trace_id);
|
|
if (name === "sessionId") return textCandidates(value.sessionId, value.session_id, event.sessionId, event.session_id, context.sessionId, context.session_id, run.sessionId, run.session_id, nestedPayload.sessionId, nestedPayload.session_id, payload.sessionId, payload.session_id, metadata.sessionId, metadata.session_id, ids.sessionId, ids.session_id, stdio.sessionId, stdio.session_id);
|
|
if (name === "runId") return textCandidates(value.runId, value.run_id, event.runId, event.run_id, context.runId, context.run_id, run.runId, run.run_id, nestedEvent.runId, nestedEvent.run_id, nestedPayload.runId, nestedPayload.run_id, payload.runId, payload.run_id, metadata.runId, metadata.run_id, ids.runId, ids.run_id, stdio.runId, stdio.run_id);
|
|
if (name === "commandId") return textCandidates(value.commandId, value.command_id, event.commandId, event.command_id, context.commandId, context.command_id, command.commandId, command.command_id, nestedEvent.commandId, nestedEvent.command_id, nestedPayload.commandId, nestedPayload.command_id, payload.commandId, payload.command_id, metadata.commandId, metadata.command_id, ids.commandId, ids.command_id, stdio.commandId, stdio.command_id);
|
|
return [];
|
|
}
|
|
|
|
function kafkaTopicForStream(stream, env) {
|
|
const normalized = stringValue(stream) || "hwlab";
|
|
if (normalized === "stdio" || normalized === "codex-stdio") return stringValue(env.HWLAB_KAFKA_STDIO_TOPIC ?? env.AGENTRUN_KAFKA_STDIO_TOPIC) || DEFAULT_STDIO_TOPIC;
|
|
if (normalized === "agentrun") return stringValue(env.HWLAB_KAFKA_AGENTRUN_EVENT_TOPIC) || DEFAULT_AGENTRUN_EVENT_TOPIC;
|
|
if (normalized === "hwlab") return stringValue(env.HWLAB_KAFKA_EVENT_TOPIC) || DEFAULT_HWLAB_EVENT_TOPIC;
|
|
return normalized;
|
|
}
|
|
|
|
function defaultKafkaFactory(config) {
|
|
return new Kafka({ clientId: config.clientId, brokers: config.brokers, logLevel: logLevel.NOTHING });
|
|
}
|
|
|
|
async function boundedDisconnect(consumer, timeoutMs = 2000) {
|
|
let timer = null;
|
|
try {
|
|
await Promise.race([
|
|
consumer.disconnect(),
|
|
new Promise((resolve) => {
|
|
timer = setTimeout(resolve, timeoutMs);
|
|
timer.unref?.();
|
|
})
|
|
]);
|
|
} finally {
|
|
if (timer) clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
export function buildHwlabKafkaProducerMessage(projected) {
|
|
return {
|
|
key: kafkaMessageKey(projected),
|
|
value: JSON.stringify(projected),
|
|
headers: kafkaHeaders(projected)
|
|
};
|
|
}
|
|
|
|
function kafkaMessageKey(projected) {
|
|
return firstText(projected.sessionId, projected.traceId, projected.context?.commandId, projected.context?.runId) || "hwlab-event";
|
|
}
|
|
|
|
function kafkaHeaders(projected) {
|
|
return {
|
|
"x-trace-id": projected.traceId ?? "",
|
|
"x-session-id": projected.sessionId ?? "",
|
|
"x-run-id": projected.context?.runId ?? "",
|
|
"x-command-id": projected.context?.commandId ?? "",
|
|
"x-values-printed": "false"
|
|
};
|
|
}
|
|
|
|
function textPayload(payload, fallback) {
|
|
return firstText(payload.text, payload.message, payload.summary, payload.output, payload.errorMessage, payload.reason) || fallback;
|
|
}
|
|
|
|
function truthy(value) {
|
|
return TRUE_VALUES.has(String(value ?? "").trim().toLowerCase());
|
|
}
|
|
|
|
function csv(value) {
|
|
return String(value ?? "").split(",").map((entry) => entry.trim()).filter(Boolean);
|
|
}
|
|
|
|
function parseJson(text) {
|
|
try {
|
|
return JSON.parse(text);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function objectValue(value) {
|
|
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
|
}
|
|
|
|
function firstText(...values) {
|
|
for (const value of values) {
|
|
const text = stringValue(value);
|
|
if (text) return text;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function textCandidates(...values) {
|
|
return values.map(stringValue).filter(Boolean);
|
|
}
|
|
|
|
function stringValue(value) {
|
|
const text = typeof value === "string" ? value.trim() : value === null || value === undefined ? "" : String(value).trim();
|
|
return text.length > 0 ? text : null;
|
|
}
|
|
|
|
function integerValue(value) {
|
|
if (value === null || value === undefined || (typeof value === "string" && value.trim() === "")) return null;
|
|
const parsed = Number(value);
|
|
return Number.isFinite(parsed) ? Math.floor(parsed) : null;
|
|
}
|
|
|
|
function nextKafkaOffset(value) {
|
|
try {
|
|
return (BigInt(String(value)) + 1n).toString();
|
|
} catch {
|
|
throw contractError("workbench_kafka_offset_invalid", "Kafka source offset must be an integer string.");
|
|
}
|
|
}
|
|
|
|
function strictPositiveInteger(value, name) {
|
|
const parsed = Number(value);
|
|
if (!Number.isInteger(parsed) || parsed <= 0) throw contractError("hwlab_kafka_projector_config_invalid", `${name} must be a positive integer.`);
|
|
return parsed;
|
|
}
|
|
|
|
function contractError(code, message) {
|
|
const error = new Error(message);
|
|
error.code = code;
|
|
return error;
|
|
}
|
|
|
|
function timestampValue(value) {
|
|
const text = stringValue(value);
|
|
if (!text) return null;
|
|
const ms = Date.parse(text);
|
|
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
|
|
}
|
|
|
|
function sha256(text) {
|
|
return createHash("sha256").update(String(text ?? "")).digest("hex");
|
|
}
|
|
|
|
function compactObject(value) {
|
|
return Object.fromEntries(Object.entries(value).filter(([, entry]) => stringValue(entry)));
|
|
}
|
|
|
|
function errorMessage(error) {
|
|
return error instanceof Error ? error.message : String(error);
|
|
}
|
|
|
|
function logInfo(logger, code, details) {
|
|
logger?.log?.(JSON.stringify({ code, component: "hwlab-kafka-event-bridge", ...details, valuesPrinted: false }));
|
|
}
|
|
|
|
function logWarn(logger, code, details) {
|
|
logger?.warn?.(JSON.stringify({ code, component: "hwlab-kafka-event-bridge", ...details, valuesPrinted: false }));
|
|
}
|