root
f416950556
Move fresh and retry runner dispatch into AgentRun command admission, remove process-local job creation and secret persistence, and split oversized modules by responsibility. Co-Authored-By: Codex <noreply@openai.com>
988 lines
42 KiB
TypeScript
988 lines
42 KiB
TypeScript
/*
|
|
* SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-read-model-timeline-contract; draft-2026-06-28-p0-d518-session-timeline-consistency; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-api-contract; PJ2026-010401 Web工作台 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-model-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0
|
|
* 职责: Workbench SSE and Kafka realtime transport over durable read-model facts.
|
|
*/
|
|
import { createHash } from "node:crypto";
|
|
|
|
import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
|
import {
|
|
parsePositiveInteger,
|
|
safeConversationId,
|
|
safeOpaqueId,
|
|
safeSessionId,
|
|
safeTraceId,
|
|
sendJson
|
|
} from "./server-http-utils.ts";
|
|
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
|
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
|
|
import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts";
|
|
import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts";
|
|
import { openKafkaEventStream } from "./kafka-event-bridge.ts";
|
|
import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts";
|
|
import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts";
|
|
import * as workbenchFacts from "./server-workbench-facts.ts";
|
|
|
|
const DEFAULT_PAGE_LIMIT = 50;
|
|
const DEFAULT_SESSION_LIST_LIMIT = 20;
|
|
const MAX_PAGE_LIMIT = 100;
|
|
const DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS = 15000;
|
|
const WORKBENCH_REALTIME_DRAIN_TIMEOUT_MS = 2500;
|
|
const WORKBENCH_REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2";
|
|
const WORKBENCH_SESSION_LIST_PAGE_FAMILIES = Object.freeze(["sessions"]);
|
|
const WORKBENCH_SESSION_DETAIL_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]);
|
|
const WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]);
|
|
const WORKBENCH_TRACE_EVENT_METADATA_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]);
|
|
const WORKBENCH_TRACE_EVENT_PAGE_FAMILIES = Object.freeze(["traceEvents"]);
|
|
const activeWorkbenchRealtimeConnections = new Set();
|
|
|
|
const {
|
|
factSessionSummary,
|
|
factSessionDetail,
|
|
factMessagesForSession,
|
|
canonicalFactMessageGroupsForSession,
|
|
factMessageCanonicalGroupKey,
|
|
selectCanonicalFactMessage,
|
|
factMessageCanonicalScore,
|
|
factMessageHasFinalResponsePart,
|
|
workbenchLifecycleMessageId,
|
|
factMessageDto,
|
|
factMessageAuthorityText,
|
|
factPartsHaveFinalResponse,
|
|
factSyntheticTerminalFinalResponse,
|
|
isTerminalProjectionStatus,
|
|
firstFactPartText,
|
|
factMessageFinalResponseText,
|
|
factTerminalMessageForTrace,
|
|
factTerminalStatusFromMessageProjection,
|
|
factPartDto,
|
|
factTurnForTrace,
|
|
factTurnSnapshot,
|
|
factTraceSnapshot,
|
|
factTraceTimingProjection,
|
|
factTraceEventIsTerminalAuthority,
|
|
factTraceEventTerminalTimes,
|
|
factTraceEventDto,
|
|
factProjectionForTrace,
|
|
factTimingProjection,
|
|
factCombinedTimingProjection,
|
|
factTimingSource,
|
|
firstTimestampIso,
|
|
latestTimestampIso,
|
|
elapsedFactMs,
|
|
timestampIso,
|
|
factCheckpointForTrace,
|
|
workbenchProjectionStoreError,
|
|
nonNegativeIntegerOrNull,
|
|
factSessionId,
|
|
factLastTraceId,
|
|
factLatestTraceIdForSession,
|
|
factCurrentTraceForSession,
|
|
factMessageTraceCandidate,
|
|
factTraceCandidate,
|
|
compareTraceCandidatesDesc,
|
|
nonUnknownStatus,
|
|
factTurnId,
|
|
factUpdatedAt,
|
|
factSeq,
|
|
factProjectedSeq,
|
|
normalizeProjectionStatus,
|
|
normalizeProjectionHealth,
|
|
compareFactSessionsDesc,
|
|
compareFactMessagesAsc,
|
|
compareFactMessagesForSessionAsc,
|
|
factMessageTimelineKey,
|
|
factTimelineAnchorMessageForTrace,
|
|
factMessageTimelineRoleRank,
|
|
firstFiniteNumber,
|
|
compareOptionalNumberAsc,
|
|
compareFactPartsAsc,
|
|
compareFactTraceEventsAsc,
|
|
compareFactRecordsDesc,
|
|
compareTimestampAsc,
|
|
compareTimestampDesc,
|
|
compareOptionalTimestampAsc,
|
|
compareNumberAsc,
|
|
compareNumberDesc,
|
|
compareText,
|
|
timestampMs,
|
|
optionalTimestampMs,
|
|
numericValue,
|
|
mergeFactSets,
|
|
emptyFactSet,
|
|
factArray,
|
|
uniqueText,
|
|
eventSeq,
|
|
normalizeStatus,
|
|
normalizeTerminalStatus,
|
|
firstUserPreview,
|
|
latestUserPreview,
|
|
latestValidMessagePreview,
|
|
latestAssistantPreview,
|
|
sessionTitleFromMessages,
|
|
sessionPreviewFromMessages,
|
|
boundedPreviewText,
|
|
isAssistantLikeRole,
|
|
objectValue,
|
|
textValue,
|
|
messageAuthorityTextValue,
|
|
safeTurnId,
|
|
safeMessageId,
|
|
safePartId,
|
|
hash
|
|
} = workbenchFacts;
|
|
|
|
import {
|
|
authenticateWorkbenchRead,
|
|
blockedTraceEventProjection,
|
|
classifyWorkbenchReadModelFailure,
|
|
isAgentRunAliasSessionId,
|
|
logWorkbenchReadModelFailure,
|
|
methodNotAllowed,
|
|
nonNegativeInteger,
|
|
queryFactsByRouteId,
|
|
traceEventsReadModelBlocker,
|
|
visibleFactSessionForTrace,
|
|
visibleFactSessions,
|
|
workbenchError
|
|
} from "./server-workbench-read-http.ts";
|
|
|
|
export async function drainWorkbenchRealtimeConnections(options = {}) {
|
|
const reason = textValue(options.reason) || "server_shutdown";
|
|
const signal = textValue(options.signal) || null;
|
|
const timeoutMs = parsePositiveInteger(options.timeoutMs, WORKBENCH_REALTIME_DRAIN_TIMEOUT_MS);
|
|
const connections = [...activeWorkbenchRealtimeConnections].filter((connection) => connection?.isActive?.());
|
|
for (const connection of connections) {
|
|
try {
|
|
connection.close({ reason, signal });
|
|
} catch {
|
|
// Best effort: one broken client must not block shutdown drain for the rest.
|
|
}
|
|
}
|
|
const deadline = Date.now() + timeoutMs;
|
|
while (Date.now() < deadline) {
|
|
const activeAfterClose = connections.filter((connection) => activeWorkbenchRealtimeConnections.has(connection) && connection?.isActive?.()).length;
|
|
if (activeAfterClose === 0) break;
|
|
await sleepMs(Math.min(50, Math.max(1, deadline - Date.now())));
|
|
}
|
|
const activeAfter = connections.filter((connection) => activeWorkbenchRealtimeConnections.has(connection) && connection?.isActive?.()).length;
|
|
return {
|
|
ok: activeAfter === 0,
|
|
activeBefore: connections.length,
|
|
closed: Math.max(0, connections.length - activeAfter),
|
|
activeAfter,
|
|
timeout: activeAfter > 0,
|
|
reason,
|
|
signal
|
|
};
|
|
}
|
|
|
|
|
|
export async function handleWorkbenchRealtimeHttp(request, response, url, options = {}) {
|
|
try {
|
|
if (request.method !== "GET") return methodNotAllowed(response, "GET");
|
|
const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId"));
|
|
const requestedTraceId = safeTraceId(url.searchParams.get("traceId"));
|
|
const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS);
|
|
attachWorkbenchRealtimeOtelContext(request, {
|
|
sessionId: requestedSessionId,
|
|
traceId: requestedTraceId,
|
|
heartbeatMs,
|
|
realtimeSource: "kafka"
|
|
});
|
|
const perf = options.backendPerformance;
|
|
const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options);
|
|
if (!auth) return;
|
|
|
|
if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) {
|
|
sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench read model is keyed by sessionId/threadId/traceId only."));
|
|
return;
|
|
}
|
|
|
|
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
|
const readModel = createWorkbenchReadModel(options, auth.actor);
|
|
const requestedAfterSeq = workbenchRealtimeAfterSeq(request, url);
|
|
let closed = false;
|
|
let realtimeCloseReason = "client_close";
|
|
let realtimeCloseSignal = null;
|
|
let realtimeSessionId = requestedSessionId ?? null;
|
|
let realtimeTraceId = requestedTraceId ?? null;
|
|
let realtimeThreadId = null;
|
|
const realtimeStartedAtMs = Date.now();
|
|
const cleanup = [];
|
|
|
|
response.writeHead(200, {
|
|
"content-type": "text/event-stream; charset=utf-8",
|
|
"cache-control": "no-store, no-transform",
|
|
connection: "keep-alive",
|
|
"x-accel-buffering": "no",
|
|
"x-content-type-options": "nosniff"
|
|
});
|
|
if (typeof response.flushHeaders === "function") response.flushHeaders();
|
|
|
|
const writeEvent = (name, payload = {}) => {
|
|
if (closed || response.destroyed) return;
|
|
const startedAt = nowMs();
|
|
const eventCreatedAt = realtimeEventCreatedAt(payload);
|
|
const traceSeq = realtimeTraceSeq(payload);
|
|
const eventId = realtimeEventId(payload);
|
|
response.write(`event: ${name}\n`);
|
|
if (eventId) response.write(`id: ${eventId}\n`);
|
|
response.write(`data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), eventCreatedAt, traceSeq, ...payload })}\n\n`);
|
|
perf?.recordPhase({ phase: "sse_write", durationMs: nowMs() - startedAt, outcome: "ok" });
|
|
};
|
|
|
|
const realtimeConnection = {
|
|
close(fields = {}) {
|
|
if (closed || response.destroyed || response.writableEnded) return false;
|
|
const reason = textValue(fields.reason) || "server_shutdown";
|
|
realtimeCloseReason = reason;
|
|
realtimeCloseSignal = textValue(fields.signal) || null;
|
|
try {
|
|
writeEvent("workbench.server_draining", {
|
|
type: "server.draining",
|
|
status: "closing",
|
|
reason,
|
|
signal: realtimeCloseSignal,
|
|
sessionId: realtimeSessionId,
|
|
threadId: realtimeThreadId,
|
|
traceId: realtimeTraceId,
|
|
observedAt: new Date().toISOString()
|
|
});
|
|
response.end();
|
|
return true;
|
|
} catch {
|
|
try { response.destroy(); } catch {}
|
|
return false;
|
|
}
|
|
},
|
|
isActive() {
|
|
return !closed && !response.destroyed && !response.writableEnded;
|
|
}
|
|
};
|
|
activeWorkbenchRealtimeConnections.add(realtimeConnection);
|
|
cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection));
|
|
|
|
writeEvent("workbench.connected", {
|
|
type: "connected",
|
|
status: "connected",
|
|
realtimeSource: "kafka",
|
|
snapshotSource: "read-model",
|
|
heartbeatMs,
|
|
cursor: { outboxSeq: requestedAfterSeq },
|
|
filters: {
|
|
sessionId: requestedSessionId,
|
|
traceId: requestedTraceId
|
|
}
|
|
});
|
|
|
|
const resolveInitialSession = requestedSessionId && requestedAfterSeq <= 0;
|
|
const initialContext = resolveInitialSession
|
|
? await realtimeInitialSessionContext(readModel, auth.actor, requestedSessionId)
|
|
: { facts: emptyFactSet(), session: null, blocker: null };
|
|
if (initialContext.blocker) {
|
|
writeEvent("workbench.error", {
|
|
type: "error",
|
|
reason: "initial-session",
|
|
sessionId: requestedSessionId,
|
|
traceId: requestedTraceId,
|
|
error: initialContext.blocker
|
|
});
|
|
}
|
|
const initialFactSession = initialContext.session;
|
|
const streamSessionId = factSessionId(initialFactSession) ?? requestedSessionId ?? null;
|
|
const streamThreadId = safeOpaqueId(initialFactSession?.threadId) ?? (textValue(initialFactSession?.threadId) || null);
|
|
const activeTraceId = requestedTraceId
|
|
?? factLastTraceId(initialFactSession)
|
|
?? factLatestTraceIdForSession(initialContext.facts, streamSessionId)
|
|
?? null;
|
|
realtimeSessionId = streamSessionId ?? requestedSessionId ?? null;
|
|
realtimeTraceId = activeTraceId ?? requestedTraceId ?? null;
|
|
realtimeThreadId = streamThreadId ?? null;
|
|
attachWorkbenchRealtimeOtelContext(request, {
|
|
sessionId: streamSessionId ?? requestedSessionId,
|
|
traceId: activeTraceId,
|
|
threadId: streamThreadId,
|
|
heartbeatMs,
|
|
realtimeSource: "kafka"
|
|
});
|
|
emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env);
|
|
if (activeTraceId && requestedAfterSeq <= 0) {
|
|
await (perf ? perf.measure("workbench_initial_trace", () => writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })) : writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" }));
|
|
}
|
|
const kafkaFilters = resolveWorkbenchRealtimeKafkaFilters({ traceId: requestedTraceId, sessionId: streamSessionId, traceStore });
|
|
try {
|
|
const kafkaStream = await openKafkaEventStream({
|
|
env: options.env ?? process.env,
|
|
stream: "hwlab",
|
|
fromBeginning: false,
|
|
...kafkaFilters,
|
|
kafkaFactory: options.kafkaFactory,
|
|
onEvent: async (record) => {
|
|
if (closed || response.destroyed) return;
|
|
const realtime = workbenchRealtimeEventFromKafka(record, { sessionId: streamSessionId, threadId: streamThreadId, traceId: activeTraceId });
|
|
if (!realtime) return;
|
|
writeEvent("workbench.trace.event", realtime.traceEvent);
|
|
if (realtime.turnSnapshot) writeEvent("workbench.turn.snapshot", realtime.turnSnapshot);
|
|
},
|
|
onError: (error) => {
|
|
writeEvent("workbench.error", {
|
|
type: "error",
|
|
realtimeSource: "kafka",
|
|
sessionId: streamSessionId,
|
|
threadId: streamThreadId,
|
|
traceId: activeTraceId,
|
|
error: { code: "workbench_kafka_stream_failed", message: error?.message ?? "Workbench Kafka realtime stream failed." }
|
|
});
|
|
}
|
|
});
|
|
cleanup.push(() => { void kafkaStream.stop?.(); });
|
|
} catch (kafkaError) {
|
|
writeEvent("workbench.error", {
|
|
type: "error",
|
|
realtimeSource: "kafka",
|
|
sessionId: streamSessionId,
|
|
threadId: streamThreadId,
|
|
traceId: activeTraceId,
|
|
error: { code: "workbench_kafka_stream_unavailable", message: kafkaError?.message ?? "Workbench Kafka realtime stream is unavailable." }
|
|
});
|
|
}
|
|
|
|
const heartbeat = setInterval(async () => {
|
|
try {
|
|
writeEvent("workbench.heartbeat", {
|
|
type: "heartbeat",
|
|
sessionId: requestedSessionId ?? null,
|
|
traceId: activeTraceId ?? null,
|
|
observedAt: new Date().toISOString()
|
|
});
|
|
} catch (error) {
|
|
writeEvent("workbench.error", {
|
|
type: "error",
|
|
error: { code: "workbench_realtime_heartbeat_failed", message: error?.message ?? "Workbench realtime heartbeat failed." }
|
|
});
|
|
}
|
|
}, Math.max(1000, heartbeatMs));
|
|
cleanup.push(() => clearInterval(heartbeat));
|
|
|
|
response.on("close", () => {
|
|
if (closed) return;
|
|
closed = true;
|
|
emitWorkbenchRealtimeClosedOtelSpan(request, options.env, {
|
|
reason: realtimeCloseReason,
|
|
signal: realtimeCloseSignal,
|
|
startedAtMs: realtimeStartedAtMs,
|
|
sessionId: realtimeSessionId,
|
|
threadId: realtimeThreadId,
|
|
traceId: realtimeTraceId,
|
|
activeConnectionCount: activeWorkbenchRealtimeConnections.size
|
|
});
|
|
for (const item of cleanup.splice(0)) item();
|
|
});
|
|
} catch (error) {
|
|
handleWorkbenchRealtimeFailure(request, response, url, options, error);
|
|
}
|
|
}
|
|
|
|
export function attachWorkbenchRealtimeOtelContext(request, fields = {}) {
|
|
const context = request?.hwlabHttpRequestContext;
|
|
if (!context) return;
|
|
context.route = "/v1/workbench/events";
|
|
context.otelAttributes = {
|
|
...(context.otelAttributes ?? {}),
|
|
"workbench.route": "events",
|
|
"workbench.session_id": fields.sessionId ?? null,
|
|
traceId: fields.traceId ?? null,
|
|
"workbench.trace_id": fields.traceId ?? null,
|
|
"workbench.thread_id": fields.threadId ?? null,
|
|
"workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null,
|
|
"workbench.sse.realtime_source": fields.realtimeSource ?? null,
|
|
"workbench.sse.outbox_mode": false
|
|
};
|
|
}
|
|
|
|
export function resolveWorkbenchRealtimeKafkaFilters({ traceId = null, sessionId = null, traceStore = null } = {}) {
|
|
const resolved = traceId && typeof traceStore?.snapshot === "function" ? collectTraceLinkedIds(traceStore.snapshot(traceId)) : {};
|
|
const hasAgentRunKey = Boolean(resolved.runId || resolved.commandId);
|
|
const fallbackSessionId = resolved.sessionId || agentRunScopedSessionIdFromWorkbenchSession(sessionId) || sessionId;
|
|
return compactObject({
|
|
traceId: hasAgentRunKey ? null : traceId,
|
|
sessionId: hasAgentRunKey ? null : fallbackSessionId,
|
|
runId: resolved.runId,
|
|
commandId: resolved.commandId
|
|
});
|
|
}
|
|
|
|
export function agentRunScopedSessionIdFromWorkbenchSession(sessionId) {
|
|
const safeId = safeSessionId(sessionId);
|
|
if (!safeId || isAgentRunAliasSessionId(safeId)) return safeId;
|
|
const base = safeId.replace(/^ses_/u, "").replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "") || "session";
|
|
return safeSessionId(`ses_agentrun_${base}`);
|
|
}
|
|
|
|
export function collectTraceLinkedIds(snapshot) {
|
|
const out = { sessionId: null, runId: null, commandId: null };
|
|
const events = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
|
for (const event of events) collectTraceLinkedIdsFromRecord(event, out);
|
|
collectTraceLinkedIdsFromRecord(snapshot?.lastEvent, out);
|
|
return out;
|
|
}
|
|
|
|
export function collectTraceLinkedIdsFromRecord(record, out) {
|
|
const value = record && typeof record === "object" && !Array.isArray(record) ? record : {};
|
|
const agentRun = value.agentRun && typeof value.agentRun === "object" && !Array.isArray(value.agentRun) ? value.agentRun : {};
|
|
const payload = value.payload && typeof value.payload === "object" && !Array.isArray(value.payload) ? value.payload : {};
|
|
out.sessionId ||= safeSessionId(value.sessionId ?? value.sourceSessionId ?? agentRun.sessionId ?? payload.sessionId);
|
|
out.runId ||= textValue(value.runId ?? value.sourceRunId ?? agentRun.runId ?? payload.runId);
|
|
out.commandId ||= textValue(value.commandId ?? value.sourceCommandId ?? agentRun.commandId ?? payload.commandId);
|
|
}
|
|
|
|
export function workbenchRealtimeEventFromKafka(record, context = {}) {
|
|
const value = record?.value && typeof record.value === "object" && !Array.isArray(record.value) ? record.value : null;
|
|
if (!value) return null;
|
|
const hwlabEvent = value.event && typeof value.event === "object" && !Array.isArray(value.event) ? value.event : {};
|
|
const sourceContext = value.context && typeof value.context === "object" && !Array.isArray(value.context) ? value.context : {};
|
|
const traceId = safeTraceId(value.traceId ?? hwlabEvent.traceId ?? context.traceId) ?? textValue(value.traceId ?? hwlabEvent.traceId ?? context.traceId);
|
|
const sessionId = safeSessionId(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId) ?? textValue(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId);
|
|
const threadId = safeOpaqueId(context.threadId ?? sourceContext.threadId) ?? textValue(context.threadId ?? sourceContext.threadId);
|
|
if (!traceId && !sessionId) return null;
|
|
const sourceSeq = integerValue(hwlabEvent.sourceSeq ?? sourceContext.sourceSeq);
|
|
const kafka = {
|
|
topic: textValue(record.topic),
|
|
partition: integerValue(record.partition),
|
|
offset: textValue(record.offset),
|
|
key: textValue(record.key),
|
|
timestamp: textValue(record.timestamp),
|
|
valuesRedacted: true
|
|
};
|
|
const event = {
|
|
...hwlabEvent,
|
|
traceId: traceId ?? hwlabEvent.traceId ?? null,
|
|
sessionId: sessionId ?? hwlabEvent.sessionId ?? null,
|
|
threadId: threadId ?? sourceContext.threadId ?? null,
|
|
runId: textValue(hwlabEvent.runId ?? sourceContext.runId),
|
|
commandId: textValue(hwlabEvent.commandId ?? sourceContext.commandId),
|
|
source: textValue(hwlabEvent.source) || "hwlab.kafka",
|
|
sourceSeq,
|
|
kafka,
|
|
valuesPrinted: false
|
|
};
|
|
const status = normalizeStatus(event.status);
|
|
const terminal = event.terminal === true || TERMINAL_STATUSES.has(status);
|
|
const cursor = {
|
|
traceSeq: sourceSeq ?? integerValue(record.offset),
|
|
kafkaOffset: textValue(record.offset),
|
|
kafkaPartition: integerValue(record.partition)
|
|
};
|
|
const entityVersion = sourceSeq ?? integerValue(record.offset) ?? 0;
|
|
const projectionRevision = ["kafka", kafka.topic, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":");
|
|
const traceEntity = {
|
|
family: "traceEvents",
|
|
id: [traceId ?? "trace", event.runId, event.commandId, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":"),
|
|
version: entityVersion,
|
|
traceSeq: cursor.traceSeq,
|
|
projectionRevision,
|
|
authority: WORKBENCH_REALTIME_AUTHORITY_VERSION
|
|
};
|
|
const traceEvent = {
|
|
id: kafka.topic && kafka.partition !== null && kafka.offset ? `${kafka.topic}:${kafka.partition}:${kafka.offset}` : null,
|
|
type: "trace.event",
|
|
contractVersion: "workbench-sync-v1",
|
|
realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION,
|
|
realtimeSource: "kafka",
|
|
sessionId,
|
|
threadId,
|
|
traceId,
|
|
event,
|
|
snapshot: { traceId, sessionId, threadId, status: status || event.status || null, events: [event], eventCount: 1 },
|
|
cursor,
|
|
entity: traceEntity,
|
|
projectionRevision,
|
|
kafka,
|
|
valuesPrinted: false
|
|
};
|
|
const turnSnapshot = terminal ? {
|
|
type: "turn.snapshot",
|
|
contractVersion: "workbench-sync-v1",
|
|
realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION,
|
|
realtimeSource: "kafka",
|
|
sessionId,
|
|
threadId,
|
|
traceId,
|
|
reason: "kafka-terminal",
|
|
cursor,
|
|
turn: {
|
|
traceId,
|
|
sessionId,
|
|
threadId,
|
|
status: status || event.status || "completed",
|
|
running: false,
|
|
terminal: true,
|
|
finalResponse: terminalFinalResponse(status || event.status || "completed", {
|
|
traceId,
|
|
status: status || event.status || "completed",
|
|
finalResponse: event.text ? { text: event.text, status: status || event.status || "completed", traceId, source: "kafka-terminal-event", valuesPrinted: false } : null,
|
|
finalText: event.text ?? event.message ?? null,
|
|
error: event.errorCode ? { message: event.message ?? event.errorCode } : null
|
|
}),
|
|
agentRun: compactObject({ runId: event.runId, commandId: event.commandId }),
|
|
valuesPrinted: false
|
|
},
|
|
entity: {
|
|
family: "turns",
|
|
id: traceId ?? event.runId ?? event.commandId ?? traceEntity.id,
|
|
version: entityVersion,
|
|
traceSeq: cursor.traceSeq,
|
|
projectionRevision,
|
|
authority: WORKBENCH_REALTIME_AUTHORITY_VERSION
|
|
},
|
|
projectionRevision,
|
|
kafka,
|
|
valuesPrinted: false
|
|
} : null;
|
|
return { traceEvent, turnSnapshot };
|
|
}
|
|
|
|
export function compactObject(value) {
|
|
return Object.fromEntries(Object.entries(value || {}).filter(([, entry]) => textValue(entry)));
|
|
}
|
|
|
|
export function integerValue(value) {
|
|
const number = Number(value);
|
|
return Number.isFinite(number) ? Math.trunc(number) : null;
|
|
}
|
|
|
|
export function attachWorkbenchReadModelDiagnosticOtel(request, fields = {}) {
|
|
const context = request?.hwlabHttpRequestContext;
|
|
if (!context) return;
|
|
const code = textValue(fields.code) || "workbench_read_model_not_found";
|
|
const route = textValue(fields.route) || context.route || "/v1/workbench";
|
|
context.route = route;
|
|
context.lastHttpError = {
|
|
code,
|
|
category: "workbench_read_model",
|
|
layer: "workbench.read_model"
|
|
};
|
|
context.otelAttributes = {
|
|
...(context.otelAttributes ?? {}),
|
|
"workbench.route": route,
|
|
"workbench.read_model.route": route,
|
|
"workbench.read_model.result": code,
|
|
"workbench.read_model.count": Number.isFinite(Number(fields.count)) ? Math.trunc(Number(fields.count)) : null,
|
|
"workbench.session_id": fields.sessionId ?? null,
|
|
"workbench.trace_id": fields.traceId ?? null,
|
|
"workbench.turn_id": fields.turnId ?? null,
|
|
traceId: fields.traceId ?? null,
|
|
"error.code": code,
|
|
"error.category": "workbench_read_model",
|
|
"error.layer": "workbench.read_model"
|
|
};
|
|
}
|
|
|
|
export function emitWorkbenchRealtimeAcceptedOtelSpan(request, env = process.env) {
|
|
const context = request?.hwlabHttpRequestContext;
|
|
if (!context?.traceId) return;
|
|
const endedAtMs = Date.now();
|
|
void emitHttpServerRequestSpan(context, env, {
|
|
startTimeMs: context.startedAtMs,
|
|
endTimeMs: endedAtMs,
|
|
statusCode: 200,
|
|
method: "GET",
|
|
route: "/v1/workbench/events",
|
|
attributes: {
|
|
...(context.otelAttributes ?? {}),
|
|
"http.closed_by": "sse-accepted",
|
|
"workbench.sse.lifecycle": "accepted",
|
|
"workbench.sse.accept_latency_ms": Math.max(0, endedAtMs - Number(context.startedAtMs ?? endedAtMs))
|
|
}
|
|
}).catch(() => undefined);
|
|
}
|
|
|
|
export function emitWorkbenchRealtimeClosedOtelSpan(request, env = process.env, fields = {}) {
|
|
const context = request?.hwlabHttpRequestContext;
|
|
if (!context?.traceId) return;
|
|
const endedAtMs = Date.now();
|
|
const startedAtMs = Number(fields.startedAtMs ?? context.startedAtMs ?? endedAtMs);
|
|
const reason = textValue(fields.reason) || "client_close";
|
|
void emitHttpServerRequestSpan(context, env, {
|
|
startTimeMs: Number.isFinite(startedAtMs) ? startedAtMs : endedAtMs,
|
|
endTimeMs: endedAtMs,
|
|
statusCode: 200,
|
|
method: "GET",
|
|
route: "/v1/workbench/events",
|
|
attributes: {
|
|
...(context.otelAttributes ?? {}),
|
|
"http.closed_by": reason,
|
|
"workbench.sse.lifecycle": "closed",
|
|
"workbench.sse.close_reason": reason,
|
|
"workbench.sse.close_signal": fields.signal ?? null,
|
|
"workbench.sse.duration_ms": Math.max(0, endedAtMs - (Number.isFinite(startedAtMs) ? startedAtMs : endedAtMs)),
|
|
"workbench.sse.session_id": fields.sessionId ?? null,
|
|
"workbench.sse.thread_id": fields.threadId ?? null,
|
|
"workbench.sse.trace_id": fields.traceId ?? null,
|
|
"workbench.sse.active_connection_count": Number.isFinite(Number(fields.activeConnectionCount)) ? Number(fields.activeConnectionCount) : null
|
|
}
|
|
}).catch(() => undefined);
|
|
}
|
|
|
|
export function handleWorkbenchRealtimeFailure(request, response, url, options = {}, error) {
|
|
const classified = classifyWorkbenchReadModelFailure(error);
|
|
const context = request?.hwlabHttpRequestContext;
|
|
if (context) {
|
|
context.route = "/v1/workbench/events";
|
|
context.otelAttributes = {
|
|
...(context.otelAttributes ?? {}),
|
|
"workbench.route": "events",
|
|
"workbench.sse.lifecycle": "failed",
|
|
"error.code": classified.code,
|
|
"error.layer": "workbench.realtime"
|
|
};
|
|
}
|
|
logWorkbenchReadModelFailure(options.logger ?? console, {
|
|
event: "workbench_realtime_route_failed",
|
|
ok: false,
|
|
route: "/v1/workbench/events",
|
|
statusCode: classified.statusCode,
|
|
errorCode: classified.code,
|
|
errorName: error?.name ?? "Error",
|
|
message: error instanceof Error ? error.message : String(error ?? "unknown"),
|
|
valuesRedacted: true
|
|
});
|
|
if (response.headersSent && !response.destroyed) {
|
|
try {
|
|
response.write(`event: workbench.error\n`);
|
|
response.write(`data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), type: "error", error: { code: classified.code, message: classified.message, valuesRedacted: true } })}\n\n`);
|
|
response.end();
|
|
} catch {
|
|
try { response.destroy(); } catch {}
|
|
}
|
|
return;
|
|
}
|
|
if (!response.destroyed) sendJson(response, classified.statusCode, workbenchError(classified.code, classified.message, { route: "/v1/workbench/events", errorName: error?.name ?? "Error" }));
|
|
}
|
|
|
|
export async function writeMessageRealtimeSnapshotSafe({ writeEvent, readModel, actor, sessionId, threadId = null, traceId = null, reason = "message", cursor = null } = {}) {
|
|
const safeSession = safeSessionId(sessionId);
|
|
if (!safeSession || typeof readModel?.queryFacts !== "function") return;
|
|
try {
|
|
const result = await queryFactsByRouteId(readModel, safeSession, { families: WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES });
|
|
if (result.error) return;
|
|
const session = visibleFactSessions(result.facts, actor)[0] ?? null;
|
|
if (!session) return;
|
|
const messages = factMessagesForSession(session, result.facts);
|
|
const message = [...messages].reverse().find((item) => {
|
|
if (!isAssistantLikeRole(item?.role)) return false;
|
|
return !traceId || item.traceId === traceId;
|
|
}) ?? null;
|
|
if (!message) return;
|
|
writeEvent("workbench.message.snapshot", {
|
|
type: "message.snapshot",
|
|
sessionId: factSessionId(session),
|
|
threadId: threadId ?? session.threadId ?? null,
|
|
traceId: message.traceId ?? traceId ?? null,
|
|
reason,
|
|
cursor,
|
|
message
|
|
});
|
|
} catch (error) {
|
|
writeEvent("workbench.error", {
|
|
type: "error",
|
|
error: { code: "workbench_message_snapshot_failed", message: error?.message ?? "Workbench realtime message snapshot failed." }
|
|
});
|
|
}
|
|
}
|
|
|
|
export function realtimeEventCreatedAt(payload) {
|
|
const candidates = [payload?.event?.createdAt, payload?.snapshot?.lastEvent?.createdAt, payload?.turn?.updatedAt];
|
|
for (const value of candidates) {
|
|
const text = textValue(value);
|
|
if (text) return text;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export function realtimeTraceSeq(payload) {
|
|
const seq = Number(payload?.cursor?.traceSeq ?? payload?.event?.seq ?? payload?.snapshot?.lastEvent?.seq);
|
|
return Number.isFinite(seq) && seq >= 0 ? seq : null;
|
|
}
|
|
|
|
export function realtimeEventId(payload) {
|
|
const raw = payload?.cursor?.outboxSeq ?? payload?.outboxSeq ?? payload?.id;
|
|
const text = textValue(raw);
|
|
if (!text || /[\r\n]/u.test(text)) return null;
|
|
return text.slice(0, 128);
|
|
}
|
|
|
|
export function workbenchRealtimeAfterSeq(request, url) {
|
|
for (const value of [url.searchParams.get("afterSeq"), url.searchParams.get("afterOutboxSeq"), request?.headers?.["last-event-id"]]) {
|
|
const parsed = nonNegativeInteger(value, 0);
|
|
if (parsed > 0) return parsed;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
export function nowMs() {
|
|
if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now();
|
|
return Date.now();
|
|
}
|
|
|
|
export function sleepMs(ms) {
|
|
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
}
|
|
|
|
|
|
export async function realtimeInitialSessionContext(readModel, actor, sessionId) {
|
|
const safeId = safeSessionId(sessionId);
|
|
if (!safeId || typeof readModel?.queryFacts !== "function") {
|
|
const blocker = traceEventsReadModelBlocker("workbench_facts_query_unavailable", "Workbench realtime initial session must be read from durable facts; the facts query interface is unavailable.", { session: { sessionId: safeId }, route: "/v1/workbench/events" });
|
|
return { facts: emptyFactSet(), session: null, blocker };
|
|
}
|
|
try {
|
|
const result = await readModel.queryFacts({ sessionId: safeId, families: WORKBENCH_SESSION_DETAIL_FAMILIES, limit: MAX_PAGE_LIMIT });
|
|
if (result.error || result.durable === false) {
|
|
const blocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench realtime initial session facts query failed; realtime must surface the read-model error instead of falling back to legacy session snapshots.", { session: { sessionId: safeId }, route: "/v1/workbench/events" });
|
|
blocker.causeCode = result.error?.code ?? null;
|
|
blocker.causeName = result.error?.name ?? null;
|
|
blocker.message = result.error?.message ? `${blocker.message} Cause: ${result.error.message}` : result.durable === false ? `${blocker.message} Cause: durable facts reader unavailable.` : blocker.message;
|
|
return { facts: emptyFactSet(), session: null, blocker };
|
|
}
|
|
const facts = result.facts ?? emptyFactSet();
|
|
const session = visibleFactSessions(facts, actor).find((item) => factSessionId(item) === safeId) ?? null;
|
|
return { facts, session, blocker: null };
|
|
} catch (error) {
|
|
const blocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench realtime initial session facts query failed; realtime must surface the read-model error instead of falling back to legacy session snapshots.", { session: { sessionId: safeId }, route: "/v1/workbench/events" });
|
|
blocker.causeCode = error?.code ?? null;
|
|
blocker.causeName = error?.name ?? null;
|
|
blocker.message = error?.message ? `${blocker.message} Cause: ${error.message}` : blocker.message;
|
|
return { facts: emptyFactSet(), session: null, blocker };
|
|
}
|
|
}
|
|
|
|
export async function writeTraceRealtimeSnapshot({ writeEvent, options, actor, traceId, reason }) {
|
|
const context = await visibleTraceContext(options, actor, traceId);
|
|
if (!context.visible) {
|
|
writeEvent("workbench.trace.unavailable", {
|
|
type: "trace.unavailable",
|
|
reason,
|
|
traceId,
|
|
error: { code: "workbench_trace_not_found", message: "Workbench trace is not visible to the current actor." }
|
|
});
|
|
return;
|
|
}
|
|
if (context.factsReadBlocker) {
|
|
writeEvent("workbench.error", {
|
|
type: "error",
|
|
traceId,
|
|
reason,
|
|
error: context.factsReadBlocker
|
|
});
|
|
}
|
|
const sessionId = factSessionId(context.factSession) ?? safeSessionId(context.session?.id) ?? null;
|
|
const threadId = safeOpaqueId(context.factSession?.threadId ?? context.session?.threadId) ?? (textValue(context.factSession?.threadId ?? context.session?.threadId) || null);
|
|
const realtimeTraceSnapshot = context.factsReadBlocker ? blockedRealtimeTraceSnapshot(context) : traceSnapshotForRealtime(context.trace);
|
|
const traceSeq = context.factsReadBlocker ? 0 : traceSnapshotLastSeq(context.trace);
|
|
writeEvent("workbench.trace.snapshot", {
|
|
type: "trace.snapshot",
|
|
reason,
|
|
sessionId,
|
|
threadId,
|
|
traceId,
|
|
snapshot: { ...realtimeTraceSnapshot, sessionId, threadId },
|
|
cursor: { traceSeq }
|
|
});
|
|
writeEvent("workbench.turn.snapshot", {
|
|
type: "turn.snapshot",
|
|
reason,
|
|
sessionId,
|
|
threadId,
|
|
traceId,
|
|
turn: realtimeTurnSnapshot(context),
|
|
cursor: { traceSeq }
|
|
});
|
|
}
|
|
|
|
export function realtimeTurnSnapshot(context) {
|
|
if (context?.factsReadBlocker) return blockedRealtimeTurnSnapshot(context, context.factsReadBlocker);
|
|
const facts = context?.facts;
|
|
const traceId = safeTraceId(context?.traceId);
|
|
if (traceId && facts && context?.factSession) {
|
|
const factTurn = factTurnForTrace(facts, traceId);
|
|
const projected = factTurnSnapshot({ turn: factTurn, session: context.factSession, facts, traceId, turnId: context.turnId });
|
|
if (projected) return projected;
|
|
}
|
|
const blocker = traceEventsReadModelBlocker("workbench_facts_turn_missing", "Workbench durable facts did not contain a turn snapshot; realtime must surface the read-model gap instead of falling back to legacy trace projection.", { traceId, session: context?.factSession, route: "/v1/workbench/events" });
|
|
return blockedRealtimeTurnSnapshot(context, blocker);
|
|
}
|
|
|
|
export function blockedRealtimeTraceSnapshot(context) {
|
|
const traceId = safeTraceId(context?.traceId ?? context?.factsReadBlocker?.traceId) ?? null;
|
|
const blocker = context?.factsReadBlocker ?? traceEventsReadModelBlocker("workbench_facts_unavailable", "Workbench durable facts are unavailable for realtime trace snapshot.", { traceId, route: "/v1/workbench/events" });
|
|
const projection = blockedTraceEventProjection({}, blocker);
|
|
return {
|
|
traceId,
|
|
status: "unknown",
|
|
traceStatus: "unknown",
|
|
events: [],
|
|
eventCount: 0,
|
|
fullTraceLoaded: true,
|
|
hasMore: false,
|
|
nextProjectedSeq: 0,
|
|
updatedAt: null,
|
|
projection,
|
|
projectionStatus: projection.projectionStatus,
|
|
projectionHealth: projection.projectionHealth,
|
|
staleMs: projection.staleMs ?? null,
|
|
blocker,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
};
|
|
}
|
|
|
|
export function blockedRealtimeTurnSnapshot(context, blocker) {
|
|
const traceId = safeTraceId(context?.traceId ?? blocker?.traceId) ?? null;
|
|
const turnId = safeTurnId(context?.turnId ?? blocker?.turnId) || traceId;
|
|
const sessionId = safeSessionId(blocker?.sessionId ?? factSessionId(context?.factSession) ?? context?.session?.id) ?? null;
|
|
const threadId = safeOpaqueId(context?.factSession?.threadId ?? context?.session?.threadId) ?? (textValue(context?.session?.threadId) || null);
|
|
const projection = blockedTraceEventProjection({}, blocker);
|
|
return {
|
|
turnId,
|
|
traceId,
|
|
status: "unknown",
|
|
running: false,
|
|
terminal: false,
|
|
sessionId,
|
|
threadId,
|
|
userMessageId: null,
|
|
assistantMessageId: null,
|
|
assistantText: null,
|
|
finalResponse: null,
|
|
timing: null,
|
|
startedAt: null,
|
|
lastEventAt: null,
|
|
finishedAt: null,
|
|
durationMs: null,
|
|
agentRun: null,
|
|
trace: {
|
|
traceId,
|
|
status: "unknown",
|
|
eventCount: 0,
|
|
updatedAt: null
|
|
},
|
|
urls: {
|
|
self: turnId ? `/v1/workbench/turns/${encodeURIComponent(turnId)}` : null,
|
|
traceEvents: traceId ? `/v1/workbench/traces/${encodeURIComponent(traceId)}/events` : null
|
|
},
|
|
projectionStatus: projection.projectionStatus,
|
|
projectionHealth: projection.projectionHealth,
|
|
staleMs: projection.staleMs ?? null,
|
|
blocker,
|
|
projection,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
};
|
|
}
|
|
|
|
export async function writeTraceRealtimeSnapshotSafe(input) {
|
|
try {
|
|
await writeTraceRealtimeSnapshot(input);
|
|
} catch (error) {
|
|
input.writeEvent("workbench.error", {
|
|
type: "error",
|
|
traceId: input.traceId,
|
|
reason: input.reason,
|
|
error: {
|
|
code: "workbench_trace_snapshot_failed",
|
|
causeCode: error?.code ?? null,
|
|
retryable: error?.data?.retryable === true,
|
|
message: error?.message ?? "Workbench trace snapshot failed.",
|
|
valuesRedacted: true
|
|
}
|
|
});
|
|
}
|
|
}
|
|
|
|
export async function visibleTraceContext(options, actor, traceId) {
|
|
const readModel = createWorkbenchReadModel(options, actor);
|
|
if (typeof readModel.queryFacts !== "function") {
|
|
const projection = factProjectionForTrace(emptyFactSet(), traceId);
|
|
const factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_unavailable", "Workbench durable facts query is unavailable; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" });
|
|
return { visible: true, turnId: traceId, traceId, status: "unknown", projection, result: null, session: null, trace: factTraceSnapshot(emptyFactSet(), traceId), facts: emptyFactSet(), factSession: null, factsReadBlocker };
|
|
}
|
|
|
|
let facts = emptyFactSet();
|
|
let factsReadBlocker = null;
|
|
try {
|
|
const factResult = await readModel.queryFacts({ traceId, families: [...WORKBENCH_SESSION_DETAIL_FAMILIES, ...WORKBENCH_TRACE_EVENT_PAGE_FAMILIES], limit: MAX_PAGE_LIMIT });
|
|
if (factResult.error || factResult.durable === false) {
|
|
const projection = factProjectionForTrace(facts, traceId);
|
|
factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts query failed; realtime must surface the read-model error instead of silently falling back to legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" });
|
|
factsReadBlocker.causeCode = factResult.error?.code ?? null;
|
|
factsReadBlocker.causeName = factResult.error?.name ?? null;
|
|
factsReadBlocker.message = factResult.error?.message ? `${factsReadBlocker.message} Cause: ${factResult.error.message}` : factResult.durable === false ? `${factsReadBlocker.message} Cause: durable facts reader unavailable.` : factsReadBlocker.message;
|
|
} else {
|
|
facts = factResult?.facts ?? facts;
|
|
}
|
|
} catch (error) {
|
|
const projection = factProjectionForTrace(facts, traceId);
|
|
factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts query failed; realtime must surface the read-model error instead of silently falling back to legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" });
|
|
factsReadBlocker.causeCode = error?.code ?? null;
|
|
factsReadBlocker.causeName = error?.name ?? null;
|
|
factsReadBlocker.message = error?.message ? `${factsReadBlocker.message} Cause: ${error.message}` : factsReadBlocker.message;
|
|
}
|
|
|
|
const resolved = factsReadBlocker ? { session: null, facts } : await visibleFactSessionForTrace(readModel, facts, actor, traceId);
|
|
if (resolved.error) {
|
|
const projection = factProjectionForTrace(facts, traceId);
|
|
factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts session lookup failed; realtime must surface the read-model error instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" });
|
|
factsReadBlocker.causeCode = resolved.error?.code ?? null;
|
|
factsReadBlocker.causeName = resolved.error?.name ?? null;
|
|
factsReadBlocker.message = resolved.error?.message ? `${factsReadBlocker.message} Cause: ${resolved.error.message}` : factsReadBlocker.message;
|
|
} else if (resolved.facts) {
|
|
facts = resolved.facts;
|
|
}
|
|
|
|
const factSession = resolved.session ?? null;
|
|
const trace = factTraceSnapshot(facts, traceId);
|
|
const turn = factTurnForTrace(facts, traceId, traceId);
|
|
let projection = factsReadBlocker ? blockedTraceEventProjection(factProjectionForTrace(facts, traceId), factsReadBlocker) : factProjectionForTrace(facts, traceId);
|
|
const visible = Boolean(factsReadBlocker || factSession || turn || factCheckpointForTrace(facts, traceId) || trace.eventCount > 0);
|
|
if (!visible) {
|
|
factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_trace_missing", "Workbench realtime could not find durable facts for the requested trace; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" });
|
|
projection = blockedTraceEventProjection(projection, factsReadBlocker);
|
|
}
|
|
if (!factsReadBlocker && !factSession) {
|
|
factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_session_missing", "Workbench durable facts did not contain the trace session; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, turn, route: "/v1/workbench/events" });
|
|
projection = blockedTraceEventProjection(projection, factsReadBlocker);
|
|
}
|
|
const snapshot = factTurnSnapshot({ turn, session: factSession, facts, traceId, turnId: traceId });
|
|
return { visible: true, turnId: snapshot.turnId ?? traceId, traceId, status: snapshot.status, projection, result: null, session: null, trace, facts, factSession, factsReadBlocker };
|
|
}
|
|
|
|
export function traceSnapshotForRealtime(snapshot) {
|
|
const events = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
|
return {
|
|
...snapshot,
|
|
events,
|
|
eventCount: Number.isFinite(Number(snapshot?.eventCount)) ? Number(snapshot.eventCount) : events.length,
|
|
lastEvent: snapshot?.lastEvent ?? events.at(-1) ?? null,
|
|
valuesRedacted: true,
|
|
secretMaterialStored: false
|
|
};
|
|
}
|
|
|
|
export function traceSnapshotSummary(snapshot) {
|
|
const events = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
|
return {
|
|
traceId: snapshot?.traceId ?? null,
|
|
status: snapshot?.status ?? "missing",
|
|
eventCount: Number.isFinite(Number(snapshot?.eventCount)) ? Number(snapshot.eventCount) : events.length,
|
|
lastEvent: snapshot?.lastEvent ?? events.at(-1) ?? null,
|
|
updatedAt: snapshot?.updatedAt ?? null,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
|
|
export function traceSnapshotLastSeq(snapshot) {
|
|
const events = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
|
return events.reduce((max, event, index) => Math.max(max, eventSeq(event, index)), 0);
|
|
}
|