Merge pull request #2465 from pikasTech/fix/2463-durable-admission
Pipelines as Code CI / hwlab-nc01-v03-ci-poll- Success
Pipelines as Code CI / hwlab-nc01-v03-ci-poll- Success
fix: AgentRun 持久调度准入替代进程内 runner Job
This commit is contained in:
Lyon
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,361 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { submitAgentRunChatTurn, syncAgentRunChatResult } from "./code-agent-agentrun-adapter.ts";
|
||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
|
||||
const sourceCommit = "0123456789abcdef0123456789abcdef01234567";
|
||||
|
||||
test("fresh gpt.pika turn returns running only after durable dispatch intent is confirmed", async () => {
|
||||
const traceId = "trc_gpt_pika_durable_admission";
|
||||
const calls = [];
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const result = await submitAgentRunChatTurn({
|
||||
traceId,
|
||||
traceStore,
|
||||
params: {
|
||||
ownerUserId: "usr_gpt_pika",
|
||||
projectId: "pikasTech/HWLAB",
|
||||
conversationId: "cnv_gpt_pika_durable",
|
||||
sessionId: "ses_gpt_pika_durable",
|
||||
message: "verify durable admission",
|
||||
providerProfile: "gpt.pika"
|
||||
},
|
||||
options: {
|
||||
env: agentRunTestEnv(),
|
||||
accessController: {
|
||||
async codeAgentToolCapabilitiesForOwner() {
|
||||
return { tools: { hwpod: { allowed: true } } };
|
||||
},
|
||||
store: {
|
||||
async findActiveDefaultApiKeyForUser() {
|
||||
return { displaySecret: "hwl_live_must_not_be_persisted" };
|
||||
}
|
||||
}
|
||||
},
|
||||
fetchImpl: createAgentRunFetch(calls, {
|
||||
dispatchIntent: {
|
||||
id: "dispatch_gpt_pika_durable",
|
||||
state: "pending",
|
||||
runnerJobId: "rjob_gpt_pika_durable",
|
||||
attemptCount: 0,
|
||||
durable: true
|
||||
}
|
||||
})
|
||||
}
|
||||
});
|
||||
|
||||
assert.equal(result.status, "running");
|
||||
assert.equal(result.agentRun.runId, "run_gpt_pika_durable");
|
||||
assert.equal(result.agentRun.commandId, "cmd_gpt_pika_durable");
|
||||
assert.equal(result.agentRun.dispatchIntentId, "dispatch_gpt_pika_durable");
|
||||
assert.equal(result.agentRun.dispatchIntentState, "pending");
|
||||
assert.equal(result.agentRun.runnerJobId, "rjob_gpt_pika_durable");
|
||||
assert.equal(result.agentRun.durableDispatch, true);
|
||||
assert.equal(result.agentRun.status, "dispatch-intent-pending");
|
||||
|
||||
const commandCall = calls.find((call) => call.path.endsWith("/commands"));
|
||||
assert.ok(commandCall);
|
||||
assert.equal(commandCall.body.idempotencyKey, traceId);
|
||||
assert.equal(commandCall.body.payload.traceId, traceId);
|
||||
assert.equal(commandCall.body.payload.sessionId, "ses_agentrun_gpt_pika_durable");
|
||||
assert.equal(commandCall.body.payload.hwlabSessionId, "ses_gpt_pika_durable");
|
||||
assert.equal(commandCall.body.payload.conversationId, "cnv_gpt_pika_durable");
|
||||
assert.equal(commandCall.body.dispatch.kind, "kubernetes-job");
|
||||
assert.equal(Object.hasOwn(commandCall.body.dispatch.input, "commandId"), false);
|
||||
assert.equal(commandCall.body.dispatch.input.idempotencyKey, traceId);
|
||||
assert.equal(Object.hasOwn(commandCall.body.dispatch.input, "namespace"), false);
|
||||
assert.equal(Object.hasOwn(commandCall.body.dispatch.input, "image"), false);
|
||||
assert.equal(Object.hasOwn(commandCall.body.dispatch.input, "serviceAccountName"), false);
|
||||
assert.equal(commandCall.body.dispatch.input.transientEnv.every((entry) => entry.sensitive === false), true);
|
||||
assert.equal(commandCall.body.dispatch.input.transientEnv.every((entry) => !entry.name.startsWith("AGENTRUN_") && !entry.name.startsWith("KUBERNETES_")), true);
|
||||
assert.equal(JSON.stringify(commandCall.body).includes("hwl_live_must_not_be_persisted"), false);
|
||||
assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false);
|
||||
assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:dispatch-intent:admitted"));
|
||||
});
|
||||
|
||||
test("fresh turn rejects an unconfirmed dispatch intent instead of returning false running", async () => {
|
||||
const calls = [];
|
||||
await assert.rejects(
|
||||
submitAgentRunChatTurn({
|
||||
traceId: "trc_dispatch_intent_missing",
|
||||
traceStore: createCodeAgentTraceStore(),
|
||||
params: {
|
||||
sessionId: "ses_dispatch_intent_missing",
|
||||
conversationId: "cnv_dispatch_intent_missing",
|
||||
message: "must fail admission",
|
||||
providerProfile: "gpt.pika"
|
||||
},
|
||||
options: {
|
||||
env: agentRunTestEnv(),
|
||||
fetchImpl: createAgentRunFetch(calls, { dispatchIntent: null })
|
||||
}
|
||||
}),
|
||||
(error) => error?.code === "agentrun_dispatch_intent_unconfirmed" && error?.statusCode === 502
|
||||
);
|
||||
assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false);
|
||||
});
|
||||
|
||||
test("fresh turn rejects a durable dispatch intent with missing contract fields", async () => {
|
||||
await assert.rejects(
|
||||
submitAgentRunChatTurn({
|
||||
traceId: "trc_dispatch_intent_attempt_missing",
|
||||
traceStore: createCodeAgentTraceStore(),
|
||||
params: {
|
||||
sessionId: "ses_dispatch_intent_attempt_missing",
|
||||
message: "must require explicit attempt count",
|
||||
providerProfile: "gpt.pika"
|
||||
},
|
||||
options: {
|
||||
env: agentRunTestEnv(),
|
||||
fetchImpl: createAgentRunFetch([], {
|
||||
dispatchIntent: {
|
||||
id: "dispatch_attempt_missing",
|
||||
state: "pending",
|
||||
runnerJobId: "rjob_attempt_missing",
|
||||
durable: true
|
||||
}
|
||||
})
|
||||
}
|
||||
}),
|
||||
(error) => error?.code === "agentrun_dispatch_intent_unconfirmed" && error?.statusCode === 502
|
||||
);
|
||||
});
|
||||
|
||||
for (const terminal of [
|
||||
{ state: "failed", code: "agentrun_dispatch_intent_failed", statusCode: 503 },
|
||||
{ state: "cancelled", code: "agentrun_dispatch_intent_cancelled", statusCode: 409 }
|
||||
]) {
|
||||
test(`fresh turn exposes ${terminal.state} durable dispatch replay instead of returning running`, async () => {
|
||||
await assert.rejects(
|
||||
submitAgentRunChatTurn({
|
||||
traceId: `trc_dispatch_intent_${terminal.state}`,
|
||||
traceStore: createCodeAgentTraceStore(),
|
||||
params: {
|
||||
sessionId: `ses_dispatch_intent_${terminal.state}`,
|
||||
message: `surface ${terminal.state} admission`,
|
||||
providerProfile: "gpt.pika"
|
||||
},
|
||||
options: {
|
||||
env: agentRunTestEnv(),
|
||||
fetchImpl: createAgentRunFetch([], {
|
||||
dispatchIntent: {
|
||||
id: `dispatch_terminal_${terminal.state}`,
|
||||
state: terminal.state,
|
||||
runnerJobId: `rjob_terminal_${terminal.state}`,
|
||||
attemptCount: 2,
|
||||
durable: true
|
||||
}
|
||||
})
|
||||
}
|
||||
}),
|
||||
(error) => error?.code === terminal.code && error?.statusCode === terminal.statusCode
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
test("gpt.pika session-eviction retry uses the same durable dispatch admission", async () => {
|
||||
const traceId = "trc_gpt_pika_durable_retry";
|
||||
const calls = [];
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const synced = await syncAgentRunChatResult({
|
||||
traceId,
|
||||
traceStore,
|
||||
refreshEvents: false,
|
||||
forceResultSync: true,
|
||||
retryParams: {
|
||||
ownerUserId: "usr_gpt_pika_retry",
|
||||
projectId: "pikasTech/HWLAB",
|
||||
conversationId: "cnv_gpt_pika_retry",
|
||||
sessionId: "ses_gpt_pika_retry",
|
||||
threadId: "thread_evicted",
|
||||
message: "retry the evicted gpt.pika turn",
|
||||
providerProfile: "gpt.pika"
|
||||
},
|
||||
currentResult: {
|
||||
traceId,
|
||||
status: "running",
|
||||
conversationId: "cnv_gpt_pika_retry",
|
||||
sessionId: "ses_gpt_pika_retry",
|
||||
threadId: "thread_evicted",
|
||||
providerProfile: "gpt.pika",
|
||||
agentRun: {
|
||||
adapter: "agentrun-v01",
|
||||
runId: "run_gpt_pika_evicted",
|
||||
commandId: "cmd_gpt_pika_evicted",
|
||||
sessionId: "ses_agentrun_gpt_pika_retry",
|
||||
threadId: "thread_evicted",
|
||||
backendProfile: "gpt.pika",
|
||||
managerUrl: "http://127.0.0.1:65535",
|
||||
status: "running",
|
||||
traceId,
|
||||
lastSeq: 0,
|
||||
providerTrace: { traceId, valuesPrinted: false },
|
||||
valuesPrinted: false
|
||||
},
|
||||
valuesPrinted: false
|
||||
},
|
||||
options: {
|
||||
env: agentRunTestEnv(),
|
||||
fetchImpl: createAgentRunRetryFetch(calls, traceId)
|
||||
}
|
||||
});
|
||||
|
||||
assert.equal(synced.freshSessionRetried, true);
|
||||
assert.equal(synced.result.status, "completed");
|
||||
assert.equal(synced.result.agentRun.runId, "run_gpt_pika_retry");
|
||||
assert.equal(synced.result.agentRun.commandId, "cmd_gpt_pika_retry");
|
||||
assert.equal(synced.result.agentRun.freshSessionRetryOf.runId, "run_gpt_pika_evicted");
|
||||
const retryCommand = calls.find((call) => call.path === "/api/v1/runs/run_gpt_pika_retry/commands");
|
||||
assert.ok(retryCommand);
|
||||
assert.equal(retryCommand.body.idempotencyKey, traceId);
|
||||
assert.equal(retryCommand.body.payload.threadId, null);
|
||||
assert.equal(retryCommand.body.dispatch.kind, "kubernetes-job");
|
||||
assert.equal(Object.hasOwn(retryCommand.body.dispatch.input, "commandId"), false);
|
||||
assert.equal(retryCommand.body.dispatch.input.transientEnv.every((entry) => entry.sensitive === false), true);
|
||||
assert.equal(JSON.stringify(retryCommand.body).includes("HWLAB_API_KEY"), false);
|
||||
assert.equal(calls.some((call) => call.path.endsWith("/runner-jobs")), false);
|
||||
assert.ok(traceStore.snapshot(traceId).events.some((event) => event.label === "agentrun:dispatch-intent:admitted-after-session-reset"));
|
||||
});
|
||||
|
||||
function agentRunTestEnv() {
|
||||
return {
|
||||
AGENTRUN_MGR_URL: "http://127.0.0.1:65535",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_DISPATCH_RETRY_MAX: "0",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "NC01",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_SOURCE_COMMIT: sourceCommit,
|
||||
HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE: "gpt.pika",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE: "caller-controlled-namespace-must-be-ignored",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_RUNNER_IMAGE: "caller-controlled-image-must-be-ignored",
|
||||
HWLAB_CODE_AGENT_AGENTRUN_RUNNER_SERVICE_ACCOUNT: "caller-controlled-service-account-must-be-ignored",
|
||||
HWLAB_RUNTIME_API_URL: "http://hwlab-cloud-api.hwlab-v03.svc.cluster.local:6667",
|
||||
HWLAB_RUNTIME_WEB_URL: "http://hwlab-cloud-web.hwlab-v03.svc.cluster.local:8080",
|
||||
HWLAB_RUNTIME_NAMESPACE: "hwlab-v03",
|
||||
HWLAB_RUNTIME_LANE: "v03",
|
||||
HWLAB_RUNTIME_ENDPOINT_LOCKED: "1",
|
||||
HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME: "1",
|
||||
UNIDESK_MAIN_SERVER_IP: "http://127.0.0.1:18081"
|
||||
};
|
||||
}
|
||||
|
||||
function createAgentRunFetch(calls, { dispatchIntent }) {
|
||||
return async (url, init = {}) => {
|
||||
const parsed = new URL(url);
|
||||
const method = init.method ?? "GET";
|
||||
const body = init.body ? JSON.parse(String(init.body)) : null;
|
||||
calls.push({ method, path: parsed.pathname, body });
|
||||
const send = (data) => new Response(JSON.stringify({ ok: true, data }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" }
|
||||
});
|
||||
if (method === "POST" && parsed.pathname === "/api/v1/sessions") {
|
||||
return send({ session: { id: "ses_agentrun_gpt_pika_durable", metadata: {} } });
|
||||
}
|
||||
if (method === "POST" && parsed.pathname === "/api/v1/runs") {
|
||||
return send({
|
||||
id: "run_gpt_pika_durable",
|
||||
status: "pending",
|
||||
sessionRef: body.sessionRef,
|
||||
backendProfile: body.backendProfile
|
||||
});
|
||||
}
|
||||
if (method === "POST" && parsed.pathname === "/api/v1/runs/run_gpt_pika_durable/commands") {
|
||||
return send({
|
||||
id: "cmd_gpt_pika_durable",
|
||||
runId: "run_gpt_pika_durable",
|
||||
state: "pending",
|
||||
type: "turn",
|
||||
seq: 1,
|
||||
dispatchIntent
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected AgentRun call ${method} ${parsed.pathname}`);
|
||||
};
|
||||
}
|
||||
|
||||
function createAgentRunRetryFetch(calls, traceId) {
|
||||
return async (url, init = {}) => {
|
||||
const parsed = new URL(url);
|
||||
const method = init.method ?? "GET";
|
||||
const body = init.body ? JSON.parse(String(init.body)) : null;
|
||||
calls.push({ method, path: parsed.pathname, body });
|
||||
const send = (data) => new Response(JSON.stringify({ ok: true, data }), {
|
||||
status: 200,
|
||||
headers: { "content-type": "application/json" }
|
||||
});
|
||||
if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_evicted/commands/cmd_gpt_pika_evicted/result") {
|
||||
return send({
|
||||
runId: "run_gpt_pika_evicted",
|
||||
commandId: "cmd_gpt_pika_evicted",
|
||||
status: "failed",
|
||||
runStatus: "failed",
|
||||
commandState: "failed",
|
||||
terminalStatus: "failed",
|
||||
failureKind: "session-store-evicted",
|
||||
failureMessage: "session storage was evicted while resuming the thread",
|
||||
completed: false,
|
||||
lastSeq: 1
|
||||
});
|
||||
}
|
||||
if (method === "POST" && parsed.pathname === "/api/v1/sessions") {
|
||||
return send({ session: { id: body.sessionId, metadata: {} } });
|
||||
}
|
||||
if (method === "POST" && parsed.pathname === "/api/v1/runs") {
|
||||
return send({
|
||||
id: "run_gpt_pika_retry",
|
||||
status: "pending",
|
||||
sessionRef: body.sessionRef,
|
||||
backendProfile: body.backendProfile
|
||||
});
|
||||
}
|
||||
if (method === "POST" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands") {
|
||||
return send({
|
||||
id: "cmd_gpt_pika_retry",
|
||||
runId: "run_gpt_pika_retry",
|
||||
state: "pending",
|
||||
type: "turn",
|
||||
seq: 1,
|
||||
dispatchIntent: {
|
||||
id: "dispatch_gpt_pika_retry",
|
||||
state: "pending",
|
||||
runnerJobId: "rjob_gpt_pika_retry",
|
||||
attemptCount: 0,
|
||||
durable: true
|
||||
}
|
||||
});
|
||||
}
|
||||
if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/events") {
|
||||
return send({ items: [] });
|
||||
}
|
||||
if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands") {
|
||||
return send({ items: [{
|
||||
id: "cmd_gpt_pika_retry",
|
||||
runId: "run_gpt_pika_retry",
|
||||
state: "completed",
|
||||
idempotencyKey: traceId,
|
||||
payload: { traceId }
|
||||
}] });
|
||||
}
|
||||
if (method === "GET" && parsed.pathname === "/api/v1/runs/run_gpt_pika_retry/commands/cmd_gpt_pika_retry/result") {
|
||||
return send({
|
||||
runId: "run_gpt_pika_retry",
|
||||
commandId: "cmd_gpt_pika_retry",
|
||||
status: "completed",
|
||||
runStatus: "completed",
|
||||
commandState: "completed",
|
||||
terminalStatus: "completed",
|
||||
completed: true,
|
||||
reply: "GPT_PIKA_RETRY_OK",
|
||||
lastSeq: 2,
|
||||
sessionRef: {
|
||||
sessionId: "ses_agentrun_gpt_pika_retry-reset-trcgptpikadu",
|
||||
conversationId: "cnv_gpt_pika_retry",
|
||||
threadId: "thread_gpt_pika_retry"
|
||||
},
|
||||
traceId
|
||||
});
|
||||
}
|
||||
throw new Error(`unexpected AgentRun retry call ${method} ${parsed.pathname}`);
|
||||
};
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,990 @@
|
||||
/*
|
||||
* SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-read-model-timeline-contract; draft-2026-06-28-p0-d518-session-timeline-consistency; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-api-contract; PJ2026-010401 Web工作台 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-model-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0
|
||||
* 职责: Workbench durable fact normalization and DTO projection helpers; no HTTP or realtime transport ownership.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import {
|
||||
parsePositiveInteger,
|
||||
safeConversationId,
|
||||
safeOpaqueId,
|
||||
safeSessionId,
|
||||
safeTraceId,
|
||||
sendJson
|
||||
} from "./server-http-utils.ts";
|
||||
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
||||
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
|
||||
import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts";
|
||||
import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts";
|
||||
import { openKafkaEventStream } from "./kafka-event-bridge.ts";
|
||||
import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts";
|
||||
import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts";
|
||||
|
||||
function projectionText(...values) {
|
||||
for (const value of values) {
|
||||
if (value && typeof value === "object") {
|
||||
const nested = messageAuthorityTextValue(value.text ?? value.content ?? value.message ?? value.summary ?? value.preview ?? value.title);
|
||||
if (nested) return nested;
|
||||
continue;
|
||||
}
|
||||
const text = messageAuthorityTextValue(value);
|
||||
if (text) return text;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function workbenchError(code, message, extra = {}) {
|
||||
return {
|
||||
ok: false,
|
||||
status: "failed",
|
||||
error: { code, message, ...extra },
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeRole(role) {
|
||||
const value = textValue(role).toLowerCase();
|
||||
return value === "agent" ? "assistant" : value;
|
||||
}
|
||||
|
||||
function partFact(part, index, messageId, traceId) {
|
||||
const type = textValue(part?.type) || "text";
|
||||
const text = messageAuthorityTextValue(part?.text ?? part?.content ?? part?.message);
|
||||
return {
|
||||
partId: safePartId(part?.partId ?? part?.id) || `prt_${hash(`${messageId}:${index}:${type}`).slice(0, 24)}`,
|
||||
messageId,
|
||||
traceId,
|
||||
type,
|
||||
text: text || null,
|
||||
status: normalizeStatus(part?.status ?? "completed"),
|
||||
toolName: textValue(part?.toolName ?? part?.name) || null,
|
||||
createdAt: textValue(part?.createdAt ?? part?.timestamp) || null,
|
||||
valuesRedacted: part?.valuesRedacted !== false
|
||||
};
|
||||
}
|
||||
|
||||
export function factSessionSummary(session, facts = {}) {
|
||||
const sessionId = factSessionId(session);
|
||||
if (!sessionId) return null;
|
||||
const currentTrace = factCurrentTraceForSession(facts, sessionId, session);
|
||||
const traceId = currentTrace?.traceId ?? factLastTraceId(session) ?? factLatestTraceIdForSession(facts, sessionId);
|
||||
const projection = traceId ? factProjectionForTrace(facts, traceId) : null;
|
||||
const turn = traceId ? factTurnForTrace(facts, traceId) : null;
|
||||
const checkpoint = traceId ? factCheckpointForTrace(facts, traceId) : null;
|
||||
const trace = traceId ? factTraceSnapshot(facts, traceId) : null;
|
||||
const messages = factMessagesForSession(session, facts);
|
||||
const terminalMessage = traceId ? factTerminalMessageForTrace(messages, traceId) : null;
|
||||
const traceStatus = normalizeTerminalStatus(trace?.status);
|
||||
const checkpointStatus = normalizeTerminalStatus(checkpoint?.status);
|
||||
const messageTerminalStatus = factTerminalStatusFromMessageProjection(terminalMessage);
|
||||
const turnStatus = normalizeStatus(turn?.status);
|
||||
const launchContext = compactLaunchContext(session?.sessionJson?.launchContext);
|
||||
const status = normalizeStatus(checkpointStatus ?? traceStatus ?? messageTerminalStatus ?? nonUnknownStatus(turnStatus) ?? nonUnknownStatus(currentTrace?.status) ?? session?.status);
|
||||
const timing = traceId
|
||||
? factCombinedTimingProjection(status, checkpoint, messageTerminalStatus ? terminalMessage : null, turn, trace)
|
||||
: factTimingProjection(session, status);
|
||||
const title = sessionTitleFromMessages(messages);
|
||||
const preview = sessionPreviewFromMessages(messages);
|
||||
return {
|
||||
sessionId,
|
||||
threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null),
|
||||
agentId: session?.agentId ?? "hwlab-code-agent",
|
||||
title,
|
||||
preview,
|
||||
titleSource: title ? "message-projection" : null,
|
||||
previewSource: preview ? "message-projection" : null,
|
||||
status,
|
||||
running: RUNNING_STATUSES.has(status),
|
||||
terminal: TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status),
|
||||
lastTraceId: traceId,
|
||||
projection,
|
||||
projectionStatus: projection?.projectionStatus ?? null,
|
||||
projectionHealth: projection?.projectionHealth ?? null,
|
||||
staleMs: projection?.staleMs ?? null,
|
||||
blocker: projection?.blocker ?? null,
|
||||
providerProfile: textValue(session?.providerProfile ?? session?.sessionJson?.providerProfile) || null,
|
||||
launchContext,
|
||||
messageCount: messages.length,
|
||||
firstUserMessagePreview: firstUserPreview(messages),
|
||||
updatedAt: factUpdatedAt(session),
|
||||
turnSummary: turn ? {
|
||||
turnId: factTurnId(turn, traceId),
|
||||
traceId,
|
||||
status,
|
||||
running: RUNNING_STATUSES.has(status),
|
||||
terminal: TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status),
|
||||
eventCount: projection?.lastProjectedSeq ?? trace?.eventCount,
|
||||
projection,
|
||||
projectionStatus: projection?.projectionStatus ?? null,
|
||||
projectionHealth: projection?.projectionHealth ?? null,
|
||||
staleMs: projection?.staleMs ?? null,
|
||||
blocker: projection?.blocker ?? null,
|
||||
timing,
|
||||
startedAt: timing.startedAt,
|
||||
lastEventAt: timing.lastEventAt,
|
||||
finishedAt: timing.finishedAt,
|
||||
durationMs: timing.durationMs,
|
||||
updatedAt: factUpdatedAt(turn)
|
||||
} : null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
export function factSessionDetail(session, facts = {}, options = {}) {
|
||||
const sessionId = factSessionId(session);
|
||||
return buildWorkbenchSessionDetail({
|
||||
summary: factSessionSummary(session, facts),
|
||||
session,
|
||||
sessionId,
|
||||
includeMessages: options.includeMessages === true,
|
||||
messages: factMessagesForSession(session, facts)
|
||||
});
|
||||
}
|
||||
|
||||
export function factMessagesForSession(session, facts = {}) {
|
||||
const sessionId = factSessionId(session);
|
||||
if (!sessionId) return [];
|
||||
const partsByMessageId = new Map();
|
||||
for (const part of factArray(facts.parts)) {
|
||||
if (part.sessionId !== sessionId) continue;
|
||||
const messageId = textValue(part.messageId);
|
||||
if (!messageId) continue;
|
||||
const existing = partsByMessageId.get(messageId) ?? [];
|
||||
existing.push(part);
|
||||
partsByMessageId.set(messageId, existing);
|
||||
}
|
||||
const messageGroups = canonicalFactMessageGroupsForSession(
|
||||
factArray(facts.messages)
|
||||
.filter((message) => message.sessionId === sessionId)
|
||||
.sort((left, right) => compareFactMessagesForSessionAsc(left, right, facts)),
|
||||
partsByMessageId,
|
||||
facts
|
||||
);
|
||||
return messageGroups
|
||||
.sort((left, right) => compareFactMessagesForSessionAsc(left.message, right.message, facts))
|
||||
.map((group) => factMessageDto(group.message, group.parts, facts))
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
export function canonicalFactMessageGroupsForSession(messages = [], partsByMessageId = new Map(), facts = {}) {
|
||||
const groups = new Map();
|
||||
for (const message of messages) {
|
||||
const key = factMessageCanonicalGroupKey(message);
|
||||
const existing = groups.get(key) ?? [];
|
||||
existing.push(message);
|
||||
groups.set(key, existing);
|
||||
}
|
||||
return [...groups.values()].map((groupMessages) => {
|
||||
const message = selectCanonicalFactMessage(groupMessages, partsByMessageId, facts);
|
||||
const aliasIds = uniqueText(groupMessages.map((item) => item?.messageId));
|
||||
const parts = aliasIds.flatMap((messageId) => partsByMessageId.get(messageId) ?? []);
|
||||
return { message, parts };
|
||||
});
|
||||
}
|
||||
|
||||
export function factMessageCanonicalGroupKey(message) {
|
||||
const messageId = textValue(message?.messageId) || textValue(message?.id);
|
||||
const role = textValue(message?.role) || "agent";
|
||||
const traceId = safeTraceId(message?.traceId);
|
||||
const sessionId = factSessionId(message) ?? textValue(message?.sessionId) ?? "session";
|
||||
if (traceId && isAssistantLikeRole(role)) return `assistant:${sessionId}:${traceId}`;
|
||||
return `message:${messageId || sessionId}:${role}:${traceId || "none"}`;
|
||||
}
|
||||
|
||||
export function selectCanonicalFactMessage(messages = [], partsByMessageId = new Map(), facts = {}) {
|
||||
if (messages.length <= 1) return messages[0] ?? null;
|
||||
const traceId = safeTraceId(messages[0]?.traceId);
|
||||
const preferredMessageId = traceId ? workbenchLifecycleMessageId(traceId, "agent") : null;
|
||||
const turnMessageId = traceId ? textValue(factTurnForTrace(facts, traceId)?.messageId) || null : null;
|
||||
return [...messages].sort((left, right) => {
|
||||
const score = factMessageCanonicalScore(right, partsByMessageId, preferredMessageId, turnMessageId) - factMessageCanonicalScore(left, partsByMessageId, preferredMessageId, turnMessageId);
|
||||
return score || compareFactRecordsDesc(left, right);
|
||||
})[0] ?? messages[0] ?? null;
|
||||
}
|
||||
|
||||
export function factMessageCanonicalScore(message, partsByMessageId = new Map(), preferredMessageId = null, turnMessageId = null) {
|
||||
const messageId = textValue(message?.messageId) || textValue(message?.id);
|
||||
let score = 0;
|
||||
if (preferredMessageId && messageId === preferredMessageId) score += 10000;
|
||||
if (turnMessageId && messageId === turnMessageId) score += 1000;
|
||||
if (factMessageHasFinalResponsePart(message, partsByMessageId)) score += 100;
|
||||
if (message?.sealed === true || message?.terminal === true) score += 50;
|
||||
if (isTerminalProjectionStatus(normalizeStatus(message?.status))) score += 25;
|
||||
score += Math.min(10, factSeq(message) ?? 0);
|
||||
return score;
|
||||
}
|
||||
|
||||
export function factMessageHasFinalResponsePart(message, partsByMessageId = new Map()) {
|
||||
const messageId = textValue(message?.messageId) || textValue(message?.id);
|
||||
if (!messageId) return false;
|
||||
return (partsByMessageId.get(messageId) ?? []).some((part) => (textValue(part?.partType ?? part?.type) === "final_response") && Boolean(projectionText(part?.text, part?.content, part?.message)));
|
||||
}
|
||||
|
||||
export function workbenchLifecycleMessageId(traceId, role = "agent") {
|
||||
const traceSuffix = (safeTraceId(traceId) || String(traceId || "trace"))
|
||||
.replace(/^trc_/u, "")
|
||||
.replace(/[^A-Za-z0-9_.:-]/gu, "_")
|
||||
.slice(0, 48) || "trace";
|
||||
return `msg_${traceSuffix}_${role === "user" ? "user" : "agent"}`;
|
||||
}
|
||||
|
||||
export function factMessageDto(message, parts = [], facts = {}) {
|
||||
const messageId = safeMessageId(message?.messageId) || textValue(message?.messageId);
|
||||
if (!messageId) return null;
|
||||
const traceId = safeTraceId(message?.traceId) ?? null;
|
||||
const role = textValue(message?.role) || "agent";
|
||||
const turn = traceId ? factTurnForTrace(facts, traceId) : null;
|
||||
const checkpoint = traceId ? factCheckpointForTrace(facts, traceId) : null;
|
||||
const messageStatus = normalizeStatus(message?.status);
|
||||
const assistantLike = isAssistantLikeRole(role);
|
||||
const legacyText = projectionText(message?.text, message?.content, message?.message, message?.finalResponse);
|
||||
const baseParts = parts.length > 0
|
||||
? [...parts].sort(compareFactPartsAsc).map((part) => factPartDto(part, messageId, traceId)).filter(Boolean)
|
||||
: !assistantLike && legacyText ? [partFact({ type: "text", text: legacyText, status: message?.status }, 0, messageId, traceId)] : [];
|
||||
const checkpointStatus = assistantLike ? normalizeTerminalStatus(checkpoint?.status) : null;
|
||||
const preliminaryStatus = normalizeStatus(checkpointStatus ?? (assistantLike ? turn?.status : null) ?? messageStatus);
|
||||
const syntheticTerminalFinalResponse = assistantLike && !factPartsHaveFinalResponse(baseParts)
|
||||
? factSyntheticTerminalFinalResponse(preliminaryStatus, traceId, message, checkpoint, turn)
|
||||
: null;
|
||||
const normalizedParts = syntheticTerminalFinalResponse
|
||||
? [...baseParts, partFact({ type: "final_response", text: syntheticTerminalFinalResponse.text, status: syntheticTerminalFinalResponse.status }, baseParts.length, messageId, traceId)]
|
||||
: baseParts;
|
||||
const messageTerminalStatus = assistantLike ? factTerminalStatusFromMessageProjection(message, normalizedParts) : null;
|
||||
const status = normalizeStatus(checkpointStatus ?? messageTerminalStatus ?? (assistantLike ? turn?.status : null) ?? messageStatus);
|
||||
const timing = assistantLike
|
||||
? factCombinedTimingProjection(status, checkpoint, messageTerminalStatus ? { ...message, parts: normalizedParts } : null, turn, message)
|
||||
: factTimingProjection(message, status);
|
||||
const text = factMessageAuthorityText({ role, status, parts: normalizedParts, legacyText });
|
||||
return {
|
||||
messageId,
|
||||
role,
|
||||
sessionId: (safeSessionId(message?.sessionId) ?? textValue(message?.sessionId)) || null,
|
||||
traceId,
|
||||
turnId: safeTurnId(message?.turnId) || traceId,
|
||||
status,
|
||||
parts: normalizedParts,
|
||||
text: text || "",
|
||||
textPreview: text ? text.slice(0, 240) : null,
|
||||
createdAt: textValue(message?.createdAt) || null,
|
||||
updatedAt: factUpdatedAt(message),
|
||||
timing,
|
||||
startedAt: timing.startedAt,
|
||||
lastEventAt: timing.lastEventAt,
|
||||
finishedAt: timing.finishedAt,
|
||||
durationMs: timing.durationMs,
|
||||
projectionStatus: null,
|
||||
projectionHealth: null,
|
||||
valuesRedacted: message?.valuesRedacted !== false
|
||||
};
|
||||
}
|
||||
|
||||
export function factMessageAuthorityText({ role, status, parts = [], legacyText = null } = {}) {
|
||||
if (!isAssistantLikeRole(role)) return firstFactPartText(parts) ?? legacyText ?? "";
|
||||
if (!isTerminalProjectionStatus(status)) return "";
|
||||
return firstFactPartText(parts, "final_response") ?? "";
|
||||
}
|
||||
|
||||
export function factPartsHaveFinalResponse(parts = []) {
|
||||
return parts.some((part) => part?.type === "final_response" && Boolean(projectionText(part?.text)));
|
||||
}
|
||||
|
||||
export function factSyntheticTerminalFinalResponse(status, traceId = null, ...records) {
|
||||
const terminalStatus = normalizeTerminalStatus(status);
|
||||
if (!terminalStatus || terminalStatus === "completed") return null;
|
||||
return terminalFinalResponse(terminalStatus, { traceId, status: terminalStatus, records }, { evidence: records });
|
||||
}
|
||||
|
||||
export function isTerminalProjectionStatus(status) {
|
||||
return TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status);
|
||||
}
|
||||
|
||||
export function firstFactPartText(parts = [], type = null) {
|
||||
for (const part of parts) {
|
||||
if (type && part?.type !== type) continue;
|
||||
const text = projectionText(part?.text);
|
||||
if (text) return text;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function factMessageFinalResponseText(message) {
|
||||
if (!message || !isAssistantLikeRole(message.role) || !isTerminalProjectionStatus(message.status)) return null;
|
||||
return firstFactPartText(message.parts, "final_response");
|
||||
}
|
||||
|
||||
export function factTerminalMessageForTrace(messages = [], traceId = null) {
|
||||
const safeTrace = safeTraceId(traceId);
|
||||
return [...factArray(messages)].reverse().find((message) => {
|
||||
if (!message || !isAssistantLikeRole(message.role)) return false;
|
||||
if (safeTrace && message.traceId !== safeTrace) return false;
|
||||
return Boolean(factTerminalStatusFromMessageProjection(message));
|
||||
}) ?? null;
|
||||
}
|
||||
|
||||
export function factTerminalStatusFromMessageProjection(message = null, parts = null) {
|
||||
if (!message || !isAssistantLikeRole(message.role)) return null;
|
||||
const sourceParts = Array.isArray(parts) ? parts : factArray(message.parts);
|
||||
if (!firstFactPartText(sourceParts, "final_response")) return null;
|
||||
const messageStatus = normalizeTerminalStatus(message.status);
|
||||
if (messageStatus) return messageStatus;
|
||||
for (const part of sourceParts) {
|
||||
if (textValue(part?.partType ?? part?.type) !== "final_response") continue;
|
||||
if (!projectionText(part?.text, part?.content, part?.message)) continue;
|
||||
const partStatus = normalizeTerminalStatus(part?.status);
|
||||
if (partStatus) return partStatus;
|
||||
if (part?.terminal === true || part?.sealed === true) return "completed";
|
||||
}
|
||||
return message.terminal === true || message.sealed === true ? "completed" : null;
|
||||
}
|
||||
|
||||
export function factPartDto(part, messageId, traceId) {
|
||||
const partId = safePartId(part?.partId) || textValue(part?.partId) || `prt_${hash(`${messageId}:${part?.partIndex ?? 0}:${part?.partType ?? part?.type ?? "text"}`).slice(0, 24)}`;
|
||||
const text = projectionText(part?.text, part?.content, part?.message);
|
||||
return {
|
||||
partId,
|
||||
messageId,
|
||||
traceId: safeTraceId(part?.traceId) ?? traceId,
|
||||
type: textValue(part?.partType ?? part?.type) || "text",
|
||||
text: text || null,
|
||||
status: normalizeStatus(part?.status),
|
||||
toolName: textValue(part?.toolName ?? part?.name) || null,
|
||||
createdAt: textValue(part?.createdAt ?? part?.occurredAt) || null,
|
||||
valuesRedacted: part?.valuesRedacted !== false
|
||||
};
|
||||
}
|
||||
|
||||
export function factTurnForTrace(facts = {}, traceId, turnId = null) {
|
||||
const safeTrace = safeTraceId(traceId);
|
||||
if (!safeTrace) return null;
|
||||
const requestedTurn = safeTurnId(turnId);
|
||||
const matches = factArray(facts.turns)
|
||||
.filter((turn) => turn.traceId === safeTrace && (!requestedTurn || turn.turnId === requestedTurn || turn.turnId === safeTrace))
|
||||
.sort(compareFactRecordsDesc);
|
||||
return matches[0] ?? null;
|
||||
}
|
||||
|
||||
export function factTurnSnapshot({ turn = null, session = null, facts = {}, traceId, turnId = null } = {}) {
|
||||
const safeTrace = safeTraceId(traceId);
|
||||
const resolvedTurnId = factTurnId(turn, safeTrace) ?? safeTurnId(turnId) ?? safeTrace;
|
||||
const messages = session ? factMessagesForSession(session, facts) : [];
|
||||
const userMessage = messages.find((message) => message.role === "user") ?? null;
|
||||
const assistantMessage = [...messages].reverse().find((message) => isAssistantLikeRole(message.role)) ?? null;
|
||||
const terminalMessage = safeTrace ? factTerminalMessageForTrace(messages, safeTrace) : null;
|
||||
const checkpoint = safeTrace ? factCheckpointForTrace(facts, safeTrace) : null;
|
||||
const trace = factTraceSnapshot(facts, safeTrace);
|
||||
const checkpointStatus = normalizeTerminalStatus(checkpoint?.status);
|
||||
const traceStatus = normalizeTerminalStatus(trace.status);
|
||||
const messageTerminalStatus = factTerminalStatusFromMessageProjection(terminalMessage);
|
||||
const status = normalizeStatus(checkpointStatus ?? traceStatus ?? messageTerminalStatus ?? turn?.status ?? session?.status);
|
||||
const terminal = isTerminalProjectionStatus(status);
|
||||
const assistantText = terminal ? factMessageFinalResponseText(assistantMessage) : null;
|
||||
const timing = factCombinedTimingProjection(status, checkpoint, messageTerminalStatus ? terminalMessage : null, turn, trace);
|
||||
return {
|
||||
turnId: resolvedTurnId,
|
||||
traceId: safeTrace,
|
||||
status,
|
||||
running: RUNNING_STATUSES.has(status),
|
||||
terminal,
|
||||
sessionId: factSessionId(session) ?? turn?.sessionId ?? checkpoint?.sessionId ?? null,
|
||||
threadId: safeOpaqueId(session?.threadId) ?? (textValue(session?.threadId) || null),
|
||||
userMessageId: userMessage?.messageId ?? null,
|
||||
assistantMessageId: assistantMessage?.messageId ?? turn?.messageId ?? null,
|
||||
assistantText: assistantText ?? null,
|
||||
finalResponse: assistantText ? { text: assistantText, sealed: true, source: "message-part" } : null,
|
||||
timing,
|
||||
startedAt: timing.startedAt,
|
||||
lastEventAt: timing.lastEventAt,
|
||||
finishedAt: timing.finishedAt,
|
||||
durationMs: timing.durationMs,
|
||||
agentRun: checkpoint ? {
|
||||
runId: textValue(checkpoint.runId) || null,
|
||||
commandId: textValue(checkpoint.commandId) || null,
|
||||
status,
|
||||
lastSeq: factSeq(checkpoint),
|
||||
valuesRedacted: true
|
||||
} : null,
|
||||
trace: {
|
||||
traceId: safeTrace,
|
||||
status: trace.status,
|
||||
eventCount: trace.eventCount,
|
||||
timing: trace.timing,
|
||||
startedAt: trace.startedAt,
|
||||
lastEventAt: trace.lastEventAt,
|
||||
finishedAt: trace.finishedAt,
|
||||
durationMs: trace.durationMs,
|
||||
updatedAt: trace.updatedAt
|
||||
},
|
||||
urls: {
|
||||
self: resolvedTurnId ? `/v1/workbench/turns/${encodeURIComponent(resolvedTurnId)}` : null,
|
||||
traceEvents: safeTrace ? `/v1/workbench/traces/${encodeURIComponent(safeTrace)}/events` : null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function factTraceSnapshot(facts = {}, traceId) {
|
||||
const safeTrace = safeTraceId(traceId);
|
||||
const events = factArray(facts.traceEvents)
|
||||
.filter((event) => !safeTrace || event.traceId === safeTrace)
|
||||
.sort(compareFactTraceEventsAsc)
|
||||
.map(factTraceEventDto)
|
||||
.filter(Boolean);
|
||||
const status = durableTraceStatus(events);
|
||||
const lastEvent = events.at(-1) ?? null;
|
||||
const checkpoint = factCheckpointForTrace(facts, safeTrace);
|
||||
const timing = factTraceTimingProjection(events, checkpoint, status);
|
||||
return {
|
||||
traceId: safeTrace,
|
||||
status,
|
||||
eventCount: events.length,
|
||||
events,
|
||||
lastEvent,
|
||||
timing,
|
||||
startedAt: timing.startedAt,
|
||||
lastEventAt: timing.lastEventAt,
|
||||
finishedAt: timing.finishedAt,
|
||||
durationMs: timing.durationMs,
|
||||
updatedAt: lastEvent?.updatedAt ?? lastEvent?.createdAt ?? checkpoint?.updatedAt ?? null,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
};
|
||||
}
|
||||
|
||||
export function factTraceTimingProjection(events = [], checkpoint = null, status = null) {
|
||||
const checkpointTiming = factTimingProjection(checkpoint, status);
|
||||
const normalizedStatus = normalizeStatus(status ?? checkpoint?.status);
|
||||
const terminal = TERMINAL_STATUSES.has(normalizedStatus) && !RUNNING_STATUSES.has(normalizedStatus);
|
||||
const observedAt = new Date().toISOString();
|
||||
const eventStartTimes = factArray(events).flatMap((event) => [event?.createdAt, event?.occurredAt]);
|
||||
const eventActivityTimes = factArray(events).flatMap((event) => [event?.createdAt, event?.occurredAt, event?.updatedAt]);
|
||||
const terminalEventTimes = factArray(events)
|
||||
.filter(factTraceEventIsTerminalAuthority)
|
||||
.flatMap(factTraceEventTerminalTimes);
|
||||
const startedAt = firstTimestampIso(checkpointTiming.startedAt, ...eventStartTimes);
|
||||
const finishedAt = terminal ? latestTimestampIso(checkpointTiming.finishedAt, ...terminalEventTimes) : null;
|
||||
const lastEventAt = terminal && finishedAt ? finishedAt : latestTimestampIso(checkpointTiming.lastEventAt, ...eventActivityTimes);
|
||||
const durationMs = elapsedFactMs(startedAt, terminal ? finishedAt : observedAt);
|
||||
const lastEventAgeMs = terminal ? null : elapsedFactMs(lastEventAt, observedAt);
|
||||
return {
|
||||
...checkpointTiming,
|
||||
startedAt,
|
||||
lastEventAt,
|
||||
finishedAt,
|
||||
durationMs,
|
||||
observedAt: terminal ? null : observedAt,
|
||||
lastEventAgeMs,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
export function factTraceEventIsTerminalAuthority(event = null) {
|
||||
const eventType = textValue(event?.eventType ?? event?.type);
|
||||
return event?.terminal === true || event?.sealed === true || eventType === "terminal";
|
||||
}
|
||||
|
||||
export function factTraceEventTerminalTimes(event = null) {
|
||||
const source = objectValue(event?.timing);
|
||||
return [source?.finishedAt, event?.finishedAt, source?.lastEventAt, event?.lastEventAt, event?.occurredAt, event?.createdAt];
|
||||
}
|
||||
|
||||
export function factTraceEventDto(event, index) {
|
||||
const seq = factProjectedSeq(event);
|
||||
if (!seq) return null;
|
||||
const sourceSeq = Number.isFinite(Number(event?.sourceSeq)) && Number(event.sourceSeq) >= 0 ? Math.trunc(Number(event.sourceSeq)) : null;
|
||||
return {
|
||||
...event,
|
||||
id: textValue(event?.id) || textValue(event?.sourceEventId) || null,
|
||||
seq,
|
||||
projectedSeq: seq,
|
||||
sourceSeq,
|
||||
type: textValue(event?.type ?? event?.eventType) || "event",
|
||||
label: textValue(event?.label ?? event?.eventType ?? event?.type) || null,
|
||||
status: normalizeStatus(event?.status),
|
||||
createdAt: textValue(event?.createdAt ?? event?.occurredAt) || null,
|
||||
updatedAt: factUpdatedAt(event),
|
||||
valuesRedacted: event?.valuesRedacted !== false
|
||||
};
|
||||
}
|
||||
|
||||
export function factProjectionForTrace(facts = {}, traceId) {
|
||||
const checkpoint = factCheckpointForTrace(facts, traceId);
|
||||
if (!checkpoint) {
|
||||
return {
|
||||
projectionStatus: "unknown",
|
||||
projectionHealth: "unknown",
|
||||
lastProjectedSeq: null,
|
||||
sourceRunId: null,
|
||||
sourceCommandId: null,
|
||||
staleMs: null,
|
||||
blocker: null,
|
||||
updatedAt: null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
const projectionStatus = normalizeProjectionStatus(checkpoint.projectionStatus);
|
||||
const projectionHealth = normalizeProjectionHealth(checkpoint.projectionHealth, projectionStatus);
|
||||
const diagnostic = objectValue(checkpoint.diagnostic);
|
||||
const timing = factTimingProjection(checkpoint, normalizeStatus(checkpoint.status));
|
||||
return {
|
||||
projectionStatus,
|
||||
projectionHealth,
|
||||
lastProjectedSeq: factSeq(checkpoint),
|
||||
sourceRunId: textValue(checkpoint.runId ?? checkpoint.sourceRunId) || null,
|
||||
sourceCommandId: textValue(checkpoint.commandId ?? checkpoint.sourceCommandId) || null,
|
||||
staleMs: null,
|
||||
blocker: diagnostic.blocker ?? checkpoint.blocker ?? null,
|
||||
timing,
|
||||
startedAt: timing.startedAt,
|
||||
lastEventAt: timing.lastEventAt,
|
||||
finishedAt: timing.finishedAt,
|
||||
durationMs: timing.durationMs,
|
||||
updatedAt: factUpdatedAt(checkpoint),
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
export function factTimingProjection(record = null, status = null) {
|
||||
const source = objectValue(record?.timing);
|
||||
const observedAt = new Date().toISOString();
|
||||
const startedAt = timestampIso(source?.startedAt ?? record?.startedAt ?? record?.createdAt);
|
||||
const lastEventAt = timestampIso(source?.lastEventAt ?? record?.lastEventAt ?? record?.updatedAt ?? record?.createdAt);
|
||||
const normalizedStatus = normalizeStatus(status ?? record?.status);
|
||||
const terminal = record?.terminal === true || TERMINAL_STATUSES.has(normalizedStatus) && !RUNNING_STATUSES.has(normalizedStatus);
|
||||
const finishedAt = terminal ? timestampIso(source?.finishedAt ?? record?.finishedAt ?? record?.completedAt) : null;
|
||||
const durationMs = elapsedFactMs(startedAt, terminal ? finishedAt : observedAt);
|
||||
const lastEventAgeMs = terminal ? null : elapsedFactMs(lastEventAt, observedAt);
|
||||
return { startedAt, lastEventAt, finishedAt, durationMs, observedAt: terminal ? null : observedAt, lastEventAgeMs, valuesRedacted: source?.valuesRedacted !== false };
|
||||
}
|
||||
|
||||
export function factCombinedTimingProjection(status = null, ...records) {
|
||||
const normalizedStatus = normalizeStatus(status ?? records.find((record) => record?.status)?.status);
|
||||
const terminal = TERMINAL_STATUSES.has(normalizedStatus) && !RUNNING_STATUSES.has(normalizedStatus);
|
||||
const observedAt = new Date().toISOString();
|
||||
const timings = records.map((record) => factTimingSource(record)).filter(Boolean);
|
||||
const startedAt = firstTimestampIso(...timings.map((timing) => timing.startedAt));
|
||||
const finishedAt = terminal ? latestTimestampIso(...timings.map((timing) => timing.finishedAt)) : null;
|
||||
const lastEventAt = terminal && finishedAt ? finishedAt : latestTimestampIso(...timings.map((timing) => timing.lastEventAt));
|
||||
const durationMs = elapsedFactMs(startedAt, terminal ? finishedAt : observedAt);
|
||||
const lastEventAgeMs = terminal ? null : elapsedFactMs(lastEventAt, observedAt);
|
||||
return { startedAt, lastEventAt, finishedAt, durationMs, observedAt: terminal ? null : observedAt, lastEventAgeMs, valuesRedacted: true };
|
||||
}
|
||||
|
||||
export function factTimingSource(record = null) {
|
||||
if (!record) return null;
|
||||
const source = objectValue(record?.timing);
|
||||
return {
|
||||
startedAt: timestampIso(source?.startedAt ?? record?.startedAt ?? record?.createdAt),
|
||||
lastEventAt: timestampIso(source?.lastEventAt ?? record?.lastEventAt ?? record?.updatedAt ?? record?.createdAt),
|
||||
finishedAt: timestampIso(source?.finishedAt ?? record?.finishedAt ?? record?.completedAt)
|
||||
};
|
||||
}
|
||||
|
||||
export function firstTimestampIso(...values) {
|
||||
for (const value of values) {
|
||||
const timestamp = timestampIso(value);
|
||||
if (timestamp) return timestamp;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function latestTimestampIso(...values) {
|
||||
let latest = null;
|
||||
let latestMs = Number.NEGATIVE_INFINITY;
|
||||
for (const value of values) {
|
||||
const timestamp = timestampIso(value);
|
||||
if (!timestamp) continue;
|
||||
const ms = Date.parse(timestamp);
|
||||
if (!Number.isFinite(ms) || ms < latestMs) continue;
|
||||
latest = timestamp;
|
||||
latestMs = ms;
|
||||
}
|
||||
return latest;
|
||||
}
|
||||
|
||||
export function elapsedFactMs(startedAt, endedAt) {
|
||||
const start = Date.parse(String(startedAt ?? ""));
|
||||
const end = Date.parse(String(endedAt ?? ""));
|
||||
if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return null;
|
||||
return Math.trunc(end - start);
|
||||
}
|
||||
|
||||
export function timestampIso(value) {
|
||||
const ms = Date.parse(String(value ?? ""));
|
||||
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
|
||||
}
|
||||
|
||||
export function factCheckpointForTrace(facts = {}, traceId) {
|
||||
const safeTrace = safeTraceId(traceId);
|
||||
if (!safeTrace) return null;
|
||||
return factArray(facts.checkpoints)
|
||||
.filter((checkpoint) => checkpoint.traceId === safeTrace)
|
||||
.sort(compareFactRecordsDesc)[0] ?? null;
|
||||
}
|
||||
|
||||
export function workbenchProjectionStoreError(error) {
|
||||
const data = objectValue(error?.data) ?? {};
|
||||
const retryAfterNumber = Number(data.retryAfterMs);
|
||||
const retryAfterMs = Number.isFinite(retryAfterNumber) && retryAfterNumber >= 0 ? Math.trunc(retryAfterNumber) : null;
|
||||
const retryAttempt = nonNegativeIntegerOrNull(data.retryAttempt ?? data.retryAttempts);
|
||||
const retryMax = nonNegativeIntegerOrNull(data.retryMax);
|
||||
const retryDelaysMs = Array.isArray(data.retryDelaysMs)
|
||||
? data.retryDelaysMs.map(nonNegativeIntegerOrNull).filter((value) => value !== null)
|
||||
: [];
|
||||
return workbenchError("projection_store_unavailable", "Workbench durable projection store is unavailable.", {
|
||||
causeCode: error?.code ?? null,
|
||||
queryResult: textValue(data.queryResult) || null,
|
||||
blockedLayer: textValue(data.blockedLayer) || null,
|
||||
retryable: data.retryable !== false,
|
||||
transient: data.transient === true || data.retryable === true,
|
||||
retryAfterMs: retryAfterMs ?? null,
|
||||
retryAttempt,
|
||||
retryMax,
|
||||
retryAttempts: retryAttempt,
|
||||
retryExhausted: data.retryExhausted === true,
|
||||
retryDelaysMs: retryDelaysMs.length > 0 ? retryDelaysMs : null
|
||||
});
|
||||
}
|
||||
|
||||
export function nonNegativeIntegerOrNull(value) {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) && parsed >= 0 ? Math.trunc(parsed) : null;
|
||||
}
|
||||
|
||||
export function factSessionId(session) {
|
||||
return safeSessionId(session?.sessionId ?? session?.id) ?? null;
|
||||
}
|
||||
|
||||
export function factLastTraceId(session) {
|
||||
return safeTraceId(session?.lastTraceId ?? session?.traceId) ?? null;
|
||||
}
|
||||
|
||||
export function factLatestTraceIdForSession(facts = {}, sessionId) {
|
||||
const turn = factArray(facts.turns).filter((item) => item.sessionId === sessionId).sort(compareFactRecordsDesc)[0] ?? null;
|
||||
return safeTraceId(turn?.traceId) ?? null;
|
||||
}
|
||||
|
||||
export function factCurrentTraceForSession(facts = {}, sessionId, session = null) {
|
||||
const candidates = [];
|
||||
const sessionTraceId = factLastTraceId(session);
|
||||
if (sessionTraceId) candidates.push(factTraceCandidate(session, sessionTraceId, normalizeStatus(session?.status), 0));
|
||||
for (const message of factArray(facts.messages)) {
|
||||
if (message?.sessionId !== sessionId || !safeTraceId(message?.traceId)) continue;
|
||||
candidates.push(factMessageTraceCandidate(message));
|
||||
}
|
||||
for (const turn of factArray(facts.turns)) {
|
||||
if (turn?.sessionId !== sessionId || !safeTraceId(turn?.traceId)) continue;
|
||||
candidates.push(factTraceCandidate(turn, safeTraceId(turn.traceId), normalizeStatus(turn?.status), 3));
|
||||
}
|
||||
return candidates.sort(compareTraceCandidatesDesc)[0] ?? null;
|
||||
}
|
||||
|
||||
export function factMessageTraceCandidate(message) {
|
||||
const role = textValue(message?.role);
|
||||
let status = normalizeStatus(message?.status);
|
||||
let priority = 1;
|
||||
if (isAssistantLikeRole(role)) {
|
||||
priority = 2;
|
||||
if (status === "unknown") status = "running";
|
||||
} else if (safeTraceId(message?.traceId)) {
|
||||
status = "running";
|
||||
}
|
||||
return factTraceCandidate(message, safeTraceId(message?.traceId), status, priority);
|
||||
}
|
||||
|
||||
export function factTraceCandidate(record, traceId, status, priority) {
|
||||
return { traceId: safeTraceId(traceId) ?? null, status: normalizeStatus(status), seq: factSeq(record), updatedAt: factUpdatedAt(record), priority };
|
||||
}
|
||||
|
||||
export function compareTraceCandidatesDesc(left, right) {
|
||||
return compareNumberDesc(left?.seq, right?.seq) || compareTimestampDesc(left?.updatedAt, right?.updatedAt) || compareNumberDesc(left?.priority, right?.priority);
|
||||
}
|
||||
|
||||
export function nonUnknownStatus(status) {
|
||||
return normalizeStatus(status) === "unknown" ? null : status;
|
||||
}
|
||||
|
||||
export function factTurnId(turn, fallbackTraceId = null) {
|
||||
return safeTurnId(turn?.turnId) ?? safeTraceId(turn?.turnId) ?? safeTraceId(fallbackTraceId) ?? null;
|
||||
}
|
||||
|
||||
export function factUpdatedAt(record) {
|
||||
return textValue(record?.updatedAt ?? record?.occurredAt ?? record?.createdAt) || null;
|
||||
}
|
||||
|
||||
export function factSeq(record) {
|
||||
for (const value of [record?.projectedSeq, record?.sourceSeq, record?.seq]) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isFinite(parsed) && parsed >= 0) return Math.trunc(parsed);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function factProjectedSeq(record) {
|
||||
const parsed = Number(record?.projectedSeq);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? Math.trunc(parsed) : null;
|
||||
}
|
||||
|
||||
export function normalizeProjectionStatus(value) {
|
||||
const status = normalizeStatus(value);
|
||||
if (status === "caught-up") return "caught-up";
|
||||
if (status === "caughtup") return "caught-up";
|
||||
return ["projecting", "blocked", "stalled", "unknown"].includes(status) ? status : "unknown";
|
||||
}
|
||||
|
||||
export function normalizeProjectionHealth(value, projectionStatus = "unknown") {
|
||||
const health = normalizeStatus(value);
|
||||
if (health === "healthy" && projectionStatus === "caught-up") return "caught-up";
|
||||
if (health === "healthy" && projectionStatus === "projecting") return "projecting";
|
||||
if (["caught-up", "projecting", "degraded", "stalled", "unavailable", "unknown"].includes(health)) return health;
|
||||
return projectionStatus === "caught-up" || projectionStatus === "projecting" ? projectionStatus : "unknown";
|
||||
}
|
||||
|
||||
export function compareFactSessionsDesc(left, right) {
|
||||
return compareTimestampDesc(factUpdatedAt(left), factUpdatedAt(right)) || compareText(factSessionId(left), factSessionId(right));
|
||||
}
|
||||
|
||||
export function compareFactMessagesAsc(left, right) {
|
||||
return compareNumberAsc(factSeq(left), factSeq(right)) || compareTimestampAsc(left?.createdAt ?? left?.updatedAt, right?.createdAt ?? right?.updatedAt) || compareText(left?.messageId, right?.messageId);
|
||||
}
|
||||
|
||||
export function compareFactMessagesForSessionAsc(left, right, facts = {}) {
|
||||
const leftKey = factMessageTimelineKey(left, facts);
|
||||
const rightKey = factMessageTimelineKey(right, facts);
|
||||
return compareOptionalTimestampAsc(leftKey.timelineAnchorAt, rightKey.timelineAnchorAt)
|
||||
|| compareOptionalNumberAsc(leftKey.turnSeq, rightKey.turnSeq)
|
||||
|| compareTimestampAsc(leftKey.turnStartedAt, rightKey.turnStartedAt)
|
||||
|| compareText(leftKey.traceId, rightKey.traceId)
|
||||
|| compareOptionalNumberAsc(leftKey.roleRank, rightKey.roleRank)
|
||||
|| compareTimestampAsc(leftKey.messageCreatedAt, rightKey.messageCreatedAt)
|
||||
|| compareOptionalNumberAsc(leftKey.messageSeq, rightKey.messageSeq)
|
||||
|| compareText(left?.messageId, right?.messageId);
|
||||
}
|
||||
|
||||
export function factMessageTimelineKey(message, facts = {}) {
|
||||
const traceId = safeTraceId(message?.traceId) ?? null;
|
||||
const turn = traceId ? factTurnForTrace(facts, traceId) : null;
|
||||
const checkpoint = traceId ? factCheckpointForTrace(facts, traceId) : null;
|
||||
const anchorMessage = traceId ? factTimelineAnchorMessageForTrace(facts, traceId) : null;
|
||||
return {
|
||||
traceId: traceId ?? "",
|
||||
timelineAnchorAt: firstTimestampIso(anchorMessage?.createdAt, anchorMessage?.updatedAt),
|
||||
turnSeq: firstFiniteNumber(factSeq(turn), factSeq(checkpoint)),
|
||||
turnStartedAt: firstTimestampIso(turn?.startedAt, checkpoint?.startedAt, message?.createdAt, message?.updatedAt),
|
||||
roleRank: factMessageTimelineRoleRank(message?.role),
|
||||
messageCreatedAt: timestampIso(message?.createdAt ?? message?.updatedAt),
|
||||
messageSeq: factSeq(message)
|
||||
};
|
||||
}
|
||||
|
||||
export function factTimelineAnchorMessageForTrace(facts = {}, traceId) {
|
||||
const safeTrace = safeTraceId(traceId);
|
||||
if (!safeTrace) return null;
|
||||
const messages = factArray(facts.messages).filter((message) => message.traceId === safeTrace);
|
||||
const userMessages = messages.filter((message) => normalizeRole(message?.role) === "user");
|
||||
const candidates = userMessages.length > 0 ? userMessages : messages;
|
||||
return [...candidates].sort((left, right) => compareOptionalTimestampAsc(left?.createdAt ?? left?.updatedAt, right?.createdAt ?? right?.updatedAt) || compareOptionalNumberAsc(factSeq(left), factSeq(right)) || compareText(left?.messageId, right?.messageId))[0] ?? null;
|
||||
}
|
||||
|
||||
export function factMessageTimelineRoleRank(role) {
|
||||
const normalized = normalizeRole(role);
|
||||
if (normalized === "user") return 0;
|
||||
if (isAssistantLikeRole(normalized)) return 1;
|
||||
return 2;
|
||||
}
|
||||
|
||||
export function firstFiniteNumber(...values) {
|
||||
for (const value of values) {
|
||||
const parsed = Number(value);
|
||||
if (Number.isFinite(parsed)) return Math.trunc(parsed);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function compareOptionalNumberAsc(left, right) {
|
||||
const leftNumber = Number(left);
|
||||
const rightNumber = Number(right);
|
||||
const leftComparable = Number.isFinite(leftNumber) ? leftNumber : Number.MAX_SAFE_INTEGER;
|
||||
const rightComparable = Number.isFinite(rightNumber) ? rightNumber : Number.MAX_SAFE_INTEGER;
|
||||
return leftComparable - rightComparable;
|
||||
}
|
||||
|
||||
export function compareFactPartsAsc(left, right) {
|
||||
return compareNumberAsc(left?.partIndex, right?.partIndex) || compareNumberAsc(factSeq(left), factSeq(right)) || compareText(left?.partId, right?.partId);
|
||||
}
|
||||
|
||||
export function compareFactTraceEventsAsc(left, right) {
|
||||
return compareNumberAsc(factProjectedSeq(left), factProjectedSeq(right));
|
||||
}
|
||||
|
||||
export function compareFactRecordsDesc(left, right) {
|
||||
return compareNumberDesc(factSeq(left), factSeq(right)) || compareTimestampDesc(factUpdatedAt(left), factUpdatedAt(right));
|
||||
}
|
||||
|
||||
export function compareTimestampAsc(left, right) {
|
||||
return timestampMs(left) - timestampMs(right);
|
||||
}
|
||||
|
||||
export function compareTimestampDesc(left, right) {
|
||||
return timestampMs(right) - timestampMs(left);
|
||||
}
|
||||
|
||||
export function compareOptionalTimestampAsc(left, right) {
|
||||
const leftMs = optionalTimestampMs(left);
|
||||
const rightMs = optionalTimestampMs(right);
|
||||
return (leftMs ?? Number.MAX_SAFE_INTEGER) - (rightMs ?? Number.MAX_SAFE_INTEGER);
|
||||
}
|
||||
|
||||
export function compareNumberAsc(left, right) {
|
||||
return numericValue(left, Number.MAX_SAFE_INTEGER) - numericValue(right, Number.MAX_SAFE_INTEGER);
|
||||
}
|
||||
|
||||
export function compareNumberDesc(left, right) {
|
||||
return numericValue(right, -1) - numericValue(left, -1);
|
||||
}
|
||||
|
||||
export function compareText(left, right) {
|
||||
return textValue(left).localeCompare(textValue(right));
|
||||
}
|
||||
|
||||
export function timestampMs(value) {
|
||||
const parsed = Date.parse(String(value ?? ""));
|
||||
return Number.isFinite(parsed) ? parsed : 0;
|
||||
}
|
||||
|
||||
export function optionalTimestampMs(value) {
|
||||
const parsed = Date.parse(String(value ?? ""));
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
|
||||
export function numericValue(value, fallback) {
|
||||
const parsed = Number(value);
|
||||
return Number.isFinite(parsed) ? parsed : fallback;
|
||||
}
|
||||
|
||||
export function mergeFactSets(...sets) {
|
||||
const merged = emptyFactSet();
|
||||
const keys = {
|
||||
sessions: "sessionId",
|
||||
messages: "messageId",
|
||||
parts: "partId",
|
||||
turns: "turnId",
|
||||
traceEvents: "id",
|
||||
checkpoints: "traceId"
|
||||
};
|
||||
for (const set of sets) {
|
||||
for (const [key, idKey] of Object.entries(keys)) {
|
||||
const seen = new Set(merged[key].map((item) => textValue(item?.[idKey])));
|
||||
for (const item of factArray(set?.[key])) {
|
||||
const id = textValue(item?.[idKey]);
|
||||
if (id && seen.has(id)) continue;
|
||||
merged[key].push(item);
|
||||
if (id) seen.add(id);
|
||||
}
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
}
|
||||
|
||||
export function emptyFactSet() {
|
||||
return {
|
||||
sessions: [],
|
||||
messages: [],
|
||||
parts: [],
|
||||
turns: [],
|
||||
traceEvents: [],
|
||||
checkpoints: []
|
||||
};
|
||||
}
|
||||
|
||||
export function factArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
export function uniqueText(values = []) {
|
||||
return [...new Set(values.map((value) => textValue(value)).filter(Boolean))];
|
||||
}
|
||||
|
||||
|
||||
export function eventSeq(event, index) {
|
||||
const seq = Number(event?.seq);
|
||||
return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1;
|
||||
}
|
||||
|
||||
export function normalizeStatus(value) {
|
||||
const text = textValue(value).toLowerCase().replace(/_/gu, "-");
|
||||
if (text === "cancelled") return "canceled";
|
||||
return text || "unknown";
|
||||
}
|
||||
|
||||
export function normalizeTerminalStatus(value) {
|
||||
const status = normalizeStatus(value);
|
||||
return TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status) ? status : null;
|
||||
}
|
||||
|
||||
export function firstUserPreview(messages) {
|
||||
return messages.find((message) => message.role === "user")?.textPreview ?? null;
|
||||
}
|
||||
|
||||
export function latestUserPreview(messages) {
|
||||
return [...messages].reverse().find((message) => message.role === "user")?.textPreview ?? null;
|
||||
}
|
||||
|
||||
export function latestValidMessagePreview(messages) {
|
||||
return [...messages].reverse().find((message) => message.role === "user" || isAssistantLikeRole(message.role))?.textPreview ?? null;
|
||||
}
|
||||
|
||||
export function latestAssistantPreview(messages) {
|
||||
return [...messages].reverse().find((message) => isAssistantLikeRole(message.role))?.textPreview ?? null;
|
||||
}
|
||||
|
||||
export function sessionTitleFromMessages(messages = []) {
|
||||
return boundedPreviewText(latestUserPreview(messages) ?? firstUserPreview(messages) ?? latestAssistantPreview(messages));
|
||||
}
|
||||
|
||||
export function sessionPreviewFromMessages(messages = []) {
|
||||
return boundedPreviewText(latestValidMessagePreview(messages) ?? firstUserPreview(messages) ?? latestAssistantPreview(messages));
|
||||
}
|
||||
|
||||
export function boundedPreviewText(value, maxLength = 160) {
|
||||
const text = textValue(value).replace(/\s+/gu, " ").trim();
|
||||
if (!text) return null;
|
||||
return text.length > maxLength ? `${text.slice(0, maxLength - 3)}...` : text;
|
||||
}
|
||||
|
||||
export function isAssistantLikeRole(role) {
|
||||
return role === "assistant" || role === "agent";
|
||||
}
|
||||
|
||||
export function objectValue(value) {
|
||||
return value && typeof value === "object" && !Array.isArray(value) ? value : {};
|
||||
}
|
||||
|
||||
export function textValue(value) {
|
||||
return String(value ?? "").trim();
|
||||
}
|
||||
|
||||
export function messageAuthorityTextValue(value) {
|
||||
const text = String(value ?? "").replace(/\r\n?/gu, "\n");
|
||||
if (!text.trim() || text.trim() === "[object Object]") return "";
|
||||
return text;
|
||||
}
|
||||
|
||||
export function safeTurnId(value) {
|
||||
const text = textValue(value);
|
||||
return /^turn_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null;
|
||||
}
|
||||
|
||||
export function safeMessageId(value) {
|
||||
const text = textValue(value);
|
||||
return /^msg_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null;
|
||||
}
|
||||
|
||||
export function safePartId(value) {
|
||||
const text = textValue(value);
|
||||
return /^prt_[A-Za-z0-9_.:-]+$/u.test(text) ? text : null;
|
||||
}
|
||||
|
||||
export function hash(value) {
|
||||
return createHash("sha256").update(String(value)).digest("hex");
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
export const ACTOR = { id: "usr_workbench_reader", username: "reader", displayName: "Reader", role: "user", status: "active" };
|
||||
|
||||
export async function getJson(port, path) {
|
||||
const response = await fetch(`http://127.0.0.1:${port}${path}`);
|
||||
return {
|
||||
status: response.status,
|
||||
body: await response.json()
|
||||
};
|
||||
}
|
||||
|
||||
export async function waitForCondition(predicate, timeoutMs = 500) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (predicate()) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 5));
|
||||
}
|
||||
assert.fail("condition was not met before timeout");
|
||||
}
|
||||
|
||||
export function createDurableFactsRuntimeStore({ sessions = [], queryError = null, queries = [] } = {}) {
|
||||
const facts = emptyFacts();
|
||||
for (const item of sessions) mergeFacts(facts, buildDurableFactsForSession(item));
|
||||
return createRuntimeStoreFromFacts(facts, { queryError, queries });
|
||||
}
|
||||
|
||||
export function createRuntimeStoreFromFacts(facts, { queryError = null, queries = [] } = {}) {
|
||||
return {
|
||||
queries,
|
||||
async queryWorkbenchFacts(params = {}) {
|
||||
queries.push(params);
|
||||
if (queryError) throw queryError;
|
||||
const filtered = filterFacts(facts, params);
|
||||
return {
|
||||
facts: filtered,
|
||||
count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0),
|
||||
persistence: { adapter: "test-durable-workbench-facts", durable: true }
|
||||
};
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export function buildDurableFactsForSession({ session, events = [], status = null, finalText = null, assistantText = null, runId = null, commandId = null, projectionStatus = null, projectionHealth = "healthy", lastProjectedSeq = null, omitSessions = false } = {}) {
|
||||
const traceId = session.lastTraceId ?? session.session?.lastTraceId;
|
||||
const projectedStatus = normalizeTestStatus(status ?? session.status);
|
||||
const terminal = testTerminalStatuses.has(projectedStatus);
|
||||
const normalizedEvents = normalizeTestEvents(events, session, { traceId });
|
||||
const timing = testTimingProjection({ session, events: normalizedEvents, status: projectedStatus, terminal });
|
||||
const normalizedMessages = normalizeTestMessages(session, { traceId, status: projectedStatus, finalText, terminal, timing });
|
||||
const explicitLastProjectedSeq = lastProjectedSeq !== null && lastProjectedSeq !== undefined ? Number(lastProjectedSeq) : NaN;
|
||||
const projectedSeq = Number.isFinite(explicitLastProjectedSeq) ? explicitLastProjectedSeq : Math.max(normalizedEvents.length, normalizedMessages.length, 0);
|
||||
const checkpointStatus = projectionStatus ?? (terminal ? "caught_up" : "projecting");
|
||||
return {
|
||||
sessions: omitSessions ? [] : [{
|
||||
sessionId: session.id,
|
||||
ownerUserId: session.ownerUserId,
|
||||
ownerRole: session.ownerRole ?? null,
|
||||
projectId: session.projectId ?? null,
|
||||
conversationId: session.conversationId ?? null,
|
||||
threadId: session.threadId ?? null,
|
||||
agentId: session.agentId ?? "hwlab-code-agent",
|
||||
status: projectedStatus,
|
||||
lastTraceId: traceId,
|
||||
providerProfile: session.session?.providerProfile ?? null,
|
||||
projectedSeq,
|
||||
sourceSeq: projectedSeq,
|
||||
sourceEventId: traceId,
|
||||
terminal,
|
||||
sealed: terminal,
|
||||
timing,
|
||||
startedAt: timing.startedAt,
|
||||
lastEventAt: timing.lastEventAt,
|
||||
finishedAt: timing.finishedAt,
|
||||
durationMs: timing.durationMs,
|
||||
createdAt: session.startedAt ?? session.createdAt ?? session.updatedAt ?? null,
|
||||
updatedAt: session.updatedAt ?? session.endedAt ?? session.startedAt ?? null,
|
||||
valuesRedacted: true
|
||||
}],
|
||||
messages: normalizedMessages,
|
||||
parts: normalizedMessages.filter((message) => message.text).map((message, index) => {
|
||||
const partType = message.role !== "user" && message.terminal ? "final_response" : "text";
|
||||
return {
|
||||
partId: `prt_${session.id}_${index}`,
|
||||
messageId: message.messageId,
|
||||
sessionId: session.id,
|
||||
turnId: message.turnId,
|
||||
traceId: message.traceId,
|
||||
partIndex: 0,
|
||||
partType,
|
||||
status: message.status,
|
||||
text: message.text,
|
||||
projectedSeq: message.projectedSeq,
|
||||
sourceSeq: message.sourceSeq,
|
||||
sourceEventId: message.sourceEventId,
|
||||
terminal: message.terminal,
|
||||
sealed: partType === "final_response" ? true : message.sealed,
|
||||
updatedAt: message.updatedAt,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}),
|
||||
turns: traceId ? [{
|
||||
turnId: traceId,
|
||||
sessionId: session.id,
|
||||
traceId,
|
||||
messageId: normalizedMessages.find((message) => message.role !== "user")?.messageId ?? null,
|
||||
status: projectedStatus,
|
||||
projectedSeq,
|
||||
sourceSeq: projectedSeq,
|
||||
sourceEventId: traceId,
|
||||
terminal,
|
||||
sealed: terminal,
|
||||
assistantText: assistantText ?? null,
|
||||
finalResponse: finalText ? { text: finalText, status: projectedStatus, traceId, valuesPrinted: false } : null,
|
||||
diagnostic: { projectionStatus: checkpointStatus, projectionHealth, valuesRedacted: true },
|
||||
timing,
|
||||
startedAt: timing.startedAt,
|
||||
lastEventAt: timing.lastEventAt,
|
||||
finishedAt: timing.finishedAt,
|
||||
durationMs: timing.durationMs,
|
||||
createdAt: session.startedAt ?? session.updatedAt ?? null,
|
||||
updatedAt: session.updatedAt ?? null,
|
||||
valuesRedacted: true
|
||||
}] : [],
|
||||
traceEvents: normalizedEvents,
|
||||
checkpoints: traceId ? [{
|
||||
traceId,
|
||||
sessionId: session.id,
|
||||
turnId: traceId,
|
||||
runId,
|
||||
commandId,
|
||||
projectedSeq,
|
||||
sourceSeq: projectedSeq,
|
||||
sourceEventId: traceId,
|
||||
projectionStatus: checkpointStatus,
|
||||
projectionHealth,
|
||||
terminal,
|
||||
sealed: terminal,
|
||||
diagnostic: { projectionStatus: checkpointStatus, projectionHealth, valuesRedacted: true },
|
||||
timing,
|
||||
startedAt: timing.startedAt,
|
||||
lastEventAt: timing.lastEventAt,
|
||||
finishedAt: timing.finishedAt,
|
||||
durationMs: timing.durationMs,
|
||||
createdAt: session.startedAt ?? session.updatedAt ?? null,
|
||||
updatedAt: session.updatedAt ?? null,
|
||||
valuesRedacted: true
|
||||
}] : []
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizeTestMessages(session, { traceId, status, finalText, terminal, timing }) {
|
||||
const raw = Array.isArray(session.session?.messages) ? session.session.messages : [];
|
||||
return raw.map((message, index) => {
|
||||
const role = message.role ?? (index % 2 === 0 ? "user" : "agent");
|
||||
const isAssistant = role === "agent" || role === "assistant";
|
||||
const text = isAssistant && finalText ? finalText : String(message.text ?? message.content ?? "");
|
||||
const messageStatus = isAssistant && finalText ? status : normalizeTestStatus(message.status ?? (role === "user" ? "sent" : status));
|
||||
const messageTraceId = message.traceId ?? traceId;
|
||||
const projectedSeq = Number.isFinite(Number(message.projectedSeq)) ? Number(message.projectedSeq) : index + 1;
|
||||
const sourceSeq = Number.isFinite(Number(message.sourceSeq)) ? Number(message.sourceSeq) : projectedSeq;
|
||||
return {
|
||||
messageId: message.messageId ?? `msg_${session.id}_${index}`,
|
||||
sessionId: session.id,
|
||||
turnId: message.turnId ?? messageTraceId,
|
||||
traceId: messageTraceId,
|
||||
role,
|
||||
status: messageStatus,
|
||||
projectedSeq,
|
||||
sourceSeq,
|
||||
sourceEventId: `${session.id}:message:${index}`,
|
||||
terminal: isAssistant && terminal,
|
||||
sealed: isAssistant && terminal,
|
||||
text,
|
||||
...(isAssistant ? { timing, startedAt: timing.startedAt, lastEventAt: timing.lastEventAt, finishedAt: timing.finishedAt, durationMs: timing.durationMs } : {}),
|
||||
createdAt: message.createdAt ?? session.startedAt ?? session.updatedAt ?? null,
|
||||
updatedAt: message.updatedAt ?? session.updatedAt ?? null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function testTimingProjection({ session, events = [], status, terminal }) {
|
||||
const startedAt = timestampIso(session.startedAt ?? session.createdAt ?? events[0]?.createdAt ?? events[0]?.occurredAt);
|
||||
const lastEvent = events.at(-1) ?? null;
|
||||
const lastEventAt = timestampIso(lastEvent?.updatedAt ?? lastEvent?.createdAt ?? lastEvent?.occurredAt ?? session.updatedAt);
|
||||
const finishedAt = terminal ? timestampIso(session.endedAt ?? lastEventAt ?? session.updatedAt) : null;
|
||||
const durationMs = terminal ? elapsedMs(startedAt, finishedAt ?? lastEventAt) : null;
|
||||
return { startedAt, lastEventAt, finishedAt, durationMs, status, valuesRedacted: true };
|
||||
}
|
||||
|
||||
export function timestampIso(value) {
|
||||
const ms = Date.parse(String(value ?? ""));
|
||||
return Number.isFinite(ms) ? new Date(ms).toISOString() : null;
|
||||
}
|
||||
|
||||
export function elapsedMs(startedAt, endedAt) {
|
||||
const start = Date.parse(String(startedAt ?? ""));
|
||||
const end = Date.parse(String(endedAt ?? ""));
|
||||
if (!Number.isFinite(start) || !Number.isFinite(end) || end < start) return null;
|
||||
return end - start;
|
||||
}
|
||||
|
||||
export function normalizeTestEvents(events, session, { traceId }) {
|
||||
return events.map((event, index) => {
|
||||
const projectedSeq = Number.isFinite(Number(event.projectedSeq)) ? Number(event.projectedSeq) : Number.isFinite(Number(event.seq)) ? Number(event.seq) : index + 1;
|
||||
const sourceSeq = Number.isFinite(Number(event.sourceSeq)) ? Number(event.sourceSeq) : null;
|
||||
return {
|
||||
...event,
|
||||
id: event.id ?? `wte_${session.id}_${index}`,
|
||||
traceId: event.traceId ?? traceId,
|
||||
sessionId: event.sessionId ?? session.id,
|
||||
turnId: event.turnId ?? traceId,
|
||||
messageId: event.messageId ?? null,
|
||||
sourceSeq,
|
||||
sourceEventId: event.sourceEventId ?? `${traceId}:${sourceSeq ?? projectedSeq}:${index}`,
|
||||
projectedSeq,
|
||||
eventType: event.eventType ?? event.type ?? event.label ?? "event",
|
||||
terminal: event.terminal === true,
|
||||
sealed: event.terminal === true,
|
||||
occurredAt: event.occurredAt ?? event.createdAt ?? session.updatedAt ?? null,
|
||||
updatedAt: event.updatedAt ?? event.createdAt ?? session.updatedAt ?? null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
});
|
||||
}
|
||||
|
||||
export function filterFacts(facts, params = {}) {
|
||||
const families = testFactFamilySet(params);
|
||||
const afterProjectedSeq = Number.parseInt(params.afterProjectedSeq ?? "", 10);
|
||||
const filtered = {
|
||||
sessions: families.has("sessions") ? facts.sessions.filter((record) => matchesFact(record, params, ["sessionId", "ownerUserId", "projectId", "conversationId", "threadId", "status"]) && matchesTraceFact(record.lastTraceId, params)) : [],
|
||||
messages: families.has("messages") ? facts.messages.filter((record) => matchesFact(record, params, ["messageId", "sessionId", "turnId", "traceId", "role", "status"])) : [],
|
||||
parts: families.has("parts") ? facts.parts.filter((record) => matchesFact(record, params, ["partId", "messageId", "sessionId", "turnId", "traceId", "partType", "status"])) : [],
|
||||
turns: families.has("turns") ? facts.turns.filter((record) => matchesFact(record, params, ["turnId", "sessionId", "traceId", "messageId", "status"])) : [],
|
||||
traceEvents: families.has("traceEvents") ? facts.traceEvents
|
||||
.filter((record) => matchesFact(record, params, ["id", "traceId", "sessionId", "turnId", "messageId", "eventType"]))
|
||||
.filter((record) => !Number.isInteger(afterProjectedSeq) || afterProjectedSeq <= 0 || Number(record.projectedSeq) > afterProjectedSeq)
|
||||
.sort((left, right) => Number(left.projectedSeq) - Number(right.projectedSeq)) : [],
|
||||
checkpoints: families.has("checkpoints") ? facts.checkpoints.filter((record) => matchesFact(record, params, ["traceId", "sessionId", "turnId", "runId", "commandId", "projectionStatus", "projectionHealth"])) : []
|
||||
};
|
||||
if (workbenchTestFactOrder(params, "sessions") === "updated_desc") {
|
||||
filtered.sessions.sort((left, right) => String(right.updatedAt ?? "").localeCompare(String(left.updatedAt ?? "")) || String(right.sessionId ?? "").localeCompare(String(left.sessionId ?? "")));
|
||||
}
|
||||
const limit = Number.parseInt(params.limit ?? "", 10);
|
||||
if (!Number.isInteger(limit) || limit <= 0) return filtered;
|
||||
return Object.fromEntries(Object.entries(filtered).map(([key, value]) => [key, value.slice(0, limit)]));
|
||||
}
|
||||
|
||||
export function workbenchTestFactOrder(params = {}, family = "") {
|
||||
const value = params[`${family}Order`] ?? params.order;
|
||||
return value === "updated_desc" ? "updated_desc" : "updated_asc";
|
||||
}
|
||||
|
||||
export function matchesFact(record, params, fields) {
|
||||
return fields.every((field) => {
|
||||
if (params[field] !== undefined && params[field] !== null && record[field] !== params[field]) return false;
|
||||
const values = Array.isArray(params[`${field}s`]) ? params[`${field}s`].map((item) => String(item ?? "")).filter(Boolean) : [];
|
||||
return values.length === 0 || values.includes(String(record[field] ?? ""));
|
||||
});
|
||||
}
|
||||
|
||||
export function matchesTraceFact(value, params) {
|
||||
if (params.traceId && value !== params.traceId) return false;
|
||||
const traceIds = Array.isArray(params.traceIds) ? params.traceIds.map((item) => String(item ?? "")).filter(Boolean) : [];
|
||||
return traceIds.length === 0 || traceIds.includes(String(value ?? ""));
|
||||
}
|
||||
|
||||
export function testFactFamilySet(params = {}) {
|
||||
const all = ["sessions", "messages", "parts", "turns", "traceEvents", "checkpoints"];
|
||||
const raw = params.families ?? params.factFamilies;
|
||||
if (!raw) return new Set(all);
|
||||
const values = Array.isArray(raw) ? raw : String(raw).split(",");
|
||||
const selected = values.map((item) => String(item ?? "").trim()).filter((item) => all.includes(item));
|
||||
return new Set(selected.length ? selected : all);
|
||||
}
|
||||
|
||||
export function mergeFacts(target, source) {
|
||||
for (const key of Object.keys(target)) target[key].push(...source[key]);
|
||||
return target;
|
||||
}
|
||||
|
||||
export function emptyFacts() {
|
||||
return { sessions: [], messages: [], parts: [], turns: [], traceEvents: [], checkpoints: [] };
|
||||
}
|
||||
|
||||
export function normalizeTestStatus(value) {
|
||||
const status = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-");
|
||||
if (status === "active" || status === "busy" || status === "processing") return "running";
|
||||
if (status === "cancelled") return "canceled";
|
||||
return status || "unknown";
|
||||
}
|
||||
|
||||
const testTerminalStatuses = new Set(["completed", "failed", "blocked", "timeout", "canceled", "idle"]);
|
||||
|
||||
export async function getSseEvents(port, path, count) {
|
||||
const controller = new AbortController();
|
||||
const response = await fetch(`http://127.0.0.1:${port}${path}`, { signal: controller.signal });
|
||||
assert.equal(response.status, 200);
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder();
|
||||
let buffer = "";
|
||||
const events = [];
|
||||
try {
|
||||
while (events.length < count) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) break;
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
for (;;) {
|
||||
const index = buffer.indexOf("\n\n");
|
||||
if (index < 0) break;
|
||||
const block = buffer.slice(0, index);
|
||||
buffer = buffer.slice(index + 2);
|
||||
events.push(parseSseBlock(block));
|
||||
if (events.length >= count) break;
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
controller.abort();
|
||||
}
|
||||
return events;
|
||||
}
|
||||
|
||||
export function parseSseBlock(block) {
|
||||
const lines = block.split("\n");
|
||||
const event = lines.find((line) => line.startsWith("event:"))?.slice(6).trim() ?? "message";
|
||||
const id = lines.find((line) => line.startsWith("id:"))?.slice(3).trim() ?? null;
|
||||
const data = lines.filter((line) => line.startsWith("data:")).map((line) => line.slice(5).trim()).join("\n");
|
||||
return { event, id, data: JSON.parse(data) };
|
||||
}
|
||||
|
||||
export function createFakeKafkaFactory() {
|
||||
let eachMessage = null;
|
||||
let offset = 0;
|
||||
let subscribedTopic = "hwlab.event.v1";
|
||||
const consumer = {
|
||||
connect: async () => undefined,
|
||||
subscribe: async (input = {}) => { subscribedTopic = String(input.topic ?? subscribedTopic); },
|
||||
run: async (input = {}) => { eachMessage = input.eachMessage; },
|
||||
stop: async () => undefined,
|
||||
disconnect: async () => undefined
|
||||
};
|
||||
return {
|
||||
factory: () => ({ consumer: () => consumer }),
|
||||
emit: async (value = {}) => {
|
||||
await waitFor(() => typeof eachMessage === "function");
|
||||
await eachMessage({
|
||||
topic: subscribedTopic,
|
||||
partition: 0,
|
||||
message: {
|
||||
offset: String(offset++),
|
||||
key: Buffer.from(value.traceId ?? value.sessionId ?? "fake"),
|
||||
timestamp: String(Date.now()),
|
||||
value: Buffer.from(JSON.stringify(value))
|
||||
}
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export async function waitFor(predicate, timeoutMs = 1000) {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
if (predicate()) return;
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
}
|
||||
throw new Error("waitFor timed out");
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,800 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery.
|
||||
// Responsibility: Workbench durable state and timeline regression tests.
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { createCloudApiServer } from "./server.ts";
|
||||
import { createBackendPerformanceStore } from "./backend-performance.ts";
|
||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { classifyWorkbenchReadModelFailure } from "./server-workbench-http.ts";
|
||||
import { codeAgentTurnStatusPayload, createCodeAgentChatResultStore } from "./server-code-agent-http.ts";
|
||||
import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts";
|
||||
|
||||
import {
|
||||
ACTOR,
|
||||
getJson,
|
||||
waitForCondition,
|
||||
createDurableFactsRuntimeStore,
|
||||
createRuntimeStoreFromFacts,
|
||||
buildDurableFactsForSession,
|
||||
normalizeTestMessages,
|
||||
testTimingProjection,
|
||||
timestampIso,
|
||||
elapsedMs,
|
||||
normalizeTestEvents,
|
||||
filterFacts,
|
||||
workbenchTestFactOrder,
|
||||
matchesFact,
|
||||
matchesTraceFact,
|
||||
testFactFamilySet,
|
||||
mergeFacts,
|
||||
emptyFacts,
|
||||
normalizeTestStatus,
|
||||
getSseEvents,
|
||||
parseSseBlock,
|
||||
createFakeKafkaFactory,
|
||||
waitFor
|
||||
} from "./server-workbench-http-test-helpers.ts";
|
||||
|
||||
test("workbench trace events reports catch-up page when session projection is visible", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const traceId = "trc_workbench_trace_events_gap";
|
||||
const session = {
|
||||
id: "ses_workbench_trace_events_gap",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "completed",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_trace_events_gap",
|
||||
threadId: "thread-workbench-trace-events-gap",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-20T13:05:00.000Z",
|
||||
session: { sessionStatus: "completed", lastTraceId: traceId, messages: [{ role: "user", text: "events gap", traceId }, { role: "agent", text: "OK", traceId }] }
|
||||
};
|
||||
const accessController = {
|
||||
store: {},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const runtimeStore = createDurableFactsRuntimeStore({
|
||||
sessions: [{ session, events: [], status: "completed", finalText: "OK", runId: "run_workbench_trace_events_gap", commandId: "cmd_workbench_trace_events_gap", lastProjectedSeq: 2 }]
|
||||
});
|
||||
const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`);
|
||||
assert.equal(turn.status, 200);
|
||||
assert.equal(turn.body.turn.status, "completed");
|
||||
|
||||
const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`);
|
||||
assert.equal(trace.status, 200);
|
||||
assert.equal(trace.body.status, "projecting");
|
||||
assert.equal(trace.body.diagnostic.code, "workbench_trace_events_missing");
|
||||
assert.equal(trace.body.sessionId, session.id);
|
||||
assert.equal(trace.body.projectionStatus, "projecting");
|
||||
assert.equal(trace.body.projectionHealth, "projecting");
|
||||
assert.equal(trace.body.lastProjectedSeq, 2);
|
||||
assert.equal(trace.body.sourceRunId, "run_workbench_trace_events_gap");
|
||||
assert.equal(trace.body.sourceCommandId, "cmd_workbench_trace_events_gap");
|
||||
assert.equal(trace.body.blocker, null);
|
||||
assert.deepEqual(trace.body.events, []);
|
||||
assert.equal(trace.body.fullTraceLoaded, false);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench read model projects current turn running state from trace projection", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const traceId = "trc_workbench_active_state";
|
||||
const session = {
|
||||
id: "ses_workbench_active_state",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "active",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_active_state",
|
||||
threadId: "thread-workbench-active-state",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-17T00:01:00.000Z",
|
||||
session: { sessionStatus: "active", lastTraceId: traceId, messages: [{ role: "user", text: "still running", traceId }] }
|
||||
};
|
||||
traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false });
|
||||
const accessController = {
|
||||
store: {
|
||||
async listAgentSessionsForUser() { return [session]; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const runtimeStore = createDurableFactsRuntimeStore({
|
||||
sessions: [{
|
||||
session,
|
||||
events: [{ seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: session.updatedAt }],
|
||||
status: "running",
|
||||
lastProjectedSeq: 1
|
||||
}]
|
||||
});
|
||||
const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const sessions = await getJson(port, "/v1/workbench/sessions");
|
||||
assert.equal(sessions.status, 200);
|
||||
assert.equal(sessions.body.sessions[0].status, "running");
|
||||
assert.equal(sessions.body.sessions[0].running, true);
|
||||
assert.equal(sessions.body.sessions[0].terminal, false);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench read model projects current turn running state for idle session summary", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const traceId = "trc_workbench_idle_with_running_trace";
|
||||
const session = {
|
||||
id: "ses_workbench_idle_with_running_trace",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "idle",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_idle_with_running_trace",
|
||||
threadId: "thread-workbench-idle-with-running-trace",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-17T00:02:00.000Z",
|
||||
session: { sessionStatus: "idle", lastTraceId: traceId, messages: [{ role: "user", text: "still running despite idle row", traceId, status: "sent" }] }
|
||||
};
|
||||
traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false });
|
||||
const accessController = {
|
||||
store: {
|
||||
async listAgentSessionsForUser() { return [session]; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const runtimeStore = createDurableFactsRuntimeStore({
|
||||
sessions: [{
|
||||
session,
|
||||
events: [{ seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: session.updatedAt }],
|
||||
status: "running",
|
||||
lastProjectedSeq: 1
|
||||
}]
|
||||
});
|
||||
const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`);
|
||||
assert.equal(sessions.status, 200);
|
||||
assert.equal(sessions.body.sessions[0].status, "running");
|
||||
assert.equal(sessions.body.sessions[0].running, true);
|
||||
assert.equal(sessions.body.sessions[0].terminal, false);
|
||||
assert.equal(sessions.body.sessions[0].turnSummary.status, "running");
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench read model projects newest running turn over stale completed session lastTraceId", async () => {
|
||||
const oldTraceId = "trc_workbench_stale_completed_old";
|
||||
const newTraceId = "trc_workbench_new_running_current";
|
||||
const session = {
|
||||
id: "ses_workbench_stale_completed_new_running",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "completed",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_stale_completed_new_running",
|
||||
threadId: "thread-workbench-stale-completed-new-running",
|
||||
lastTraceId: oldTraceId,
|
||||
updatedAt: "2026-07-01T00:00:02.000Z",
|
||||
session: {
|
||||
sessionStatus: "completed",
|
||||
lastTraceId: oldTraceId,
|
||||
messages: [
|
||||
{ messageId: "msg_workbench_old_user", role: "user", text: "first", traceId: oldTraceId, status: "sent", projectedSeq: 10 },
|
||||
{ messageId: "msg_workbench_old_agent", role: "agent", text: "first done", traceId: oldTraceId, status: "completed", projectedSeq: 20 }
|
||||
]
|
||||
}
|
||||
};
|
||||
const facts = emptyFacts();
|
||||
mergeFacts(facts, buildDurableFactsForSession({ session, status: "completed", finalText: "first done", lastProjectedSeq: 20 }));
|
||||
facts.messages.push(
|
||||
{ messageId: "msg_workbench_new_user", sessionId: session.id, turnId: newTraceId, traceId: newTraceId, role: "user", status: "sent", text: "second", projectedSeq: 30, sourceSeq: 30, sourceEventId: "evt_new_user", terminal: false, sealed: false, updatedAt: "2026-07-01T00:00:03.000Z", valuesRedacted: true },
|
||||
{ messageId: "msg_workbench_new_agent", sessionId: session.id, turnId: newTraceId, traceId: newTraceId, role: "agent", status: "running", text: "", projectedSeq: 40, sourceSeq: 40, sourceEventId: "evt_new_agent", terminal: false, sealed: false, updatedAt: "2026-07-01T00:00:04.000Z", valuesRedacted: true }
|
||||
);
|
||||
facts.turns.push({ turnId: newTraceId, sessionId: session.id, traceId: newTraceId, messageId: "msg_workbench_new_agent", status: "running", projectedSeq: 40, sourceSeq: 40, sourceEventId: "evt_new_turn", terminal: false, sealed: false, diagnostic: { projectionStatus: "projecting", projectionHealth: "healthy", valuesRedacted: true }, timing: { startedAt: "2026-07-01T00:00:03.000Z", lastEventAt: "2026-07-01T00:00:04.000Z", finishedAt: null, durationMs: null, valuesRedacted: true }, startedAt: "2026-07-01T00:00:03.000Z", lastEventAt: "2026-07-01T00:00:04.000Z", finishedAt: null, durationMs: null, updatedAt: "2026-07-01T00:00:04.000Z", valuesRedacted: true });
|
||||
facts.checkpoints.push({ traceId: newTraceId, sessionId: session.id, turnId: newTraceId, projectedSeq: 40, sourceSeq: 40, sourceEventId: "evt_new_checkpoint", projectionStatus: "projecting", projectionHealth: "healthy", terminal: false, sealed: false, diagnostic: { projectionStatus: "projecting", projectionHealth: "healthy", valuesRedacted: true }, updatedAt: "2026-07-01T00:00:04.000Z", valuesRedacted: true });
|
||||
const runtimeStore = createRuntimeStoreFromFacts(facts);
|
||||
const accessController = {
|
||||
store: { async listAgentSessionsForUser() { return [session]; } },
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const server = createCloudApiServer({ accessController, traceStore: createCodeAgentTraceStore(), runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`);
|
||||
assert.equal(sessions.status, 200);
|
||||
assert.equal(sessions.body.sessions[0].lastTraceId, newTraceId);
|
||||
assert.equal(sessions.body.sessions[0].status, "running");
|
||||
assert.equal(sessions.body.sessions[0].running, true);
|
||||
assert.equal(sessions.body.sessions[0].terminal, false);
|
||||
assert.equal(sessions.body.sessions[0].turnSummary.traceId, newTraceId);
|
||||
assert.equal(sessions.body.sessions[0].turnSummary.status, "running");
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench read model projects completed current turn for idle session summary", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const results = createCodeAgentChatResultStore();
|
||||
const traceId = "trc_workbench_idle_with_completed_trace";
|
||||
const finalText = "OK";
|
||||
const session = {
|
||||
id: "ses_workbench_idle_with_completed_trace",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "idle",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_idle_with_completed_trace",
|
||||
threadId: "thread-workbench-idle-with-completed-trace",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-17T00:03:00.000Z",
|
||||
session: {
|
||||
sessionStatus: "idle",
|
||||
lastTraceId: traceId,
|
||||
messages: [
|
||||
{ role: "user", text: "reply OK", traceId, status: "sent" },
|
||||
{ role: "agent", text: "partial assistant text", traceId, status: "running" }
|
||||
]
|
||||
}
|
||||
};
|
||||
traceStore.append(traceId, { type: "request", status: "accepted", label: "request:accepted" });
|
||||
traceStore.append(traceId, { type: "result", status: "completed", label: "result:completed", terminal: true });
|
||||
results.set(traceId, {
|
||||
status: "completed",
|
||||
traceId,
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: session.conversationId,
|
||||
sessionId: session.id,
|
||||
threadId: session.threadId,
|
||||
finalResponse: finalText,
|
||||
agentRun: { runId: "run_workbench_idle_completed", commandId: "cmd_workbench_idle_completed", status: "completed" }
|
||||
});
|
||||
const accessController = {
|
||||
store: {
|
||||
async listAgentSessionsForUser() { return [session]; },
|
||||
async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
|
||||
async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const backendPerformanceStore = createBackendPerformanceStore({ env: { POD_NAMESPACE: "hwlab-v03", HWLAB_GITOPS_TARGET: "v03", HWLAB_RUNTIME_LANE: "v03", HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: "D601" } });
|
||||
const runtimeStore = createDurableFactsRuntimeStore({
|
||||
sessions: [{
|
||||
session,
|
||||
events: [
|
||||
{ seq: 1, type: "request", status: "accepted", label: "request:accepted", createdAt: "2026-06-17T00:02:58.000Z" },
|
||||
{ seq: 2, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-17T00:03:00.000Z" }
|
||||
],
|
||||
status: "completed",
|
||||
finalText,
|
||||
runId: "run_workbench_idle_completed",
|
||||
commandId: "cmd_workbench_idle_completed",
|
||||
lastProjectedSeq: 2
|
||||
}]
|
||||
});
|
||||
const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results, backendPerformanceStore });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`);
|
||||
assert.equal(sessions.status, 200);
|
||||
assert.equal(sessions.body.sessions[0].status, "completed");
|
||||
assert.equal(sessions.body.sessions[0].running, false);
|
||||
assert.equal(sessions.body.sessions[0].terminal, true);
|
||||
assert.equal(sessions.body.sessions[0].turnSummary.status, "completed");
|
||||
|
||||
const messages = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}/messages?limit=10`);
|
||||
assert.equal(messages.status, 200);
|
||||
assert.equal(messages.body.messages[1].status, "completed");
|
||||
assert.equal(messages.body.messages[1].text, finalText);
|
||||
|
||||
const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`);
|
||||
assert.equal(turn.status, 200);
|
||||
assert.equal(turn.body.turn.status, "completed");
|
||||
assert.equal(turn.body.turn.running, false);
|
||||
assert.equal(turn.body.turn.terminal, true);
|
||||
assert.equal(turn.body.turn.assistantText, finalText);
|
||||
assert.equal(turn.body.projectionStatus, "caught-up");
|
||||
assert.equal(turn.body.sourceRunId, "run_workbench_idle_completed");
|
||||
assert.equal(turn.body.sourceCommandId, "cmd_workbench_idle_completed");
|
||||
const metricsText = backendPerformanceStore.metricsText();
|
||||
const lagCountLine = metricsText.split("\n").find((line) => line.startsWith("hwlab_workbench_projection_lag_seconds_count") && line.includes('projection_status="caught_up"'));
|
||||
const turnGetCountLine = metricsText.split("\n").find((line) => line.startsWith("hwlab_workbench_turn_get_duration_seconds_count") && line.includes('route="/v1/workbench/turns/:traceId"'));
|
||||
assert.ok(lagCountLine?.endsWith(" 1"));
|
||||
assert.ok(turnGetCountLine?.endsWith(" 1"));
|
||||
|
||||
const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`);
|
||||
assert.equal(trace.status, 200);
|
||||
assert.equal(trace.body.projectionStatus, "caught-up");
|
||||
assert.equal(trace.body.sourceRunId, "run_workbench_idle_completed");
|
||||
assert.equal(trace.body.sourceCommandId, "cmd_workbench_idle_completed");
|
||||
assert.equal(trace.body.lastProjectedSeq, 2);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench read model lets terminal result override stale running session state", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const results = createCodeAgentChatResultStore();
|
||||
const traceId = "trc_workbench_stale_running_result_completed";
|
||||
const finalText = "OK";
|
||||
const session = {
|
||||
id: "ses_workbench_stale_running_result_completed",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_stale_running_result_completed",
|
||||
threadId: "thread-workbench-stale-running-result-completed",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-18T16:31:04.000Z",
|
||||
session: {
|
||||
sessionStatus: "running",
|
||||
lastTraceId: traceId,
|
||||
messages: [
|
||||
{ role: "user", text: "reply OK", traceId, status: "sent" },
|
||||
{ role: "agent", text: "", traceId, status: "running" }
|
||||
]
|
||||
}
|
||||
};
|
||||
traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false });
|
||||
results.set(traceId, {
|
||||
status: "completed",
|
||||
traceId,
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: session.conversationId,
|
||||
sessionId: session.id,
|
||||
threadId: session.threadId,
|
||||
finalResponse: finalText,
|
||||
agentRun: { runId: "run_workbench_stale_running_completed", commandId: "cmd_workbench_stale_running_completed", status: "completed" }
|
||||
});
|
||||
const accessController = {
|
||||
store: {
|
||||
async listAgentSessionsForUser() { return [session]; },
|
||||
async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
|
||||
async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const runtimeStore = createDurableFactsRuntimeStore({
|
||||
sessions: [{
|
||||
session,
|
||||
events: [
|
||||
{ seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: "2026-06-18T16:30:00.000Z" },
|
||||
{ seq: 2, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-18T16:31:04.000Z" }
|
||||
],
|
||||
status: "completed",
|
||||
finalText,
|
||||
runId: "run_workbench_stale_running_completed",
|
||||
commandId: "cmd_workbench_stale_running_completed",
|
||||
lastProjectedSeq: 2
|
||||
}]
|
||||
});
|
||||
const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`);
|
||||
assert.equal(sessions.status, 200);
|
||||
assert.equal(sessions.body.sessions[0].status, "completed");
|
||||
assert.equal(sessions.body.sessions[0].running, false);
|
||||
assert.equal(sessions.body.sessions[0].terminal, true);
|
||||
assert.equal(sessions.body.sessions[0].turnSummary.status, "completed");
|
||||
|
||||
const detail = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}`);
|
||||
assert.equal(detail.status, 200);
|
||||
assert.equal(detail.body.session.status, "completed");
|
||||
assert.equal(detail.body.session.running, false);
|
||||
assert.equal(detail.body.session.terminal, true);
|
||||
|
||||
const messages = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}/messages?limit=10`);
|
||||
assert.equal(messages.status, 200);
|
||||
assert.equal(messages.body.messages[1].status, "completed");
|
||||
assert.equal(messages.body.messages[1].text, finalText);
|
||||
|
||||
const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`);
|
||||
assert.equal(turn.status, 200);
|
||||
assert.equal(turn.body.status, "completed");
|
||||
assert.equal(turn.body.turn.status, "completed");
|
||||
assert.equal(turn.body.turn.running, false);
|
||||
assert.equal(turn.body.turn.terminal, true);
|
||||
assert.equal(turn.body.turn.assistantText, finalText);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench session list uses durable projection checkpoint without trace/result hydration", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const traceId = "trc_workbench_compact_list_summary";
|
||||
const finalText = "compact list summary OK";
|
||||
const session = {
|
||||
id: "ses_workbench_compact_list_summary",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_compact_list_summary",
|
||||
threadId: "thread-workbench-compact-list-summary",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-19T15:40:00.000Z",
|
||||
session: {
|
||||
sessionStatus: "running",
|
||||
lastTraceId: traceId,
|
||||
messages: [
|
||||
{ role: "user", text: "compact summary", traceId, status: "sent" },
|
||||
{ role: "agent", text: "", traceId, status: "running" }
|
||||
],
|
||||
traceResults: {
|
||||
[traceId]: {
|
||||
traceId,
|
||||
status: "completed",
|
||||
finalResponse: { text: finalText, status: "completed", traceId, valuesPrinted: false },
|
||||
traceSummary: {
|
||||
traceId,
|
||||
source: "agent-session-compact-summary",
|
||||
sourceEventCount: 42,
|
||||
terminalStatus: "completed",
|
||||
agentRun: { runId: "run_workbench_compact_list_summary", commandId: "cmd_workbench_compact_list_summary", lastSeq: 42, valuesPrinted: false },
|
||||
valuesPrinted: false
|
||||
},
|
||||
agentRun: { runId: "run_workbench_compact_list_summary", commandId: "cmd_workbench_compact_list_summary", status: "completed", terminalStatus: "completed", lastSeq: 42, valuesPrinted: false },
|
||||
updatedAt: "2026-06-19T15:40:00.000Z",
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
}
|
||||
},
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
}
|
||||
};
|
||||
const accessController = {
|
||||
store: {
|
||||
async listAgentSessionsForUser() { return [session]; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const factQueries = [];
|
||||
const runtimeStore = createDurableFactsRuntimeStore({
|
||||
sessions: [{
|
||||
session,
|
||||
events: [
|
||||
{ seq: 1, sourceSeq: 1, type: "request", status: "accepted", label: "request:accepted", createdAt: "2026-06-19T15:39:10.000Z" },
|
||||
{ seq: 42, sourceSeq: 42, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-19T15:40:00.000Z" }
|
||||
],
|
||||
status: "completed",
|
||||
finalText,
|
||||
runId: "run_workbench_compact_list_summary",
|
||||
commandId: "cmd_workbench_compact_list_summary",
|
||||
lastProjectedSeq: 42
|
||||
}],
|
||||
queries: factQueries
|
||||
});
|
||||
const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`);
|
||||
assert.equal(sessions.status, 200);
|
||||
assert.equal(sessions.body.sessions[0].status, "completed");
|
||||
assert.equal(sessions.body.sessions[0].running, false);
|
||||
assert.equal(sessions.body.sessions[0].terminal, true);
|
||||
assert.equal(sessions.body.sessions[0].turnSummary.status, "completed");
|
||||
assert.equal(sessions.body.sessions[0].turnSummary.eventCount, 42);
|
||||
assert.equal(sessions.body.sessions[0].projectionStatus, "caught-up");
|
||||
assert.ok(factQueries.length >= 1);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench session list overlaps include lookup with the page query", async () => {
|
||||
const makeSession = (id, traceId, updatedAt) => ({
|
||||
id,
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: id.replace(/^ses_/u, "cnv_"),
|
||||
threadId: `thread-${id}`,
|
||||
lastTraceId: traceId,
|
||||
updatedAt,
|
||||
session: {
|
||||
sessionStatus: "running",
|
||||
lastTraceId: traceId,
|
||||
messages: [{ role: "user", text: id, traceId, status: "sent" }]
|
||||
}
|
||||
});
|
||||
const included = makeSession("ses_parallel_include", "trc_parallel_include", "2026-06-19T14:00:00.000Z");
|
||||
const firstPage = makeSession("ses_parallel_page_a", "trc_parallel_page_a", "2026-06-19T16:00:00.000Z");
|
||||
const secondPage = makeSession("ses_parallel_page_b", "trc_parallel_page_b", "2026-06-19T15:00:00.000Z");
|
||||
const facts = emptyFacts();
|
||||
for (const session of [included, firstPage, secondPage]) mergeFacts(facts, buildDurableFactsForSession({ session, status: "running" }));
|
||||
const accessController = {
|
||||
store: {},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
let pageReleased = false;
|
||||
let releasePage = () => {};
|
||||
const pageGate = new Promise((resolve) => { releasePage = resolve; });
|
||||
const queries = [];
|
||||
const runtimeStore = {
|
||||
async queryWorkbenchFacts(params = {}) {
|
||||
queries.push({ ...params, pageReleased });
|
||||
if (params.sessionsOrder === "updated_desc") await pageGate;
|
||||
const filtered = filterFacts(facts, params);
|
||||
return {
|
||||
facts: filtered,
|
||||
count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0),
|
||||
persistence: { adapter: "test-durable-workbench-facts", durable: true }
|
||||
};
|
||||
}
|
||||
};
|
||||
const server = createCloudApiServer({ accessController, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const pending = getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(included.id)}&limit=1`);
|
||||
await waitForCondition(() => queries.some((query) => query.sessionId === included.id) && queries.some((query) => query.sessionsOrder === "updated_desc"));
|
||||
const includeIndex = queries.findIndex((query) => query.sessionId === included.id);
|
||||
assert.ok(includeIndex >= 0);
|
||||
assert.equal(queries[includeIndex].pageReleased, false);
|
||||
pageReleased = true;
|
||||
releasePage();
|
||||
const sessions = await pending;
|
||||
assert.equal(sessions.status, 200);
|
||||
assert.equal(sessions.body.sessions[0].sessionId, included.id);
|
||||
assert.equal(sessions.body.hasMore, true);
|
||||
assert.equal(queries.find((query) => query.sessionsOrder === "updated_desc")?.sessionProjection, "summary");
|
||||
assert.equal(queries.find((query) => query.sessionId === included.id)?.sessionProjection, undefined);
|
||||
} finally {
|
||||
pageReleased = true;
|
||||
releasePage();
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench read model reads terminal session status from durable facts", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const traceId = "trc_workbench_durable_terminal_after_memory_running";
|
||||
const session = {
|
||||
id: "ses_workbench_durable_terminal_after_memory_running",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_durable_terminal_after_memory_running",
|
||||
threadId: "thread-workbench-durable-terminal-after-memory-running",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-18T17:22:00.000Z",
|
||||
session: {
|
||||
sessionStatus: "running",
|
||||
lastTraceId: traceId,
|
||||
messages: [
|
||||
{ role: "user", text: "reply OK", traceId, status: "sent" },
|
||||
{ role: "agent", text: "", traceId, status: "running" }
|
||||
]
|
||||
}
|
||||
};
|
||||
traceStore.append(traceId, { type: "backend", status: "running", label: "runner:created", terminal: false, seq: 1 });
|
||||
const durableEvents = [
|
||||
{ traceId, seq: 1, type: "request", status: "accepted", label: "request:accepted", createdAt: "2026-06-18T17:21:40.000Z", valuesPrinted: false },
|
||||
{ traceId, seq: 2, type: "result", status: "completed", label: "result:completed", terminal: true, createdAt: "2026-06-18T17:22:00.000Z", valuesPrinted: false }
|
||||
];
|
||||
const accessController = {
|
||||
store: {
|
||||
async listAgentSessionsForUser() { return [session]; },
|
||||
async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
|
||||
async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const runtimeStore = createDurableFactsRuntimeStore({
|
||||
sessions: [{ session, events: durableEvents, status: "completed", finalText: "OK", lastProjectedSeq: 2 }]
|
||||
});
|
||||
const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const sessions = await getJson(port, `/v1/workbench/sessions?includeSessionId=${encodeURIComponent(session.id)}`);
|
||||
assert.equal(sessions.status, 200);
|
||||
assert.equal(sessions.body.sessions[0].status, "completed");
|
||||
assert.equal(sessions.body.sessions[0].running, false);
|
||||
assert.equal(sessions.body.sessions[0].terminal, true);
|
||||
|
||||
const detail = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}`);
|
||||
assert.equal(detail.status, 200);
|
||||
assert.equal(detail.body.session.status, "completed");
|
||||
assert.equal(detail.body.session.running, false);
|
||||
assert.equal(detail.body.session.terminal, true);
|
||||
|
||||
const messages = await getJson(port, `/v1/workbench/sessions/${encodeURIComponent(session.id)}/messages?limit=10`);
|
||||
assert.equal(messages.status, 200);
|
||||
assert.equal(messages.body.messages[1].status, "completed");
|
||||
|
||||
const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`);
|
||||
assert.equal(turn.status, 200);
|
||||
assert.equal(turn.body.status, "completed");
|
||||
assert.equal(turn.body.turn.status, "completed");
|
||||
assert.equal(turn.body.turn.running, false);
|
||||
assert.equal(turn.body.turn.terminal, true);
|
||||
assert.equal(turn.body.turn.trace.eventCount, 2);
|
||||
assert.equal(turn.body.projectionStatus, "caught-up");
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench read model keeps non-terminal durable tool completed events out of turn status", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const results = createCodeAgentChatResultStore();
|
||||
const traceId = "trc_workbench_durable_tool_completed_nonterminal";
|
||||
const session = {
|
||||
id: "ses_workbench_durable_tool_completed_nonterminal",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_durable_tool_completed_nonterminal",
|
||||
threadId: "thread-workbench-durable-tool-completed-nonterminal",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-18T19:35:51.787Z",
|
||||
session: {
|
||||
sessionStatus: "running",
|
||||
lastTraceId: traceId,
|
||||
messages: [
|
||||
{ role: "user", text: "write benchmark", traceId, status: "sent" },
|
||||
{ role: "agent", text: "", traceId, status: "running" }
|
||||
],
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
}
|
||||
};
|
||||
traceStore.append(traceId, { seq: 1, type: "backend_status", status: "running", label: "runner:created", terminal: false });
|
||||
const durableEvents = [
|
||||
{ traceId, seq: 1, type: "assistant_message", status: "running", label: "agentrun:assistant:message", terminal: false, message: "正在安装 Python。", createdAt: "2026-06-18T19:35:40.000Z", valuesPrinted: false },
|
||||
{ traceId, seq: 2, type: "commandExecution", status: "completed", label: "item/commandExecution:completed", terminal: false, command: "ls -ld .", stdout: "OK\n", createdAt: "2026-06-18T19:35:51.787Z", valuesPrinted: false }
|
||||
];
|
||||
results.set(traceId, {
|
||||
status: "running",
|
||||
traceId,
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: session.conversationId,
|
||||
sessionId: session.id,
|
||||
threadId: session.threadId,
|
||||
finalResponse: null,
|
||||
assistantText: null,
|
||||
agentRun: { runId: "run_workbench_nonterminal_tool_completed", commandId: "cmd_workbench_nonterminal_tool_completed", status: "runner-job-created", runStatus: "pending", commandState: "pending" }
|
||||
});
|
||||
const accessController = {
|
||||
store: {
|
||||
async listAgentSessionsForUser() { return [session]; },
|
||||
async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
|
||||
async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const runtimeStore = createDurableFactsRuntimeStore({
|
||||
sessions: [{
|
||||
session,
|
||||
events: durableEvents,
|
||||
status: "running",
|
||||
assistantText: "partial assistant text",
|
||||
runId: "run_workbench_nonterminal_tool_completed",
|
||||
commandId: "cmd_workbench_nonterminal_tool_completed",
|
||||
projectionStatus: "projecting",
|
||||
lastProjectedSeq: 2
|
||||
}]
|
||||
});
|
||||
const workbenchRuntime = {
|
||||
async queryWorkbenchFacts(query) { return runtimeStore.queryWorkbenchFacts(query); }
|
||||
};
|
||||
const server = createCloudApiServer({ accessController, traceStore, runtimeStore, workbenchRuntime, codeAgentChatResults: results });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`);
|
||||
assert.equal(turn.status, 200);
|
||||
assert.equal(turn.body.status, "running");
|
||||
assert.equal(turn.body.turn.status, "running");
|
||||
assert.equal(turn.body.turn.running, true);
|
||||
assert.equal(turn.body.turn.terminal, false);
|
||||
assert.equal(turn.body.turn.assistantText, null);
|
||||
assert.equal(turn.body.turn.finalResponse, null);
|
||||
assert.equal(turn.body.projectionStatus, "projecting");
|
||||
|
||||
const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`);
|
||||
assert.equal(trace.status, 200);
|
||||
assert.equal(trace.body.events.at(-1).status, "completed");
|
||||
assert.equal(trace.body.events.at(-1).terminal, false);
|
||||
assert.equal(trace.body.projectionStatus, "projecting");
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench trace events API blocks historical projectedSeq collisions instead of handing them to renderer", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const traceId = "trc_workbench_projected_seq_collision";
|
||||
const session = {
|
||||
id: "ses_workbench_projected_seq_collision",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_projected_seq_collision",
|
||||
threadId: "thread-workbench-projected-seq-collision",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-20T12:20:00.000Z",
|
||||
session: { sessionStatus: "running", lastTraceId: traceId, messages: [{ role: "user", text: "collision", traceId, status: "sent" }] }
|
||||
};
|
||||
const accessController = {
|
||||
store: {
|
||||
async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
|
||||
async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const runtimeStore = createDurableFactsRuntimeStore({
|
||||
sessions: [{
|
||||
session,
|
||||
events: [
|
||||
{ id: "wte_collision_a", projectedSeq: 1, sourceSeq: 1, sourceEventId: "source-a", type: "backend", status: "running", label: "projected:1:a", createdAt: "2026-06-20T12:19:58.000Z" },
|
||||
{ id: "wte_collision_b", projectedSeq: 1, sourceSeq: 2, sourceEventId: "source-b", type: "assistant", status: "running", label: "projected:1:b", message: "later", createdAt: "2026-06-20T12:19:59.000Z" }
|
||||
],
|
||||
status: "running",
|
||||
projectionStatus: "projecting",
|
||||
lastProjectedSeq: 1
|
||||
}]
|
||||
});
|
||||
const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: createCodeAgentChatResultStore() });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`);
|
||||
assert.equal(trace.status, 200);
|
||||
assert.equal(trace.body.status, "blocked");
|
||||
assert.equal(trace.body.projectionStatus, "blocked");
|
||||
assert.equal(trace.body.projectionHealth, "degraded");
|
||||
assert.equal(trace.body.blocker.code, "projected_seq_collision");
|
||||
assert.deepEqual(trace.body.events, []);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,448 @@
|
||||
// SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery.
|
||||
// Responsibility: Workbench realtime and projection blocker regression tests.
|
||||
|
||||
import assert from "node:assert/strict";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { createCloudApiServer } from "./server.ts";
|
||||
import { createBackendPerformanceStore } from "./backend-performance.ts";
|
||||
import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import { classifyWorkbenchReadModelFailure } from "./server-workbench-http.ts";
|
||||
import { codeAgentTurnStatusPayload, createCodeAgentChatResultStore } from "./server-code-agent-http.ts";
|
||||
import { createWorkbenchTurnProjection, durableTraceStatus, projectionDiagnostics, traceTerminalEvidence } from "./workbench-turn-projection.ts";
|
||||
|
||||
import {
|
||||
ACTOR,
|
||||
getJson,
|
||||
waitForCondition,
|
||||
createDurableFactsRuntimeStore,
|
||||
createRuntimeStoreFromFacts,
|
||||
buildDurableFactsForSession,
|
||||
normalizeTestMessages,
|
||||
testTimingProjection,
|
||||
timestampIso,
|
||||
elapsedMs,
|
||||
normalizeTestEvents,
|
||||
filterFacts,
|
||||
workbenchTestFactOrder,
|
||||
matchesFact,
|
||||
matchesTraceFact,
|
||||
testFactFamilySet,
|
||||
mergeFacts,
|
||||
emptyFacts,
|
||||
normalizeTestStatus,
|
||||
getSseEvents,
|
||||
parseSseBlock,
|
||||
createFakeKafkaFactory,
|
||||
waitFor
|
||||
} from "./server-workbench-http-test-helpers.ts";
|
||||
|
||||
test("workbench realtime stream surfaces facts blocker instead of legacy trace fallback", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const results = createCodeAgentChatResultStore();
|
||||
const traceId = "trc_workbench_realtime";
|
||||
const session = {
|
||||
id: "ses_workbench_realtime",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_realtime",
|
||||
threadId: "thread-workbench-realtime",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-17T02:00:00.000Z",
|
||||
session: { sessionStatus: "running", lastTraceId: traceId, messages: [{ role: "user", text: "stream", traceId }] }
|
||||
};
|
||||
traceStore.append(traceId, { type: "assistant", status: "completed", label: "assistant:completed", terminal: true, message: "legacy trace answer" });
|
||||
results.set(traceId, { status: "completed", traceId, ownerUserId: ACTOR.id, sessionId: session.id, threadId: session.threadId, finalResponse: { text: "legacy result answer" } });
|
||||
const accessController = {
|
||||
store: {
|
||||
async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
|
||||
async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const server = createCloudApiServer({ accessController, traceStore, codeAgentChatResults: results, env: { HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000" } });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const eventsPromise = getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}`, 4);
|
||||
const events = await eventsPromise;
|
||||
assert.deepEqual(events.map((event) => event.event), [
|
||||
"workbench.connected",
|
||||
"workbench.error",
|
||||
"workbench.trace.snapshot",
|
||||
"workbench.turn.snapshot"
|
||||
]);
|
||||
assert.equal(events[0].data.filters.sessionId, session.id);
|
||||
assert.equal(events[1].data.error.code, "workbench_facts_trace_missing");
|
||||
assert.equal(events[2].data.traceId, traceId);
|
||||
assert.equal(events[2].data.snapshot.status, "unknown");
|
||||
assert.equal(events[2].data.snapshot.eventCount, 0);
|
||||
assert.equal(events[3].data.turn.traceId, traceId);
|
||||
assert.equal(events[3].data.turn.status, "unknown");
|
||||
assert.equal(events[3].data.turn.terminal, false);
|
||||
assert.equal(events[3].data.turn.finalResponse, null);
|
||||
assert.equal(JSON.stringify(events).includes("legacy result answer"), false);
|
||||
assert.equal(JSON.stringify(events).includes("legacy trace answer"), false);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
test("workbench realtime stream forwards HWLAB Kafka events after initial connection", async () => {
|
||||
const fakeKafka = createFakeKafkaFactory();
|
||||
const traceId = "trc_workbench_realtime_after_seq";
|
||||
const session = {
|
||||
id: "ses_workbench_realtime_after_seq",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_realtime_after_seq",
|
||||
threadId: "thread-workbench-realtime-after-seq",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-24T14:00:00.000Z",
|
||||
session: { sessionStatus: "running", lastTraceId: traceId }
|
||||
};
|
||||
const outboxQueries = [];
|
||||
const workbenchRuntime = {
|
||||
async readWorkbenchProjectionOutbox(params = {}) {
|
||||
outboxQueries.push({ ...params });
|
||||
return [
|
||||
{ outboxSeq: 11, projectedSeq: 7, traceId, sessionId: session.id, turnId: traceId, commitType: "event", terminal: false, sealed: false, createdAt: "2026-06-24T14:00:01.000Z" }
|
||||
];
|
||||
}
|
||||
};
|
||||
const accessController = {
|
||||
store: {
|
||||
async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
|
||||
async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const serverWithKafka = createCloudApiServer({
|
||||
accessController,
|
||||
workbenchRuntime,
|
||||
kafkaFactory: fakeKafka.factory,
|
||||
env: {
|
||||
HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000",
|
||||
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092",
|
||||
HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api",
|
||||
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = serverWithKafka.address();
|
||||
setTimeout(() => { void fakeKafka.emit({
|
||||
eventType: "hwlab.trace.event.projected",
|
||||
sessionId: "ses_agentrun_workbench_realtime_after_seq",
|
||||
traceId,
|
||||
context: { sourceSeq: 7, runId: "run_workbench_realtime_after_seq", commandId: "cmd_workbench_realtime_after_seq" },
|
||||
event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:test", message: "Kafka realtime event", sourceSeq: 7 }
|
||||
}); }, 25);
|
||||
const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}&traceId=${encodeURIComponent(traceId)}&afterSeq=10`, 2);
|
||||
assert.deepEqual(events.map((event) => event.event), ["workbench.connected", "workbench.trace.event"]);
|
||||
assert.equal(events[0].id, "10");
|
||||
assert.equal(events[1].id, "hwlab.event.v1:0:0");
|
||||
assert.equal(events[1].data.realtimeSource, "kafka");
|
||||
assert.equal(events[1].data.realtimeAuthority, "workbench-realtime-authority-v2");
|
||||
assert.equal(events[1].data.entity.family, "traceEvents");
|
||||
assert.equal(events[1].data.entity.version, 7);
|
||||
assert.equal(events[1].data.entity.projectionRevision, "kafka:hwlab.event.v1:0:0");
|
||||
assert.equal(events[1].data.event.message, "Kafka realtime event");
|
||||
assert.equal(events[1].data.kafka.topic, "hwlab.event.v1");
|
||||
assert.equal(events[1].data.cursor.traceSeq, 7);
|
||||
assert.deepEqual(outboxQueries, []);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench session realtime Kafka stream does not pin subscription to stale lastTraceId", async () => {
|
||||
const fakeKafka = createFakeKafkaFactory();
|
||||
const staleTraceId = "trc_workbench_realtime_stale_trace";
|
||||
const liveTraceId = "trc_workbench_realtime_live_trace";
|
||||
const session = {
|
||||
id: "ses_workbench_realtime_session_wide",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_realtime_session_wide",
|
||||
threadId: "thread-workbench-realtime-session-wide",
|
||||
lastTraceId: staleTraceId,
|
||||
updatedAt: "2026-06-24T14:00:00.000Z",
|
||||
session: { sessionStatus: "running", lastTraceId: staleTraceId }
|
||||
};
|
||||
const accessController = {
|
||||
store: {
|
||||
async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
|
||||
async getAgentSessionByTraceId() { return null; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const serverWithKafka = createCloudApiServer({
|
||||
accessController,
|
||||
workbenchRuntime: {
|
||||
async queryWorkbenchFacts(params = {}) {
|
||||
return {
|
||||
facts: {
|
||||
sessions: [{ sessionId: session.id, ownerUserId: ACTOR.id, threadId: session.threadId, lastTraceId: staleTraceId, status: "running", valuesRedacted: true }],
|
||||
messages: [],
|
||||
parts: [],
|
||||
turns: [],
|
||||
checkpoints: []
|
||||
},
|
||||
count: 1,
|
||||
persistence: { adapter: "test-session-wide-kafka", durable: true },
|
||||
params
|
||||
};
|
||||
}
|
||||
},
|
||||
kafkaFactory: fakeKafka.factory,
|
||||
env: {
|
||||
HWLAB_WORKBENCH_SSE_HEARTBEAT_MS: "60000",
|
||||
HWLAB_KAFKA_BOOTSTRAP_SERVERS: "127.0.0.1:9092",
|
||||
HWLAB_KAFKA_CLIENT_ID: "test-hwlab-cloud-api",
|
||||
HWLAB_KAFKA_EVENT_TOPIC: "hwlab.event.v1"
|
||||
}
|
||||
});
|
||||
await new Promise((resolve) => serverWithKafka.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = serverWithKafka.address();
|
||||
setTimeout(() => { void fakeKafka.emit({
|
||||
eventType: "hwlab.trace.event.projected",
|
||||
sessionId: "ses_agentrun_workbench_realtime_session_wide",
|
||||
traceId: liveTraceId,
|
||||
context: { runId: "run_workbench_realtime_session_wide", commandId: "cmd_workbench_realtime_session_wide" },
|
||||
event: { type: "backend", eventType: "backend", status: "running", label: "agentrun:event:live", message: "Kafka live trace event" }
|
||||
}); }, 100);
|
||||
const events = await getSseEvents(port, `/v1/workbench/events?sessionId=${encodeURIComponent(session.id)}`, 4);
|
||||
assert.equal(events[0].event, "workbench.connected");
|
||||
const liveEvent = events.find((event) => event.event === "workbench.trace.event" && event.data?.traceId === liveTraceId);
|
||||
assert.ok(liveEvent, JSON.stringify(events.map((event) => ({ event: event.event, traceId: event.data?.traceId, message: event.data?.event?.message }))));
|
||||
assert.equal(liveEvent.data.event.message, "Kafka live trace event");
|
||||
assert.equal(liveEvent.data.realtimeAuthority, "workbench-realtime-authority-v2");
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => serverWithKafka.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench read model exposes runtime trace projection query failures as projection blockers", async () => {
|
||||
const traceStore = createCodeAgentTraceStore();
|
||||
const results = createCodeAgentChatResultStore();
|
||||
const traceId = "trc_workbench_projection_store_unavailable";
|
||||
const session = {
|
||||
id: "ses_workbench_projection_store_unavailable",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_projection_store_unavailable",
|
||||
threadId: "thread-workbench-projection-store-unavailable",
|
||||
lastTraceId: traceId,
|
||||
updatedAt: "2026-06-18T01:10:00.000Z",
|
||||
session: {
|
||||
sessionStatus: "running",
|
||||
lastTraceId: traceId,
|
||||
messages: [
|
||||
{ role: "user", text: "projection store unavailable", traceId, status: "sent", createdAt: "2026-06-18T01:09:58.000Z" },
|
||||
{ role: "agent", text: "", traceId, status: "running", createdAt: "2026-06-18T01:09:59.000Z" }
|
||||
],
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
}
|
||||
};
|
||||
traceStore.append(traceId, { seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: "2026-06-18T01:10:00.000Z" });
|
||||
results.set(traceId, { status: "running", traceId, ownerUserId: ACTOR.id, sessionId: session.id, threadId: session.threadId });
|
||||
const accessController = {
|
||||
store: {
|
||||
async getAgentSession(sessionId) { return sessionId === session.id ? session : null; },
|
||||
async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; }
|
||||
},
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const runtimeStore = {
|
||||
async queryWorkbenchFacts() {
|
||||
const error = new Error("runtime query failed");
|
||||
error.code = "TEST_RUNTIME_QUERY_FAILED";
|
||||
throw error;
|
||||
}
|
||||
};
|
||||
const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`);
|
||||
assert.equal(turn.status, 503);
|
||||
assert.equal(turn.body.error.code, "projection_store_unavailable");
|
||||
|
||||
const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`);
|
||||
assert.equal(trace.status, 503);
|
||||
assert.equal(trace.body.error.code, "projection_store_unavailable");
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench trace events remain visible after session lastTraceId moves", async () => {
|
||||
const oldTraceId = "trc_workbench_history_old_trace";
|
||||
const newTraceId = "trc_workbench_history_new_trace";
|
||||
const session = {
|
||||
id: "ses_workbench_history_trace",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "running",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_history_trace",
|
||||
threadId: "thread-workbench-history-trace",
|
||||
lastTraceId: newTraceId,
|
||||
updatedAt: "2026-06-20T10:10:00.000Z",
|
||||
session: {
|
||||
messages: [
|
||||
{ role: "user", text: "old turn", traceId: oldTraceId, status: "sent", createdAt: "2026-06-20T10:00:00.000Z" },
|
||||
{ role: "agent", text: "old final", traceId: oldTraceId, status: "completed", createdAt: "2026-06-20T10:00:01.000Z" }
|
||||
],
|
||||
valuesRedacted: true
|
||||
}
|
||||
};
|
||||
const oldFacts = buildDurableFactsForSession({
|
||||
session: { ...session, status: "completed", lastTraceId: oldTraceId, updatedAt: "2026-06-20T10:01:00.000Z" },
|
||||
status: "completed",
|
||||
finalText: "old final",
|
||||
runId: "run_workbench_history_old_trace",
|
||||
commandId: "cmd_workbench_history_old_trace",
|
||||
events: [
|
||||
{ projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:00:00.000Z" },
|
||||
{ projectedSeq: 2, sourceSeq: 2, type: "result", status: "completed", label: "runner:completed", terminal: true, createdAt: "2026-06-20T10:01:00.000Z" }
|
||||
],
|
||||
lastProjectedSeq: 2
|
||||
});
|
||||
const newFacts = buildDurableFactsForSession({
|
||||
session: { ...session, session: { messages: [{ role: "user", text: "new turn", traceId: newTraceId, status: "sent" }] } },
|
||||
status: "running",
|
||||
runId: "run_workbench_history_new_trace",
|
||||
commandId: "cmd_workbench_history_new_trace",
|
||||
events: [{ projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:10:00.000Z" }],
|
||||
lastProjectedSeq: 1
|
||||
});
|
||||
const facts = emptyFacts();
|
||||
mergeFacts(facts, oldFacts);
|
||||
mergeFacts(facts, newFacts);
|
||||
facts.sessions = newFacts.sessions;
|
||||
const queries = [];
|
||||
const runtimeStore = {
|
||||
async queryWorkbenchFacts(params = {}) {
|
||||
queries.push(params);
|
||||
const filtered = filterFacts(facts, params);
|
||||
return {
|
||||
facts: filtered,
|
||||
count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0),
|
||||
persistence: { adapter: "test-durable-workbench-facts", durable: true }
|
||||
};
|
||||
}
|
||||
};
|
||||
const accessController = {
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const server = createCloudApiServer({ accessController, runtimeStore });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const response = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(oldTraceId)}/events?limit=10`);
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(response.body.status, "succeeded");
|
||||
assert.equal(response.body.sessionId, session.id);
|
||||
assert.equal(response.body.traceId, oldTraceId);
|
||||
assert.equal(response.body.events.length, 2);
|
||||
assert.equal(response.body.events.at(-1).status, "completed");
|
||||
assert.equal(response.body.error, undefined);
|
||||
assert.equal(queries.some((query) => query.sessionId === session.id && Array.isArray(query.families) && query.families.includes("sessions")), true);
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
|
||||
test("workbench trace event page keeps per-trace terminal status after later session cancel", async () => {
|
||||
const completedTraceId = "trc_workbench_completed_before_cancel";
|
||||
const canceledTraceId = "trc_workbench_later_cancel";
|
||||
const session = {
|
||||
id: "ses_workbench_completed_before_cancel",
|
||||
projectId: "prj_hwpod_workbench",
|
||||
agentId: "hwlab-code-agent",
|
||||
status: "canceled",
|
||||
ownerUserId: ACTOR.id,
|
||||
conversationId: "cnv_workbench_completed_before_cancel",
|
||||
threadId: "thread-workbench-completed-before-cancel",
|
||||
lastTraceId: canceledTraceId,
|
||||
updatedAt: "2026-06-20T10:20:30.000Z",
|
||||
session: {
|
||||
messages: [
|
||||
{ role: "user", text: "completed first", traceId: completedTraceId, status: "sent", createdAt: "2026-06-20T10:20:00.000Z" },
|
||||
{ role: "agent", text: "completed final", traceId: completedTraceId, status: "completed", createdAt: "2026-06-20T10:20:10.000Z" },
|
||||
{ role: "user", text: "cancel later", traceId: canceledTraceId, status: "sent", createdAt: "2026-06-20T10:20:20.000Z" },
|
||||
{ role: "agent", text: "hwlab-user-cancel", traceId: canceledTraceId, status: "canceled", createdAt: "2026-06-20T10:20:30.000Z" }
|
||||
],
|
||||
valuesRedacted: true
|
||||
}
|
||||
};
|
||||
const facts = emptyFacts();
|
||||
mergeFacts(facts, buildDurableFactsForSession({
|
||||
session: { ...session, status: "completed", lastTraceId: completedTraceId, updatedAt: "2026-06-20T10:20:10.000Z" },
|
||||
status: "completed",
|
||||
finalText: "completed final",
|
||||
events: [
|
||||
{ projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:20:00.000Z" },
|
||||
{ projectedSeq: 2, sourceSeq: 2, type: "result", status: "completed", label: "runner:completed", terminal: true, createdAt: "2026-06-20T10:20:10.000Z" }
|
||||
],
|
||||
lastProjectedSeq: 2
|
||||
}));
|
||||
mergeFacts(facts, buildDurableFactsForSession({
|
||||
session: { ...session, status: "canceled", lastTraceId: canceledTraceId, session: { messages: session.session.messages.slice(2), valuesRedacted: true } },
|
||||
status: "canceled",
|
||||
finalText: "hwlab-user-cancel",
|
||||
events: [
|
||||
{ projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:20:20.000Z" },
|
||||
{ projectedSeq: 2, sourceSeq: 2, type: "cancel", status: "canceled", label: "runner:canceled", terminal: true, createdAt: "2026-06-20T10:20:30.000Z" }
|
||||
],
|
||||
lastProjectedSeq: 2
|
||||
}));
|
||||
facts.sessions = facts.sessions.filter((item) => item.lastTraceId === canceledTraceId);
|
||||
const runtimeStore = {
|
||||
async queryWorkbenchFacts(params = {}) {
|
||||
const filtered = filterFacts(facts, params);
|
||||
return {
|
||||
facts: filtered,
|
||||
count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0),
|
||||
persistence: { adapter: "test-durable-workbench-facts", durable: true }
|
||||
};
|
||||
}
|
||||
};
|
||||
const accessController = {
|
||||
async ensureBootstrap() {},
|
||||
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
|
||||
};
|
||||
const server = createCloudApiServer({ accessController, workbenchRuntime: runtimeStore });
|
||||
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
|
||||
|
||||
try {
|
||||
const { port } = server.address();
|
||||
const response = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(completedTraceId)}/events?limit=10`);
|
||||
assert.equal(response.status, 200);
|
||||
assert.equal(response.body.traceStatus, "completed");
|
||||
assert.equal(response.body.events.at(-1).status, "completed");
|
||||
} finally {
|
||||
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,987 @@
|
||||
/*
|
||||
* SPEC: PJ2026-0104010803 Workbench唯一投影 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-read-model-timeline-contract; draft-2026-06-28-p0-d518-session-timeline-consistency; PJ2026-010403 API契约 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-api-contract; PJ2026-010401 Web工作台 draft-2026-06-20-p2-terminal-outbox-recovery; draft-2026-06-27-p0-workbench-read-model-contract; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0
|
||||
* 职责: Workbench SSE and Kafka realtime transport over durable read-model facts.
|
||||
*/
|
||||
import { createHash } from "node:crypto";
|
||||
|
||||
import { defaultCodeAgentTraceStore } from "./code-agent-trace-store.ts";
|
||||
import {
|
||||
parsePositiveInteger,
|
||||
safeConversationId,
|
||||
safeOpaqueId,
|
||||
safeSessionId,
|
||||
safeTraceId,
|
||||
sendJson
|
||||
} from "./server-http-utils.ts";
|
||||
import { createWorkbenchReadModel } from "./workbench-read-model.ts";
|
||||
import { createWorkbenchRuntimeClient } from "./workbench-runtime-client.ts";
|
||||
import { buildWorkbenchSessionDetail, compactLaunchContext, includeMessagesForSessionDetail } from "./workbench-session-detail-response.ts";
|
||||
import { handleWorkbenchSyncHttp } from "./workbench-realtime-authority.ts";
|
||||
import { openKafkaEventStream } from "./kafka-event-bridge.ts";
|
||||
import { durableTraceStatus, RUNNING_STATUSES, terminalFinalResponse, TERMINAL_STATUSES } from "./workbench-turn-projection.ts";
|
||||
import { emitCodeAgentOtelSpan, emitHttpServerRequestSpan } from "./otel-trace.ts";
|
||||
import * as workbenchFacts from "./server-workbench-facts.ts";
|
||||
|
||||
const DEFAULT_PAGE_LIMIT = 50;
|
||||
const DEFAULT_SESSION_LIST_LIMIT = 20;
|
||||
const MAX_PAGE_LIMIT = 100;
|
||||
const DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS = 15000;
|
||||
const WORKBENCH_REALTIME_DRAIN_TIMEOUT_MS = 2500;
|
||||
const WORKBENCH_REALTIME_AUTHORITY_VERSION = "workbench-realtime-authority-v2";
|
||||
const WORKBENCH_SESSION_LIST_PAGE_FAMILIES = Object.freeze(["sessions"]);
|
||||
const WORKBENCH_SESSION_DETAIL_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]);
|
||||
const WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]);
|
||||
const WORKBENCH_TRACE_EVENT_METADATA_FAMILIES = Object.freeze(["sessions", "messages", "parts", "turns", "checkpoints"]);
|
||||
const WORKBENCH_TRACE_EVENT_PAGE_FAMILIES = Object.freeze(["traceEvents"]);
|
||||
const activeWorkbenchRealtimeConnections = new Set();
|
||||
|
||||
const {
|
||||
factSessionSummary,
|
||||
factSessionDetail,
|
||||
factMessagesForSession,
|
||||
canonicalFactMessageGroupsForSession,
|
||||
factMessageCanonicalGroupKey,
|
||||
selectCanonicalFactMessage,
|
||||
factMessageCanonicalScore,
|
||||
factMessageHasFinalResponsePart,
|
||||
workbenchLifecycleMessageId,
|
||||
factMessageDto,
|
||||
factMessageAuthorityText,
|
||||
factPartsHaveFinalResponse,
|
||||
factSyntheticTerminalFinalResponse,
|
||||
isTerminalProjectionStatus,
|
||||
firstFactPartText,
|
||||
factMessageFinalResponseText,
|
||||
factTerminalMessageForTrace,
|
||||
factTerminalStatusFromMessageProjection,
|
||||
factPartDto,
|
||||
factTurnForTrace,
|
||||
factTurnSnapshot,
|
||||
factTraceSnapshot,
|
||||
factTraceTimingProjection,
|
||||
factTraceEventIsTerminalAuthority,
|
||||
factTraceEventTerminalTimes,
|
||||
factTraceEventDto,
|
||||
factProjectionForTrace,
|
||||
factTimingProjection,
|
||||
factCombinedTimingProjection,
|
||||
factTimingSource,
|
||||
firstTimestampIso,
|
||||
latestTimestampIso,
|
||||
elapsedFactMs,
|
||||
timestampIso,
|
||||
factCheckpointForTrace,
|
||||
workbenchProjectionStoreError,
|
||||
nonNegativeIntegerOrNull,
|
||||
factSessionId,
|
||||
factLastTraceId,
|
||||
factLatestTraceIdForSession,
|
||||
factCurrentTraceForSession,
|
||||
factMessageTraceCandidate,
|
||||
factTraceCandidate,
|
||||
compareTraceCandidatesDesc,
|
||||
nonUnknownStatus,
|
||||
factTurnId,
|
||||
factUpdatedAt,
|
||||
factSeq,
|
||||
factProjectedSeq,
|
||||
normalizeProjectionStatus,
|
||||
normalizeProjectionHealth,
|
||||
compareFactSessionsDesc,
|
||||
compareFactMessagesAsc,
|
||||
compareFactMessagesForSessionAsc,
|
||||
factMessageTimelineKey,
|
||||
factTimelineAnchorMessageForTrace,
|
||||
factMessageTimelineRoleRank,
|
||||
firstFiniteNumber,
|
||||
compareOptionalNumberAsc,
|
||||
compareFactPartsAsc,
|
||||
compareFactTraceEventsAsc,
|
||||
compareFactRecordsDesc,
|
||||
compareTimestampAsc,
|
||||
compareTimestampDesc,
|
||||
compareOptionalTimestampAsc,
|
||||
compareNumberAsc,
|
||||
compareNumberDesc,
|
||||
compareText,
|
||||
timestampMs,
|
||||
optionalTimestampMs,
|
||||
numericValue,
|
||||
mergeFactSets,
|
||||
emptyFactSet,
|
||||
factArray,
|
||||
uniqueText,
|
||||
eventSeq,
|
||||
normalizeStatus,
|
||||
normalizeTerminalStatus,
|
||||
firstUserPreview,
|
||||
latestUserPreview,
|
||||
latestValidMessagePreview,
|
||||
latestAssistantPreview,
|
||||
sessionTitleFromMessages,
|
||||
sessionPreviewFromMessages,
|
||||
boundedPreviewText,
|
||||
isAssistantLikeRole,
|
||||
objectValue,
|
||||
textValue,
|
||||
messageAuthorityTextValue,
|
||||
safeTurnId,
|
||||
safeMessageId,
|
||||
safePartId,
|
||||
hash
|
||||
} = workbenchFacts;
|
||||
|
||||
import {
|
||||
authenticateWorkbenchRead,
|
||||
blockedTraceEventProjection,
|
||||
classifyWorkbenchReadModelFailure,
|
||||
isAgentRunAliasSessionId,
|
||||
logWorkbenchReadModelFailure,
|
||||
methodNotAllowed,
|
||||
nonNegativeInteger,
|
||||
queryFactsByRouteId,
|
||||
traceEventsReadModelBlocker,
|
||||
visibleFactSessionForTrace,
|
||||
visibleFactSessions,
|
||||
workbenchError
|
||||
} from "./server-workbench-read-http.ts";
|
||||
|
||||
export async function drainWorkbenchRealtimeConnections(options = {}) {
|
||||
const reason = textValue(options.reason) || "server_shutdown";
|
||||
const signal = textValue(options.signal) || null;
|
||||
const timeoutMs = parsePositiveInteger(options.timeoutMs, WORKBENCH_REALTIME_DRAIN_TIMEOUT_MS);
|
||||
const connections = [...activeWorkbenchRealtimeConnections].filter((connection) => connection?.isActive?.());
|
||||
for (const connection of connections) {
|
||||
try {
|
||||
connection.close({ reason, signal });
|
||||
} catch {
|
||||
// Best effort: one broken client must not block shutdown drain for the rest.
|
||||
}
|
||||
}
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
while (Date.now() < deadline) {
|
||||
const activeAfterClose = connections.filter((connection) => activeWorkbenchRealtimeConnections.has(connection) && connection?.isActive?.()).length;
|
||||
if (activeAfterClose === 0) break;
|
||||
await sleepMs(Math.min(50, Math.max(1, deadline - Date.now())));
|
||||
}
|
||||
const activeAfter = connections.filter((connection) => activeWorkbenchRealtimeConnections.has(connection) && connection?.isActive?.()).length;
|
||||
return {
|
||||
ok: activeAfter === 0,
|
||||
activeBefore: connections.length,
|
||||
closed: Math.max(0, connections.length - activeAfter),
|
||||
activeAfter,
|
||||
timeout: activeAfter > 0,
|
||||
reason,
|
||||
signal
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
export async function handleWorkbenchRealtimeHttp(request, response, url, options = {}) {
|
||||
try {
|
||||
if (request.method !== "GET") return methodNotAllowed(response, "GET");
|
||||
const requestedSessionId = safeSessionId(url.searchParams.get("sessionId") ?? url.searchParams.get("includeSessionId"));
|
||||
const requestedTraceId = safeTraceId(url.searchParams.get("traceId"));
|
||||
const heartbeatMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_HEARTBEAT_MS, DEFAULT_WORKBENCH_SSE_HEARTBEAT_MS);
|
||||
attachWorkbenchRealtimeOtelContext(request, {
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId,
|
||||
heartbeatMs,
|
||||
realtimeSource: "kafka"
|
||||
});
|
||||
const perf = options.backendPerformance;
|
||||
const auth = perf ? await perf.measure("workbench_auth", () => authenticateWorkbenchRead(request, response, options)) : await authenticateWorkbenchRead(request, response, options);
|
||||
if (!auth) return;
|
||||
|
||||
if (url.searchParams.has("projectId") || url.searchParams.has("workspaceId")) {
|
||||
sendJson(response, 400, workbenchError("workbench_authority_removed", "Workbench read model is keyed by sessionId/threadId/traceId only."));
|
||||
return;
|
||||
}
|
||||
|
||||
const traceStore = options.traceStore ?? defaultCodeAgentTraceStore;
|
||||
const readModel = createWorkbenchReadModel(options, auth.actor);
|
||||
const requestedAfterSeq = workbenchRealtimeAfterSeq(request, url);
|
||||
let closed = false;
|
||||
let realtimeCloseReason = "client_close";
|
||||
let realtimeCloseSignal = null;
|
||||
let realtimeSessionId = requestedSessionId ?? null;
|
||||
let realtimeTraceId = requestedTraceId ?? null;
|
||||
let realtimeThreadId = null;
|
||||
const realtimeStartedAtMs = Date.now();
|
||||
const cleanup = [];
|
||||
|
||||
response.writeHead(200, {
|
||||
"content-type": "text/event-stream; charset=utf-8",
|
||||
"cache-control": "no-store, no-transform",
|
||||
connection: "keep-alive",
|
||||
"x-accel-buffering": "no",
|
||||
"x-content-type-options": "nosniff"
|
||||
});
|
||||
if (typeof response.flushHeaders === "function") response.flushHeaders();
|
||||
|
||||
const writeEvent = (name, payload = {}) => {
|
||||
if (closed || response.destroyed) return;
|
||||
const startedAt = nowMs();
|
||||
const eventCreatedAt = realtimeEventCreatedAt(payload);
|
||||
const traceSeq = realtimeTraceSeq(payload);
|
||||
const eventId = realtimeEventId(payload);
|
||||
response.write(`event: ${name}\n`);
|
||||
if (eventId) response.write(`id: ${eventId}\n`);
|
||||
response.write(`data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), eventCreatedAt, traceSeq, ...payload })}\n\n`);
|
||||
perf?.recordPhase({ phase: "sse_write", durationMs: nowMs() - startedAt, outcome: "ok" });
|
||||
};
|
||||
|
||||
const realtimeConnection = {
|
||||
close(fields = {}) {
|
||||
if (closed || response.destroyed || response.writableEnded) return false;
|
||||
const reason = textValue(fields.reason) || "server_shutdown";
|
||||
realtimeCloseReason = reason;
|
||||
realtimeCloseSignal = textValue(fields.signal) || null;
|
||||
try {
|
||||
writeEvent("workbench.server_draining", {
|
||||
type: "server.draining",
|
||||
status: "closing",
|
||||
reason,
|
||||
signal: realtimeCloseSignal,
|
||||
sessionId: realtimeSessionId,
|
||||
threadId: realtimeThreadId,
|
||||
traceId: realtimeTraceId,
|
||||
observedAt: new Date().toISOString()
|
||||
});
|
||||
response.end();
|
||||
return true;
|
||||
} catch {
|
||||
try { response.destroy(); } catch {}
|
||||
return false;
|
||||
}
|
||||
},
|
||||
isActive() {
|
||||
return !closed && !response.destroyed && !response.writableEnded;
|
||||
}
|
||||
};
|
||||
activeWorkbenchRealtimeConnections.add(realtimeConnection);
|
||||
cleanup.push(() => activeWorkbenchRealtimeConnections.delete(realtimeConnection));
|
||||
|
||||
writeEvent("workbench.connected", {
|
||||
type: "connected",
|
||||
status: "connected",
|
||||
realtimeSource: "kafka",
|
||||
snapshotSource: "read-model",
|
||||
heartbeatMs,
|
||||
cursor: { outboxSeq: requestedAfterSeq },
|
||||
filters: {
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId
|
||||
}
|
||||
});
|
||||
|
||||
const resolveInitialSession = requestedSessionId && requestedAfterSeq <= 0;
|
||||
const initialContext = resolveInitialSession
|
||||
? await realtimeInitialSessionContext(readModel, auth.actor, requestedSessionId)
|
||||
: { facts: emptyFactSet(), session: null, blocker: null };
|
||||
if (initialContext.blocker) {
|
||||
writeEvent("workbench.error", {
|
||||
type: "error",
|
||||
reason: "initial-session",
|
||||
sessionId: requestedSessionId,
|
||||
traceId: requestedTraceId,
|
||||
error: initialContext.blocker
|
||||
});
|
||||
}
|
||||
const initialFactSession = initialContext.session;
|
||||
const streamSessionId = factSessionId(initialFactSession) ?? requestedSessionId ?? null;
|
||||
const streamThreadId = safeOpaqueId(initialFactSession?.threadId) ?? (textValue(initialFactSession?.threadId) || null);
|
||||
const activeTraceId = requestedTraceId
|
||||
?? factLastTraceId(initialFactSession)
|
||||
?? factLatestTraceIdForSession(initialContext.facts, streamSessionId)
|
||||
?? null;
|
||||
realtimeSessionId = streamSessionId ?? requestedSessionId ?? null;
|
||||
realtimeTraceId = activeTraceId ?? requestedTraceId ?? null;
|
||||
realtimeThreadId = streamThreadId ?? null;
|
||||
attachWorkbenchRealtimeOtelContext(request, {
|
||||
sessionId: streamSessionId ?? requestedSessionId,
|
||||
traceId: activeTraceId,
|
||||
threadId: streamThreadId,
|
||||
heartbeatMs,
|
||||
realtimeSource: "kafka"
|
||||
});
|
||||
emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env);
|
||||
if (activeTraceId && requestedAfterSeq <= 0) {
|
||||
await (perf ? perf.measure("workbench_initial_trace", () => writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })) : writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" }));
|
||||
}
|
||||
const kafkaFilters = resolveWorkbenchRealtimeKafkaFilters({ traceId: requestedTraceId, sessionId: streamSessionId, traceStore });
|
||||
try {
|
||||
const kafkaStream = await openKafkaEventStream({
|
||||
env: options.env ?? process.env,
|
||||
stream: "hwlab",
|
||||
fromBeginning: false,
|
||||
...kafkaFilters,
|
||||
kafkaFactory: options.kafkaFactory,
|
||||
onEvent: async (record) => {
|
||||
if (closed || response.destroyed) return;
|
||||
const realtime = workbenchRealtimeEventFromKafka(record, { sessionId: streamSessionId, threadId: streamThreadId, traceId: activeTraceId });
|
||||
if (!realtime) return;
|
||||
writeEvent("workbench.trace.event", realtime.traceEvent);
|
||||
if (realtime.turnSnapshot) writeEvent("workbench.turn.snapshot", realtime.turnSnapshot);
|
||||
},
|
||||
onError: (error) => {
|
||||
writeEvent("workbench.error", {
|
||||
type: "error",
|
||||
realtimeSource: "kafka",
|
||||
sessionId: streamSessionId,
|
||||
threadId: streamThreadId,
|
||||
traceId: activeTraceId,
|
||||
error: { code: "workbench_kafka_stream_failed", message: error?.message ?? "Workbench Kafka realtime stream failed." }
|
||||
});
|
||||
}
|
||||
});
|
||||
cleanup.push(() => { void kafkaStream.stop?.(); });
|
||||
} catch (kafkaError) {
|
||||
writeEvent("workbench.error", {
|
||||
type: "error",
|
||||
realtimeSource: "kafka",
|
||||
sessionId: streamSessionId,
|
||||
threadId: streamThreadId,
|
||||
traceId: activeTraceId,
|
||||
error: { code: "workbench_kafka_stream_unavailable", message: kafkaError?.message ?? "Workbench Kafka realtime stream is unavailable." }
|
||||
});
|
||||
}
|
||||
|
||||
const heartbeat = setInterval(async () => {
|
||||
try {
|
||||
writeEvent("workbench.heartbeat", {
|
||||
type: "heartbeat",
|
||||
sessionId: requestedSessionId ?? null,
|
||||
traceId: activeTraceId ?? null,
|
||||
observedAt: new Date().toISOString()
|
||||
});
|
||||
} catch (error) {
|
||||
writeEvent("workbench.error", {
|
||||
type: "error",
|
||||
error: { code: "workbench_realtime_heartbeat_failed", message: error?.message ?? "Workbench realtime heartbeat failed." }
|
||||
});
|
||||
}
|
||||
}, Math.max(1000, heartbeatMs));
|
||||
cleanup.push(() => clearInterval(heartbeat));
|
||||
|
||||
response.on("close", () => {
|
||||
if (closed) return;
|
||||
closed = true;
|
||||
emitWorkbenchRealtimeClosedOtelSpan(request, options.env, {
|
||||
reason: realtimeCloseReason,
|
||||
signal: realtimeCloseSignal,
|
||||
startedAtMs: realtimeStartedAtMs,
|
||||
sessionId: realtimeSessionId,
|
||||
threadId: realtimeThreadId,
|
||||
traceId: realtimeTraceId,
|
||||
activeConnectionCount: activeWorkbenchRealtimeConnections.size
|
||||
});
|
||||
for (const item of cleanup.splice(0)) item();
|
||||
});
|
||||
} catch (error) {
|
||||
handleWorkbenchRealtimeFailure(request, response, url, options, error);
|
||||
}
|
||||
}
|
||||
|
||||
export function attachWorkbenchRealtimeOtelContext(request, fields = {}) {
|
||||
const context = request?.hwlabHttpRequestContext;
|
||||
if (!context) return;
|
||||
context.route = "/v1/workbench/events";
|
||||
context.otelAttributes = {
|
||||
...(context.otelAttributes ?? {}),
|
||||
"workbench.route": "events",
|
||||
"workbench.session_id": fields.sessionId ?? null,
|
||||
traceId: fields.traceId ?? null,
|
||||
"workbench.trace_id": fields.traceId ?? null,
|
||||
"workbench.thread_id": fields.threadId ?? null,
|
||||
"workbench.sse.heartbeat_ms": fields.heartbeatMs ?? null,
|
||||
"workbench.sse.realtime_source": fields.realtimeSource ?? null,
|
||||
"workbench.sse.outbox_mode": false
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveWorkbenchRealtimeKafkaFilters({ traceId = null, sessionId = null, traceStore = null } = {}) {
|
||||
const resolved = traceId && typeof traceStore?.snapshot === "function" ? collectTraceLinkedIds(traceStore.snapshot(traceId)) : {};
|
||||
const hasAgentRunKey = Boolean(resolved.runId || resolved.commandId);
|
||||
const fallbackSessionId = resolved.sessionId || agentRunScopedSessionIdFromWorkbenchSession(sessionId) || sessionId;
|
||||
return compactObject({
|
||||
traceId: hasAgentRunKey ? null : traceId,
|
||||
sessionId: hasAgentRunKey ? null : fallbackSessionId,
|
||||
runId: resolved.runId,
|
||||
commandId: resolved.commandId
|
||||
});
|
||||
}
|
||||
|
||||
export function agentRunScopedSessionIdFromWorkbenchSession(sessionId) {
|
||||
const safeId = safeSessionId(sessionId);
|
||||
if (!safeId || isAgentRunAliasSessionId(safeId)) return safeId;
|
||||
const base = safeId.replace(/^ses_/u, "").replace(/[^A-Za-z0-9_]+/gu, "_").replace(/^_+|_+$/gu, "") || "session";
|
||||
return safeSessionId(`ses_agentrun_${base}`);
|
||||
}
|
||||
|
||||
export function collectTraceLinkedIds(snapshot) {
|
||||
const out = { sessionId: null, runId: null, commandId: null };
|
||||
const events = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
||||
for (const event of events) collectTraceLinkedIdsFromRecord(event, out);
|
||||
collectTraceLinkedIdsFromRecord(snapshot?.lastEvent, out);
|
||||
return out;
|
||||
}
|
||||
|
||||
export function collectTraceLinkedIdsFromRecord(record, out) {
|
||||
const value = record && typeof record === "object" && !Array.isArray(record) ? record : {};
|
||||
const agentRun = value.agentRun && typeof value.agentRun === "object" && !Array.isArray(value.agentRun) ? value.agentRun : {};
|
||||
const payload = value.payload && typeof value.payload === "object" && !Array.isArray(value.payload) ? value.payload : {};
|
||||
out.sessionId ||= safeSessionId(value.sessionId ?? value.sourceSessionId ?? agentRun.sessionId ?? payload.sessionId);
|
||||
out.runId ||= textValue(value.runId ?? value.sourceRunId ?? agentRun.runId ?? payload.runId);
|
||||
out.commandId ||= textValue(value.commandId ?? value.sourceCommandId ?? agentRun.commandId ?? payload.commandId);
|
||||
}
|
||||
|
||||
export function workbenchRealtimeEventFromKafka(record, context = {}) {
|
||||
const value = record?.value && typeof record.value === "object" && !Array.isArray(record.value) ? record.value : null;
|
||||
if (!value) return null;
|
||||
const hwlabEvent = value.event && typeof value.event === "object" && !Array.isArray(value.event) ? value.event : {};
|
||||
const sourceContext = value.context && typeof value.context === "object" && !Array.isArray(value.context) ? value.context : {};
|
||||
const traceId = safeTraceId(value.traceId ?? hwlabEvent.traceId ?? context.traceId) ?? textValue(value.traceId ?? hwlabEvent.traceId ?? context.traceId);
|
||||
const sessionId = safeSessionId(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId) ?? textValue(context.sessionId ?? value.sessionId ?? hwlabEvent.sessionId);
|
||||
const threadId = safeOpaqueId(context.threadId ?? sourceContext.threadId) ?? textValue(context.threadId ?? sourceContext.threadId);
|
||||
if (!traceId && !sessionId) return null;
|
||||
const sourceSeq = integerValue(hwlabEvent.sourceSeq ?? sourceContext.sourceSeq);
|
||||
const kafka = {
|
||||
topic: textValue(record.topic),
|
||||
partition: integerValue(record.partition),
|
||||
offset: textValue(record.offset),
|
||||
key: textValue(record.key),
|
||||
timestamp: textValue(record.timestamp),
|
||||
valuesRedacted: true
|
||||
};
|
||||
const event = {
|
||||
...hwlabEvent,
|
||||
traceId: traceId ?? hwlabEvent.traceId ?? null,
|
||||
sessionId: sessionId ?? hwlabEvent.sessionId ?? null,
|
||||
threadId: threadId ?? sourceContext.threadId ?? null,
|
||||
runId: textValue(hwlabEvent.runId ?? sourceContext.runId),
|
||||
commandId: textValue(hwlabEvent.commandId ?? sourceContext.commandId),
|
||||
source: textValue(hwlabEvent.source) || "hwlab.kafka",
|
||||
sourceSeq,
|
||||
kafka,
|
||||
valuesPrinted: false
|
||||
};
|
||||
const status = normalizeStatus(event.status);
|
||||
const terminal = event.terminal === true || TERMINAL_STATUSES.has(status);
|
||||
const cursor = {
|
||||
traceSeq: sourceSeq ?? integerValue(record.offset),
|
||||
kafkaOffset: textValue(record.offset),
|
||||
kafkaPartition: integerValue(record.partition)
|
||||
};
|
||||
const entityVersion = sourceSeq ?? integerValue(record.offset) ?? 0;
|
||||
const projectionRevision = ["kafka", kafka.topic, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":");
|
||||
const traceEntity = {
|
||||
family: "traceEvents",
|
||||
id: [traceId ?? "trace", event.runId, event.commandId, kafka.partition, kafka.offset].filter((part) => textValue(part)).join(":"),
|
||||
version: entityVersion,
|
||||
traceSeq: cursor.traceSeq,
|
||||
projectionRevision,
|
||||
authority: WORKBENCH_REALTIME_AUTHORITY_VERSION
|
||||
};
|
||||
const traceEvent = {
|
||||
id: kafka.topic && kafka.partition !== null && kafka.offset ? `${kafka.topic}:${kafka.partition}:${kafka.offset}` : null,
|
||||
type: "trace.event",
|
||||
contractVersion: "workbench-sync-v1",
|
||||
realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION,
|
||||
realtimeSource: "kafka",
|
||||
sessionId,
|
||||
threadId,
|
||||
traceId,
|
||||
event,
|
||||
snapshot: { traceId, sessionId, threadId, status: status || event.status || null, events: [event], eventCount: 1 },
|
||||
cursor,
|
||||
entity: traceEntity,
|
||||
projectionRevision,
|
||||
kafka,
|
||||
valuesPrinted: false
|
||||
};
|
||||
const turnSnapshot = terminal ? {
|
||||
type: "turn.snapshot",
|
||||
contractVersion: "workbench-sync-v1",
|
||||
realtimeAuthority: WORKBENCH_REALTIME_AUTHORITY_VERSION,
|
||||
realtimeSource: "kafka",
|
||||
sessionId,
|
||||
threadId,
|
||||
traceId,
|
||||
reason: "kafka-terminal",
|
||||
cursor,
|
||||
turn: {
|
||||
traceId,
|
||||
sessionId,
|
||||
threadId,
|
||||
status: status || event.status || "completed",
|
||||
running: false,
|
||||
terminal: true,
|
||||
finalResponse: terminalFinalResponse(status || event.status || "completed", {
|
||||
traceId,
|
||||
status: status || event.status || "completed",
|
||||
finalResponse: event.text ? { text: event.text, status: status || event.status || "completed", traceId, source: "kafka-terminal-event", valuesPrinted: false } : null,
|
||||
finalText: event.text ?? event.message ?? null,
|
||||
error: event.errorCode ? { message: event.message ?? event.errorCode } : null
|
||||
}),
|
||||
agentRun: compactObject({ runId: event.runId, commandId: event.commandId }),
|
||||
valuesPrinted: false
|
||||
},
|
||||
entity: {
|
||||
family: "turns",
|
||||
id: traceId ?? event.runId ?? event.commandId ?? traceEntity.id,
|
||||
version: entityVersion,
|
||||
traceSeq: cursor.traceSeq,
|
||||
projectionRevision,
|
||||
authority: WORKBENCH_REALTIME_AUTHORITY_VERSION
|
||||
},
|
||||
projectionRevision,
|
||||
kafka,
|
||||
valuesPrinted: false
|
||||
} : null;
|
||||
return { traceEvent, turnSnapshot };
|
||||
}
|
||||
|
||||
export function compactObject(value) {
|
||||
return Object.fromEntries(Object.entries(value || {}).filter(([, entry]) => textValue(entry)));
|
||||
}
|
||||
|
||||
export function integerValue(value) {
|
||||
const number = Number(value);
|
||||
return Number.isFinite(number) ? Math.trunc(number) : null;
|
||||
}
|
||||
|
||||
export function attachWorkbenchReadModelDiagnosticOtel(request, fields = {}) {
|
||||
const context = request?.hwlabHttpRequestContext;
|
||||
if (!context) return;
|
||||
const code = textValue(fields.code) || "workbench_read_model_not_found";
|
||||
const route = textValue(fields.route) || context.route || "/v1/workbench";
|
||||
context.route = route;
|
||||
context.lastHttpError = {
|
||||
code,
|
||||
category: "workbench_read_model",
|
||||
layer: "workbench.read_model"
|
||||
};
|
||||
context.otelAttributes = {
|
||||
...(context.otelAttributes ?? {}),
|
||||
"workbench.route": route,
|
||||
"workbench.read_model.route": route,
|
||||
"workbench.read_model.result": code,
|
||||
"workbench.read_model.count": Number.isFinite(Number(fields.count)) ? Math.trunc(Number(fields.count)) : null,
|
||||
"workbench.session_id": fields.sessionId ?? null,
|
||||
"workbench.trace_id": fields.traceId ?? null,
|
||||
"workbench.turn_id": fields.turnId ?? null,
|
||||
traceId: fields.traceId ?? null,
|
||||
"error.code": code,
|
||||
"error.category": "workbench_read_model",
|
||||
"error.layer": "workbench.read_model"
|
||||
};
|
||||
}
|
||||
|
||||
export function emitWorkbenchRealtimeAcceptedOtelSpan(request, env = process.env) {
|
||||
const context = request?.hwlabHttpRequestContext;
|
||||
if (!context?.traceId) return;
|
||||
const endedAtMs = Date.now();
|
||||
void emitHttpServerRequestSpan(context, env, {
|
||||
startTimeMs: context.startedAtMs,
|
||||
endTimeMs: endedAtMs,
|
||||
statusCode: 200,
|
||||
method: "GET",
|
||||
route: "/v1/workbench/events",
|
||||
attributes: {
|
||||
...(context.otelAttributes ?? {}),
|
||||
"http.closed_by": "sse-accepted",
|
||||
"workbench.sse.lifecycle": "accepted",
|
||||
"workbench.sse.accept_latency_ms": Math.max(0, endedAtMs - Number(context.startedAtMs ?? endedAtMs))
|
||||
}
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
|
||||
export function emitWorkbenchRealtimeClosedOtelSpan(request, env = process.env, fields = {}) {
|
||||
const context = request?.hwlabHttpRequestContext;
|
||||
if (!context?.traceId) return;
|
||||
const endedAtMs = Date.now();
|
||||
const startedAtMs = Number(fields.startedAtMs ?? context.startedAtMs ?? endedAtMs);
|
||||
const reason = textValue(fields.reason) || "client_close";
|
||||
void emitHttpServerRequestSpan(context, env, {
|
||||
startTimeMs: Number.isFinite(startedAtMs) ? startedAtMs : endedAtMs,
|
||||
endTimeMs: endedAtMs,
|
||||
statusCode: 200,
|
||||
method: "GET",
|
||||
route: "/v1/workbench/events",
|
||||
attributes: {
|
||||
...(context.otelAttributes ?? {}),
|
||||
"http.closed_by": reason,
|
||||
"workbench.sse.lifecycle": "closed",
|
||||
"workbench.sse.close_reason": reason,
|
||||
"workbench.sse.close_signal": fields.signal ?? null,
|
||||
"workbench.sse.duration_ms": Math.max(0, endedAtMs - (Number.isFinite(startedAtMs) ? startedAtMs : endedAtMs)),
|
||||
"workbench.sse.session_id": fields.sessionId ?? null,
|
||||
"workbench.sse.thread_id": fields.threadId ?? null,
|
||||
"workbench.sse.trace_id": fields.traceId ?? null,
|
||||
"workbench.sse.active_connection_count": Number.isFinite(Number(fields.activeConnectionCount)) ? Number(fields.activeConnectionCount) : null
|
||||
}
|
||||
}).catch(() => undefined);
|
||||
}
|
||||
|
||||
export function handleWorkbenchRealtimeFailure(request, response, url, options = {}, error) {
|
||||
const classified = classifyWorkbenchReadModelFailure(error);
|
||||
const context = request?.hwlabHttpRequestContext;
|
||||
if (context) {
|
||||
context.route = "/v1/workbench/events";
|
||||
context.otelAttributes = {
|
||||
...(context.otelAttributes ?? {}),
|
||||
"workbench.route": "events",
|
||||
"workbench.sse.lifecycle": "failed",
|
||||
"error.code": classified.code,
|
||||
"error.layer": "workbench.realtime"
|
||||
};
|
||||
}
|
||||
logWorkbenchReadModelFailure(options.logger ?? console, {
|
||||
event: "workbench_realtime_route_failed",
|
||||
ok: false,
|
||||
route: "/v1/workbench/events",
|
||||
statusCode: classified.statusCode,
|
||||
errorCode: classified.code,
|
||||
errorName: error?.name ?? "Error",
|
||||
message: error instanceof Error ? error.message : String(error ?? "unknown"),
|
||||
valuesRedacted: true
|
||||
});
|
||||
if (response.headersSent && !response.destroyed) {
|
||||
try {
|
||||
response.write(`event: workbench.error\n`);
|
||||
response.write(`data: ${JSON.stringify({ contractVersion: "workbench-events-v1", serverSentAt: new Date().toISOString(), type: "error", error: { code: classified.code, message: classified.message, valuesRedacted: true } })}\n\n`);
|
||||
response.end();
|
||||
} catch {
|
||||
try { response.destroy(); } catch {}
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (!response.destroyed) sendJson(response, classified.statusCode, workbenchError(classified.code, classified.message, { route: "/v1/workbench/events", errorName: error?.name ?? "Error" }));
|
||||
}
|
||||
|
||||
export async function writeMessageRealtimeSnapshotSafe({ writeEvent, readModel, actor, sessionId, threadId = null, traceId = null, reason = "message", cursor = null } = {}) {
|
||||
const safeSession = safeSessionId(sessionId);
|
||||
if (!safeSession || typeof readModel?.queryFacts !== "function") return;
|
||||
try {
|
||||
const result = await queryFactsByRouteId(readModel, safeSession, { families: WORKBENCH_SESSION_MESSAGE_PAGE_FAMILIES });
|
||||
if (result.error) return;
|
||||
const session = visibleFactSessions(result.facts, actor)[0] ?? null;
|
||||
if (!session) return;
|
||||
const messages = factMessagesForSession(session, result.facts);
|
||||
const message = [...messages].reverse().find((item) => {
|
||||
if (!isAssistantLikeRole(item?.role)) return false;
|
||||
return !traceId || item.traceId === traceId;
|
||||
}) ?? null;
|
||||
if (!message) return;
|
||||
writeEvent("workbench.message.snapshot", {
|
||||
type: "message.snapshot",
|
||||
sessionId: factSessionId(session),
|
||||
threadId: threadId ?? session.threadId ?? null,
|
||||
traceId: message.traceId ?? traceId ?? null,
|
||||
reason,
|
||||
cursor,
|
||||
message
|
||||
});
|
||||
} catch (error) {
|
||||
writeEvent("workbench.error", {
|
||||
type: "error",
|
||||
error: { code: "workbench_message_snapshot_failed", message: error?.message ?? "Workbench realtime message snapshot failed." }
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export function realtimeEventCreatedAt(payload) {
|
||||
const candidates = [payload?.event?.createdAt, payload?.snapshot?.lastEvent?.createdAt, payload?.turn?.updatedAt];
|
||||
for (const value of candidates) {
|
||||
const text = textValue(value);
|
||||
if (text) return text;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function realtimeTraceSeq(payload) {
|
||||
const seq = Number(payload?.cursor?.traceSeq ?? payload?.event?.seq ?? payload?.snapshot?.lastEvent?.seq);
|
||||
return Number.isFinite(seq) && seq >= 0 ? seq : null;
|
||||
}
|
||||
|
||||
export function realtimeEventId(payload) {
|
||||
const raw = payload?.cursor?.outboxSeq ?? payload?.outboxSeq ?? payload?.id;
|
||||
const text = textValue(raw);
|
||||
if (!text || /[\r\n]/u.test(text)) return null;
|
||||
return text.slice(0, 128);
|
||||
}
|
||||
|
||||
export function workbenchRealtimeAfterSeq(request, url) {
|
||||
for (const value of [url.searchParams.get("afterSeq"), url.searchParams.get("afterOutboxSeq"), request?.headers?.["last-event-id"]]) {
|
||||
const parsed = nonNegativeInteger(value, 0);
|
||||
if (parsed > 0) return parsed;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export function nowMs() {
|
||||
if (typeof performance !== "undefined" && typeof performance.now === "function") return performance.now();
|
||||
return Date.now();
|
||||
}
|
||||
|
||||
export function sleepMs(ms) {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
|
||||
export async function realtimeInitialSessionContext(readModel, actor, sessionId) {
|
||||
const safeId = safeSessionId(sessionId);
|
||||
if (!safeId || typeof readModel?.queryFacts !== "function") {
|
||||
const blocker = traceEventsReadModelBlocker("workbench_facts_query_unavailable", "Workbench realtime initial session must be read from durable facts; the facts query interface is unavailable.", { session: { sessionId: safeId }, route: "/v1/workbench/events" });
|
||||
return { facts: emptyFactSet(), session: null, blocker };
|
||||
}
|
||||
try {
|
||||
const result = await readModel.queryFacts({ sessionId: safeId, families: WORKBENCH_SESSION_DETAIL_FAMILIES, limit: MAX_PAGE_LIMIT });
|
||||
if (result.error || result.durable === false) {
|
||||
const blocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench realtime initial session facts query failed; realtime must surface the read-model error instead of falling back to legacy session snapshots.", { session: { sessionId: safeId }, route: "/v1/workbench/events" });
|
||||
blocker.causeCode = result.error?.code ?? null;
|
||||
blocker.causeName = result.error?.name ?? null;
|
||||
blocker.message = result.error?.message ? `${blocker.message} Cause: ${result.error.message}` : result.durable === false ? `${blocker.message} Cause: durable facts reader unavailable.` : blocker.message;
|
||||
return { facts: emptyFactSet(), session: null, blocker };
|
||||
}
|
||||
const facts = result.facts ?? emptyFactSet();
|
||||
const session = visibleFactSessions(facts, actor).find((item) => factSessionId(item) === safeId) ?? null;
|
||||
return { facts, session, blocker: null };
|
||||
} catch (error) {
|
||||
const blocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench realtime initial session facts query failed; realtime must surface the read-model error instead of falling back to legacy session snapshots.", { session: { sessionId: safeId }, route: "/v1/workbench/events" });
|
||||
blocker.causeCode = error?.code ?? null;
|
||||
blocker.causeName = error?.name ?? null;
|
||||
blocker.message = error?.message ? `${blocker.message} Cause: ${error.message}` : blocker.message;
|
||||
return { facts: emptyFactSet(), session: null, blocker };
|
||||
}
|
||||
}
|
||||
|
||||
export async function writeTraceRealtimeSnapshot({ writeEvent, options, actor, traceId, reason }) {
|
||||
const context = await visibleTraceContext(options, actor, traceId);
|
||||
if (!context.visible) {
|
||||
writeEvent("workbench.trace.unavailable", {
|
||||
type: "trace.unavailable",
|
||||
reason,
|
||||
traceId,
|
||||
error: { code: "workbench_trace_not_found", message: "Workbench trace is not visible to the current actor." }
|
||||
});
|
||||
return;
|
||||
}
|
||||
if (context.factsReadBlocker) {
|
||||
writeEvent("workbench.error", {
|
||||
type: "error",
|
||||
traceId,
|
||||
reason,
|
||||
error: context.factsReadBlocker
|
||||
});
|
||||
}
|
||||
const sessionId = factSessionId(context.factSession) ?? safeSessionId(context.session?.id) ?? null;
|
||||
const threadId = safeOpaqueId(context.factSession?.threadId ?? context.session?.threadId) ?? (textValue(context.factSession?.threadId ?? context.session?.threadId) || null);
|
||||
const realtimeTraceSnapshot = context.factsReadBlocker ? blockedRealtimeTraceSnapshot(context) : traceSnapshotForRealtime(context.trace);
|
||||
const traceSeq = context.factsReadBlocker ? 0 : traceSnapshotLastSeq(context.trace);
|
||||
writeEvent("workbench.trace.snapshot", {
|
||||
type: "trace.snapshot",
|
||||
reason,
|
||||
sessionId,
|
||||
threadId,
|
||||
traceId,
|
||||
snapshot: { ...realtimeTraceSnapshot, sessionId, threadId },
|
||||
cursor: { traceSeq }
|
||||
});
|
||||
writeEvent("workbench.turn.snapshot", {
|
||||
type: "turn.snapshot",
|
||||
reason,
|
||||
sessionId,
|
||||
threadId,
|
||||
traceId,
|
||||
turn: realtimeTurnSnapshot(context),
|
||||
cursor: { traceSeq }
|
||||
});
|
||||
}
|
||||
|
||||
export function realtimeTurnSnapshot(context) {
|
||||
if (context?.factsReadBlocker) return blockedRealtimeTurnSnapshot(context, context.factsReadBlocker);
|
||||
const facts = context?.facts;
|
||||
const traceId = safeTraceId(context?.traceId);
|
||||
if (traceId && facts && context?.factSession) {
|
||||
const factTurn = factTurnForTrace(facts, traceId);
|
||||
const projected = factTurnSnapshot({ turn: factTurn, session: context.factSession, facts, traceId, turnId: context.turnId });
|
||||
if (projected) return projected;
|
||||
}
|
||||
const blocker = traceEventsReadModelBlocker("workbench_facts_turn_missing", "Workbench durable facts did not contain a turn snapshot; realtime must surface the read-model gap instead of falling back to legacy trace projection.", { traceId, session: context?.factSession, route: "/v1/workbench/events" });
|
||||
return blockedRealtimeTurnSnapshot(context, blocker);
|
||||
}
|
||||
|
||||
export function blockedRealtimeTraceSnapshot(context) {
|
||||
const traceId = safeTraceId(context?.traceId ?? context?.factsReadBlocker?.traceId) ?? null;
|
||||
const blocker = context?.factsReadBlocker ?? traceEventsReadModelBlocker("workbench_facts_unavailable", "Workbench durable facts are unavailable for realtime trace snapshot.", { traceId, route: "/v1/workbench/events" });
|
||||
const projection = blockedTraceEventProjection({}, blocker);
|
||||
return {
|
||||
traceId,
|
||||
status: "unknown",
|
||||
traceStatus: "unknown",
|
||||
events: [],
|
||||
eventCount: 0,
|
||||
fullTraceLoaded: true,
|
||||
hasMore: false,
|
||||
nextProjectedSeq: 0,
|
||||
updatedAt: null,
|
||||
projection,
|
||||
projectionStatus: projection.projectionStatus,
|
||||
projectionHealth: projection.projectionHealth,
|
||||
staleMs: projection.staleMs ?? null,
|
||||
blocker,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
};
|
||||
}
|
||||
|
||||
export function blockedRealtimeTurnSnapshot(context, blocker) {
|
||||
const traceId = safeTraceId(context?.traceId ?? blocker?.traceId) ?? null;
|
||||
const turnId = safeTurnId(context?.turnId ?? blocker?.turnId) || traceId;
|
||||
const sessionId = safeSessionId(blocker?.sessionId ?? factSessionId(context?.factSession) ?? context?.session?.id) ?? null;
|
||||
const threadId = safeOpaqueId(context?.factSession?.threadId ?? context?.session?.threadId) ?? (textValue(context?.session?.threadId) || null);
|
||||
const projection = blockedTraceEventProjection({}, blocker);
|
||||
return {
|
||||
turnId,
|
||||
traceId,
|
||||
status: "unknown",
|
||||
running: false,
|
||||
terminal: false,
|
||||
sessionId,
|
||||
threadId,
|
||||
userMessageId: null,
|
||||
assistantMessageId: null,
|
||||
assistantText: null,
|
||||
finalResponse: null,
|
||||
timing: null,
|
||||
startedAt: null,
|
||||
lastEventAt: null,
|
||||
finishedAt: null,
|
||||
durationMs: null,
|
||||
agentRun: null,
|
||||
trace: {
|
||||
traceId,
|
||||
status: "unknown",
|
||||
eventCount: 0,
|
||||
updatedAt: null
|
||||
},
|
||||
urls: {
|
||||
self: turnId ? `/v1/workbench/turns/${encodeURIComponent(turnId)}` : null,
|
||||
traceEvents: traceId ? `/v1/workbench/traces/${encodeURIComponent(traceId)}/events` : null
|
||||
},
|
||||
projectionStatus: projection.projectionStatus,
|
||||
projectionHealth: projection.projectionHealth,
|
||||
staleMs: projection.staleMs ?? null,
|
||||
blocker,
|
||||
projection,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
};
|
||||
}
|
||||
|
||||
export async function writeTraceRealtimeSnapshotSafe(input) {
|
||||
try {
|
||||
await writeTraceRealtimeSnapshot(input);
|
||||
} catch (error) {
|
||||
input.writeEvent("workbench.error", {
|
||||
type: "error",
|
||||
traceId: input.traceId,
|
||||
reason: input.reason,
|
||||
error: {
|
||||
code: "workbench_trace_snapshot_failed",
|
||||
causeCode: error?.code ?? null,
|
||||
retryable: error?.data?.retryable === true,
|
||||
message: error?.message ?? "Workbench trace snapshot failed.",
|
||||
valuesRedacted: true
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
export async function visibleTraceContext(options, actor, traceId) {
|
||||
const readModel = createWorkbenchReadModel(options, actor);
|
||||
if (typeof readModel.queryFacts !== "function") {
|
||||
const projection = factProjectionForTrace(emptyFactSet(), traceId);
|
||||
const factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_unavailable", "Workbench durable facts query is unavailable; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" });
|
||||
return { visible: true, turnId: traceId, traceId, status: "unknown", projection, result: null, session: null, trace: factTraceSnapshot(emptyFactSet(), traceId), facts: emptyFactSet(), factSession: null, factsReadBlocker };
|
||||
}
|
||||
|
||||
let facts = emptyFactSet();
|
||||
let factsReadBlocker = null;
|
||||
try {
|
||||
const factResult = await readModel.queryFacts({ traceId, families: [...WORKBENCH_SESSION_DETAIL_FAMILIES, ...WORKBENCH_TRACE_EVENT_PAGE_FAMILIES], limit: MAX_PAGE_LIMIT });
|
||||
if (factResult.error || factResult.durable === false) {
|
||||
const projection = factProjectionForTrace(facts, traceId);
|
||||
factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts query failed; realtime must surface the read-model error instead of silently falling back to legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" });
|
||||
factsReadBlocker.causeCode = factResult.error?.code ?? null;
|
||||
factsReadBlocker.causeName = factResult.error?.name ?? null;
|
||||
factsReadBlocker.message = factResult.error?.message ? `${factsReadBlocker.message} Cause: ${factResult.error.message}` : factResult.durable === false ? `${factsReadBlocker.message} Cause: durable facts reader unavailable.` : factsReadBlocker.message;
|
||||
} else {
|
||||
facts = factResult?.facts ?? facts;
|
||||
}
|
||||
} catch (error) {
|
||||
const projection = factProjectionForTrace(facts, traceId);
|
||||
factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts query failed; realtime must surface the read-model error instead of silently falling back to legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" });
|
||||
factsReadBlocker.causeCode = error?.code ?? null;
|
||||
factsReadBlocker.causeName = error?.name ?? null;
|
||||
factsReadBlocker.message = error?.message ? `${factsReadBlocker.message} Cause: ${error.message}` : factsReadBlocker.message;
|
||||
}
|
||||
|
||||
const resolved = factsReadBlocker ? { session: null, facts } : await visibleFactSessionForTrace(readModel, facts, actor, traceId);
|
||||
if (resolved.error) {
|
||||
const projection = factProjectionForTrace(facts, traceId);
|
||||
factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_query_failed", "Workbench durable facts session lookup failed; realtime must surface the read-model error instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" });
|
||||
factsReadBlocker.causeCode = resolved.error?.code ?? null;
|
||||
factsReadBlocker.causeName = resolved.error?.name ?? null;
|
||||
factsReadBlocker.message = resolved.error?.message ? `${factsReadBlocker.message} Cause: ${resolved.error.message}` : factsReadBlocker.message;
|
||||
} else if (resolved.facts) {
|
||||
facts = resolved.facts;
|
||||
}
|
||||
|
||||
const factSession = resolved.session ?? null;
|
||||
const trace = factTraceSnapshot(facts, traceId);
|
||||
const turn = factTurnForTrace(facts, traceId, traceId);
|
||||
let projection = factsReadBlocker ? blockedTraceEventProjection(factProjectionForTrace(facts, traceId), factsReadBlocker) : factProjectionForTrace(facts, traceId);
|
||||
const visible = Boolean(factsReadBlocker || factSession || turn || factCheckpointForTrace(facts, traceId) || trace.eventCount > 0);
|
||||
if (!visible) {
|
||||
factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_trace_missing", "Workbench realtime could not find durable facts for the requested trace; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, route: "/v1/workbench/events" });
|
||||
projection = blockedTraceEventProjection(projection, factsReadBlocker);
|
||||
}
|
||||
if (!factsReadBlocker && !factSession) {
|
||||
factsReadBlocker = traceEventsReadModelBlocker("workbench_facts_session_missing", "Workbench durable facts did not contain the trace session; realtime must surface the read-model gap instead of using legacy trace projection.", { traceId, projection, turn, route: "/v1/workbench/events" });
|
||||
projection = blockedTraceEventProjection(projection, factsReadBlocker);
|
||||
}
|
||||
const snapshot = factTurnSnapshot({ turn, session: factSession, facts, traceId, turnId: traceId });
|
||||
return { visible: true, turnId: snapshot.turnId ?? traceId, traceId, status: snapshot.status, projection, result: null, session: null, trace, facts, factSession, factsReadBlocker };
|
||||
}
|
||||
|
||||
export function traceSnapshotForRealtime(snapshot) {
|
||||
const events = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
||||
return {
|
||||
...snapshot,
|
||||
events,
|
||||
eventCount: Number.isFinite(Number(snapshot?.eventCount)) ? Number(snapshot.eventCount) : events.length,
|
||||
lastEvent: snapshot?.lastEvent ?? events.at(-1) ?? null,
|
||||
valuesRedacted: true,
|
||||
secretMaterialStored: false
|
||||
};
|
||||
}
|
||||
|
||||
export function traceSnapshotSummary(snapshot) {
|
||||
const events = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
||||
return {
|
||||
traceId: snapshot?.traceId ?? null,
|
||||
status: snapshot?.status ?? "missing",
|
||||
eventCount: Number.isFinite(Number(snapshot?.eventCount)) ? Number(snapshot.eventCount) : events.length,
|
||||
lastEvent: snapshot?.lastEvent ?? events.at(-1) ?? null,
|
||||
updatedAt: snapshot?.updatedAt ?? null,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
export function traceSnapshotLastSeq(snapshot) {
|
||||
const events = Array.isArray(snapshot?.events) ? snapshot.events : [];
|
||||
return events.reduce((max, event, index) => Math.max(max, eventSeq(event, index)), 0);
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,815 @@
|
||||
/*
|
||||
* In-memory Cloud runtime store implementation.
|
||||
* Public construction remains in runtime-store.ts.
|
||||
*/
|
||||
import { createHash, randomUUID } from "node:crypto";
|
||||
|
||||
import {
|
||||
ENVIRONMENT_DEV,
|
||||
ERROR_CODES,
|
||||
HwlabProtocolError,
|
||||
assertProtocolRecord
|
||||
} from "../protocol/index.mjs";
|
||||
import {
|
||||
CLOUD_API_SERVICE_ID,
|
||||
createProtocolAuditEvent,
|
||||
deriveProtocolActorFromMeta
|
||||
} from "../audit/index.mjs";
|
||||
import {
|
||||
CLOUD_CORE_MIGRATION_ID,
|
||||
CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION,
|
||||
CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE,
|
||||
CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS
|
||||
} from "./schema.ts";
|
||||
|
||||
|
||||
import * as runtimeCore from "./runtime-store-core.ts";
|
||||
|
||||
const {
|
||||
RUNTIME_STORE_KIND,
|
||||
RUNTIME_STORE_KIND_POSTGRES,
|
||||
RUNTIME_DURABLE_ADAPTER_MISSING,
|
||||
RUNTIME_DURABLE_ADAPTER_UNCONFIGURED,
|
||||
RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING,
|
||||
RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED,
|
||||
RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
|
||||
RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
|
||||
RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED,
|
||||
RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
|
||||
RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
|
||||
RUNTIME_ADAPTER_ENV,
|
||||
RUNTIME_DURABLE_ENV,
|
||||
RUNTIME_DB_POOL_MAX_ENV,
|
||||
requiredPostgresSchema,
|
||||
postgresCountTables,
|
||||
postgresRuntimeReadIndexes,
|
||||
buildPostgresPoolConfig,
|
||||
RUNTIME_DB_QUERY_TIMEOUT_MS_ENV,
|
||||
RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV,
|
||||
RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV,
|
||||
RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV,
|
||||
RUNTIME_DB_POOL_RESET_COOLDOWN_MS_ENV,
|
||||
DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS,
|
||||
DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS,
|
||||
DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS,
|
||||
DEFAULT_RUNTIME_DB_POOL_RESET_COOLDOWN_MS,
|
||||
normalizePositiveInteger,
|
||||
delayRuntimeDbQueryRetry,
|
||||
isReadyForDurableRuntimeRead,
|
||||
normalizeCapabilityInputs,
|
||||
normalizeShellInput,
|
||||
normalizeAgentTraceEvent,
|
||||
normalizeWorkbenchProjectionState,
|
||||
normalizeWorkbenchProjectionAllocation,
|
||||
normalizeWorkbenchFacts,
|
||||
normalizeWorkbenchAggregateEventsForFacts,
|
||||
appendWorkbenchFactEvent,
|
||||
workbenchAggregateForFact,
|
||||
normalizeWorkbenchAggregateEvent,
|
||||
workbenchAggregateEventType,
|
||||
workbenchInputDelivery,
|
||||
workbenchInputStatus,
|
||||
indexWorkbenchAggregateEvents,
|
||||
normalizeWorkbenchSessionInputFact,
|
||||
memoryWorkbenchSessionInputWithSeq,
|
||||
inputFactWithAggregateSeq,
|
||||
workbenchSessionInputWithSeq,
|
||||
workbenchSessionInputPromotedSeq,
|
||||
normalizeWorkbenchSessionFact,
|
||||
normalizeWorkbenchMessageFact,
|
||||
normalizeWorkbenchPartFact,
|
||||
normalizeWorkbenchTurnFact,
|
||||
assertWorkbenchTurnFinalResponseInvariant,
|
||||
normalizeWorkbenchFactStatus,
|
||||
workbenchFactFinalResponseText,
|
||||
normalizeWorkbenchTraceEventFact,
|
||||
normalizeWorkbenchProjectionCheckpoint,
|
||||
arrayInput,
|
||||
stableWorkbenchFactId,
|
||||
requiredWorkbenchFactError,
|
||||
compareTraceEventRecords,
|
||||
compareProjectionStateRecords,
|
||||
compareWorkbenchFactRecords,
|
||||
compareWorkbenchInputFactRecords,
|
||||
compareWorkbenchPartFactRecords,
|
||||
compareWorkbenchTraceEventFacts,
|
||||
sortWorkbenchFactRows,
|
||||
WORKBENCH_FACT_FAMILIES,
|
||||
workbenchFactFamilySet,
|
||||
workbenchFactFamilyForTable,
|
||||
workbenchSessionSummarySelectClause,
|
||||
workbenchSessionSummaryFactFromRow,
|
||||
workbenchFactOrder,
|
||||
traceEventLevel,
|
||||
positiveIntegerOrNull,
|
||||
nonNegativeInteger,
|
||||
nonNegativeIntegerOrNull,
|
||||
projectionStatusOr,
|
||||
projectionHealthOr,
|
||||
resultSyncStateOr,
|
||||
timestampOr,
|
||||
textOr,
|
||||
textList,
|
||||
asObject,
|
||||
requireString,
|
||||
requireProtocolId,
|
||||
normalizeJsonObject,
|
||||
pruneUndefined,
|
||||
makeId,
|
||||
stableJson,
|
||||
sha256Hex,
|
||||
sortJson,
|
||||
matchesQuery,
|
||||
matchesWorkbenchSessionFactQuery,
|
||||
limitResults,
|
||||
normalizeRuntimeAdapter,
|
||||
normalizePostgresSslMode,
|
||||
postgresSslModeFromConnectionString,
|
||||
postgresSslOption,
|
||||
normalizePostgresConnectionStringForSslMode,
|
||||
postgresAdapterContract,
|
||||
addRuntimeDurabilityContract,
|
||||
durabilityBlockedLayer,
|
||||
runtimeSafety,
|
||||
classifyRuntimeDbError,
|
||||
isRuntimeDbConnectTimeout,
|
||||
durableRuntimeReadBlockedError,
|
||||
durableRuntimeReadBlockedData,
|
||||
isRuntimeDbSslError,
|
||||
summarizeRuntimeSchema,
|
||||
notCheckedRuntimeMigration,
|
||||
blockedRuntimeMigration,
|
||||
runtimeGates,
|
||||
defaultRuntimeGates,
|
||||
defaultSchemaGate,
|
||||
defaultMigrationGate,
|
||||
readyGate,
|
||||
blockedGate,
|
||||
notCheckedGate,
|
||||
gateFromReadiness,
|
||||
snapshotMemory,
|
||||
newRecords,
|
||||
changedRecords,
|
||||
withPersistence,
|
||||
parseJsonColumn,
|
||||
postgresPoolStats,
|
||||
toCountKey,
|
||||
normalizeRuntimeTimeoutMs,
|
||||
normalizeRuntimePoolMax
|
||||
} = runtimeCore;
|
||||
|
||||
export class CloudRuntimeStore {
|
||||
constructor({ now = () => new Date().toISOString() } = {}) {
|
||||
this.now = now;
|
||||
this.gatewaySessions = new Map();
|
||||
this.boxResources = new Map();
|
||||
this.boxCapabilities = new Map();
|
||||
this.hardwareOperations = new Map();
|
||||
this.auditEvents = new Map();
|
||||
this.evidenceRecords = new Map();
|
||||
this.agentTraceEvents = new Map();
|
||||
this.workbenchProjectionStates = new Map();
|
||||
this.workbenchSessions = new Map();
|
||||
this.workbenchMessages = new Map();
|
||||
this.workbenchParts = new Map();
|
||||
this.workbenchTurns = new Map();
|
||||
this.workbenchTraceEvents = new Map();
|
||||
this.workbenchEventSequences = new Map();
|
||||
this.workbenchEvents = new Map();
|
||||
this.workbenchSessionInputs = new Map();
|
||||
this.workbenchProjectionCheckpoints = new Map();
|
||||
}
|
||||
|
||||
summary() {
|
||||
return addRuntimeDurabilityContract({
|
||||
adapter: RUNTIME_STORE_KIND,
|
||||
durable: false,
|
||||
status: "degraded",
|
||||
blocker: RUNTIME_DURABLE_ADAPTER_MISSING,
|
||||
reason: "L1 runtime writes are process-local only; no DB-backed durable runtime adapter is configured",
|
||||
adapterContract: {
|
||||
required: "DB-backed durable runtime adapter using HWLAB_CLOUD_DB_URL",
|
||||
sourceIssue: "pikasTech/HWLAB#164",
|
||||
secretMaterialRequiredInSource: false
|
||||
},
|
||||
counts: {
|
||||
gatewaySessions: this.gatewaySessions.size,
|
||||
boxResources: this.boxResources.size,
|
||||
boxCapabilities: this.boxCapabilities.size,
|
||||
hardwareOperations: this.hardwareOperations.size,
|
||||
auditEvents: this.auditEvents.size,
|
||||
evidenceRecords: this.evidenceRecords.size,
|
||||
agentTraceEvents: this.agentTraceEvents.size,
|
||||
workbenchProjectionStates: this.workbenchProjectionStates.size,
|
||||
workbenchSessions: this.workbenchSessions.size,
|
||||
workbenchMessages: this.workbenchMessages.size,
|
||||
workbenchParts: this.workbenchParts.size,
|
||||
workbenchTurns: this.workbenchTurns.size,
|
||||
workbenchTraceEvents: this.workbenchTraceEvents.size,
|
||||
workbenchEventSequences: this.workbenchEventSequences.size,
|
||||
workbenchEvents: this.workbenchEvents.size,
|
||||
workbenchSessionInputs: this.workbenchSessionInputs.size,
|
||||
workbenchProjectionCheckpoints: this.workbenchProjectionCheckpoints.size
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
registerGatewaySession(params = {}, requestMeta = {}) {
|
||||
const now = this.now();
|
||||
const input = asObject(params.gatewaySession ?? params, "gatewaySession");
|
||||
const gatewaySession = pruneUndefined({
|
||||
gatewaySessionId: input.gatewaySessionId || makeId("gws"),
|
||||
projectId: requireProtocolId(input, "projectId"),
|
||||
serviceId: input.serviceId || "hwlab-gateway",
|
||||
gatewayId: input.gatewayId || makeId("gtw"),
|
||||
endpoint: input.endpoint,
|
||||
status: input.status || "connected",
|
||||
environment: ENVIRONMENT_DEV,
|
||||
startedAt: input.startedAt || now,
|
||||
lastSeenAt: input.lastSeenAt || now,
|
||||
stoppedAt: input.stoppedAt,
|
||||
labels: normalizeJsonObject(input.labels)
|
||||
});
|
||||
|
||||
assertProtocolRecord("gatewaySession", gatewaySession);
|
||||
this.gatewaySessions.set(gatewaySession.gatewaySessionId, gatewaySession);
|
||||
|
||||
const auditEvent = this.recordAuditEvent({
|
||||
requestMeta,
|
||||
params,
|
||||
action: "gateway.session.register",
|
||||
targetType: "gateway_session",
|
||||
targetId: gatewaySession.gatewaySessionId,
|
||||
projectId: gatewaySession.projectId,
|
||||
gatewaySessionId: gatewaySession.gatewaySessionId,
|
||||
outcome: "accepted",
|
||||
metadata: {
|
||||
gatewayId: gatewaySession.gatewayId,
|
||||
serviceId: gatewaySession.serviceId
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
registered: true,
|
||||
gatewaySession,
|
||||
auditEvent,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
registerBoxResource(params = {}, requestMeta = {}) {
|
||||
const now = this.now();
|
||||
const input = asObject(params.resource ?? params.boxResource ?? params, "boxResource");
|
||||
const gatewaySessionId = requireProtocolId(input, "gatewaySessionId");
|
||||
const gatewaySession = this.requireGatewaySession(gatewaySessionId);
|
||||
const projectId = input.projectId || gatewaySession.projectId;
|
||||
const resource = pruneUndefined({
|
||||
resourceId: input.resourceId || makeId("res"),
|
||||
projectId,
|
||||
gatewaySessionId,
|
||||
boxId: input.boxId || makeId("box"),
|
||||
resourceType: input.resourceType || "simulator_endpoint",
|
||||
name: input.name,
|
||||
state: input.state || "available",
|
||||
environment: ENVIRONMENT_DEV,
|
||||
metadata: normalizeJsonObject(input.metadata),
|
||||
createdAt: input.createdAt || now,
|
||||
updatedAt: input.updatedAt || now
|
||||
});
|
||||
|
||||
assertProtocolRecord("boxResource", resource);
|
||||
this.boxResources.set(resource.resourceId, resource);
|
||||
|
||||
const auditEvent = this.recordAuditEvent({
|
||||
requestMeta,
|
||||
params,
|
||||
action: "box.resource.register",
|
||||
targetType: "box_resource",
|
||||
targetId: resource.resourceId,
|
||||
projectId: resource.projectId,
|
||||
gatewaySessionId: resource.gatewaySessionId,
|
||||
outcome: "accepted",
|
||||
metadata: {
|
||||
boxId: resource.boxId,
|
||||
resourceType: resource.resourceType
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
registered: true,
|
||||
resource,
|
||||
auditEvent,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
reportBoxCapabilities(params = {}, requestMeta = {}) {
|
||||
const now = this.now();
|
||||
const inputs = normalizeCapabilityInputs(params);
|
||||
const capabilities = [];
|
||||
|
||||
for (const input of inputs) {
|
||||
const resourceId = requireProtocolId(input, "resourceId");
|
||||
const resource = this.requireBoxResource(resourceId);
|
||||
const capability = pruneUndefined({
|
||||
capabilityId: input.capabilityId || makeId("cap"),
|
||||
resourceId,
|
||||
projectId: input.projectId || resource.projectId,
|
||||
name: requireString(input, "name"),
|
||||
description: input.description,
|
||||
direction: input.direction || "bidirectional",
|
||||
valueType: input.valueType || "object",
|
||||
unit: input.unit,
|
||||
constraints: normalizeJsonObject(input.constraints),
|
||||
mutatesState: input.mutatesState ?? true,
|
||||
createdAt: input.createdAt || now,
|
||||
updatedAt: input.updatedAt || now
|
||||
});
|
||||
|
||||
assertProtocolRecord("boxCapability", capability);
|
||||
this.boxCapabilities.set(capability.capabilityId, capability);
|
||||
capabilities.push(capability);
|
||||
}
|
||||
|
||||
const first = capabilities[0];
|
||||
const auditEvent = this.recordAuditEvent({
|
||||
requestMeta,
|
||||
params,
|
||||
action: "box.capability.report",
|
||||
targetType: "box_capability",
|
||||
targetId: first.capabilityId,
|
||||
projectId: first.projectId,
|
||||
gatewaySessionId: this.boxResources.get(first.resourceId)?.gatewaySessionId,
|
||||
outcome: "accepted",
|
||||
metadata: {
|
||||
capabilityIds: capabilities.map((capability) => capability.capabilityId)
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
reported: true,
|
||||
capabilities,
|
||||
auditEvent,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
requestHardwareOperation(params = {}, requestMeta = {}) {
|
||||
const refs = this.requireOperationRefs(params, requestMeta);
|
||||
const now = this.now();
|
||||
const operation = pruneUndefined({
|
||||
operationId: params.operationId || makeId("op"),
|
||||
projectId: refs.projectId,
|
||||
gatewaySessionId: refs.gatewaySession.gatewaySessionId,
|
||||
agentSessionId: params.agentSessionId,
|
||||
workerSessionId: params.workerSessionId,
|
||||
resourceId: refs.resource.resourceId,
|
||||
capabilityId: refs.capability.capabilityId,
|
||||
requestedBy: refs.actor.actorId,
|
||||
input: normalizeJsonObject(params.input),
|
||||
status: "accepted",
|
||||
environment: ENVIRONMENT_DEV,
|
||||
requestedAt: params.requestedAt || now,
|
||||
updatedAt: now
|
||||
});
|
||||
|
||||
assertProtocolRecord("hardwareOperation", operation);
|
||||
this.hardwareOperations.set(operation.operationId, operation);
|
||||
|
||||
const auditEvent = this.recordAuditEvent({
|
||||
requestMeta,
|
||||
params,
|
||||
action: "hardware.operation.request",
|
||||
targetType: "hardware_operation",
|
||||
targetId: operation.operationId,
|
||||
projectId: operation.projectId,
|
||||
gatewaySessionId: operation.gatewaySessionId,
|
||||
workerSessionId: operation.workerSessionId,
|
||||
operationId: operation.operationId,
|
||||
outcome: "accepted",
|
||||
metadata: {
|
||||
resourceId: operation.resourceId,
|
||||
capabilityId: operation.capabilityId
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
accepted: true,
|
||||
status: operation.status,
|
||||
operationId: operation.operationId,
|
||||
operation,
|
||||
auditEvent,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
invokeHardwareShell(params = {}, requestMeta = {}) {
|
||||
const shellInput = normalizeShellInput(params);
|
||||
const operationResult = this.requestHardwareOperation(
|
||||
{
|
||||
...params,
|
||||
input: {
|
||||
...normalizeJsonObject(params.input),
|
||||
shell: shellInput
|
||||
}
|
||||
},
|
||||
requestMeta
|
||||
);
|
||||
const operation = {
|
||||
...operationResult.operation,
|
||||
output: {
|
||||
shellExecuted: false,
|
||||
dispatchStatus: "not_connected",
|
||||
message: "cloud-api recorded the shell invoke request; no gateway shell adapter is connected in L1"
|
||||
},
|
||||
updatedAt: this.now()
|
||||
};
|
||||
assertProtocolRecord("hardwareOperation", operation);
|
||||
this.hardwareOperations.set(operation.operationId, operation);
|
||||
|
||||
const evidenceRecord = this.writeEvidenceRecord(
|
||||
{
|
||||
projectId: operation.projectId,
|
||||
operationId: operation.operationId,
|
||||
kind: "trace",
|
||||
mimeType: "application/json",
|
||||
serviceId: CLOUD_API_SERVICE_ID,
|
||||
metadata: {
|
||||
traceId: requestMeta.traceId,
|
||||
gatewaySessionId: operation.gatewaySessionId,
|
||||
resourceId: operation.resourceId,
|
||||
capabilityId: operation.capabilityId,
|
||||
shellExecuted: false,
|
||||
dispatchStatus: "not_connected"
|
||||
},
|
||||
content: {
|
||||
operationId: operation.operationId,
|
||||
shell: shellInput,
|
||||
dispatchStatus: "not_connected"
|
||||
}
|
||||
},
|
||||
requestMeta
|
||||
).evidenceRecord;
|
||||
|
||||
const auditEvent = this.recordAuditEvent({
|
||||
requestMeta,
|
||||
params,
|
||||
action: "hardware.invoke.shell",
|
||||
targetType: "hardware_operation",
|
||||
targetId: operation.operationId,
|
||||
projectId: operation.projectId,
|
||||
gatewaySessionId: operation.gatewaySessionId,
|
||||
workerSessionId: operation.workerSessionId,
|
||||
operationId: operation.operationId,
|
||||
outcome: "accepted",
|
||||
metadata: {
|
||||
evidenceId: evidenceRecord.evidenceId,
|
||||
shellExecuted: false,
|
||||
dispatchStatus: "not_connected"
|
||||
}
|
||||
});
|
||||
|
||||
return {
|
||||
accepted: true,
|
||||
status: operation.status,
|
||||
operationId: operation.operationId,
|
||||
operation,
|
||||
auditEvent,
|
||||
evidenceRecord,
|
||||
dispatch: operation.output,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
writeAuditEvent(params = {}, requestMeta = {}) {
|
||||
const eventInput = params.event ?? params.auditEvent ?? params;
|
||||
const auditEvent = eventInput.auditId
|
||||
? pruneUndefined({
|
||||
...eventInput,
|
||||
environment: ENVIRONMENT_DEV
|
||||
})
|
||||
: this.buildAuditEventFromInput(eventInput, requestMeta);
|
||||
|
||||
assertProtocolRecord("auditEvent", auditEvent);
|
||||
this.auditEvents.set(auditEvent.auditId, auditEvent);
|
||||
|
||||
return {
|
||||
written: true,
|
||||
auditEvent,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
queryAuditEvents(params = {}) {
|
||||
const events = [...this.auditEvents.values()].filter((event) => matchesQuery(event, params, [
|
||||
"auditId",
|
||||
"traceId",
|
||||
"projectId",
|
||||
"gatewaySessionId",
|
||||
"operationId",
|
||||
"action",
|
||||
"targetId"
|
||||
]));
|
||||
|
||||
return {
|
||||
events: limitResults(events, params.limit),
|
||||
count: events.length,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
writeEvidenceRecord(params = {}, requestMeta = {}) {
|
||||
const now = this.now();
|
||||
const input = asObject(params.record ?? params.evidenceRecord ?? params, "evidenceRecord");
|
||||
const content = input.content ?? input.metadata ?? {};
|
||||
const sha256 = input.sha256 || sha256Hex(content);
|
||||
const evidenceId = input.evidenceId || makeId("evd");
|
||||
const evidenceRecord = pruneUndefined({
|
||||
evidenceId,
|
||||
projectId: requireProtocolId(input, "projectId"),
|
||||
operationId: requireProtocolId(input, "operationId"),
|
||||
agentSessionId: input.agentSessionId,
|
||||
workerSessionId: input.workerSessionId,
|
||||
kind: input.kind || "trace",
|
||||
uri: input.uri || `memory://hwlab/evidence/${evidenceId}`,
|
||||
mimeType: input.mimeType,
|
||||
sha256,
|
||||
sizeBytes: input.sizeBytes ?? Buffer.byteLength(stableJson(content), "utf8"),
|
||||
serviceId: input.serviceId || CLOUD_API_SERVICE_ID,
|
||||
environment: ENVIRONMENT_DEV,
|
||||
metadata: {
|
||||
...normalizeJsonObject(input.metadata),
|
||||
...(requestMeta.traceId ? { traceId: requestMeta.traceId } : {})
|
||||
},
|
||||
createdAt: input.createdAt || now
|
||||
});
|
||||
|
||||
assertProtocolRecord("evidenceRecord", evidenceRecord);
|
||||
this.evidenceRecords.set(evidenceRecord.evidenceId, evidenceRecord);
|
||||
|
||||
return {
|
||||
written: true,
|
||||
evidenceRecord,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
queryEvidenceRecords(params = {}) {
|
||||
const records = [...this.evidenceRecords.values()].filter((record) => matchesQuery(record, params, [
|
||||
"evidenceId",
|
||||
"projectId",
|
||||
"operationId",
|
||||
"agentSessionId",
|
||||
"workerSessionId",
|
||||
"kind",
|
||||
"serviceId"
|
||||
]));
|
||||
|
||||
return {
|
||||
records: limitResults(records, params.limit),
|
||||
count: records.length,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
writeAgentTraceEvent(params = {}, requestMeta = {}) {
|
||||
const traceEvent = normalizeAgentTraceEvent(params.traceEvent ?? params.event ?? params, requestMeta, this.now());
|
||||
this.agentTraceEvents.set(traceEvent.id, traceEvent);
|
||||
return {
|
||||
written: true,
|
||||
traceEvent,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
queryAgentTraceEvents(params = {}) {
|
||||
const records = [...this.agentTraceEvents.values()].filter((record) => matchesQuery(record, params, [
|
||||
"id",
|
||||
"traceId",
|
||||
"agentSessionId",
|
||||
"workerSessionId",
|
||||
"level"
|
||||
]));
|
||||
records.sort(compareTraceEventRecords);
|
||||
return {
|
||||
events: limitResults(records.map((record) => record.event), params.limit),
|
||||
count: records.length,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
writeWorkbenchProjectionState(params = {}, requestMeta = {}) {
|
||||
const state = normalizeWorkbenchProjectionState(params.projectionState ?? params.state ?? params, requestMeta, this.now());
|
||||
this.workbenchProjectionStates.set(state.traceId, state);
|
||||
return {
|
||||
written: true,
|
||||
projectionState: state,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
getWorkbenchProjectionState(params = {}) {
|
||||
const traceId = textOr(params.traceId, "");
|
||||
const state = traceId ? this.workbenchProjectionStates.get(traceId) ?? null : null;
|
||||
return {
|
||||
projectionState: state,
|
||||
found: Boolean(state),
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
allocateWorkbenchProjectedSeq(params = {}, requestMeta = {}) {
|
||||
const allocation = normalizeWorkbenchProjectionAllocation(params, requestMeta, this.now());
|
||||
const existing = [...this.workbenchTraceEvents.values()].find((record) =>
|
||||
record.traceId === allocation.traceId && record.sourceEventId === allocation.sourceEventId
|
||||
);
|
||||
if (existing) {
|
||||
return {
|
||||
...allocation,
|
||||
projectedSeq: nonNegativeInteger(existing.projectedSeq),
|
||||
reused: true,
|
||||
persistence: this.summary(),
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
const eventMax = [...this.workbenchTraceEvents.values()]
|
||||
.filter((record) => record.traceId === allocation.traceId)
|
||||
.reduce((max, record) => Math.max(max, nonNegativeInteger(record.projectedSeq)), 0);
|
||||
const checkpointMax = nonNegativeInteger(this.workbenchProjectionCheckpoints.get(allocation.traceId)?.projectedSeq);
|
||||
return {
|
||||
...allocation,
|
||||
projectedSeq: Math.max(eventMax, checkpointMax) + 1,
|
||||
reused: false,
|
||||
persistence: this.summary(),
|
||||
valuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
queryWorkbenchProjectionStates(params = {}) {
|
||||
const dueAt = textOr(params.dueAt, "");
|
||||
const statuses = Array.isArray(params.projectionStatuses) ? new Set(params.projectionStatuses.map((item) => textOr(item, "")).filter(Boolean)) : null;
|
||||
const records = [...this.workbenchProjectionStates.values()]
|
||||
.filter((record) => matchesQuery(record, params, ["traceId", "sessionId", "runId", "commandId", "sourceRunId", "sourceCommandId"]))
|
||||
.filter((record) => !statuses || statuses.has(record.projectionStatus))
|
||||
.filter((record) => !dueAt || !record.nextRetryAt || String(record.nextRetryAt) <= dueAt)
|
||||
.sort(compareProjectionStateRecords);
|
||||
return {
|
||||
states: limitResults(records, params.limit),
|
||||
count: records.length,
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
writeWorkbenchFacts(params = {}, requestMeta = {}) {
|
||||
const facts = normalizeWorkbenchFacts(params.facts ?? params, requestMeta, this.now());
|
||||
const inputFacts = facts.inputs.map((fact) => memoryWorkbenchSessionInputWithSeq(fact, this.workbenchSessionInputs));
|
||||
for (const fact of inputFacts) this.workbenchSessionInputs.set(fact.inputId, fact);
|
||||
for (const fact of facts.sessions) this.workbenchSessions.set(fact.sessionId, fact);
|
||||
for (const fact of facts.messages) this.workbenchMessages.set(fact.messageId, fact);
|
||||
for (const fact of facts.parts) this.workbenchParts.set(fact.partId, fact);
|
||||
for (const fact of facts.turns) this.workbenchTurns.set(fact.turnId, fact);
|
||||
for (const fact of facts.traceEvents) this.workbenchTraceEvents.set(fact.id, fact);
|
||||
for (const fact of facts.checkpoints) {
|
||||
const previous = this.workbenchProjectionCheckpoints.get(fact.traceId) ?? null;
|
||||
if (!previous || nonNegativeInteger(fact.projectedSeq) >= nonNegativeInteger(previous.projectedSeq)) {
|
||||
this.workbenchProjectionCheckpoints.set(fact.traceId, fact);
|
||||
}
|
||||
}
|
||||
return {
|
||||
written: true,
|
||||
facts: { ...facts, inputs: inputFacts },
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
queryWorkbenchFacts(params = {}) {
|
||||
const families = workbenchFactFamilySet(params);
|
||||
const sessions = [...this.workbenchSessions.values()].filter((record) => matchesWorkbenchSessionFactQuery(record, params));
|
||||
const inputs = [...this.workbenchSessionInputs.values()].filter((record) => matchesQuery(record, params, ["inputId", "sessionId", "turnId", "traceId", "messageId", "commandId", "delivery", "status"]));
|
||||
const messages = [...this.workbenchMessages.values()].filter((record) => matchesQuery(record, params, ["messageId", "sessionId", "turnId", "traceId", "role", "status"]));
|
||||
const parts = [...this.workbenchParts.values()].filter((record) => matchesQuery(record, params, ["partId", "messageId", "sessionId", "turnId", "traceId", "partType", "status"]));
|
||||
const turns = [...this.workbenchTurns.values()].filter((record) => matchesQuery(record, params, ["turnId", "sessionId", "traceId", "messageId", "status"]));
|
||||
const afterProjectedSeq = nonNegativeInteger(params.afterProjectedSeq);
|
||||
const traceEvents = [...this.workbenchTraceEvents.values()].filter((record) => matchesQuery(record, params, ["id", "traceId", "sessionId", "turnId", "messageId", "eventType"]) && (afterProjectedSeq <= 0 || nonNegativeInteger(record.projectedSeq) > afterProjectedSeq));
|
||||
const checkpoints = [...this.workbenchProjectionCheckpoints.values()].filter((record) => matchesQuery(record, params, ["traceId", "sessionId", "turnId", "runId", "commandId", "projectionStatus", "projectionHealth"]));
|
||||
const facts = {
|
||||
inputs: families.has("inputs") ? limitResults(sortWorkbenchFactRows(inputs, params, "inputs", compareWorkbenchFactRecords), params.limit) : [],
|
||||
sessions: families.has("sessions") ? limitResults(sortWorkbenchFactRows(sessions, params, "sessions", compareWorkbenchFactRecords), params.limit) : [],
|
||||
messages: families.has("messages") ? limitResults(sortWorkbenchFactRows(messages, params, "messages", compareWorkbenchFactRecords), params.limit) : [],
|
||||
parts: families.has("parts") ? limitResults(sortWorkbenchFactRows(parts, params, "parts", compareWorkbenchPartFactRecords), params.limit) : [],
|
||||
turns: families.has("turns") ? limitResults(sortWorkbenchFactRows(turns, params, "turns", compareWorkbenchFactRecords), params.limit) : [],
|
||||
traceEvents: families.has("traceEvents") ? limitResults(sortWorkbenchFactRows(traceEvents, params, "traceEvents", compareWorkbenchTraceEventFacts), params.limit) : [],
|
||||
checkpoints: families.has("checkpoints") ? limitResults(sortWorkbenchFactRows(checkpoints, params, "checkpoints", compareWorkbenchFactRecords), params.limit) : []
|
||||
};
|
||||
return {
|
||||
facts,
|
||||
count: Object.values(facts).reduce((sum, rows) => sum + rows.length, 0),
|
||||
persistence: this.summary()
|
||||
};
|
||||
}
|
||||
|
||||
recordAuditEvent(input) {
|
||||
return this.writeAuditEvent(this.buildAuditEventFromInput(input, input.requestMeta), input.requestMeta).auditEvent;
|
||||
}
|
||||
|
||||
buildAuditEventFromInput(input = {}, requestMeta = {}) {
|
||||
const actor = deriveProtocolActorFromMeta(requestMeta, input.params?.audit ?? input);
|
||||
return createProtocolAuditEvent({
|
||||
auditId: input.auditId,
|
||||
traceId: requestMeta.traceId || input.traceId,
|
||||
actorType: actor.actorType,
|
||||
actorId: actor.actorId,
|
||||
action: input.action,
|
||||
targetType: input.targetType,
|
||||
targetId: input.targetId,
|
||||
projectId: input.projectId,
|
||||
gatewaySessionId: input.gatewaySessionId,
|
||||
workerSessionId: input.workerSessionId,
|
||||
operationId: input.operationId,
|
||||
serviceId: CLOUD_API_SERVICE_ID,
|
||||
outcome: input.outcome,
|
||||
reason: input.reason || input.params?.audit?.reason,
|
||||
metadata: input.metadata,
|
||||
occurredAt: input.occurredAt
|
||||
});
|
||||
}
|
||||
|
||||
requireOperationRefs(params, requestMeta = {}) {
|
||||
const projectId = requireProtocolId(params, "projectId");
|
||||
const gatewaySession = this.requireGatewaySession(requireProtocolId(params, "gatewaySessionId"));
|
||||
const resource = this.requireBoxResource(requireProtocolId(params, "resourceId"));
|
||||
const capability = this.requireBoxCapability(requireProtocolId(params, "capabilityId"));
|
||||
|
||||
if (gatewaySession.projectId !== projectId || resource.projectId !== projectId || capability.projectId !== projectId) {
|
||||
throw new HwlabProtocolError("hardware operation references must belong to one project", {
|
||||
code: ERROR_CODES.invalidParams,
|
||||
data: {
|
||||
projectId,
|
||||
gatewaySessionId: gatewaySession.gatewaySessionId,
|
||||
resourceId: resource.resourceId,
|
||||
capabilityId: capability.capabilityId
|
||||
}
|
||||
});
|
||||
}
|
||||
if (resource.gatewaySessionId !== gatewaySession.gatewaySessionId) {
|
||||
throw new HwlabProtocolError("resource is not registered under the requested gateway session", {
|
||||
code: ERROR_CODES.resourceLocked,
|
||||
data: {
|
||||
gatewaySessionId: gatewaySession.gatewaySessionId,
|
||||
resourceGatewaySessionId: resource.gatewaySessionId,
|
||||
resourceId: resource.resourceId
|
||||
}
|
||||
});
|
||||
}
|
||||
if (capability.resourceId !== resource.resourceId) {
|
||||
throw new HwlabProtocolError("capability is not registered on the requested resource", {
|
||||
code: ERROR_CODES.capabilityUnavailable,
|
||||
data: {
|
||||
resourceId: resource.resourceId,
|
||||
capabilityId: capability.capabilityId
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
projectId,
|
||||
gatewaySession,
|
||||
resource,
|
||||
capability,
|
||||
actor: deriveProtocolActorFromMeta(requestMeta, params.audit ?? {})
|
||||
};
|
||||
}
|
||||
|
||||
requireGatewaySession(gatewaySessionId) {
|
||||
const gatewaySession = this.gatewaySessions.get(gatewaySessionId);
|
||||
if (!gatewaySession) {
|
||||
throw new HwlabProtocolError(`gateway session ${gatewaySessionId} is not registered`, {
|
||||
code: ERROR_CODES.sessionNotFound,
|
||||
data: { gatewaySessionId }
|
||||
});
|
||||
}
|
||||
return gatewaySession;
|
||||
}
|
||||
|
||||
requireBoxResource(resourceId) {
|
||||
const resource = this.boxResources.get(resourceId);
|
||||
if (!resource) {
|
||||
throw new HwlabProtocolError(`box resource ${resourceId} is not registered`, {
|
||||
code: ERROR_CODES.capabilityUnavailable,
|
||||
data: { resourceId }
|
||||
});
|
||||
}
|
||||
return resource;
|
||||
}
|
||||
|
||||
requireBoxCapability(capabilityId) {
|
||||
const capability = this.boxCapabilities.get(capabilityId);
|
||||
if (!capability) {
|
||||
throw new HwlabProtocolError(`capability ${capabilityId} is not reported`, {
|
||||
code: ERROR_CODES.capabilityUnavailable,
|
||||
data: { capabilityId }
|
||||
});
|
||||
}
|
||||
return capability;
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
+28
-3585
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user