fix: keep workbench trace terminal facts scoped per trace

This commit is contained in:
root
2026-06-25 11:02:32 +00:00
parent 50a99fc159
commit db9da40ee2
4 changed files with 385 additions and 81 deletions
@@ -1890,6 +1890,79 @@ test("workbench trace events remain visible after session lastTraceId moves", as
}
});
test("workbench trace event page keeps per-trace terminal status after later session cancel", async () => {
const completedTraceId = "trc_workbench_completed_before_cancel";
const canceledTraceId = "trc_workbench_later_cancel";
const session = {
id: "ses_workbench_completed_before_cancel",
projectId: "prj_hwpod_workbench",
agentId: "hwlab-code-agent",
status: "canceled",
ownerUserId: ACTOR.id,
conversationId: "cnv_workbench_completed_before_cancel",
threadId: "thread-workbench-completed-before-cancel",
lastTraceId: canceledTraceId,
updatedAt: "2026-06-20T10:20:30.000Z",
session: {
messages: [
{ role: "user", text: "completed first", traceId: completedTraceId, status: "sent", createdAt: "2026-06-20T10:20:00.000Z" },
{ role: "agent", text: "completed final", traceId: completedTraceId, status: "completed", createdAt: "2026-06-20T10:20:10.000Z" },
{ role: "user", text: "cancel later", traceId: canceledTraceId, status: "sent", createdAt: "2026-06-20T10:20:20.000Z" },
{ role: "agent", text: "hwlab-user-cancel", traceId: canceledTraceId, status: "canceled", createdAt: "2026-06-20T10:20:30.000Z" }
],
valuesRedacted: true
}
};
const facts = emptyFacts();
mergeFacts(facts, buildDurableFactsForSession({
session: { ...session, status: "completed", lastTraceId: completedTraceId, updatedAt: "2026-06-20T10:20:10.000Z" },
status: "completed",
finalText: "completed final",
events: [
{ projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:20:00.000Z" },
{ projectedSeq: 2, sourceSeq: 2, type: "result", status: "completed", label: "runner:completed", terminal: true, createdAt: "2026-06-20T10:20:10.000Z" }
],
lastProjectedSeq: 2
}));
mergeFacts(facts, buildDurableFactsForSession({
session: { ...session, status: "canceled", lastTraceId: canceledTraceId, session: { messages: session.session.messages.slice(2), valuesRedacted: true } },
status: "canceled",
finalText: "hwlab-user-cancel",
events: [
{ projectedSeq: 1, sourceSeq: 1, type: "backend", status: "running", label: "runner:started", createdAt: "2026-06-20T10:20:20.000Z" },
{ projectedSeq: 2, sourceSeq: 2, type: "cancel", status: "canceled", label: "runner:canceled", terminal: true, createdAt: "2026-06-20T10:20:30.000Z" }
],
lastProjectedSeq: 2
}));
facts.sessions = facts.sessions.filter((item) => item.lastTraceId === canceledTraceId);
const runtimeStore = {
async queryWorkbenchFacts(params = {}) {
const filtered = filterFacts(facts, params);
return {
facts: filtered,
count: Object.values(filtered).reduce((sum, rows) => sum + rows.length, 0),
persistence: { adapter: "test-durable-workbench-facts", durable: true }
};
}
};
const accessController = {
async ensureBootstrap() {},
async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; }
};
const server = createCloudApiServer({ accessController, workbenchRuntime: runtimeStore });
await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));
try {
const { port } = server.address();
const response = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(completedTraceId)}/events?limit=10`);
assert.equal(response.status, 200);
assert.equal(response.body.traceStatus, "completed");
assert.equal(response.body.events.at(-1).status, "completed");
} finally {
await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve()));
}
});
async function getJson(port, path) {
const response = await fetch(`http://127.0.0.1:${port}${path}`);
return {
+4 -1
View File
@@ -1708,7 +1708,10 @@ async function handleWorkbenchTraceEventPage(request, response, url, options, ac
const projection = factProjectionForTrace(metadataFacts, traceId);
const pageResult = await readModel.queryFacts({ traceId, families: WORKBENCH_TRACE_EVENT_PAGE_FAMILIES, afterProjectedSeq: pageOptions.afterProjectedSeq, limit: pageOptions.limit + 1 });
if (pageResult.error) return sendJson(response, 503, workbenchProjectionStoreError(pageResult.error));
const page = traceEventPageFromFacts(pageResult.facts.traceEvents, pageOptions, { total: projection.lastProjectedSeq, traceStatus: session.status });
const traceTurn = factTurnForTrace(metadataFacts, traceId, traceId);
const turnTraceStatus = normalizeStatus(traceTurn?.status);
const traceStatus = turnTraceStatus !== "unknown" ? turnTraceStatus : durableTraceStatus(pageResult.facts.traceEvents);
const page = traceEventPageFromFacts(pageResult.facts.traceEvents, pageOptions, { total: projection.lastProjectedSeq, traceStatus });
const missingTraceEvents = traceEventPageMissing(page, projection, pageOptions);
if (missingTraceEvents) {
const blocker = traceEventsReadModelBlocker("workbench_trace_events_missing", "Workbench trace events are missing from the durable read model.", { traceId, projection, session, route: url.pathname });
@@ -196,7 +196,7 @@ test("workbench projection writer keeps event writes durable when checkpoint rea
assert.equal(factWrites[0].params.facts.checkpoints[0].projectionStatus, "projecting");
});
test("workbench projection writer keeps terminal session and trace facts durable when reads are blocked", async () => {
test("workbench projection writer keeps terminal session facts durable and skips unsafe trace backfill when reads are blocked", async () => {
const factWrites = [];
let allocationSeq = 0;
let queryCount = 0;
@@ -261,15 +261,12 @@ test("workbench projection writer keeps terminal session and trace facts durable
});
assert.equal(owner.id, "ses_writer_blocked_backfill");
assert.equal(queryCount, 1);
assert.equal(factWrites.length, 3);
assert.equal(queryCount, 2);
assert.equal(factWrites.length, 1);
assert.equal(factWrites[0].params.facts.checkpoints[0].traceId, "trc_writer_blocked_backfill");
assert.equal(factWrites[0].params.facts.turns[0].finalResponse.text, "BLOCKED_BACKFILL_DONE");
assert.equal(factWrites[1].params.facts.traceEvents[0].label, "agentrun:assistant:message");
assert.equal(factWrites[1].params.facts.traceEvents[0].text, "BLOCKED_BACKFILL_DONE");
assert.equal(factWrites[2].params.facts.traceEvents[0].label, "agentrun:result:completed");
assert.equal(factWrites[2].params.facts.traceEvents[0].terminal, true);
assert.equal(allocationSeq, 2);
assert.equal(factWrites[0].params.facts.traceEvents?.length ?? 0, 0);
assert.equal(allocationSeq, 0);
});
test("workbench projection writer backfills terminal final and completion trace events from sealed facts", async () => {
@@ -361,6 +358,110 @@ test("workbench projection writer backfills terminal final and completion trace
assert.equal(loaded.facts.checkpoints[0].projectionStatus, "caught_up");
});
test("workbench projection writer does not duplicate terminal trace events that already exist", async () => {
const runtimeStore = createCloudRuntimeStore({ now: () => "2026-06-20T11:25:00.000Z" });
const traceId = "trc_writer_terminal_backfill_existing";
const accessController = {
async recordAgentSessionOwner(input) {
return {
id: input.sessionId,
projectId: input.projectId,
ownerUserId: input.ownerUserId,
conversationId: input.conversationId,
threadId: input.threadId,
lastTraceId: input.traceId,
status: input.status,
session: input.session,
updatedAt: "2026-06-20T11:25:00.000Z"
};
}
};
await writeWorkbenchProjectionEvent({
runtimeStore,
event: {
traceId,
sessionId: "ses_writer_terminal_backfill_existing",
sourceSeq: 1,
type: "backend",
status: "running",
label: "agentrun:backend:admitted",
createdAt: "2026-06-20T11:24:01.000Z"
}
});
await writeWorkbenchProjectionEvent({
runtimeStore,
event: {
traceId,
sessionId: "ses_writer_terminal_backfill_existing",
sourceSeq: 2,
type: "assistant",
eventType: "assistant",
status: "completed",
label: "agentrun:assistant:message",
text: "ALREADY_DONE",
terminal: true,
createdAt: "2026-06-20T11:24:12.000Z"
}
});
await writeWorkbenchProjectionEvent({
runtimeStore,
event: {
traceId,
sessionId: "ses_writer_terminal_backfill_existing",
sourceSeq: 3,
type: "result",
eventType: "terminal",
status: "completed",
label: "agentrun:result:completed",
terminal: true,
createdAt: "2026-06-20T11:24:13.000Z"
}
});
await writeWorkbenchProjectionSession({
accessController,
runtimeStore,
traceId,
ownerUserId: "usr_writer",
ownerRole: "user",
sessionId: "ses_writer_terminal_backfill_existing",
projectId: "prj_writer",
conversationId: "cnv_writer_terminal_backfill_existing",
threadId: "thread-writer-terminal-backfill-existing",
status: "completed",
payload: {
traceId,
status: "completed",
finalResponse: { text: "ALREADY_DONE", status: "completed", traceId },
runnerTrace: {
traceId,
status: "completed",
startedAt: "2026-06-20T11:24:01.000Z",
lastEventAt: "2026-06-20T11:24:13.000Z",
finishedAt: "2026-06-20T11:24:13.000Z",
eventCount: 3,
events: []
},
agentRun: { runId: "run_writer_terminal_backfill_existing", commandId: "cmd_writer_terminal_backfill_existing", status: "completed", terminalStatus: "completed", lastSeq: 3 },
updatedAt: "2026-06-20T11:24:13.000Z"
},
session: {
sessionStatus: "completed",
messages: [
{ messageId: "msg_writer_backfill_existing_user", role: "user", text: "question", status: "sent", turnId: traceId, traceId },
{ messageId: "msg_writer_backfill_existing_agent", role: "agent", text: "ALREADY_DONE", status: "completed", turnId: traceId, traceId }
],
finalResponse: { text: "ALREADY_DONE", status: "completed", traceId }
}
});
const loaded = runtimeStore.queryWorkbenchFacts({ traceId, families: ["traceEvents", "checkpoints"], limit: 20 });
assert.deepEqual(loaded.facts.traceEvents.map((event) => event.projectedSeq), [1, 2, 3]);
assert.deepEqual(loaded.facts.traceEvents.map((event) => event.label), ["agentrun:backend:admitted", "agentrun:assistant:message", "agentrun:result:completed"]);
assert.equal(loaded.facts.checkpoints[0].projectedSeq, 3);
});
test("workbench projection writer does not rewrite prior-turn messages from merged owner session", async () => {
const factWrites = [];
const runtimeStore = {
@@ -498,6 +599,80 @@ test("workbench projection writer seals canceled AgentRun turns with canonical c
assert.equal(facts.parts.some((part) => part.messageId === "msg_writer_canceled_agent" && part.partType === "final_response" && part.text === "hwlab-user-cancel" && part.sealed === true), true);
});
test("workbench projection writer does not replace prior user prompt with cancel notice", async () => {
const runtimeStore = createCloudRuntimeStore({ now: () => "2026-06-20T11:04:00.000Z" });
const traceId = "trc_writer_canceled_prompt_preserved";
const accessController = {
async recordAgentSessionOwner(input) {
return {
id: input.sessionId,
projectId: input.projectId,
ownerUserId: input.ownerUserId,
conversationId: input.conversationId,
threadId: input.threadId,
lastTraceId: input.traceId,
status: input.status,
session: input.session,
updatedAt: input.status === "canceled" ? "2026-06-20T11:04:30.000Z" : "2026-06-20T11:04:00.000Z"
};
}
};
await writeWorkbenchProjectionSession({
accessController,
runtimeStore,
traceId,
ownerUserId: "usr_writer",
ownerRole: "user",
sessionId: "ses_writer_canceled_prompt_preserved",
projectId: "prj_writer",
conversationId: "cnv_writer_canceled_prompt_preserved",
threadId: "thread-writer-canceled-prompt-preserved",
status: "running",
payload: {
traceId,
status: "running",
prompt: "ORIGINAL CANCEL PROMPT",
updatedAt: "2026-06-20T11:04:00.000Z"
},
session: { sessionStatus: "running", lastTraceId: traceId }
});
await writeWorkbenchProjectionSession({
accessController,
runtimeStore,
traceId,
ownerUserId: "usr_writer",
ownerRole: "user",
sessionId: "ses_writer_canceled_prompt_preserved",
projectId: "prj_writer",
conversationId: "cnv_writer_canceled_prompt_preserved",
threadId: "thread-writer-canceled-prompt-preserved",
status: "canceled",
payload: {
traceId,
status: "canceled",
message: "当前 AgentRun 请求已取消;traceId/runId/commandId 已保留,可重试上一条消息。",
finalResponse: { text: "command cancelled before backend start", status: "canceled", traceId },
runnerTrace: {
traceId,
status: "canceled",
events: [{ type: "cancel", status: "canceled", terminal: true, createdAt: "2026-06-20T11:04:30.000Z" }],
updatedAt: "2026-06-20T11:04:30.000Z"
},
agentRun: { runId: "run_writer_canceled_prompt_preserved", commandId: "cmd_writer_canceled_prompt_preserved", status: "cancelled", commandState: "cancelled", terminalStatus: "cancelled", lastSeq: 12 },
updatedAt: "2026-06-20T11:04:30.000Z"
},
session: { sessionStatus: "canceled", lastTraceId: traceId }
});
const loaded = runtimeStore.queryWorkbenchFacts({ traceId, families: ["messages", "parts", "turns"], limit: 20 });
const userMessages = loaded.facts.messages.filter((message) => message.role === "user");
const agentMessage = loaded.facts.messages.find((message) => message.role !== "user");
assert.deepEqual(userMessages.map((message) => message.text), ["ORIGINAL CANCEL PROMPT"]);
assert.equal(agentMessage.text, "hwlab-user-cancel");
assert.equal(loaded.facts.parts.some((part) => part.messageId === "msg_writer_canceled_prompt_preserved_user" && part.text.includes("请求已取消")), false);
});
test("workbench projection writer does not seal running assistant text as final response", async () => {
const factWrites = [];
const runtimeStore = {
+125 -72
View File
@@ -382,79 +382,88 @@ async function backfillTerminalTraceEvents({ runtimeStore = null, traceStore = d
const commandId = textValue(checkpoint?.commandId ?? payload?.agentRun?.commandId) || null;
const assistantMessageId = textValue(turn?.messageId ?? facts?.messages?.find((message) => message.role !== "user" && message.traceId === safeId)?.messageId) || null;
try {
const existingEvents = await existingWorkbenchTraceEvents(runtimeStore, safeId, { traceStore });
if (existingEvents === null) return { written: 0, skipped: true, errorCode: "workbench_terminal_trace_backfill_probe_failed", valuesPrinted: false };
const writeAssistantFinal = !hasTerminalAssistantFinalTraceEvent(existingEvents, finalText);
const writeCompletion = !hasTerminalResultTraceEvent(existingEvents);
if (!writeAssistantFinal && !writeCompletion) return { written: 0, idempotent: true, skippedExisting: true, valuesPrinted: false };
const timestamp = latestTimestamp(turn?.finishedAt, checkpoint?.finishedAt, turn?.lastEventAt, checkpoint?.lastEventAt, turn?.updatedAt, checkpoint?.updatedAt, payload?.updatedAt) ?? new Date().toISOString();
const baseSourceSeq = Math.max(nonNegativeInteger(checkpoint?.sourceSeq), nonNegativeInteger(turn?.sourceSeq));
let offset = 1;
let written = 0;
await writeWorkbenchProjectionEvent({
runtimeStore,
previousCheckpoint: checkpoint,
event: {
traceId: safeId,
sessionId,
turnId,
messageId: assistantMessageId,
source: "workbench-terminal-projection",
sourceSeq: baseSourceSeq + offset,
sourceEventId: `${safeId}:workbench-terminal-projection:assistant-final`,
type: "assistant",
eventType: "assistant",
status: "completed",
label: "agentrun:assistant:message",
message: finalText,
text: finalText,
finalResponse: { text: finalText, status: "completed", traceId: safeId, valuesPrinted: false },
final: true,
replyAuthority: true,
terminal: true,
runId,
commandId,
timing: checkpoint?.timing,
startedAt: checkpoint?.startedAt,
lastEventAt: checkpoint?.lastEventAt,
finishedAt: checkpoint?.finishedAt,
durationMs: checkpoint?.durationMs,
createdAt: timestamp,
occurredAt: timestamp,
updatedAt: timestamp,
valuesPrinted: false
}
});
written += 1;
offset += 1;
await writeWorkbenchProjectionEvent({
runtimeStore,
previousCheckpoint: checkpoint,
event: {
traceId: safeId,
sessionId,
turnId,
messageId: assistantMessageId,
source: "workbench-terminal-projection",
sourceSeq: baseSourceSeq + offset,
sourceEventId: `${safeId}:workbench-terminal-projection:completion`,
type: "result",
eventType: "terminal",
status: "completed",
label: "agentrun:result:completed",
message: "AgentRun result is ready for HWLAB short-connection polling.",
finalResponse: { text: finalText, status: "completed", traceId: safeId, valuesPrinted: false },
terminal: true,
sealed: true,
runId,
commandId,
timing: checkpoint?.timing,
startedAt: checkpoint?.startedAt,
lastEventAt: checkpoint?.lastEventAt,
finishedAt: checkpoint?.finishedAt,
durationMs: checkpoint?.durationMs,
createdAt: timestamp,
occurredAt: timestamp,
updatedAt: timestamp,
valuesPrinted: false
}
});
written += 1;
if (writeAssistantFinal) {
await writeWorkbenchProjectionEvent({
runtimeStore,
previousCheckpoint: checkpoint,
event: {
traceId: safeId,
sessionId,
turnId,
messageId: assistantMessageId,
source: "workbench-terminal-projection",
sourceSeq: baseSourceSeq + offset,
sourceEventId: `${safeId}:workbench-terminal-projection:assistant-final`,
type: "assistant",
eventType: "assistant",
status: "completed",
label: "agentrun:assistant:message",
message: finalText,
text: finalText,
finalResponse: { text: finalText, status: "completed", traceId: safeId, valuesPrinted: false },
final: true,
replyAuthority: true,
terminal: true,
runId,
commandId,
timing: checkpoint?.timing,
startedAt: checkpoint?.startedAt,
lastEventAt: checkpoint?.lastEventAt,
finishedAt: checkpoint?.finishedAt,
durationMs: checkpoint?.durationMs,
createdAt: timestamp,
occurredAt: timestamp,
updatedAt: timestamp,
valuesPrinted: false
}
});
written += 1;
offset += 1;
}
if (writeCompletion) {
await writeWorkbenchProjectionEvent({
runtimeStore,
previousCheckpoint: checkpoint,
event: {
traceId: safeId,
sessionId,
turnId,
messageId: assistantMessageId,
source: "workbench-terminal-projection",
sourceSeq: baseSourceSeq + offset,
sourceEventId: `${safeId}:workbench-terminal-projection:completion`,
type: "result",
eventType: "terminal",
status: "completed",
label: "agentrun:result:completed",
message: "AgentRun result is ready for HWLAB short-connection polling.",
finalResponse: { text: finalText, status: "completed", traceId: safeId, valuesPrinted: false },
terminal: true,
sealed: true,
runId,
commandId,
timing: checkpoint?.timing,
startedAt: checkpoint?.startedAt,
lastEventAt: checkpoint?.lastEventAt,
finishedAt: checkpoint?.finishedAt,
durationMs: checkpoint?.durationMs,
createdAt: timestamp,
occurredAt: timestamp,
updatedAt: timestamp,
valuesPrinted: false
}
});
written += 1;
}
return { written, idempotent: true, valuesPrinted: false };
} catch (error) {
emitWorkbenchProjectionOtel("workbench.projection.terminal_trace_backfill", safeId, "error", {
@@ -620,7 +629,7 @@ function workbenchProjectionInputMessages({ session = {}, payload = {}, params =
const assistantMessageId = eventProjectionAssistantMessageId(traceId, payload);
const createdAt = timestampValue(timestamp ?? payload?.createdAt ?? payload?.updatedAt);
const messages = [];
const userMessage = workbenchProjectionUserMessage({ payload, params, traceId, turnId, timestamp: createdAt });
const userMessage = workbenchProjectionUserMessage({ payload, params, traceId, turnId, timestamp: createdAt, terminal });
if (userMessage) messages.push(userMessage);
messages.push({
messageId: assistantMessageId,
@@ -641,8 +650,8 @@ function workbenchProjectionInputMessages({ session = {}, payload = {}, params =
return messages;
}
function workbenchProjectionUserMessage({ payload = {}, params = {}, traceId = null, turnId = null, timestamp = null } = {}) {
const userText = textValue(params?.message ?? params?.prompt ?? payload?.prompt ?? payload?.message);
function workbenchProjectionUserMessage({ payload = {}, params = {}, traceId = null, turnId = null, timestamp = null, terminal = false } = {}) {
const userText = textValue(params?.message ?? params?.prompt ?? payload?.prompt ?? payload?.userText ?? payload?.userMessage?.text ?? payload?.userMessage?.content ?? (!terminal ? payload?.message : null));
if (!traceId || !userText) return null;
const createdAt = timestampValue(timestamp ?? payload?.createdAt ?? payload?.updatedAt);
return {
@@ -697,6 +706,50 @@ function messagePartFacts(message = {}, { finalText = null } = {}) {
}];
}
async function existingWorkbenchTraceEvents(runtimeStore, traceId, { traceStore = defaultCodeAgentTraceStore } = {}) {
if (typeof runtimeStore?.queryWorkbenchFacts !== "function") return [];
const safeId = safeTraceId(traceId);
try {
const result = await runtimeStore.queryWorkbenchFacts({ traceId: safeId ?? traceId, families: ["traceEvents"], limit: 500 });
return Array.isArray(result?.facts?.traceEvents) ? result.facts.traceEvents : [];
} catch (error) {
emitWorkbenchProjectionOtel("workbench.projection.terminal_trace_backfill.probe", safeId, "error", {
"workbench.projection.phase": "terminal-trace-backfill-probe"
}, error);
if (safeId) appendProjectionDiagnostic(traceStore, safeId, {
type: "facts",
status: "degraded",
label: "projection-writer:terminal-trace-backfill-probe-failed",
errorCode: error?.code ?? "workbench_terminal_trace_backfill_probe_failed",
message: error?.message ?? "Workbench terminal trace backfill probe failed.",
terminal: false,
valuesPrinted: false
});
return null;
}
}
function hasTerminalAssistantFinalTraceEvent(events = [], finalText = null) {
const expectedText = finalResponseTextValue(finalText);
return events.some((event) => {
const type = textValue(event?.eventType ?? event?.type).toLowerCase();
const label = textValue(event?.label).toLowerCase();
if (type !== "assistant" && !/assistant:message|assistant_message/u.test(label)) return false;
if (event?.terminal !== true && normalizeWorkbenchStatus(event?.status) !== "completed") return false;
const text = finalResponseTextValue(event?.finalResponse, event?.text, event?.message, event?.content);
return !expectedText || text === expectedText;
});
}
function hasTerminalResultTraceEvent(events = []) {
return events.some((event) => {
const type = textValue(event?.eventType ?? event?.type).toLowerCase();
const label = textValue(event?.label).toLowerCase();
const status = normalizeWorkbenchStatus(event?.status);
return (type === "terminal" || type === "result" || /result:completed|result_completed/u.test(label)) && event?.terminal === true && status === "completed";
});
}
function normalizeMessageFact(message = {}, index, { traceId, sessionId, conversationId, threadId, terminal, terminalStatus = "completed", finalText = null, timestamp, timing: contextTiming }) {
const role = textValue(message.role) || "agent";
const messageId = textValue(message.messageId ?? message.id) || (role === "user" ? eventProjectionUserMessageId(traceId, message) : eventProjectionAssistantMessageId(traceId, message));