diff --git a/internal/cloud/server-code-agent-http.ts b/internal/cloud/server-code-agent-http.ts index 99ff851e..b29472cc 100644 --- a/internal/cloud/server-code-agent-http.ts +++ b/internal/cloud/server-code-agent-http.ts @@ -1162,7 +1162,7 @@ export async function handleCodeAgentChatResultHttp(request, response, url, opti return; } if (context.found) { - const body = codeAgentTurnStatusPayload({ traceId, result, snapshot: context.trace, resultPollError: null, refreshError: null, options }); + const body = codeAgentTurnStatusPayload({ traceId, result, snapshot: context.trace, resultPollError: null, refreshError: context.refreshError ?? null, options }); sendJson(response, body.terminal === true ? 200 : 202, codeAgentCompatProjectionPayload({ accepted: body.ok, shortConnection: true, @@ -1214,7 +1214,7 @@ async function resolveCodeAgentTurnStatusSnapshot(traceId, options) { result: context.result, snapshot: context.trace, resultPollError: null, - refreshError: null, + refreshError: context.refreshError ?? null, options })); return { statusCode: body.ok ? 200 : 404, body: codeAgentCompatProjectionPayload(body, context) }; @@ -1330,11 +1330,12 @@ async function readCodeAgentCompatProjection(traceId, options = {}) { const projectionTrace = await readModel.traceSnapshot(traceId); let trace = traceSnapshotWithTerminalEvidence(projectionTrace, result, traceId, null); const refreshed = await refreshAgentRunProjectionForCompatRead({ traceId, result, trace, options }); + if (refreshed.runnerTrace) trace = traceSnapshotWithTerminalEvidence(refreshed.runnerTrace, result, traceId, refreshed.refreshError ?? null); if (refreshed.result && refreshed.result !== result) { result = refreshed.result; trace = traceSnapshotWithTerminalEvidence(refreshed.runnerTrace ?? trace, result, traceId, refreshed.refreshError ?? null); } - const projection = readModel.projectionDiagnostics({ traceId, result, trace }); + const projection = readModel.projectionDiagnostics({ traceId, result, trace, refreshError: refreshed.refreshError ?? null }); const found = Boolean(result || session || (trace && trace.status !== "missing") || trace?.persisted === true); return { result, session, trace, projection, found, refreshError: refreshed.refreshError ?? null }; } @@ -1366,9 +1367,11 @@ function codeAgentCompatProjectionPayload(payload = {}, context = {}) { ...payload, projection, projectionStatus: projection.projectionStatus ?? null, + projectionHealth: projection.projectionHealth ?? null, lastProjectedSeq: projection.lastProjectedSeq ?? null, sourceRunId: projection.sourceRunId ?? null, sourceCommandId: projection.sourceCommandId ?? null, + staleMs: projection.staleMs ?? null, blocker: projection.blocker ?? null, workbench: traceId ? { turnUrl: `/v1/workbench/turns/${encodeURIComponent(traceId)}`, diff --git a/internal/cloud/server-workbench-http.test.ts b/internal/cloud/server-workbench-http.test.ts index 145ffdc3..c771cd1c 100644 --- a/internal/cloud/server-workbench-http.test.ts +++ b/internal/cloud/server-workbench-http.test.ts @@ -7,9 +7,35 @@ import { test } from "bun:test"; import { createCloudApiServer } from "./server.ts"; import { createCodeAgentTraceStore } from "./code-agent-trace-store.ts"; import { createCodeAgentChatResultStore } from "./server-code-agent-http.ts"; +import { createWorkbenchTurnProjection, projectionDiagnostics } from "./workbench-turn-projection.ts"; const ACTOR = { id: "usr_workbench_reader", username: "reader", displayName: "Reader", role: "user", status: "active" }; +test("workbench projection diagnostics exposes AgentRun refresh errors without terminal inference", () => { + const traceId = "trc_projection_refresh_degraded"; + const result = { + traceId, + status: "running", + agentRun: { runId: "run_projection_refresh_degraded", commandId: "cmd_projection_refresh_degraded", status: "running" } + }; + const trace = { traceId, status: "running", events: [], eventCount: 0, updatedAt: "2026-06-18T01:00:00.000Z" }; + const projection = createWorkbenchTurnProjection({ traceId, result, trace }); + const diagnostics = projectionDiagnostics({ + traceId, + result, + trace, + projection, + refreshError: { code: "agentrun_result_timeout", message: "状态更新超时,AgentRun 结果暂不可见。", retryable: true, timeoutMs: 2500 } + }); + assert.equal(projection.status, "running"); + assert.equal(diagnostics.projectionStatus, "blocked"); + assert.equal(diagnostics.projectionHealth, "degraded"); + assert.equal(diagnostics.blocker.code, "agentrun_result_timeout"); + assert.equal(diagnostics.blocker.message, "状态更新超时,AgentRun 结果暂不可见。"); + assert.equal(diagnostics.sourceRunId, "run_projection_refresh_degraded"); + assert.equal(diagnostics.sourceCommandId, "cmd_projection_refresh_degraded"); +}); + test("workbench read model exposes session, messages, turn, and trace without write repair", async () => { const traceStore = createCodeAgentTraceStore(); const results = createCodeAgentChatResultStore(); @@ -751,6 +777,71 @@ test("workbench realtime stream accepts session authority without project or wor } }); +test("workbench read model exposes runtime trace projection query failures as projection blockers", async () => { + const traceStore = createCodeAgentTraceStore(); + const results = createCodeAgentChatResultStore(); + const traceId = "trc_workbench_projection_store_unavailable"; + const session = { + id: "ses_workbench_projection_store_unavailable", + projectId: "prj_hwpod_workbench", + agentId: "hwlab-code-agent", + status: "running", + ownerUserId: ACTOR.id, + conversationId: "cnv_workbench_projection_store_unavailable", + threadId: "thread-workbench-projection-store-unavailable", + lastTraceId: traceId, + updatedAt: "2026-06-18T01:10:00.000Z", + session: { + sessionStatus: "running", + lastTraceId: traceId, + messages: [ + { role: "user", text: "projection store unavailable", traceId, status: "sent", createdAt: "2026-06-18T01:09:58.000Z" }, + { role: "agent", text: "", traceId, status: "running", createdAt: "2026-06-18T01:09:59.000Z" } + ], + valuesRedacted: true, + secretMaterialStored: false + } + }; + traceStore.append(traceId, { seq: 1, type: "backend", status: "running", label: "runner:created", terminal: false, createdAt: "2026-06-18T01:10:00.000Z" }); + results.set(traceId, { status: "running", traceId, ownerUserId: ACTOR.id, sessionId: session.id, threadId: session.threadId }); + const accessController = { + store: { + async getAgentSession(sessionId) { return sessionId === session.id ? session : null; }, + async getAgentSessionByTraceId(requestTraceId) { return requestTraceId === traceId ? session : null; } + }, + async ensureBootstrap() {}, + async authenticate() { return { ok: true, actor: ACTOR, session: { id: "uss_workbench_reader" } }; } + }; + const runtimeStore = { + async queryAgentTraceEvents() { + const error = new Error("runtime query failed"); + error.code = "TEST_RUNTIME_QUERY_FAILED"; + throw error; + } + }; + const server = createCloudApiServer({ accessController, traceStore, runtimeStore, codeAgentChatResults: results }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const turn = await getJson(port, `/v1/workbench/turns/${encodeURIComponent(traceId)}`); + assert.equal(turn.status, 200); + assert.equal(turn.body.turn.status, "running"); + assert.equal(turn.body.projectionHealth, "unavailable"); + assert.equal(turn.body.projection.blocker.code, "projection_store_unavailable"); + assert.equal(turn.body.blocker.code, "projection_store_unavailable"); + + const trace = await getJson(port, `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?limit=10`); + assert.equal(trace.status, 200); + assert.equal(trace.body.events.length, 1); + assert.equal(trace.body.traceStatus, "running"); + assert.equal(trace.body.projectionHealth, "unavailable"); + assert.equal(trace.body.projection.blocker.code, "projection_store_unavailable"); + } finally { + await new Promise((resolve, reject) => server.close((error) => error ? reject(error) : resolve())); + } +}); + async function getJson(port, path) { const response = await fetch(`http://127.0.0.1:${port}${path}`); return { diff --git a/internal/cloud/server-workbench-http.ts b/internal/cloud/server-workbench-http.ts index 2a1938c8..42342c6a 100644 --- a/internal/cloud/server-workbench-http.ts +++ b/internal/cloud/server-workbench-http.ts @@ -306,9 +306,11 @@ async function handleWorkbenchTurnSnapshot(response, url, options, actor, rawTur turn: turnSnapshot({ projection: turnProjection, result, session, trace }), projection, projectionStatus: projection.projectionStatus, + projectionHealth: projection.projectionHealth, lastProjectedSeq: projection.lastProjectedSeq, sourceRunId: projection.sourceRunId, sourceCommandId: projection.sourceCommandId, + staleMs: projection.staleMs, blocker: projection.blocker, valuesRedacted: true, secretMaterialStored: false @@ -342,9 +344,11 @@ async function handleWorkbenchTraceEventPage(response, url, options, actor, rawT ...page, projection, projectionStatus: projection.projectionStatus, + projectionHealth: projection.projectionHealth, lastProjectedSeq: projection.lastProjectedSeq, sourceRunId: projection.sourceRunId, sourceCommandId: projection.sourceCommandId, + staleMs: projection.staleMs, blocker: projection.blocker, valuesRedacted: true, secretMaterialStored: false @@ -385,13 +389,15 @@ async function sessionProjectionOptions(readModel, session, options) { const trace = await readModel.traceSnapshot(traceId); const result = readModel.resultForTrace(traceId); const projection = createWorkbenchTurnProjection({ traceId, result, session, trace }); - return { ...options, trace, result, projection }; + const projectionDiagnostic = readModel.projectionDiagnostics({ traceId, result, trace, projection }); + return { ...options, trace, result, projection, projectionDiagnostic }; } function sessionSummary(session, options) { if (!session?.id) return null; const snapshot = objectValue(session.session); const currentTurn = currentWorkbenchTurnProjection(session, options); + const projectionDiagnostic = options.projectionDiagnostic ?? null; const messages = sessionMessages(session, { ...options, currentTurn }); const status = currentTurn ? currentTurn.status : sessionLifecycleProjectionStatus(session); return { @@ -402,6 +408,11 @@ function sessionSummary(session, options) { running: RUNNING_STATUSES.has(status), terminal: TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status), lastTraceId: currentTurn ? currentTurn.traceId : null, + projection: projectionDiagnostic, + projectionStatus: projectionDiagnostic?.projectionStatus ?? null, + projectionHealth: projectionDiagnostic?.projectionHealth ?? null, + staleMs: projectionDiagnostic?.staleMs ?? null, + blocker: projectionDiagnostic?.blocker ?? null, providerProfile: textValue(snapshot.providerProfile) || null, messageCount: messages.length, firstUserMessagePreview: firstUserPreview(messages), @@ -413,6 +424,11 @@ function sessionSummary(session, options) { running: RUNNING_STATUSES.has(status), terminal: TERMINAL_STATUSES.has(status) && !RUNNING_STATUSES.has(status), eventCount: currentTurn.eventCount, + projection: projectionDiagnostic, + projectionStatus: projectionDiagnostic?.projectionStatus ?? null, + projectionHealth: projectionDiagnostic?.projectionHealth ?? null, + staleMs: projectionDiagnostic?.staleMs ?? null, + blocker: projectionDiagnostic?.blocker ?? null, updatedAt: currentTurn.updatedAt } : null, valuesRedacted: true @@ -442,7 +458,8 @@ function sessionMessages(session, options = {}) { const messages = raw .map((message, index) => messageFact(message, index, session, snapshot)) .filter(Boolean) - .map((message) => applyTerminalMessageProjection(message, terminalProjection)); + .map((message) => applyTerminalMessageProjection(message, terminalProjection)) + .map((message) => applyProjectionMessageDiagnostic(message, options.projectionDiagnostic, currentTurn)); const hasAssistantLikeFinal = messages.some((message) => isAssistantLikeRole(message.role) && (!finalTraceId || message.traceId === finalTraceId)); const finalText = terminalProjection?.text ?? projectionText(snapshot.finalResponse); if (!hasAssistantLikeFinal && finalText) { @@ -453,6 +470,21 @@ function sessionMessages(session, options = {}) { return messages; } +function applyProjectionMessageDiagnostic(message, diagnostic, currentTurn) { + if (!diagnostic || !isAssistantLikeRole(message.role)) return message; + if (currentTurn?.traceId && message.traceId && message.traceId !== currentTurn.traceId) return message; + const trace = message.runnerTrace && typeof message.runnerTrace === "object" ? message.runnerTrace : null; + return { + ...message, + projection: diagnostic, + projectionStatus: diagnostic.projectionStatus ?? null, + projectionHealth: diagnostic.projectionHealth ?? null, + staleMs: diagnostic.staleMs ?? null, + blocker: diagnostic.blocker ?? null, + runnerTrace: trace ? { ...trace, projection: diagnostic, projectionStatus: diagnostic.projectionStatus ?? null, projectionHealth: diagnostic.projectionHealth ?? null, staleMs: diagnostic.staleMs ?? null, blocker: diagnostic.blocker ?? null } : trace + }; +} + function terminalMessageProjection(snapshot, options, currentTurn) { if (!currentTurn) return null; const traceId = currentTurn.traceId; diff --git a/internal/cloud/workbench-facts-store.ts b/internal/cloud/workbench-facts-store.ts index d4cd6a0c..27451569 100644 --- a/internal/cloud/workbench-facts-store.ts +++ b/internal/cloud/workbench-facts-store.ts @@ -76,6 +76,7 @@ export function createWorkbenchFactsStore(options = {}, actor = null) { const memory = traceSnapshotSync(traceId); if (hasTraceProjection(memory) && TERMINAL_STATUSES.has(normalizeStatus(memory.status))) return memory; const durable = await durableTraceSnapshot(runtimeStore, traceId); + if (isProjectionDiagnosticTrace(durable) && hasTraceProjection(memory)) return mergeTraceProjectionDiagnostic(memory, durable); if (durable && shouldPreferDurableTrace(memory, durable)) return durable; if (hasTraceProjection(memory)) return memory; return durable ?? memory; @@ -103,8 +104,8 @@ async function durableTraceSnapshot(runtimeStore, traceId) { let result; try { result = await runtimeStore.queryAgentTraceEvents({ traceId }); - } catch { - return null; + } catch (error) { + return projectionStoreUnavailableTrace(traceId, error); } const events = Array.isArray(result?.events) ? result.events : []; if (events.length === 0) return null; @@ -131,6 +132,51 @@ async function durableTraceSnapshot(runtimeStore, traceId) { }; } +function projectionStoreUnavailableTrace(traceId, error) { + const message = "运行记录投影存储暂不可读,Trace 更新暂不可见。"; + const projection = { + projectionStatus: "blocked", + projectionHealth: "unavailable", + blocker: { + code: "projection_store_unavailable", + layer: "runtime-store", + category: "projection-store-unavailable", + message, + causeCode: error?.code ?? null, + retryable: true, + valuesPrinted: false + }, + valuesRedacted: true + }; + return { + traceId, + status: "missing", + eventCount: 0, + events: [], + projection, + projectionStatus: projection.projectionStatus, + projectionHealth: projection.projectionHealth, + blocker: projection.blocker, + updatedAt: new Date().toISOString(), + valuesPrinted: false + }; +} + +function isProjectionDiagnosticTrace(snapshot) { + return Boolean(snapshot?.projection?.blocker || snapshot?.projectionHealth || snapshot?.blocker); +} + +function mergeTraceProjectionDiagnostic(memory, diagnostic) { + return { + ...memory, + projection: diagnostic.projection ?? memory?.projection ?? null, + projectionStatus: diagnostic.projectionStatus ?? diagnostic.projection?.projectionStatus ?? memory?.projectionStatus ?? null, + projectionHealth: diagnostic.projectionHealth ?? diagnostic.projection?.projectionHealth ?? memory?.projectionHealth ?? null, + blocker: diagnostic.blocker ?? diagnostic.projection?.blocker ?? memory?.blocker ?? null, + valuesPrinted: false + }; +} + function hasTraceProjection(snapshot) { return Boolean(snapshot?.status !== "missing" && (Number(snapshot?.eventCount ?? 0) > 0 || snapshot?.lastEvent)); } diff --git a/internal/cloud/workbench-read-model.ts b/internal/cloud/workbench-read-model.ts index c2b3f2f2..7219bdc3 100644 --- a/internal/cloud/workbench-read-model.ts +++ b/internal/cloud/workbench-read-model.ts @@ -18,6 +18,6 @@ export function createWorkbenchReadModel(options = {}, actor = null) { traceSnapshotSync: (traceId) => facts.traceSnapshotSync(traceId), subscribeTrace: (traceId, listener) => facts.subscribeTrace(traceId, listener), canReadOwner: (ownerUserId) => facts.canReadOwner(ownerUserId), - projectionDiagnostics: ({ traceId, result = null, trace = null, projection = null } = {}) => workbenchProjectionDiagnostics({ traceId, result, trace, projection }) + projectionDiagnostics: ({ traceId, result = null, trace = null, projection = null, refreshError = null } = {}) => workbenchProjectionDiagnostics({ traceId, result, trace, projection, refreshError }) }; } diff --git a/internal/cloud/workbench-turn-projection.ts b/internal/cloud/workbench-turn-projection.ts index c60a92ac..baa65cb4 100644 --- a/internal/cloud/workbench-turn-projection.ts +++ b/internal/cloud/workbench-turn-projection.ts @@ -36,17 +36,25 @@ export function createWorkbenchTurnProjection({ turnId = null, traceId = null, r }; } -export function projectionDiagnostics({ traceId = null, projection = null, result = null, trace = null } = {}) { +export function projectionDiagnostics({ traceId = null, projection = null, result = null, trace = null, refreshError = null } = {}) { const turn = projection ?? createWorkbenchTurnProjection({ traceId, result, trace }); - const blocker = result?.blocker ?? result?.error ?? null; + const source = projectionDiagnosticSource(trace); + const blocker = refreshError ?? source?.blocker ?? trace?.blocker ?? result?.blocker ?? result?.error ?? null; const hasProjectionInput = hasTraceProjection(trace) || Boolean(result || result?.agentRun); - const status = blocker ? "blocked" : turn.terminal ? "caught-up" : hasProjectionInput ? "projecting" : "unknown"; + const sourceStatus = normalizeProjectionStatus(source?.projectionStatus ?? trace?.projectionStatus); + const status = sourceStatus ?? (blocker ? "blocked" : turn.terminal ? "caught-up" : hasProjectionInput ? "projecting" : "unknown"); + const diagnostic = blocker ? diagnosticBlocker(blocker) : null; + const projectionHealth = normalizeProjectionHealth(source?.projectionHealth ?? trace?.projectionHealth) + ?? projectionHealthFor({ status, turn, hasProjectionInput, blocker: diagnostic }); + const staleMs = projectionStaleMs(source?.staleMs ?? trace?.staleMs, turn.updatedAt ?? trace?.updatedAt ?? result?.updatedAt); return { projectionStatus: status, + projectionHealth, lastProjectedSeq: turn.lastProjectedSeq ?? null, sourceRunId: turn.sourceRunId ?? null, sourceCommandId: turn.sourceCommandId ?? null, - blocker: blocker ? diagnosticBlocker(blocker) : null, + staleMs, + blocker: diagnostic, updatedAt: turn.updatedAt ?? null, valuesRedacted: true }; @@ -166,6 +174,38 @@ function traceLastEvent(trace) { return trace?.lastEvent ?? events.at(-1) ?? null; } +function projectionDiagnosticSource(trace) { + const direct = objectValue(trace?.projection); + if (direct) return direct; + if (trace?.projectionHealth || trace?.projectionStatus || trace?.blocker) return trace; + return null; +} + +function normalizeProjectionStatus(value) { + const text = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); + return ["caught-up", "projecting", "blocked", "stalled", "unknown"].includes(text) ? text : null; +} + +function normalizeProjectionHealth(value) { + const text = String(value ?? "").trim().toLowerCase().replace(/_/gu, "-"); + return ["caught-up", "degraded", "stalled", "unavailable", "unknown"].includes(text) ? text : null; +} + +function projectionHealthFor({ status, turn, hasProjectionInput, blocker }) { + if (blocker?.code === "projection_store_unavailable") return "unavailable"; + if (blocker) return status === "stalled" ? "stalled" : "degraded"; + if (turn.terminal || status === "caught-up" || hasProjectionInput) return "caught-up"; + return "unknown"; +} + +function projectionStaleMs(value, updatedAt) { + const direct = Number(value); + if (Number.isFinite(direct) && direct >= 0) return Math.trunc(direct); + const updatedAtMs = Date.parse(String(updatedAt ?? "")); + if (!Number.isFinite(updatedAtMs)) return null; + return Math.max(0, Date.now() - updatedAtMs); +} + function eventSeq(event, index) { const seq = Number(event?.seq); return Number.isFinite(seq) && seq > 0 ? Math.trunc(seq) : index + 1; @@ -196,8 +236,15 @@ function diagnosticBlocker(value = {}) { if (!value || typeof value !== "object") return { code: "projection_blocked", summary: String(value), valuesRedacted: true }; return { code: String(value.code ?? value.errorCode ?? "projection_blocked"), + layer: textValue(value.layer ?? value.source) || null, + category: textValue(value.category ?? value.type) || null, summary: String(value.summary ?? value.userMessage ?? value.message ?? value.code ?? "Projection is blocked."), + message: textValue(value.message ?? value.summary ?? value.userMessage) || null, + userMessage: textValue(value.userMessage ?? value.zh) || null, retryable: typeof value.retryable === "boolean" ? value.retryable : null, + runId: textValue(value.runId) || null, + commandId: textValue(value.commandId) || null, + timeoutMs: Number.isFinite(Number(value.timeoutMs)) ? Math.trunc(Number(value.timeoutMs)) : null, valuesRedacted: true }; } diff --git a/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts b/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts index a50fb64d..b7879b9b 100644 --- a/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts +++ b/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts @@ -111,13 +111,14 @@ async function handleRequest(request: IncomingMessage, response: ServerResponse) const traceId = decodeURIComponent(turnMatch[1] ?? ""); if (state.scenarioId === "completed-replay-detail-404" && traceId === "trc_completed") return json(response, 404, { ok: false, status: 404, error: { code: "turn_replay_unavailable" } }); if (traceId === state.staleTraceId) return json(response, 502, { ok: false, status: 502, error: { code: "upstream_unavailable", message: "stale trace is unavailable" } }); - return json(response, 200, { ok: true, status: "ok", contractVersion: "workbench-read-model-v1", turn: turnPayload(traceId) }); + return json(response, 200, workbenchTurnPayload(traceId)); } const traceMatch = path.match(/^\/v1\/workbench\/traces\/([^/]+)\/events$/u); if (traceMatch && method === "GET") { const traceId = decodeURIComponent(traceMatch[1] ?? ""); if (state.scenarioId === "completed-replay-detail-404" && traceId === "trc_completed") return json(response, 404, { ok: false, status: 404, error: { code: "trace_replay_unavailable" } }); + if (state.scenarioId === "trace-hydration-timeout-diagnostic" && traceId === "trc_trace_hydration_timeout") return json(response, 504, { ok: false, status: 504, error: { code: "trace_hydration_timeout", message: "Trace 更新超时,运行记录暂不可见。" } }); if (traceId === state.staleTraceId) return json(response, 502, { ok: false, status: 502, error: { code: "upstream_unavailable", message: "stale trace is unavailable" } }); return json(response, 200, workbenchTracePayload(traceId, url)); } @@ -283,6 +284,18 @@ function createScenarioState(scenarioId: string): ScenarioState { if (id === "progress-only-final-response") markRunningProgressOnly(sessions, traces); if (id === "terminal-completed-no-final-response") markRunningNoFinalResponse(sessions, traces); if (id === "tool-completed-projection-running") markToolCompletedProjectionRunning(sessions, traces); + if (id === "projection-degraded-diagnostics") { + sessions.unshift(projectionDegradedSession()); + traces.trc_projection_degraded = projectionDegradedTrace(); + } + if (id === "projection-sse-error") { + sessions.unshift(projectionSseErrorSession()); + traces.trc_projection_sse_error = projectionSseErrorTrace(); + } + if (id === "trace-hydration-timeout-diagnostic") { + sessions.unshift(traceHydrationTimeoutSession()); + traces.trc_trace_hydration_timeout = traceHydrationTimeoutTrace(); + } if (id === "scroll-follow-long-trace") { const session = scrollFollowSession(); sessions.unshift(session); @@ -307,6 +320,12 @@ function createScenarioState(scenarioId: string): ScenarioState { ? "ses_terminal_empty" : id === "progress-only-final-response" || id === "terminal-completed-no-final-response" || id === "tool-completed-projection-running" ? "ses_running" + : id === "projection-degraded-diagnostics" + ? "ses_projection_degraded" + : id === "projection-sse-error" + ? "ses_projection_sse_error" + : id === "trace-hydration-timeout-diagnostic" + ? "ses_trace_hydration_timeout" : id === "create-while-route-loading" ? "ses_running" : base.selectedSessionId; @@ -413,6 +432,95 @@ function toolCompletedProjectionRunningTrace(): JsonRecord { }; } +function projectionDiagnostic(input: { code: string; message: string; health?: string; category?: string; layer?: string; traceId: string; runId?: string; commandId?: string; staleMs?: number }): JsonRecord { + return { + projectionStatus: "projecting", + projectionHealth: input.health ?? "degraded", + lastProjectedSeq: 137, + sourceRunId: input.runId ?? "run_projection_diag", + sourceCommandId: input.commandId ?? "cmd_projection_diag", + staleMs: input.staleMs ?? 12_345, + blocker: { + code: input.code, + layer: input.layer ?? "agentrun", + category: input.category ?? "upstream-timeout", + message: input.message, + retryable: true, + timeoutMs: 2500, + runId: input.runId ?? "run_projection_diag", + commandId: input.commandId ?? "cmd_projection_diag", + valuesPrinted: false + }, + valuesRedacted: true + }; +} + +function projectionDegradedTrace(): JsonRecord { + const projection = projectionDiagnostic({ traceId: "trc_projection_degraded", code: "agentrun_result_timeout", message: "状态更新超时,最近投影 seq=137,AgentRun 结果暂不可见。" }); + return { traceId: "trc_projection_degraded", status: "running", sessionId: "ses_projection_degraded", threadId: "thr_projection_degraded", turnId: "turn_projection_degraded", events: [], eventCount: 0, fullTraceLoaded: true, hasMore: false, projection, ...projection }; +} + +function projectionDegradedSession(): SessionRecord { + const now = new Date().toISOString(); + const projection = projectionDiagnostic({ traceId: "trc_projection_degraded", code: "agentrun_result_timeout", message: "状态更新超时,最近投影 seq=137,AgentRun 结果暂不可见。" }); + return { + sessionId: "ses_projection_degraded", + threadId: "thr_projection_degraded", + status: "running", + lastTraceId: "trc_projection_degraded", + updatedAt: now, + messageCount: 2, + firstUserMessagePreview: "projection degraded visibility", + turnSummary: { traceId: "trc_projection_degraded", status: "running", running: true, terminal: false, ...projection }, + messages: [ + { id: "msg_projection_degraded_user", messageId: "msg_projection_degraded_user", role: "user", title: "用户", text: "projection degraded visibility", status: "sent", createdAt: now, sessionId: "ses_projection_degraded", threadId: "thr_projection_degraded", turnId: "turn_projection_degraded" }, + { id: "msg_projection_degraded_agent", messageId: "msg_projection_degraded_agent", role: "agent", title: "Code Agent", text: "", status: "running", createdAt: now, sessionId: "ses_projection_degraded", threadId: "thr_projection_degraded", traceId: "trc_projection_degraded", turnId: "turn_projection_degraded", projection, runnerTrace: projectionDegradedTrace() } + ] + }; +} + +function projectionSseErrorTrace(): JsonRecord { + return { traceId: "trc_projection_sse_error", status: "running", sessionId: "ses_projection_sse_error", threadId: "thr_projection_sse_error", turnId: "turn_projection_sse_error", events: [], eventCount: 0, fullTraceLoaded: false, hasMore: false }; +} + +function projectionSseErrorSession(): SessionRecord { + const now = new Date().toISOString(); + return { + sessionId: "ses_projection_sse_error", + threadId: "thr_projection_sse_error", + status: "running", + lastTraceId: "trc_projection_sse_error", + updatedAt: now, + messageCount: 2, + firstUserMessagePreview: "projection SSE error visibility", + messages: [ + { id: "msg_projection_sse_error_user", messageId: "msg_projection_sse_error_user", role: "user", title: "用户", text: "projection SSE error visibility", status: "sent", createdAt: now, sessionId: "ses_projection_sse_error", threadId: "thr_projection_sse_error", turnId: "turn_projection_sse_error" }, + { id: "msg_projection_sse_error_agent", messageId: "msg_projection_sse_error_agent", role: "agent", title: "Code Agent", text: "", status: "running", createdAt: now, sessionId: "ses_projection_sse_error", threadId: "thr_projection_sse_error", traceId: "trc_projection_sse_error", turnId: "turn_projection_sse_error", runnerTrace: projectionSseErrorTrace() } + ] + }; +} + +function traceHydrationTimeoutTrace(): JsonRecord { + return { traceId: "trc_trace_hydration_timeout", status: "running", sessionId: "ses_trace_hydration_timeout", threadId: "thr_trace_hydration_timeout", turnId: "turn_trace_hydration_timeout", events: [], eventCount: 0, fullTraceLoaded: false, hasMore: true }; +} + +function traceHydrationTimeoutSession(): SessionRecord { + const now = new Date().toISOString(); + return { + sessionId: "ses_trace_hydration_timeout", + threadId: "thr_trace_hydration_timeout", + status: "running", + lastTraceId: "trc_trace_hydration_timeout", + updatedAt: now, + messageCount: 2, + firstUserMessagePreview: "trace hydration timeout visibility", + messages: [ + { id: "msg_trace_hydration_timeout_user", messageId: "msg_trace_hydration_timeout_user", role: "user", title: "用户", text: "trace hydration timeout visibility", status: "sent", createdAt: now, sessionId: "ses_trace_hydration_timeout", threadId: "thr_trace_hydration_timeout", turnId: "turn_trace_hydration_timeout" }, + { id: "msg_trace_hydration_timeout_agent", messageId: "msg_trace_hydration_timeout_agent", role: "agent", title: "Code Agent", text: "", status: "running", createdAt: now, sessionId: "ses_trace_hydration_timeout", threadId: "thr_trace_hydration_timeout", traceId: "trc_trace_hydration_timeout", turnId: "turn_trace_hydration_timeout", runnerTrace: traceHydrationTimeoutTrace() } + ] + }; +} + function noFinalRunningTrace(): JsonRecord { const createdAt = new Date().toISOString(); return { @@ -921,6 +1029,11 @@ function turnPayload(traceId: string): JsonRecord { return { ...trace, traceId, status, running: ["running", "pending", "accepted"].includes(status), terminal: ["completed", "failed", "blocked", "timeout", "canceled"].includes(status) }; } +function workbenchTurnPayload(traceId: string): JsonRecord { + const turn = turnPayload(traceId); + return { ok: true, status: "ok", contractVersion: "workbench-read-model-v1", turn, ...projectionEnvelope(turn) }; +} + function tracePayload(traceId: string, url: URL): JsonRecord { const turn = turnPayload(traceId); const events = Array.isArray(turn.events) ? turn.events as JsonRecord[] : []; @@ -939,7 +1052,22 @@ function workbenchTracePayload(traceId: string, url: URL): JsonRecord { if (state.scenarioId === "tool-completed-projection-running" && traceId === "trc_running") { return { ok: true, status: "ok", contractVersion: "workbench-read-model-v1", traceId, sessionId: payload.sessionId ?? null, threadId: payload.threadId ?? null, traceStatus: "completed", events: payload.events, eventCount: payload.eventCount, hasMore: payload.hasMore, nextSeq: payload.nextSinceSeq, range: payload.range, fullTraceLoaded: payload.fullTraceLoaded, projectionStatus: "projecting", terminalEvidence: null, finalResponse: null, traceSummary: payload.traceSummary, retention: payload.retention }; } - return { ok: true, status: "ok", contractVersion: "workbench-read-model-v1", traceId, sessionId: payload.sessionId ?? null, threadId: payload.threadId ?? null, traceStatus: payload.status, events: payload.events, eventCount: payload.eventCount, hasMore: payload.hasMore, nextSeq: payload.nextSinceSeq, range: payload.range, fullTraceLoaded: payload.fullTraceLoaded, terminalEvidence: payload.terminalEvidence, finalResponse: payload.finalResponse, traceSummary: payload.traceSummary, retention: payload.retention }; + return { ok: true, status: "ok", contractVersion: "workbench-read-model-v1", traceId, sessionId: payload.sessionId ?? null, threadId: payload.threadId ?? null, traceStatus: payload.status, events: payload.events, eventCount: payload.eventCount, hasMore: payload.hasMore, nextSeq: payload.nextSinceSeq, range: payload.range, fullTraceLoaded: payload.fullTraceLoaded, terminalEvidence: payload.terminalEvidence, finalResponse: payload.finalResponse, traceSummary: payload.traceSummary, retention: payload.retention, ...projectionEnvelope(payload) }; +} + +function projectionEnvelope(payload: JsonRecord): JsonRecord { + const projection = payload.projection && typeof payload.projection === "object" ? payload.projection as JsonRecord : null; + if (!projection) return {}; + return { + projection, + projectionStatus: projection.projectionStatus, + projectionHealth: projection.projectionHealth, + lastProjectedSeq: projection.lastProjectedSeq, + sourceRunId: projection.sourceRunId, + sourceCommandId: projection.sourceCommandId, + staleMs: projection.staleMs, + blocker: projection.blocker + }; } function sse(request: IncomingMessage, response: ServerResponse, url: URL): void { @@ -947,6 +1075,14 @@ function sse(request: IncomingMessage, response: ServerResponse, url: URL): void sseClients.add(response); request.on("close", () => sseClients.delete(response)); writeSse(response, "workbench.connected", { type: "connected", sessionId: url.searchParams.get("sessionId") }); + if (state.scenarioId === "projection-sse-error") { + const scenarioId = state.scenarioId; + setTimeout(() => { + if (state.scenarioId !== scenarioId) return; + const projection = projectionDiagnostic({ traceId: "trc_projection_sse_error", code: "workbench_sse_projection_error", message: "SSE 投影事件异常,实时状态暂不可见。", category: "realtime-error", layer: "workbench-sse", runId: "run_projection_sse_error", commandId: "cmd_projection_sse_error" }); + writeSse(response, "workbench.error", { type: "error", sessionId: "ses_projection_sse_error", threadId: "thr_projection_sse_error", traceId: "trc_projection_sse_error", projection, ...projectionEnvelope({ projection }), error: projection.blocker }); + }, 350); + } if (state.scenarioId === "cross-session-late-events") { const scenarioId = state.scenarioId; setTimeout(() => { diff --git a/web/hwlab-cloud-web/src/api/workbench-events.ts b/web/hwlab-cloud-web/src/api/workbench-events.ts index 7f8a7813..08002410 100644 --- a/web/hwlab-cloud-web/src/api/workbench-events.ts +++ b/web/hwlab-cloud-web/src/api/workbench-events.ts @@ -1,7 +1,7 @@ // SPEC: PJ2026-010403 API契约 draft-2026-06-18-r1; PJ2026-010401 Web工作台 draft-2026-06-18-r1; PJ2026-01060505 Workbench Performance draft-2026-06-17-p0. // Responsibility: Workbench SSE client. Realtime events accelerate UI projection; REST snapshots remain gap-fill authority. -import type { TraceEvent } from "@/types"; +import type { ProjectionDiagnostic, TraceEvent } from "@/types"; export interface WorkbenchRealtimeTraceSnapshot { traceId?: string | null; @@ -10,6 +10,11 @@ export interface WorkbenchRealtimeTraceSnapshot { eventCount?: number | null; lastEvent?: TraceEvent | null; updatedAt?: string | null; + projection?: ProjectionDiagnostic | null; + projectionStatus?: string | null; + projectionHealth?: string | null; + staleMs?: number | null; + blocker?: ProjectionDiagnostic["blocker"] | null; [key: string]: unknown; } @@ -25,7 +30,7 @@ export interface WorkbenchRealtimeEvent { snapshot?: WorkbenchRealtimeTraceSnapshot | null; turn?: Record | null; reason?: string | null; - error?: { code?: string; message?: string } | null; + error?: { code?: string; message?: string; [key: string]: unknown } | null; traceSeq?: number | null; cursor?: { traceSeq?: number | null; [key: string]: unknown }; [key: string]: unknown; diff --git a/web/hwlab-cloud-web/src/api/workbench.ts b/web/hwlab-cloud-web/src/api/workbench.ts index 120b3f7f..4d0f2aae 100644 --- a/web/hwlab-cloud-web/src/api/workbench.ts +++ b/web/hwlab-cloud-web/src/api/workbench.ts @@ -2,7 +2,7 @@ // Responsibility: Workbench read-model HTTP client for session, message, turn and trace projection APIs. import { fetchJson, type ApiRequestOptions } from "./client"; -import type { AgentChatResultResponse, ApiResult, ChatMessage, WorkbenchSessionRecord } from "@/types"; +import type { AgentChatResultResponse, ApiResult, ChatMessage, ProjectionDiagnostic, WorkbenchSessionRecord } from "@/types"; export interface WorkbenchSessionListResponse { sessions?: WorkbenchSessionRecord[]; @@ -92,13 +92,19 @@ function normalizeWorkbenchTurnResult(result: ApiResult> const turn = recordValue(result.data.turn) ?? result.data; const trace = recordValue(turn.trace); const status = normalizeStatus(textValue(turn.status)) ?? "unknown"; + const projection = normalizeProjectionDiagnostic(result.data.projection, turn.projection, result.data); return { ...result, data: { ...turn, - projection: recordValue(result.data.projection) ?? undefined, - projectionStatus: textValue(result.data.projectionStatus) ?? undefined, + projection, + projectionStatus: projection?.projectionStatus ?? textValue(result.data.projectionStatus) ?? undefined, + projectionHealth: projection?.projectionHealth ?? textValue(result.data.projectionHealth) ?? undefined, lastProjectedSeq: Number.isFinite(Number(result.data.lastProjectedSeq)) ? Number(result.data.lastProjectedSeq) : undefined, + sourceRunId: projection?.sourceRunId ?? textValue(result.data.sourceRunId) ?? undefined, + sourceCommandId: projection?.sourceCommandId ?? textValue(result.data.sourceCommandId) ?? undefined, + staleMs: projection?.staleMs ?? numberValue(result.data.staleMs) ?? undefined, + blocker: projection?.blocker ?? recordValue(result.data.blocker) ?? undefined, traceId: textValue(turn.traceId) ?? traceId, status, running: typeof turn.running === "boolean" ? turn.running : undefined, @@ -120,6 +126,7 @@ function normalizeWorkbenchTraceResult(result: ApiResult const normalizedNextSeq = Number.isFinite(nextSeq) ? Math.trunc(nextSeq) : null; const hasMore = result.data.hasMore === true; const fullTraceLoaded = typeof result.data.fullTraceLoaded === "boolean" ? result.data.fullTraceLoaded : undefined; + const projection = normalizeProjectionDiagnostic(result.data.projection, result.data); return { ...result, data: { @@ -132,6 +139,11 @@ function normalizeWorkbenchTraceResult(result: ApiResult eventCount, hasMore, fullTraceLoaded, + projection, + projectionStatus: projection?.projectionStatus ?? textValue(result.data.projectionStatus) ?? undefined, + projectionHealth: projection?.projectionHealth ?? textValue(result.data.projectionHealth) ?? undefined, + staleMs: projection?.staleMs ?? numberValue(result.data.staleMs) ?? undefined, + blocker: projection?.blocker ?? recordValue(result.data.blocker) ?? undefined, nextSinceSeq: normalizedNextSeq, range: recordValue(result.data.range) ?? undefined, runnerTrace: { @@ -143,6 +155,11 @@ function normalizeWorkbenchTraceResult(result: ApiResult fullTraceLoaded, nextSinceSeq: normalizedNextSeq, range: recordValue(result.data.range) ?? undefined, + projection, + projectionStatus: projection?.projectionStatus ?? textValue(result.data.projectionStatus) ?? undefined, + projectionHealth: projection?.projectionHealth ?? textValue(result.data.projectionHealth) ?? undefined, + staleMs: projection?.staleMs ?? numberValue(result.data.staleMs) ?? undefined, + blocker: projection?.blocker ?? recordValue(result.data.blocker) ?? undefined, eventSource: "trace-api" } } as AgentChatResultResponse @@ -160,6 +177,39 @@ function recordValue(value: unknown): Record | null { return value && typeof value === "object" ? value as Record : null; } +function normalizeProjectionDiagnostic(...values: unknown[]): ProjectionDiagnostic | undefined { + for (const value of values) { + const record = recordValue(value); + if (!record) continue; + const source = recordValue(record.projection) ?? record; + const blocker = recordValue(source.blocker ?? record.blocker); + const projectionStatus = textValue(source.projectionStatus ?? record.projectionStatus); + const projectionHealth = textValue(source.projectionHealth ?? record.projectionHealth); + const lastProjectedSeq = numberValue(source.lastProjectedSeq ?? record.lastProjectedSeq); + const staleMs = numberValue(source.staleMs ?? record.staleMs); + const sourceRunId = textValue(source.sourceRunId ?? record.sourceRunId); + const sourceCommandId = textValue(source.sourceCommandId ?? record.sourceCommandId); + if (!projectionStatus && !projectionHealth && !blocker && lastProjectedSeq == null && staleMs == null && !sourceRunId && !sourceCommandId) continue; + return { + ...source, + projectionStatus: projectionStatus ?? undefined, + projectionHealth: projectionHealth ?? undefined, + lastProjectedSeq, + sourceRunId: sourceRunId ?? undefined, + sourceCommandId: sourceCommandId ?? undefined, + staleMs, + blocker: blocker ?? null, + valuesRedacted: source.valuesRedacted !== false + } as ProjectionDiagnostic; + } + return undefined; +} + +function numberValue(value: unknown): number | null { + const number = Number(value); + return Number.isFinite(number) ? number : null; +} + function textValue(...values: unknown[]): string | null { for (const value of values) { if (typeof value !== "string") continue; diff --git a/web/hwlab-cloud-web/src/components/agent/TraceTimeline.vue b/web/hwlab-cloud-web/src/components/agent/TraceTimeline.vue index 95eec8da..a4fc2faa 100644 --- a/web/hwlab-cloud-web/src/components/agent/TraceTimeline.vue +++ b/web/hwlab-cloud-web/src/components/agent/TraceTimeline.vue @@ -15,7 +15,14 @@ const { following, keepBottomAfterUpdate, onScroll, scrollToBottom } = useBottom const events = computed(() => Array.isArray(props.trace?.events) ? props.trace.events : []); const eventCount = computed(() => props.trace?.eventCount ?? events.value.length); const readableRows = computed(() => traceDisplayRows(traceRecord(props.trace), events.value as Record[])); +const projectionDiagnosticLabel = computed(() => { + const projection = props.trace?.projection ?? null; + const health = projection?.projectionHealth ?? props.trace?.projectionHealth; + if (!projection || health === "caught-up") return null; + return firstNonEmptyString(projection.blocker?.userMessage, projection.blocker?.message, projection.blocker?.summary, projection.blocker?.code ? `状态更新异常:${projection.blocker.code}` : null); +}); const emptyTraceLabel = computed(() => { + if (projectionDiagnosticLabel.value) return projectionDiagnosticLabel.value; if (isTerminalTraceStatus(props.trace?.status)) return terminalTraceEmptyLabel(props.trace?.status); if (events.value.length > 0 || (props.trace?.fullTraceLoaded === true && eventCount.value > 0)) return "暂无可读 Trace"; return props.trace?.fullTraceLoaded === true ? "思考中..." : "加载中..."; @@ -106,10 +113,19 @@ function terminalTraceEmptyLabel(status: unknown): string { } } +function firstNonEmptyString(...values: unknown[]): string | null { + for (const value of values) { + if (typeof value !== "string") continue; + const text = value.trim(); + if (text) return text; + } + return null; +} +