From eca9dbaa9c89822765c73963a73d9218c3261c34 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Thu, 4 Jun 2026 20:40:36 +0800 Subject: [PATCH] fix: persist expired code agent trace summaries (#851) Co-authored-by: Codex Agent --- internal/cloud/code-agent-agentrun-adapter.ts | 2 + internal/cloud/server-agent-chat.test.ts | 88 ++++++++ internal/cloud/server-code-agent-http.ts | 192 +++++++++++++++++- tools/hwlab-cli/client.test.ts | 47 +++++ tools/src/hwlab-cli-lib.ts | 54 ++++- 5 files changed, 375 insertions(+), 8 deletions(-) diff --git a/internal/cloud/code-agent-agentrun-adapter.ts b/internal/cloud/code-agent-agentrun-adapter.ts index e3a77e73..7c0b7a0c 100644 --- a/internal/cloud/code-agent-agentrun-adapter.ts +++ b/internal/cloud/code-agent-agentrun-adapter.ts @@ -473,6 +473,8 @@ export async function loadPersistedAgentRunResult(traceId, options = {}) { capabilityLevel: CODEX_STDIO_CAPABILITY_LEVEL, sessionMode: CODEX_STDIO_SESSION_MODE, implementationType: CODEX_STDIO_IMPLEMENTATION_TYPE, + finalResponse: session.session?.finalResponse ?? null, + traceSummary: session.session?.traceSummary ?? null, agentRun: { ...agentRun, adapter: ADAPTER_ID, valuesPrinted: false }, valuesPrinted: false }; diff --git a/internal/cloud/server-agent-chat.test.ts b/internal/cloud/server-agent-chat.test.ts index d45d9af1..ab360be6 100644 --- a/internal/cloud/server-agent-chat.test.ts +++ b/internal/cloud/server-agent-chat.test.ts @@ -1463,6 +1463,94 @@ test("cloud api /v1/agent/chat/inspect returns not_found for empty conversation } }); +test("cloud api trace returns persisted summary when live trace store has expired", async () => { + const ownerSessions = new Map(); + ownerSessions.set("ses_issue842_expired", testAgentSessionRecord({ + sessionId: "ses_issue842_expired", + projectId: "prj_v02_code_agent", + conversationId: "cnv_issue842_expired", + threadId: "thread-issue-842-expired", + traceId: "trc_issue842_expired", + status: "active", + session: { + finalResponse: { + text: "历史 trace 已过期，但 final response 仍可审计。", + textChars: 31, + role: "assistant", + status: "completed", + traceId: "trc_issue842_expired", + source: "code-agent-result", + valuesPrinted: false + }, + traceSummary: { + traceId: "trc_issue842_expired", + source: "agent-session-snapshot", + sourceEventCount: 26, + terminalStatus: "completed", + noiseEventCount: 4, + omittedNoiseCount: 4, + valuesPrinted: false + }, + agentRun: { + adapter: "agentrun-v01", + runId: "run_issue842_expired", + commandId: "cmd_issue842_expired", + runnerId: "runner_issue842_expired", + jobName: "agentrun-v01-runner-issue842-expired", + namespace: "agentrun-v01", + terminalStatus: "completed", + valuesPrinted: false + }, + valuesRedacted: true + } + })); + const traceStore = createCodeAgentTraceStore(); + const server = createCloudApiServer({ + traceStore, + env: { + HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01", + AGENTRUN_MGR_URL: "http://127.0.0.1:9", + HWLAB_CODE_AGENT_AGENTRUN_ALLOW_NON_K3S_URL: "1", + HWLAB_ENVIRONMENT: "v02", + HWLAB_GITOPS_PROFILE: "v02" + }, + accessController: { + required: false, + async authenticate() { + return { ok: true, actor: TEST_AGENT_ACTOR, session: TEST_AUTH_SESSION }; + }, + async getAgentSessionByTraceId(traceId) { + return [...ownerSessions.values()].find((session) => session.lastTraceId === traceId) ?? null; + } + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat/trace/trc_issue842_expired`, { + headers: { cookie: "hwlab_session=test-stub-session" } + }); + assert.equal(response.status, 200); + const body = await response.json(); + assert.equal(body.ok, true); + assert.equal(body.status, "expired"); + assert.equal(body.traceStatus, "expired"); + assert.equal(body.eventCount, 0); + assert.equal(body.fallback.available, true); + assert.equal(body.finalResponse.text, "历史 trace 已过期，但 final response 仍可审计。"); + assert.equal(body.traceSummary.sourceEventCount, 26); + assert.equal(body.agentRun.runId, "run_issue842_expired"); + assert.equal(body.agentRun.commandId, "cmd_issue842_expired"); + assert.equal(body.retention.liveTraceStore, "expired-or-evicted"); + assert.equal(JSON.stringify(body).includes("password"), false); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + } +}); + test("cloud api result polling compacts large runnerTrace while preserving providerTrace", async () => { const workspace = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-result-compact-")); const codexHome = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-result-compact-codex-home-")); diff --git a/internal/cloud/server-code-agent-http.ts b/internal/cloud/server-code-agent-http.ts index 4c6b3e91..e2baf758 100644 --- a/internal/cloud/server-code-agent-http.ts +++ b/internal/cloud/server-code-agent-http.ts @@ -1526,6 +1526,9 @@ function textValue(value) { } function codeAgentSessionOwnerEvidence(payload = {}, params = {}) { + const traceId = payload.traceId ?? params.traceId ?? null; + const finalResponse = codeAgentFinalResponseEvidence(payload, traceId); + const traceSummary = codeAgentTraceSummaryEvidence(payload, traceId, finalResponse); return { provider: payload.provider ?? null, model: payload.model ?? null, @@ -1537,14 +1540,66 @@ function codeAgentSessionOwnerEvidence(payload = {}, params = {}) { conversationId: payload.conversationId ?? params.conversationId ?? null, sessionId: payload.sessionId ?? payload.session?.sessionId ?? payload.sessionReuse?.sessionId ?? params.sessionId ?? null, threadId: payload.session?.threadId ?? payload.sessionReuse?.threadId ?? payload.threadId ?? params.threadId ?? null, - traceId: payload.traceId ?? params.traceId ?? null, + traceId, projectId: payload.projectId ?? params.projectId ?? null, + finalResponse, + traceSummary, ...agentRunSessionEvidence(payload), secretMaterialStored: false, valuesRedacted: true }; } +function codeAgentFinalResponseEvidence(payload = {}, traceId = null) { + const text = textValue(payload.reply?.content ?? payload.message?.content ?? payload.assistantText); + if (!text) return null; + return { + text, + textChars: text.length, + messageId: payload.reply?.messageId ?? payload.messageId ?? null, + role: payload.reply?.role ?? payload.message?.role ?? "assistant", + status: payload.status ?? null, + traceId, + createdAt: payload.reply?.createdAt ?? payload.createdAt ?? null, + updatedAt: payload.updatedAt ?? null, + source: "code-agent-result", + valuesPrinted: false + }; +} + +function codeAgentTraceSummaryEvidence(payload = {}, traceId = null, finalResponse = null) { + const runnerTrace = payload.runnerTrace && typeof payload.runnerTrace === "object" ? payload.runnerTrace : null; + const events = Array.isArray(runnerTrace?.events) ? runnerTrace.events : []; + const terminalEvent = [...events].reverse().find((event) => event?.terminal === true) ?? runnerTrace?.lastEvent ?? null; + return { + traceId, + source: "agent-session-snapshot", + sourceEventCount: Number.isFinite(Number(runnerTrace?.eventCount)) ? Number(runnerTrace.eventCount) : events.length, + terminalStatus: payload.agentRun?.terminalStatus ?? payload.status ?? terminalEvent?.status ?? null, + lastEventLabel: terminalEvent?.label ?? null, + finalAssistantRow: finalResponse ? { + role: finalResponse.role, + status: finalResponse.status, + textChars: finalResponse.textChars, + textPreview: finalResponse.text.slice(0, 240), + messageId: finalResponse.messageId, + valuesPrinted: false + } : null, + agentRun: payload.agentRun ? { + runId: payload.agentRun.runId ?? null, + commandId: payload.agentRun.commandId ?? null, + attemptId: payload.agentRun.attemptId ?? null, + runnerId: payload.agentRun.runnerId ?? null, + jobName: payload.agentRun.jobName ?? null, + namespace: payload.agentRun.namespace ?? null, + lastSeq: payload.agentRun.lastSeq ?? null, + valuesPrinted: false + } : null, + updatedAt: payload.updatedAt ?? null, + valuesPrinted: false + }; +} + function annotateOwner(payload, params = {}) { if (!params.ownerUserId) return payload; return { @@ -1648,8 +1703,22 @@ export async function handleCodeAgentTraceHttp(request, response, url, options) } const agentRunResult = result?.agentRun ? result : await loadPersistedAgentRunResult(traceId, options); + let refreshError = null; + if (agentRunResult && !canAccessOwnedResult(agentRunResult, options.actor)) { + sendJson(response, 403, { + error: { + code: "agent_session_owner_required", + message: "Only the session owner or admin can read this Code Agent trace" + } + }); + return; + } if (agentRunResult?.agentRun) { - await refreshAgentRunTrace({ traceId, result: agentRunResult, options, traceStore }); + try { + await refreshAgentRunTrace({ traceId, result: agentRunResult, options, traceStore }); + } catch (error) { + refreshError = error; + } } if (parts[5] === "stream") { @@ -1657,7 +1726,124 @@ export async function handleCodeAgentTraceHttp(request, response, url, options) return; } - sendJson(response, 200, traceStore.snapshot(traceId)); + const snapshot = traceStore.snapshot(traceId); + sendJson(response, 200, traceSnapshotWithPersistentFallback(snapshot, agentRunResult, traceId, refreshError)); +} + +function traceSnapshotWithPersistentFallback(snapshot, persistedResult, traceId, refreshError = null) { + if (snapshot?.status !== "missing") return snapshot; + const fallback = persistentTraceFallback(persistedResult, traceId); + if (!fallback) return { + ...snapshot, + ok: false, + traceStatus: "missing", + retention: traceRetentionSummary("missing"), + fallback: { available: false, source: null } + }; + return { + ...snapshot, + ok: true, + status: "expired", + traceStatus: "expired", + persisted: true, + retention: traceRetentionSummary("expired"), + conversationId: fallback.conversationId, + sessionId: fallback.sessionId, + threadId: fallback.threadId, + agentRun: fallback.agentRun, + finalResponse: fallback.finalResponse, + traceSummary: fallback.traceSummary, + fallback: { + available: true, + source: fallback.source, + refresh: refreshError ? { + attempted: true, + ok: false, + code: refreshError?.code ?? "agentrun_trace_refresh_failed", + message: refreshError?.message ?? "AgentRun trace refresh failed; using persisted session summary.", + valuesPrinted: false + } : { attempted: false, ok: null }, + conversationId: fallback.conversationId, + sessionId: fallback.sessionId, + threadId: fallback.threadId, + agentRun: fallback.agentRun, + traceSummary: fallback.traceSummary, + finalResponse: fallback.finalResponse, + valuesPrinted: false + } + }; +} + +function persistentTraceFallback(result, traceId) { + if (!result || typeof result !== "object") return null; + const storedSummary = result.traceSummary && typeof result.traceSummary === "object" ? result.traceSummary : null; + const finalResponse = persistentFinalResponse(result); + const agentRun = result.agentRun && typeof result.agentRun === "object" ? { + adapter: result.agentRun.adapter ?? null, + runId: result.agentRun.runId ?? null, + commandId: result.agentRun.commandId ?? null, + attemptId: result.agentRun.attemptId ?? null, + runnerId: result.agentRun.runnerId ?? null, + jobName: result.agentRun.jobName ?? null, + namespace: result.agentRun.namespace ?? null, + terminalStatus: result.agentRun.terminalStatus ?? null, + lastSeq: result.agentRun.lastSeq ?? null, + valuesPrinted: false + } : null; + if (!storedSummary && !finalResponse && !agentRun) return null; + const traceSummary = { + traceId, + source: storedSummary?.source ?? (finalResponse ? "agent-session-final-response" : "agent-session-agentrun"), + sourceEventCount: numberOrNull(storedSummary?.sourceEventCount ?? storedSummary?.eventCount ?? result.runnerTrace?.eventCount), + renderedRowSummary: storedSummary?.renderedRowSummary ?? null, + noiseEventCount: numberOrNull(storedSummary?.noiseEventCount), + omittedNoiseCount: numberOrNull(storedSummary?.omittedNoiseCount), + terminalStatus: storedSummary?.terminalStatus ?? result.agentRun?.terminalStatus ?? result.status ?? null, + updatedAt: storedSummary?.updatedAt ?? result.updatedAt ?? null, + valuesPrinted: false + }; + return { + source: storedSummary?.source ?? (finalResponse ? "agent-session-final-response" : "agent-session-agentrun"), + conversationId: safeConversationId(result.conversationId) || null, + sessionId: safeSessionId(result.sessionId) || null, + threadId: safeOpaqueId(result.threadId) || null, + agentRun, + finalResponse, + traceSummary + }; +} + +function persistentFinalResponse(result) { + const stored = result.finalResponse && typeof result.finalResponse === "object" ? result.finalResponse : null; + const text = textValue(stored?.text ?? result.reply?.content ?? result.message?.content ?? result.assistantText); + if (!text) return null; + return { + text, + textChars: text.length, + messageId: stored?.messageId ?? result.reply?.messageId ?? result.messageId ?? null, + role: stored?.role ?? result.reply?.role ?? "assistant", + status: stored?.status ?? result.status ?? null, + traceId: safeTraceId(stored?.traceId ?? result.traceId) || null, + createdAt: stored?.createdAt ?? result.reply?.createdAt ?? result.createdAt ?? null, + updatedAt: stored?.updatedAt ?? result.updatedAt ?? null, + source: stored?.source ?? "code-agent-result", + valuesPrinted: false + }; +} + +function traceRetentionSummary(status) { + return { + traceStatus: status, + liveTraceStore: status === "missing" ? "missing" : "expired-or-evicted", + policy: "live trace events are best-effort short-term storage; completed turns persist final response and summary in the agent session snapshot", + replacementEvidence: status === "missing" ? "submit a fresh equivalent trace or inspect conversation/session if available" : "use fallback.finalResponse, fallback.traceSummary, conversationId, sessionId, threadId, and agentRun ids", + valuesPrinted: false + }; +} + +function numberOrNull(value) { + const parsed = Number(value); + return Number.isFinite(parsed) ? parsed : null; } function sendTraceSse(response, traceStore, traceId, options = {}) { diff --git a/tools/hwlab-cli/client.test.ts b/tools/hwlab-cli/client.test.ts index c13eadee..a4958a82 100644 --- a/tools/hwlab-cli/client.test.ts +++ b/tools/hwlab-cli/client.test.ts @@ -1523,6 +1523,53 @@ test("hwlab-cli client agent trace exposes missing Web render state without a bo assert.deepEqual(result.payload.data, result.payload.rendered); }); +test("hwlab-cli client agent trace renders persisted summary for expired traces", async () => { + const finalText = "历史 trace 已过期，但 final response 仍可审计。"; + const result = await runHwlabCli(["client", "agent", "trace", "trc_expired", "--base-url", "http://web.test", "--cookie", "hwlab_session=session-a", "--render", "web"], { + fetchImpl: async () => new Response(JSON.stringify({ + ok: true, + status: "expired", + traceStatus: "expired", + traceId: "trc_expired", + eventCount: 0, + events: [], + conversationId: "cnv_expired", + sessionId: "ses_expired", + threadId: "thread-expired", + retention: { traceStatus: "expired", liveTraceStore: "expired-or-evicted" }, + agentRun: { runId: "run_expired", commandId: "cmd_expired", valuesPrinted: false }, + finalResponse: { text: finalText, textChars: finalText.length, role: "assistant", status: "completed", source: "code-agent-result", valuesPrinted: false }, + traceSummary: { source: "agent-session-snapshot", sourceEventCount: 26, terminalStatus: "completed", valuesPrinted: false }, + fallback: { + available: true, + source: "agent-session-snapshot", + conversationId: "cnv_expired", + sessionId: "ses_expired", + threadId: "thread-expired", + agentRun: { runId: "run_expired", commandId: "cmd_expired", valuesPrinted: false }, + finalResponse: { text: finalText, textChars: finalText.length, role: "assistant", status: "completed", valuesPrinted: false }, + traceSummary: { source: "agent-session-snapshot", sourceEventCount: 26, terminalStatus: "completed", valuesPrinted: false }, + valuesPrinted: false + } + }), { status: 200 }) + }); + + assert.equal(result.exitCode, 0); + assert.equal(result.payload.status, "succeeded"); + assert.equal(result.payload.traceStatus, "expired"); + assert.equal(result.payload.body.traceStatus, "expired"); + assert.equal(result.payload.body.assistantText, finalText); + assert.equal(result.payload.rendered.traceStatus, "expired"); + assert.equal(result.payload.rendered.sourceEventCount, 0); + assert.equal(result.payload.rendered.renderedRowCount, 1); + assert.match(result.payload.rendered.rows[0].header, /历史 trace 已过期/u); + assert.match(result.payload.rendered.rows[0].body, /final response 仍可审计/u); + assert.match(result.payload.rendered.rows[0].body, /run_expired/u); + assert.equal(result.payload.rendered.fallback.available, true); + assert.equal(result.payload.rendered.traceSummary.sourceEventCount, 26); + assert.deepEqual(result.payload.data, result.payload.rendered); +}); + test("hwlab-cli Web trace render does not treat AgentRun result-ready as final assistant text", async () => { const result = await runHwlabCli(["client", "agent", "trace", "trc_result_ready", "--base-url", "http://web.test", "--cookie", "hwlab_session=session-a", "--render", "web"], { fetchImpl: async () => new Response(JSON.stringify({ diff --git a/tools/src/hwlab-cli-lib.ts b/tools/src/hwlab-cli-lib.ts index 8661e1ba..d1d5dbf4 100644 --- a/tools/src/hwlab-cli-lib.ts +++ b/tools/src/hwlab-cli-lib.ts @@ -2674,16 +2674,19 @@ function traceBodyForCli(body: any, parsed: ParsedArgs) { function webTraceRenderBody(traceObject: any, parsed: ParsedArgs) { const events = Array.isArray(traceObject?.events) ? traceObject.events : []; const rows = traceDisplayRows(traceObject ?? {}, events); + const fallbackRows = rows.length > 0 ? [] : persistentTraceFallbackRows(traceObject); + const allRows = rows.length > 0 ? rows : fallbackRows; const noiseEventCount = traceNoiseEventCount(events); const rowLimit = Math.max(1, Math.min(numberOption(parsed.limit) ?? 80, 500)); - const rowTail = parsed.full === true ? rows : rows.slice(-rowLimit); + const rowTail = parsed.full === true ? allRows : allRows.slice(-rowLimit); return pruneUndefined({ traceId: traceObject?.traceId ?? null, - status: traceObject?.status ?? null, + status: traceObject?.traceStatus ?? traceObject?.status ?? null, + traceStatus: traceObject?.traceStatus ?? traceObject?.status ?? null, render: "web", renderer: "tools/src/hwlab-cli/trace-renderer:traceDisplayRows", sourceEventCount: traceObject?.eventCount ?? events.length, - renderedRowCount: rows.length, + renderedRowCount: allRows.length, returnedRowCount: rowTail.length, noiseEventCount, omittedNoiseCount: noiseEventCount, @@ -2694,10 +2697,42 @@ function webTraceRenderBody(traceObject: any, parsed: ParsedArgs) { } : undefined, assistantText: assistantText(traceObject), rows: rowTail.map(compactTraceRenderRow), + retention: traceObject?.retention, + fallback: traceObject?.fallback, + finalResponse: traceObject?.finalResponse, + traceSummary: traceObject?.traceSummary, fullBodyAvailable: true }); } +function persistentTraceFallbackRows(traceObject: any) { + const fallback = traceObject?.fallback && typeof traceObject.fallback === "object" ? traceObject.fallback : null; + const finalResponse = traceObject?.finalResponse ?? fallback?.finalResponse; + const traceSummary = traceObject?.traceSummary ?? fallback?.traceSummary; + const textValue = text(finalResponse?.text ?? finalResponse?.textPreview ?? traceSummary?.finalAssistantRow?.textPreview); + if (!fallback?.available && !textValue && !traceSummary) return []; + const body = [ + textValue ? `final assistant: ${textValue}` : "final assistant: unavailable in persisted snapshot", + `source: ${text(traceSummary?.source ?? fallback?.source ?? "agent-session-snapshot")}`, + `terminalStatus: ${text(traceSummary?.terminalStatus ?? finalResponse?.status ?? traceObject?.status ?? "unknown")}`, + `sourceEventCount: ${traceSummary?.sourceEventCount ?? traceObject?.eventCount ?? 0}`, + fallback?.conversationId ? `conversationId: ${fallback.conversationId}` : null, + fallback?.sessionId ? `sessionId: ${fallback.sessionId}` : null, + fallback?.threadId ? `threadId: ${fallback.threadId}` : null, + fallback?.agentRun?.runId ? `runId: ${fallback.agentRun.runId}` : null, + fallback?.agentRun?.commandId ? `commandId: ${fallback.agentRun.commandId}` : null + ].filter(Boolean).join("\n"); + return [{ + rowId: `${traceObject?.traceId ?? "trace"}:persisted-summary`, + seq: null, + tone: "info", + header: "历史 trace 已过期，显示持久化摘要", + bodyFormat: "text", + terminal: true, + body + }]; +} + function traceResponseAliases(traceBody: any, parsed: ParsedArgs) { if (text(parsed.render) !== "web" || !traceBody || typeof traceBody !== "object") return {}; const rendered = pruneUndefined({ @@ -2713,7 +2748,11 @@ function traceResponseAliases(traceBody: any, parsed: ParsedArgs) { rowLimit: traceBody.rowLimit, rows: traceBody.rows, eventCountMismatch: traceBody.eventCountMismatch, - assistantText: traceBody.assistantText + assistantText: traceBody.assistantText, + retention: traceBody.retention, + fallback: traceBody.fallback, + finalResponse: traceBody.finalResponse, + traceSummary: traceBody.traceSummary }); return { traceStatus: traceBody.status ?? null, @@ -2808,7 +2847,7 @@ function compactAgentRun(value: any) { } function assistantText(body: any) { - const direct = text(body?.reply?.content ?? body?.message?.content ?? body?.assistantText); + const direct = text(body?.reply?.content ?? body?.message?.content ?? body?.assistantText ?? body?.finalResponse?.text ?? body?.fallback?.finalResponse?.text); if (direct) return direct; if (Array.isArray(body?.assistantStreams)) { for (const item of [...body.assistantStreams].reverse()) { @@ -2958,6 +2997,7 @@ function compactBody(body: any) { selectedDevicePodId: body.selectedDevicePodId, devicePodCount: Array.isArray(body.devicePods) ? body.devicePods.length : undefined, traceId: body.traceId, + traceStatus: body.traceStatus, conversationId: body.conversationId, sessionId: body.sessionId, threadId: body.threadId, @@ -2970,6 +3010,10 @@ function compactBody(body: any) { error: body.error, blocker: body.blocker, summary: body.summary, + retention: body.retention, + fallback: body.fallback, + finalResponse: body.finalResponse, + traceSummary: body.traceSummary, freshness: body.freshness, profileHash: body.profileHash, targetId: body.targetId,