From 2c6574e6254759882eafd9d82f2b7df286ba741c Mon Sep 17 00:00:00 2001 From: lyon Date: Mon, 15 Jun 2026 15:34:36 +0800 Subject: [PATCH] fix: expose agent trace result summary --- tools/src/hwlab-cli-lib.ts | 339 ++++++++++++++++++++++++++++++++++++- 1 file changed, 336 insertions(+), 3 deletions(-) diff --git a/tools/src/hwlab-cli-lib.ts b/tools/src/hwlab-cli-lib.ts index e258e184..1ba0b1d3 100644 --- a/tools/src/hwlab-cli-lib.ts +++ b/tools/src/hwlab-cli-lib.ts @@ -999,14 +999,25 @@ async function agentCommand(context: any) { const traceId = requiredTraceId(context.rest[1] ?? context.parsed.traceId); const pathName = `/v1/agent/chat/result/${encodeURIComponent(traceId)}`; const response = await requestJson({ ...context, method: "GET", path: pathName }); - return responsePayload("client.agent.result", response, context, { route: route("GET", pathName), traceId, body: responseBodyForCli(response.body, context.parsed) }); + return responsePayload("client.agent.result", response, context, { + route: route("GET", pathName), + traceId, + body: responseBodyForCli(response.body, context.parsed), + traceResultSummary: traceResultSummaryForCli(response.body, { traceId, command: "result" }) + }); } if (subcommand === "trace") { const traceId = requiredTraceId(context.rest[1] ?? context.parsed.traceId); const pathName = `/v1/agent/chat/trace/${encodeURIComponent(traceId)}`; const response = await requestJson({ ...context, method: "GET", path: pathName }); const traceBody = traceBodyForCli(response.body, context.parsed); - return responsePayload("client.agent.trace", response, context, { route: route("GET", pathName), traceId, body: traceBody, ...traceResponseAliases(traceBody, context.parsed) }); + return responsePayload("client.agent.trace", response, context, { + route: route("GET", pathName), + traceId, + body: traceBody, + traceResultSummary: traceResultSummaryForCli(response.body, { traceId, command: "trace", renderedTrace: traceBody }), + ...traceResponseAliases(traceBody, context.parsed) + }); } if (subcommand === "inspect") { return agentInspect(context); @@ -1023,7 +1034,329 @@ async function agentCommand(context: any) { async function agentInspect(context: any) { const pathName = agentInspectPath(context); const response = await requestJson({ ...context, method: "GET", path: pathName }); - return responsePayload("client.agent.inspect", response, context, { route: route("GET", pathName), body: responseBodyForCli(response.body, context.parsed) }); + const traceId = text(context.parsed.traceId); + return responsePayload("client.agent.inspect", response, context, { + route: route("GET", pathName), + body: responseBodyForCli(response.body, context.parsed), + traceResultSummary: traceResultSummaryForCli(response.body, { traceId, command: "inspect" }) + }); +} + +function traceResultSummaryForCli(source: any, options: { traceId?: string; command?: string; renderedTrace?: any } = {}) { + const rendered = options.renderedTrace && typeof options.renderedTrace === "object" ? options.renderedTrace : null; + const prompt = tracePromptForCli(source); + const toolCalls = traceToolCallsForCli(source, rendered); + const ids = completeTraceResultIdsFromToolCalls(traceResultIdsForCli(source, rendered, options.traceId), toolCalls); + const finalResponse = traceFinalResponseForCli(source, rendered, ids.traceId); + const agentMessages = traceAgentMessagesForCli(source, rendered, finalResponse); + const diagnostics = traceDiagnosticsForCli(source, rendered); + const counts = traceCountsForCli(source, rendered, toolCalls.length); + const upstreamGaps = traceResultUpstreamGaps({ ids, prompt, finalResponse, toolCalls, agentMessages, diagnostics }); + return pruneUndefined({ + command: text(options.command) || undefined, + ids, + prompt, + diagnostics, + counts, + toolCalls, + toolCallCount: toolCalls.length, + agentMessages, + finalResponse, + upstreamGaps: upstreamGaps.length > 0 ? upstreamGaps : undefined, + valuesPrinted: false, + fullBodyAvailable: true + }); +} + +function completeTraceResultIdsFromToolCalls(ids: any, toolCalls: any[]) { + const source = toolCalls.find((call) => call?.runId || call?.commandId || call?.runnerId) ?? null; + if (!source) return ids; + return pruneUndefined({ + ...ids, + runId: ids.runId ?? (text(source.runId) || undefined), + commandId: ids.commandId ?? (text(source.commandId) || undefined), + runnerId: ids.runnerId ?? (text(source.runnerId) || undefined) + }); +} + +function traceResultIdsForCli(source: any, rendered: any, traceId: unknown) { + const agentRun = traceAgentRunForCli(source, rendered); + const terminal = source?.terminalEvidence && typeof source.terminalEvidence === "object" ? source.terminalEvidence : null; + const session = source?.session && typeof source.session === "object" ? source.session : null; + const sessionReuse = source?.sessionReuse && typeof source.sessionReuse === "object" ? source.sessionReuse : null; + return pruneUndefined({ + promptId: text(source?.promptId ?? source?.prompt?.id) || undefined, + traceId: text(traceId ?? source?.traceId ?? rendered?.traceId ?? terminal?.traceId ?? source?.traceSummary?.traceId ?? rendered?.traceSummary?.traceId) || undefined, + conversationId: text(source?.conversationId ?? session?.conversationId ?? sessionReuse?.conversationId ?? terminal?.conversationId ?? source?.agentRun?.conversationId) || undefined, + sessionId: text(source?.sessionId ?? session?.sessionId ?? sessionReuse?.sessionId ?? terminal?.sessionId) || undefined, + threadId: text(source?.threadId ?? session?.threadId ?? sessionReuse?.threadId ?? terminal?.threadId ?? source?.agentRun?.threadId) || undefined, + runId: text(agentRun?.runId) || undefined, + commandId: text(agentRun?.commandId) || undefined, + runnerId: text(agentRun?.runnerId) || undefined, + messageId: text(source?.messageId ?? source?.finalResponse?.messageId ?? terminal?.finalResponse?.messageId) || undefined + }); +} + +function traceAgentRunForCli(source: any, rendered: any) { + const candidates = [ + source?.agentRun, + source?.runner, + source?.traceSummary?.agentRun, + source?.terminalEvidence?.agentRun, + source?.terminalEvidence?.traceSummary?.agentRun, + rendered?.traceSummary?.agentRun, + rendered?.terminalEvidence?.agentRun + ]; + return candidates.find((candidate) => candidate && typeof candidate === "object") ?? null; +} + +function tracePromptForCli(source: any) { + const events = traceEventsForCli(source); + const promptEvent = events.find((event: any) => text(event?.promptId ?? event?.promptSummary ?? event?.promptPreview ?? event?.prompt ?? event?.userPrompt ?? event?.inputPreview ?? event?.input)); + const promptText = text(source?.promptSummary ?? source?.promptPreview ?? source?.prompt?.summary ?? source?.prompt?.text ?? promptEvent?.promptSummary ?? promptEvent?.promptPreview ?? promptEvent?.prompt ?? promptEvent?.userPrompt ?? promptEvent?.inputPreview ?? promptEvent?.input); + const promptId = text(source?.promptId ?? source?.prompt?.id ?? promptEvent?.promptId ?? promptEvent?.prompt?.id); + const promptSha256 = text(source?.promptSha256 ?? source?.prompt?.sha256 ?? promptEvent?.promptSha256); + if (!promptText && !promptId && !promptSha256) return undefined; + return pruneUndefined({ + promptId: promptId || undefined, + promptSha256: promptSha256 || undefined, + textPreview: clippedText(promptText, 500), + textChars: promptText ? promptText.length : undefined, + sourceSeq: promptEvent?.seq ?? promptEvent?.sourceSeq, + valuesPrinted: false + }); +} + +function traceDiagnosticsForCli(source: any, rendered: any) { + const agentRun = traceAgentRunForCli(source, rendered); + const providerTrace = source?.providerTrace ?? source?.agentRun?.providerTrace ?? agentRun?.providerTrace; + return pruneUndefined({ + status: text(source?.status ?? rendered?.status) || undefined, + traceStatus: text(source?.traceStatus ?? rendered?.traceStatus ?? rendered?.status) || undefined, + terminalStatus: text(source?.terminalStatus ?? source?.traceSummary?.terminalStatus ?? rendered?.traceSummary?.terminalStatus ?? agentRun?.terminalStatus ?? agentRun?.status) || undefined, + provider: text(source?.provider ?? source?.runner?.provider ?? source?.longLivedSessionGate?.provider ?? providerTrace?.backendProfile) || undefined, + providerProfile: text(source?.providerProfile ?? source?.profile ?? source?.agentRun?.backendProfile ?? agentRun?.backendProfile) || undefined, + adapter: text(agentRun?.adapter ?? providerTrace?.transport) || undefined, + namespace: text(agentRun?.namespace ?? providerTrace?.namespace ?? source?.runner?.namespace) || undefined, + lane: text(agentRun?.namespace ?? providerTrace?.namespace ?? source?.runner?.namespace) || undefined, + runnerJobName: text(agentRun?.jobName ?? source?.runner?.jobName) || undefined, + runStatus: text(agentRun?.runStatus) || undefined, + commandState: text(agentRun?.commandState) || undefined, + failureKind: text(agentRun?.failureKind ?? providerTrace?.failureKind) || undefined, + valuesPrinted: false + }); +} + +function traceCountsForCli(source: any, rendered: any, toolCallCount: number) { + const events = traceEventsForCli(source); + const sourceEventCount = source?.traceSummary?.sourceEventCount ?? rendered?.traceSummary?.sourceEventCount ?? source?.eventCount ?? events.length; + return pruneUndefined({ + sourceEventCount: sourceEventCount || undefined, + renderedRowCount: rendered?.renderedRowCount, + returnedRowCount: rendered?.returnedRowCount, + noiseEventCount: rendered?.noiseEventCount, + omittedNoiseCount: rendered?.omittedNoiseCount, + toolCallCount + }); +} + +function traceFinalResponseForCli(source: any, rendered: any, traceId: unknown) { + const terminal = source?.terminalEvidence && typeof source.terminalEvidence === "object" ? source.terminalEvidence : null; + const final = firstObjectForCli(source?.finalResponse, terminal?.finalResponse, rendered?.finalResponse); + const finalText = text(final?.text ?? final?.textPreview ?? assistantText(source) ?? rendered?.assistantText); + if (!finalText && !final) return undefined; + return pruneUndefined({ + role: text(final?.role) || "assistant", + status: text(final?.status ?? source?.status ?? rendered?.status) || undefined, + traceId: text(final?.traceId ?? traceId) || undefined, + messageId: text(final?.messageId ?? source?.messageId) || undefined, + text: clippedText(finalText, 4000), + textChars: finalText ? finalText.length : final?.textChars, + valuesPrinted: false + }); +} + +function traceAgentMessagesForCli(source: any, rendered: any, finalResponse: any) { + const messages = []; + const seen = new Set(); + const pushMessage = (value: any) => { + const body = text(value?.text ?? value?.body ?? value?.message ?? value?.content); + if (!body) return; + const key = `${value?.kind ?? value?.role ?? "message"}:${body}`; + if (seen.has(key)) return; + seen.add(key); + messages.push(pruneUndefined({ + kind: text(value?.kind) || undefined, + role: text(value?.role) || "assistant", + status: text(value?.status) || undefined, + sourceSeq: value?.sourceSeq ?? value?.seq, + terminal: value?.terminal === true ? true : undefined, + text: clippedText(body, 1200), + textChars: body.length, + valuesPrinted: false + })); + }; + if (Array.isArray(source?.assistantStreams)) { + for (const item of source.assistantStreams.slice(-3)) pushMessage({ ...item, kind: "assistant-stream", text: item?.text ?? item?.lastChunk }); + } + for (const row of firstArray(rendered?.rows, source?.rows)) { + if (row?.terminal === true || /助手|assistant/iu.test(text(row?.header))) pushMessage({ ...row, kind: "trace-row", text: row?.body }); + } + if (finalResponse) pushMessage({ ...finalResponse, kind: "final-response" }); + return messages.slice(0, 8); +} + +function traceToolCallsForCli(source: any, rendered: any) { + const calls = []; + for (const event of traceEventsForCli(source)) { + const call = traceToolCallFromEvent(event); + if (call) calls.push(call); + } + for (const row of firstArray(rendered?.rows, source?.rows)) { + const call = traceToolCallFromRow(row); + if (call) calls.push(call); + } + if (calls.length === 0 && Array.isArray(source?.toolCalls)) { + for (const item of source.toolCalls) { + const call = traceToolCallFromObject(item); + if (call) calls.push(call); + } + } + return dedupeTraceToolCalls(calls).slice(0, 20); +} + +function traceEventsForCli(source: any) { + return firstArray(source?.events, source?.runnerTrace?.events, source?.terminalEvidence?.events); +} + +function traceToolCallFromObject(value: any) { + if (!value || typeof value !== "object") return null; + return pruneUndefined({ + source: "toolCalls", + status: text(value.status) || undefined, + toolName: text(value.toolName ?? value.name) || undefined, + command: text(value.command) || undefined, + runId: text(value.runId) || undefined, + commandId: text(value.commandId) || undefined, + runnerId: text(value.runnerId) || undefined, + exitCode: value.exitCode, + durationMs: value.durationMs, + outputBytes: value.outputBytes, + outputTruncated: value.outputTruncated, + outputPreview: clippedText(text(value.outputPreview ?? value.stdoutSummary ?? value.stderrSummary ?? value.message), 500), + valuesPrinted: false + }); +} + +function traceToolCallFromEvent(event: any) { + if (!event || typeof event !== "object") return null; + const command = text(event.command); + const hasCommandOutput = command || /commandExecution/iu.test(text(event.message ?? event.stdoutSummary ?? event.stderrSummary)); + if (!hasCommandOutput) return null; + const outputText = traceToolOutputText(event); + return pruneUndefined({ + source: "runnerTrace.events", + sourceSeq: event.sourceSeq ?? event.seq, + status: traceToolStatusForCli(event.status, event.exitCode), + label: text(event.label) || undefined, + toolName: text(event.toolName ?? event.name) || undefined, + command: command || undefined, + runId: text(event.runId) || undefined, + commandId: text(event.commandId) || undefined, + runnerId: text(event.runnerId) || undefined, + exitCode: event.exitCode, + durationMs: event.durationMs, + outputBytes: event.outputBytes, + outputTruncated: event.outputTruncated, + outputPreview: outputText ? clippedText(outputText, 500) : undefined, + valuesPrinted: false + }); +} + +function traceToolCallFromRow(row: any) { + if (!row || typeof row !== "object") return null; + const rowId = text(row.rowId); + const body = text(row.body); + const header = text(row.header); + const isTool = rowId.startsWith("tool:") || /commandExecution|exitCode=/iu.test(`${header} ${body}`); + if (!isTool) return null; + const parsed = parseRenderedToolBody(body); + return pruneUndefined({ + source: "trace-render-row", + rowId: rowId || undefined, + sourceSeq: row.seq, + status: traceToolStatusForCli(row.tone === "ok" ? "completed" : row.tone, parsed.exitCode), + label: header || undefined, + toolName: /commandExecution/u.test(header) ? "commandExecution" : undefined, + command: parsed.command, + exitCode: parsed.exitCode, + outputPreview: body ? clippedText(body, 500) : undefined, + valuesPrinted: false + }); +} + +function traceToolOutputText(event: any) { + const stdout = text(event.stdoutSummary ?? event.stdoutPreview ?? event.stdout); + const stderr = text(event.stderrSummary ?? event.stderrPreview ?? event.stderr); + const message = text(event.message); + const value = stdout || stderr || (/commandExecution inProgress/iu.test(message) ? "" : message); + return value; +} + +function parseRenderedToolBody(value: string) { + const exitMatch = value.match(/\bexitCode=(-?\d+)\b/u); + const streamMatch = value.match(/^(.+?)\s+(?:stdout|stderr):\s+/u); + return pruneUndefined({ + command: text(streamMatch?.[1]) || undefined, + exitCode: exitMatch ? Number.parseInt(exitMatch[1], 10) : undefined + }); +} + +function traceToolStatusForCli(status: unknown, exitCode: unknown) { + const code = typeof exitCode === "number" ? exitCode : Number.isFinite(Number(exitCode)) ? Number(exitCode) : null; + if (code !== null) return code === 0 ? "completed" : "failed"; + const value = text(status); + if (value === "ok") return "completed"; + return value || undefined; +} + +function dedupeTraceToolCalls(calls: any[]) { + const byKey = new Map(); + for (const call of calls) { + const key = [call.commandId, call.command, call.sourceSeq, call.rowId].map((part) => text(part)).filter(Boolean).join(":") || JSON.stringify(call); + const existing = byKey.get(key); + if (!existing || traceToolCallScore(call) > traceToolCallScore(existing)) byKey.set(key, call); + } + return [...byKey.values()]; +} + +function traceToolCallScore(call: any) { + return [call.command, call.exitCode !== undefined, call.outputPreview, call.runId, call.commandId].filter(Boolean).length; +} + +function traceResultUpstreamGaps({ ids, prompt, finalResponse, toolCalls, agentMessages, diagnostics }: any) { + const gaps = []; + if (!ids?.traceId) gaps.push("traceId_missing"); + if (!ids?.conversationId) gaps.push("conversationId_missing"); + if (!ids?.sessionId) gaps.push("sessionId_missing"); + if (!ids?.runId) gaps.push("runId_missing"); + if (!ids?.commandId) gaps.push("commandId_missing"); + if (!prompt?.promptId && !prompt?.textPreview) gaps.push("prompt_not_returned_by_upstream"); + if (!Array.isArray(toolCalls) || toolCalls.length === 0) gaps.push("tool_calls_missing"); + if (!Array.isArray(agentMessages) || agentMessages.length === 0) gaps.push("agent_message_missing"); + if (!finalResponse?.text) gaps.push("final_response_missing"); + if (!diagnostics?.provider && !diagnostics?.namespace) gaps.push("provider_runner_lane_missing"); + return gaps; +} + +function firstObjectForCli(...values: any[]) { + return values.find((value) => value && typeof value === "object") ?? null; +} + +function clippedText(value: unknown, max: number) { + const result = text(value); + if (!result) return undefined; + return result.length > max ? `${result.slice(0, max)}...` : result; } function agentHelp() {