diff --git a/docs/reference/spec-v02-hwlab-cli.md b/docs/reference/spec-v02-hwlab-cli.md index bf67658e..2f75f4cc 100644 --- a/docs/reference/spec-v02-hwlab-cli.md +++ b/docs/reference/spec-v02-hwlab-cli.md @@ -82,6 +82,7 @@ Code Agent session 是显式资源，不再由普通 `client agent send`、Workb | `hwlab-cli client gateway invoke` | `POST Cloud API /v1/rpc/hardware.invoke.shell` | 显式 Cloud API 诊断入口，执行一次 bounded shell dispatch 并返回结构化 dispatch 摘要。 | | `hwlab-cli client gateway pressure` | `POST Cloud API /v1/rpc/hardware.invoke.shell` | 显式 Cloud API 压测入口，真实验证大输出、长单行、stderr、timeout 和并发超容量不会造成黑洞。 | | `hwlab-cli client agent session create|select|status|list` | `POST/GET/PATCH /v1/agent/sessions*` + `GET/PATCH /v1/workbench/workspace*` | 显式创建、选择和观察 Code Agent session；session 创建或选择是 `send` 前置条件。 | +| `hwlab-cli client session final-response CONVERSATION_ID` | `GET /v1/agent/conversations/{conversation}` + `GET /v1/agent/conversations?projectId=...` + `GET /v1/agent/chat/result/{trace}` | 持久化 final response 展示回归的低噪声 closeout 验证入口；先 inspect 自动发现正确 `projectId`，再用 list/inspect/result 对齐 latest agent text、`lastTraceId` 和旧 fallback 缺失状态。 | | `hwlab-cli client agent send` | `GET /v1/agent/chat/inspect` + `POST /v1/agent/chat` + `GET /result/{trace}` | 以 short connection 向显式 session 提交 Code Agent turn 并轮询结果，默认输出 assistant 回复文本摘要；支持 `--from-trace` 显式复现 Web continuation，但不自动创建或滚动 session。 | | `hwlab-cli client agent composer status|submit` | `GET /v1/workbench/workspace` + `POST /v1/agent/chat` 或 `POST /v1/agent/chat/steer` | 使用 Web 共享 composer policy 探测输入框锁定状态和 turn/steer 分流；运行中应显示无锁 steer，并自动提交 steer。 | | `hwlab-cli client agent trace TRACE [--render web]` | `GET /v1/agent/chat/trace/{trace}` | 回放 trace，默认输出状态、事件摘要和 assistant stream 文本；`--render web` 复用 Cloud Web trace row 转换，便于 CLI 复现 Web trace 渲染问题。 | diff --git a/tools/hwlab-cli/client.test.ts b/tools/hwlab-cli/client.test.ts index dd668c0c..c13eadee 100644 --- a/tools/hwlab-cli/client.test.ts +++ b/tools/hwlab-cli/client.test.ts @@ -232,6 +232,100 @@ test("hwlab-cli client session management uses Cloud Web workspace and conversat assert.equal(deleted.payload.body.status, "deleted"); }); +test("hwlab-cli client session final-response verifies original conversation across project list and result", async () => { + const calls: any[] = []; + const finalText = "**直接答：能，但只到一半。**\n\n真实 final response"; + const conversation = { + conversationId: "cnv_issue834", + projectId: "prj_v02_code_agent", + sessionId: "ses_issue834", + threadId: "thread_issue834", + status: "idle", + lastTraceId: "trc_issue834_final", + snapshot: { source: "conversation-terminal-repair" }, + messages: [{ role: "agent", traceId: "trc_issue834_final", status: "idle", text: finalText }] + }; + const fetchImpl = async (url: string | URL) => { + calls.push(String(url)); + if (String(url).endsWith("/v1/agent/conversations/cnv_issue834")) { + return new Response(JSON.stringify({ ok: true, conversation }), { status: 200 }); + } + if (String(url).endsWith("/v1/agent/conversations?projectId=prj_v02_code_agent&limit=100")) { + return new Response(JSON.stringify({ ok: true, conversations: [conversation] }), { status: 200 }); + } + if (String(url).endsWith("/v1/agent/chat/result/trc_issue834_final")) { + return new Response(JSON.stringify({ ok: true, status: "completed", traceId: "trc_issue834_final", reply: { content: finalText } }), { status: 200 }); + } + return new Response(JSON.stringify({ ok: false, error: { code: "unexpected_route" } }), { status: 404 }); + }; + + const result = await runHwlabCli(["client", "session", "final-response", "cnv_issue834", "--base-url", "http://web.test", "--cookie", "hwlab_session=session-a"], { fetchImpl }); + + assert.equal(result.exitCode, 0); + assert.equal(result.payload.action, "client.session.final-response"); + assert.equal(result.payload.validation.state, "passed"); + assert.equal(result.payload.projectId, "prj_v02_code_agent"); + assert.equal(result.payload.validation.checks.listFound, true); + assert.equal(result.payload.validation.checks.inspectTextMatchesResult, true); + assert.equal(result.payload.validation.checks.listTextMatchesResult, true); + assert.equal(result.payload.validation.checks.fallbackTextAbsent, true); + assert.equal(result.payload.validation.repair.appearsRepaired, true); + assert.equal(calls[1], "http://web.test/v1/agent/conversations?projectId=prj_v02_code_agent&limit=100"); + assert.deepEqual(result.payload.validation.failures, []); +}); + +test("hwlab-cli client session final-response fails when old fallback is still visible", async () => { + const fallbackText = "Code Agent 仍在处理，可以继续 steer 或等待 trace 完成。"; + const conversation = { + conversationId: "cnv_issue834", + projectId: "prj_v02_code_agent", + sessionId: "ses_issue834", + status: "idle", + lastTraceId: "trc_issue834_final", + messages: [{ role: "agent", traceId: "trc_issue834_final", status: "idle", text: fallbackText }] + }; + const fetchImpl = async (url: string | URL) => { + if (String(url).endsWith("/v1/agent/conversations/cnv_issue834")) { + return new Response(JSON.stringify({ ok: true, conversation }), { status: 200 }); + } + if (String(url).endsWith("/v1/agent/conversations?projectId=prj_v02_code_agent&limit=100")) { + return new Response(JSON.stringify({ ok: true, conversations: [conversation] }), { status: 200 }); + } + if (String(url).endsWith("/v1/agent/chat/result/trc_issue834_final")) { + return new Response(JSON.stringify({ ok: true, status: "completed", traceId: "trc_issue834_final", reply: { content: "真实 final response" } }), { status: 200 }); + } + return new Response(JSON.stringify({ ok: false, error: { code: "unexpected_route" } }), { status: 404 }); + }; + + const result = await runHwlabCli(["client", "session", "final-response", "cnv_issue834", "--base-url", "http://web.test", "--cookie", "hwlab_session=session-a"], { fetchImpl }); + + assert.equal(result.exitCode, 1); + assert.equal(result.payload.action, "client.session.final-response"); + assert.equal(result.payload.validation.state, "failed"); + assert.equal(result.payload.validation.checks.fallbackTextAbsent, false); + assert.ok(result.payload.validation.failures.includes("fallback-text-present")); + assert.ok(result.payload.validation.failures.includes("inspect-text-result-mismatch")); + assert.deepEqual(result.payload.validation.fallbackText.presentIn, ["inspect", "list"]); +}); + +test("hwlab-cli client session final-response preserves inspect auth failure visibility", async () => { + const fetchImpl = async (url: string | URL) => { + if (String(url).endsWith("/v1/agent/conversations/cnv_issue834")) { + return new Response(JSON.stringify({ ok: false, status: 401, error: { code: "auth_required", message: "Authentication is required" } }), { status: 401 }); + } + return new Response(JSON.stringify({ ok: false, error: { code: "unexpected_route" } }), { status: 404 }); + }; + + const result = await runHwlabCli(["client", "session", "final-response", "cnv_issue834", "--base-url", "http://web.test", "--cookie", "hwlab_session=session-a"], { fetchImpl }); + + assert.equal(result.exitCode, 1); + assert.equal(result.payload.action, "client.session.final-response"); + assert.equal(result.payload.httpStatus, 401); + assert.equal(result.payload.validation.failures[0], "inspect-conversation-missing"); + assert.equal(result.payload.validation.error.code, "auth_required"); + assert.equal(result.payload.error?.code, undefined); +}); + test("hwlab-cli rejects manual endpoint override in locked assembled runtime", async () => { const result = await runHwlabCli(["client", "auth", "status", "--base-url", "http://74.48.78.17:17666"], { env: { diff --git a/tools/src/hwlab-cli-lib.ts b/tools/src/hwlab-cli-lib.ts index e3e51a78..8661e1ba 100644 --- a/tools/src/hwlab-cli-lib.ts +++ b/tools/src/hwlab-cli-lib.ts @@ -677,6 +677,7 @@ async function sessionCommand(context: any) { if (subcommand === "switch" || subcommand === "select") return sessionSwitchCommand(context); if (subcommand === "delete" || subcommand === "rm") return sessionDeleteCommand(context); if (subcommand === "inspect" || subcommand === "describe" || subcommand === "show") return sessionInspectCommand(context); + if (subcommand === "final-response" || subcommand === "final") return sessionFinalResponseCommand(context); throw cliError("unsupported_session_command", `unsupported session command: ${subcommand}`, { subcommand }); } @@ -691,7 +692,8 @@ function sessionHelp() { "create [--conversation-id ID] [--session-id ID] [--thread-id ID]", "switch CONVERSATION_ID|--conversation-id ID", "delete CONVERSATION_ID|--conversation-id ID --confirm", - "inspect CONVERSATION_ID|--conversation-id ID [--trace-id TRACE]" + "inspect CONVERSATION_ID|--conversation-id ID [--trace-id TRACE]", + "final-response CONVERSATION_ID|--conversation-id ID [--trace-id TRACE] [--project-id PROJECT] [--fallback-text TEXT]" ] }); } @@ -728,6 +730,132 @@ async function sessionListCommand(context: any) { }); } +async function sessionFinalResponseCommand(context: any) { + const conversationId = requiredConversationId(context.rest[1] ?? context.parsed.conversationId); + const inspectPath = `/v1/agent/conversations/${encodeURIComponent(conversationId)}`; + const inspectResponse = await requestJson({ ...context, method: "GET", path: inspectPath }); + const conversation = inspectResponse.body?.conversation ?? null; + if (!responseSucceeded(inspectResponse) || !conversation) { + const validation = finalResponseEarlyValidation({ + conversationId, + routeName: "inspect", + response: inspectResponse, + failure: conversation ? "inspect-request-failed" : "inspect-conversation-missing" + }); + return { + ...responsePayload("client.session.final-response", inspectResponse, context, { + route: route("GET", inspectPath), + conversationId, + routes: { inspect: route("GET", inspectPath) }, + httpStatuses: { inspect: inspectResponse.status }, + validation, + inspect: finalResponseConversationSummary(conversation), + body: context.parsed.full === true ? { inspect: inspectResponse.body } : compactApiBody(inspectResponse.body) + }), + ok: false, + status: "failed" + }; + } + const projectId = text(context.parsed.projectId) || text(conversation?.projectId) || DEFAULT_WORKBENCH_PROJECT_ID; + const limit = numberOption(context.parsed.limit) ?? 100; + const listPath = `/v1/agent/conversations?projectId=${encodeURIComponent(projectId)}&limit=${encodeURIComponent(String(limit))}`; + const listResponse = await requestJson({ ...context, method: "GET", path: listPath }); + if (!responseSucceeded(listResponse)) { + const validation = finalResponseEarlyValidation({ + conversationId, + projectId, + routeName: "list", + response: listResponse, + failure: "list-request-failed" + }); + return { + ...responsePayload("client.session.final-response", listResponse, context, { + route: route("GET", listPath), + conversationId, + projectId, + routes: { inspect: route("GET", inspectPath), list: route("GET", listPath) }, + httpStatuses: { inspect: inspectResponse.status, list: listResponse.status }, + validation, + inspect: finalResponseConversationSummary(conversation), + body: context.parsed.full === true ? { inspect: inspectResponse.body, list: listResponse.body } : compactApiBody(listResponse.body) + }), + ok: false, + status: "failed" + }; + } + const listedConversation = Array.isArray(listResponse.body?.conversations) + ? listResponse.body.conversations.find((item: any) => text(item?.conversationId) === conversationId) ?? null + : null; + const effectiveConversation = listedConversation ?? conversation; + const resolvedTraceId = text(context.parsed.traceId ?? effectiveConversation?.lastTraceId ?? latestAgentMessage(effectiveConversation)?.traceId); + if (!resolvedTraceId) { + const validation = finalResponseEarlyValidation({ + conversationId, + projectId, + routeName: "conversation", + response: listResponse, + failure: "trace-id-missing" + }); + return { + ...responsePayload("client.session.final-response", listResponse, context, { + route: route("GET", listPath), + conversationId, + projectId, + routes: { inspect: route("GET", inspectPath), list: route("GET", listPath) }, + httpStatuses: { inspect: inspectResponse.status, list: listResponse.status }, + validation, + inspect: finalResponseConversationSummary(conversation), + list: finalResponseConversationSummary(listedConversation), + body: context.parsed.full === true ? { inspect: inspectResponse.body, list: listResponse.body } : undefined + }), + ok: false, + status: "failed" + }; + } + const traceId = requiredTraceId(resolvedTraceId); + const resultPath = `/v1/agent/chat/result/${encodeURIComponent(traceId)}`; + const resultResponse = await requestJson({ ...context, method: "GET", path: resultPath }); + const validation = finalResponseValidation({ + conversationId, + projectId, + inspectConversation: conversation, + listedConversation, + traceId, + resultBody: resultResponse.body, + fallbackText: text(context.parsed.fallbackText) || "Code Agent 仍在处理，可以继续 steer 或等待 trace 完成。" + }); + const success = responseSucceeded(inspectResponse) && responseSucceeded(listResponse) && responseSucceeded(resultResponse) && validation.ok; + return { + ...responsePayload("client.session.final-response", resultResponse, context, { + route: route("GET", resultPath), + conversationId, + projectId, + traceId, + routes: { + inspect: route("GET", inspectPath), + list: route("GET", listPath), + result: route("GET", resultPath) + }, + httpStatuses: { + inspect: inspectResponse.status, + list: listResponse.status, + result: resultResponse.status + }, + validation, + inspect: finalResponseConversationSummary(conversation), + list: finalResponseConversationSummary(listedConversation), + result: finalResponseResultSummary(resultResponse.body), + body: context.parsed.full === true ? { + inspect: inspectResponse.body, + list: listResponse.body, + result: resultResponse.body + } : finalResponseResultSummary(resultResponse.body) + }), + ok: success, + status: success ? "succeeded" : "failed" + }; +} + async function sessionCurrentCommand(context: any) { const workspace = await restoreWorkbenchWorkspace(context, { quiet: false }); return ok("client.session.current", { @@ -2322,6 +2450,141 @@ function sessionSummary(conversation: any) { valuesRedacted: conversation.valuesRedacted === true ? true : undefined }); } + +function latestAgentMessage(conversation: any) { + const messages = Array.isArray(conversation?.messages) ? conversation.messages : []; + return [...messages].reverse().find((message: any) => text(message?.role) === "agent") ?? null; +} + +function finalResponseConversationSummary(conversation: any) { + if (!conversation || typeof conversation !== "object") return null; + const latest = latestAgentMessage(conversation); + return clean({ + found: true, + conversationId: text(conversation.conversationId), + projectId: text(conversation.projectId), + sessionId: text(conversation.sessionId ?? conversation.session?.sessionId), + threadId: text(conversation.threadId ?? conversation.session?.threadId), + status: text(conversation.status ?? conversation.snapshot?.sessionStatus), + lastTraceId: text(conversation.lastTraceId), + updatedAt: text(conversation.updatedAt), + snapshotSource: text(conversation.snapshot?.source), + latestAgent: latest ? clean({ + id: text(latest.id), + status: text(latest.status), + traceId: text(latest.traceId), + updatedAt: text(latest.updatedAt), + textPreview: preview(latest.text, 240) + }) : undefined + }); +} + +function finalResponseResultSummary(body: any) { + const resultText = assistantText(body) || ""; + return clean({ + status: text(body?.status), + traceId: text(body?.traceId), + conversationId: text(body?.conversationId), + sessionId: text(body?.sessionId), + threadId: text(body?.threadId ?? body?.session?.threadId ?? body?.providerTrace?.threadId), + assistantTextPreview: preview(resultText, 240), + assistantTextChars: resultText.length || undefined + }); +} + +function finalResponseEarlyValidation({ conversationId, projectId, routeName, response, failure }: any) { + const failures = [failure].filter(Boolean); + return { + ok: false, + state: "failed", + summary: `${routeName} step failed before final response comparison`, + conversationId, + projectId: text(projectId) || undefined, + traceId: undefined, + checks: { + inspectFound: false, + listFound: false, + resultHasFinalText: false, + fallbackTextAbsent: false + }, + httpStatus: response?.status, + error: response?.body?.error && typeof response.body.error === "object" + ? clean({ code: text(response.body.error.code), message: text(response.body.error.message) }) + : compactApiBody(response?.body), + failures, + nextCommands: [ + `hwlab-cli client session inspect ${conversationId} --full`, + text(projectId) ? `hwlab-cli client session list --project-id ${projectId} --limit 20 --full` : "hwlab-cli client auth status" + ] + }; +} + +function finalResponseValidation({ conversationId, projectId, inspectConversation, listedConversation, traceId, resultBody, fallbackText }: any) { + const inspectedLatest = latestAgentMessage(inspectConversation); + const listedLatest = latestAgentMessage(listedConversation); + const inspectText = text(inspectedLatest?.text); + const listText = text(listedLatest?.text); + const resultText = assistantText(resultBody) || ""; + const fallback = text(fallbackText); + const textPairs = [ + { source: "inspect", text: inspectText }, + { source: "list", text: listText }, + { source: "result", text: resultText } + ]; + const fallbackHits = fallback ? textPairs.filter((item) => item.text.includes(fallback)).map((item) => item.source) : []; + const failures = [ + !inspectConversation ? "inspect-conversation-missing" : "", + !listedConversation ? "list-conversation-missing" : "", + !traceId ? "trace-id-missing" : "", + !resultText ? "result-final-response-missing" : "", + text(inspectConversation?.lastTraceId) !== traceId ? "inspect-last-trace-mismatch" : "", + listedConversation && text(listedConversation?.lastTraceId) !== traceId ? "list-last-trace-mismatch" : "", + inspectedLatest && text(inspectedLatest.traceId) !== traceId ? "inspect-latest-agent-trace-mismatch" : "", + listedLatest && text(listedLatest.traceId) !== traceId ? "list-latest-agent-trace-mismatch" : "", + inspectText && resultText && inspectText !== resultText ? "inspect-text-result-mismatch" : "", + listText && resultText && listText !== resultText ? "list-text-result-mismatch" : "", + fallbackHits.length > 0 ? "fallback-text-present" : "" + ].filter(Boolean); + return { + ok: failures.length === 0, + state: failures.length === 0 ? "passed" : "failed", + summary: failures.length === 0 + ? "list, inspect, and agent result agree on the final assistant response" + : `final response validation failed with ${failures.length} issue(s)`, + conversationId, + projectId, + traceId, + checks: { + inspectFound: Boolean(inspectConversation), + listFound: Boolean(listedConversation), + resultHasFinalText: Boolean(resultText), + inspectLastTraceMatches: text(inspectConversation?.lastTraceId) === traceId, + listLastTraceMatches: listedConversation ? text(listedConversation?.lastTraceId) === traceId : false, + inspectLatestAgentTraceMatches: inspectedLatest ? text(inspectedLatest.traceId) === traceId : false, + listLatestAgentTraceMatches: listedLatest ? text(listedLatest.traceId) === traceId : false, + inspectTextMatchesResult: Boolean(inspectText && resultText && inspectText === resultText), + listTextMatchesResult: Boolean(listText && resultText && listText === resultText), + fallbackTextAbsent: fallbackHits.length === 0 + }, + fallbackText: fallback ? { checked: true, presentIn: fallbackHits } : { checked: false, presentIn: [] }, + repair: clean({ + inspectSnapshotSource: text(inspectConversation?.snapshot?.source), + listSnapshotSource: text(listedConversation?.snapshot?.source), + appearsRepaired: text(inspectConversation?.snapshot?.source).includes("repair") || text(listedConversation?.snapshot?.source).includes("repair") + }), + previews: { + inspectText: preview(inspectText, 200), + listText: preview(listText, 200), + resultText: preview(resultText, 200) + }, + failures, + nextCommands: [ + `hwlab-cli client session inspect ${conversationId} --full`, + `hwlab-cli client session list --project-id ${projectId} --limit 20 --full`, + `hwlab-cli client agent result ${traceId} --full` + ] + }; +} function numberOrZero(value: any) { const parsed = Number(value); return Number.isFinite(parsed) && parsed > 0 ? parsed : 0;