diff --git a/tools/hwlab-cli/caserun.test.ts b/tools/hwlab-cli/caserun.test.ts index 5bf0860d..725b4b10 100644 --- a/tools/hwlab-cli/caserun.test.ts +++ b/tools/hwlab-cli/caserun.test.ts @@ -328,7 +328,7 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit assert.ok(result.payload.evidence.agentTrace.commandCount > 0); assert.ok(result.payload.evidence.agentTrace.commandCount <= 30); assert.ok(result.payload.evidence.agentTrace.hwpodCommandCount >= 3); - assert.equal(result.payload.evidence.agentTrace.keilJobCandidates[0], "20260605_203835_798515c0"); + assert.equal(result.payload.evidence.agentTrace.keilJobCandidates.includes("20260605_203835_798515c0"), true); assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod-ctl spec validate --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true); assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod inspect --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true); assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod build --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true); @@ -378,7 +378,13 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit assert.equal(manifest.readableAgent.finalResponsePath, "final-response.md"); assert.equal(manifest.readableAgent.finalResponse.present, true); assert.equal(manifest.readableAgent.finalResponse.text, "done"); + assert.deepEqual(manifest.agentStage.kinds, ["spec-validate", "inspect", "workspace-edit", "build", "download", "uart-read"]); + assert.equal(manifest.agentStage.autoEvaluation, false); + assert.equal(manifest.agentStage.commands.some((item: any) => item.kind === "download" && /uvoptx file not found/u.test(item.detail)), true); + assert.equal(manifest.agentStage.commands.some((item: any) => item.kind === "uart-read" && /hwpod_uart_monitor_not_active/u.test(item.detail)), true); assert.equal(manifest.decisions.autoEvaluation, false); + assert.equal(manifest.decisions.downloadSkipped, undefined); + assert.equal(manifest.decisions.runnerPostAgentCompileCheck, "recorded"); assert.equal(manifest.files.some((item: any) => item.path === "agent-messages.json"), true); assert.equal(manifest.files.some((item: any) => item.path === "agent-trace.md"), true); assert.equal(manifest.files.some((item: any) => item.path === "final-response.md"), true); @@ -397,9 +403,17 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit assert.match(finalResponse, /done/u); assert.equal(result.payload.collect.caseRepoRunDir, registryRunDir); assert.equal(result.payload.collect.artifactManifestPath, path.join(registryRunDir, "artifact-manifest.json")); + assert.equal(result.payload.collect.agentStage.commands.some((item: any) => item.kind === "download"), true); assert.equal(result.payload.collect.caseRepoFiles.some((item: string) => item.endsWith("artifact-manifest.json")), true); const registryResult = JSON.parse(await readFile(path.join(registryRunDir, "result.json"), "utf8")); assert.equal(registryResult.collect.artifactManifestPath, path.join(registryRunDir, "artifact-manifest.json")); + const summaryMarkdown = await readFile(path.join(registryRunDir, "summary.md"), "utf8"); + assert.match(summaryMarkdown, /## Agent HWPOD Raw Steps/u); + assert.match(summaryMarkdown, /download/u); + assert.match(summaryMarkdown, /uvoptx file not found/u); + assert.match(summaryMarkdown, /uart-read/u); + assert.match(summaryMarkdown, /hwpod_uart_monitor_not_active/u); + assert.doesNotMatch(summaryMarkdown, /downloadSkipped/u); const prompt = await readFile(path.join(cwd, ".state", "hwlab-cli", "caserun", "run-agent-flow", "agent-prompt.md"), "utf8"); assert.match(prompt, /without auto-grading/u); @@ -683,7 +697,10 @@ function caseRunTraceEvents() { const events: any[] = [ { seq: 1, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "cat .agents/skills/hwpod-cli/SKILL.md", message: "use `hwpod build` for compile", exitCode: 0 }, { seq: 248, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod-ctl spec validate --spec .hwlab/hwpod-spec.yaml'", message: "validated", exitCode: 0 }, - { seq: 323, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod inspect --spec .hwlab/hwpod-spec.yaml'", message: "inspected", exitCode: 0 } + { seq: 323, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod inspect --spec .hwlab/hwpod-spec.yaml'", message: "inspected", exitCode: 0 }, + { seq: 360, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod workspace insert-after --spec .hwlab/hwpod-spec.yaml --path projects/01_baseline/User/main.c --anchor ready --line marker'", message: "inserted marker", exitCode: 0 }, + { seq: 480, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod download --spec .hwlab/hwpod-spec.yaml'", message: "uvoptx file not found; program accepted job 20260605_213000_aaaaaaaa", exitCode: 1 }, + { seq: 520, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod uart read --spec .hwlab/hwpod-spec.yaml --port uart1 --max-bytes 4096'", message: "hwpod_uart_monitor_not_active on COM9", exitCode: 1 } ]; for (let index = 0; index < 31; index += 1) { const seq = 400 + index * 2; diff --git a/tools/src/hwlab-caserun-lib.ts b/tools/src/hwlab-caserun-lib.ts index b7f77af0..245e057d 100644 --- a/tools/src/hwlab-caserun-lib.ts +++ b/tools/src/hwlab-caserun-lib.ts @@ -511,7 +511,8 @@ export async function buildCaseRun(context: CaseContext, prepared?: PreparedCase keilJob: job ? job.summary : null, artifacts: job?.summary?.artifacts ?? [], decisions: { - downloadSkipped: true, + autoEvaluation: false, + runnerPostAgentCompileCheck: "recorded", reason: "flow-only run: CaseRun records agent, diff and compile evidence without auto-grading them" } }); @@ -1035,7 +1036,7 @@ function selectAgentTraceCommands(commands: AgentTraceCommand[]) { } function selectRepresentativeHwpodCommands(commands: AgentTraceCommand[]) { - return ["spec-validate", "inspect", "build"].flatMap((kind) => { + return ["spec-validate", "inspect", "workspace-edit", "build", "download", "uart-read"].flatMap((kind) => { const candidates = commands.filter((command) => hwpodCommandKind(command) === kind); return candidates.find((command) => text(command.status) === "completed" || command.exitCode === 0) ?? candidates[0] ?? []; }); @@ -1045,7 +1046,10 @@ function hwpodCommandKind(command: AgentTraceCommand) { const raw = normalizedHwpodCommand(command.normalizedCommand || command.command || ""); if (/^(?:hwpod-ctl|hwpod-cli)\s+spec\s+validate(?:\s|$)/iu.test(raw)) return "spec-validate"; if (/^hwpod\s+inspect(?:\s|$)/iu.test(raw)) return "inspect"; + if (/^hwpod\s+workspace\s+(?:insert-after|apply-patch|write|replace|append)(?:\s|$)/iu.test(raw)) return "workspace-edit"; if (/^(?:hwpod|hwpod-cli)\s+build(?:\s|$)/iu.test(raw) || /^(?:bun\s+)?tools\/hwpod-cli\.ts\s+build(?:\s|$)/iu.test(raw)) return "build"; + if (/^(?:hwpod|hwpod-cli)\s+download(?:\s|$)/iu.test(raw) || /^(?:bun\s+)?tools\/hwpod-cli\.ts\s+download(?:\s|$)/iu.test(raw)) return "download"; + if (/^(?:hwpod|hwpod-cli)\s+(?:uart\s+read|io\s+uart\s+read)(?:\s|$)/iu.test(raw) || /^(?:bun\s+)?tools\/hwpod-cli\.ts\s+(?:uart\s+read|io\s+uart\s+read)(?:\s|$)/iu.test(raw)) return "uart-read"; return ""; } @@ -1388,11 +1392,60 @@ function prepareSummary(run: PreparedCaseRun) { } function buildSummary(evidence: any) { - return { status: evidence.status, autoEvaluation: false, agentTraceCommandCount: evidence.agentTrace?.commandCount ?? 0, agentTraceHwpodCommandCount: evidence.agentTrace?.hwpodCommandCount ?? 0, hwpodExitCode: evidence.hwpod?.exitCode ?? null, hwpodSource: evidence.hwpod?.source ?? null, jobId: evidence.keilJob?.jobId ?? null, keilStatus: evidence.keilJob?.status ?? null, artifacts: evidence.artifacts ?? [] }; + return { status: evidence.status, autoEvaluation: false, agentTraceCommandCount: evidence.agentTrace?.commandCount ?? 0, agentTraceHwpodCommandCount: evidence.agentTrace?.hwpodCommandCount ?? 0, agentStage: agentStageSummary(evidence), hwpodExitCode: evidence.hwpod?.exitCode ?? null, hwpodSource: evidence.hwpod?.source ?? null, jobId: evidence.keilJob?.jobId ?? null, keilStatus: evidence.keilJob?.status ?? null, artifacts: evidence.artifacts ?? [] }; } function renderEvidenceSummary(evidence: any) { - return `# HWPOD CaseRun ${evidence.caseId}\n\n- runId: ${evidence.runId}\n- status: ${evidence.status}\n- autoEvaluation: false\n- compileOnly: ${evidence.compileOnly}\n- subjectRepoLocalPath: ${evidence.subject?.repoLocalPath ?? ""}\n- subjectCommitId: ${evidence.subject?.commitId ?? ""}\n- subjectWorktreePath: ${evidence.subject?.worktreePath ?? ""}\n- agentTraceId: ${evidence.agent?.traceId ?? ""}\n- agentSessionId: ${evidence.agent?.sessionId ?? ""}\n- agentTraceCommandCount: ${evidence.agentTrace?.commandCount ?? 0}\n- agentTraceHwpodCommandCount: ${evidence.agentTrace?.hwpodCommandCount ?? 0}\n- agentTraceHwpodBuildCommandCount: ${evidence.agentTrace?.hwpodBuildCommandCount ?? 0}\n- diffPatchPath: ${evidence.agentDiff?.diffPatchPath ?? ""}\n- runnerHwpodSource: ${evidence.hwpod?.source ?? ""}\n- hwpodExitCode: ${evidence.hwpod?.exitCode ?? ""}\n- jobId: ${evidence.keilJob?.jobId ?? ""}\n- downloadSkipped: true\n`; + const agentStage = agentStageSummary(evidence); + const stageTable = agentStage.commands.length > 0 ? [ + ``, + `## Agent HWPOD Raw Steps`, + ``, + `| Step | Status | Exit | Command | Raw detail |`, + `|---|---:|---:|---|---|`, + ...agentStage.commands.map((command: any) => `| ${command.kind} | ${command.status || ""} | ${command.exitCode ?? ""} | \`${markdownTableText(command.command, 160)}\` | ${markdownTableText(command.detail, 180)} |`) + ].join("\n") : ""; + return `# HWPOD CaseRun ${evidence.caseId}\n\n- runId: ${evidence.runId}\n- status: ${evidence.status}\n- autoEvaluation: false\n- compileOnly: ${evidence.compileOnly}\n- subjectRepoLocalPath: ${evidence.subject?.repoLocalPath ?? ""}\n- subjectCommitId: ${evidence.subject?.commitId ?? ""}\n- subjectWorktreePath: ${evidence.subject?.worktreePath ?? ""}\n- agentTraceId: ${evidence.agent?.traceId ?? ""}\n- agentSessionId: ${evidence.agent?.sessionId ?? ""}\n- agentTraceCommandCount: ${evidence.agentTrace?.commandCount ?? 0}\n- agentTraceHwpodCommandCount: ${evidence.agentTrace?.hwpodCommandCount ?? 0}\n- agentTraceHwpodBuildCommandCount: ${evidence.agentTrace?.hwpodBuildCommandCount ?? 0}\n- agentStageCommandCount: ${agentStage.commands.length}\n- agentStageKinds: ${agentStage.kinds.join(", ")}\n- diffPatchPath: ${evidence.agentDiff?.diffPatchPath ?? ""}\n- runnerHwpodSource: ${evidence.hwpod?.source ?? ""}\n- hwpodExitCode: ${evidence.hwpod?.exitCode ?? ""}\n- jobId: ${evidence.keilJob?.jobId ?? ""}\n- runnerPostAgentCompileCheck: recorded\n${stageTable}\n`; +} + +function agentStageSummary(evidence: any) { + const commands = arrayOfObjects(evidence.agentTrace?.commands).filter(mentionsHwpod).map(agentStageCommand).filter((command) => text(command.kind)); + const selected: Record = {}; + for (const command of commands) { + if (!selected[command.kind]) selected[command.kind] = command; + if (command.status === "completed" || command.exitCode === 0) selected[command.kind] = command; + } + const ordered = ["spec-validate", "inspect", "workspace-edit", "build", "download", "uart-read"] + .map((kind) => selected[kind]) + .filter(Boolean); + const extras = commands.filter((command) => !ordered.some((item) => item.kind === command.kind && item.command === command.command)); + return clean({ + source: "agent-trace-commands", + autoEvaluation: false, + commands: [...ordered, ...extras].slice(0, MAX_AGENT_TRACE_COMMANDS), + kinds: uniqueStrings([...ordered, ...extras].map((command) => command.kind)) + }); +} + +function agentStageCommand(command: AgentTraceCommand) { + return clean({ + kind: hwpodCommandKind(command) || "hwpod-other", + seq: command.seq, + status: command.status, + exitCode: command.exitCode, + command: clipText(command.normalizedCommand || command.command, 1000), + detail: clipText(agentCommandDetail(command), 1000) + }); +} + +function agentCommandDetail(command: AgentTraceCommand) { + const body = text(command.bodyPreview); + if (!body) return ""; + return body.split(/\r?\n/u).filter((line) => !line.includes(text(command.command))).join(" ").replace(/\s+/gu, " ").trim() || body; +} + +function markdownTableText(value: unknown, maxBytes = 200) { + return clipText(value, maxBytes).replace(/\|/gu, "\\|").replace(/\r?\n/gu, " ").trim(); } async function archiveCaseRunArtifacts(context: CaseContext, run: PreparedCaseRun, evidence: any): Promise { @@ -1434,6 +1487,7 @@ function collectSummaryFromArchive(run: PreparedCaseRun, evidence: any, archive: artifactCount: archive.files.length + 1, skippedFiles: archive.skippedFiles, trace: archive.manifest.trace ?? null, + agentStage: archive.manifest.agentStage ?? agentStageSummary(evidence), status: evidence.status, autoEvaluation: false }; @@ -1666,11 +1720,12 @@ function caseRunArtifactManifest(context: CaseContext, run: PreparedCaseRun, evi }), prompt: clean({ path: "agent-prompt.md", sha256: evidence.agent?.promptSha256 ?? run.agent?.promptSha256 }), diff: clean({ path: "agent-diff.patch", sha256: evidence.agentDiff?.diffPatchSha256 ?? run.agentDiff?.diffPatchSha256, statusShort: evidence.agentDiff?.statusShort ?? run.agentDiff?.statusShort, diffStat: evidence.agentDiff?.diffStat ?? run.agentDiff?.diffStat }), + agentStage: agentStageSummary(evidence), keilJob: evidence.keilJob ?? null, artifacts: evidence.artifacts ?? [], decisions: { autoEvaluation: false, - downloadSkipped: true, + runnerPostAgentCompileCheck: "recorded", reason: "flow-only run: the manifest records artifacts and trace without auto-grading or gate decisions" }, files,