mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Merge pull request #1017 from pikasTech/fix/caserun-agent-stage-summary-1010

Browse Source 
fix: expose CaseRun agent HWPOD raw stages
This commit is contained in:
Lyon
2026-06-06 19:06:59 +08:00
committed by GitHub
parent 3580bf7b4a c71347f072
commit 942748d5e5
2 changed files with 79 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tools/hwlab-cli/caserun.test.ts
+19 -2
View File
@@ -328,7 +328,7 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
assert.ok(result.payload.evidence.agentTrace.commandCount > 0);
assert.ok(result.payload.evidence.agentTrace.commandCount <= 30);
assert.ok(result.payload.evidence.agentTrace.hwpodCommandCount >= 3);
assert.equal(result.payload.evidence.agentTrace.keilJobCandidates[0], "20260605_203835_798515c0");
assert.equal(result.payload.evidence.agentTrace.keilJobCandidates.includes("20260605_203835_798515c0"), true);
assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod-ctl spec validate --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true);
assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod inspect --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true);
assert.equal(result.payload.evidence.agentTrace.commands.some((item: any) => /^hwpod build --spec \.hwlab\/hwpod-spec\.yaml/u.test(item.normalizedCommand ?? "")), true);
@@ -378,7 +378,13 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
assert.equal(manifest.readableAgent.finalResponsePath, "final-response.md");
assert.equal(manifest.readableAgent.finalResponse.present, true);
assert.equal(manifest.readableAgent.finalResponse.text, "done");
assert.deepEqual(manifest.agentStage.kinds, ["spec-validate", "inspect", "workspace-edit", "build", "download", "uart-read"]);
assert.equal(manifest.agentStage.autoEvaluation, false);
assert.equal(manifest.agentStage.commands.some((item: any) => item.kind === "download" && /uvoptx file not found/u.test(item.detail)), true);
assert.equal(manifest.agentStage.commands.some((item: any) => item.kind === "uart-read" && /hwpod_uart_monitor_not_active/u.test(item.detail)), true);
assert.equal(manifest.decisions.autoEvaluation, false);
assert.equal(manifest.decisions.downloadSkipped, undefined);
assert.equal(manifest.decisions.runnerPostAgentCompileCheck, "recorded");
assert.equal(manifest.files.some((item: any) => item.path === "agent-messages.json"), true);
assert.equal(manifest.files.some((item: any) => item.path === "agent-trace.md"), true);
assert.equal(manifest.files.some((item: any) => item.path === "final-response.md"), true);
@@ -397,9 +403,17 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
assert.match(finalResponse, /done/u);
assert.equal(result.payload.collect.caseRepoRunDir, registryRunDir);
assert.equal(result.payload.collect.artifactManifestPath, path.join(registryRunDir, "artifact-manifest.json"));
assert.equal(result.payload.collect.agentStage.commands.some((item: any) => item.kind === "download"), true);
assert.equal(result.payload.collect.caseRepoFiles.some((item: string) => item.endsWith("artifact-manifest.json")), true);
const registryResult = JSON.parse(await readFile(path.join(registryRunDir, "result.json"), "utf8"));
assert.equal(registryResult.collect.artifactManifestPath, path.join(registryRunDir, "artifact-manifest.json"));
const summaryMarkdown = await readFile(path.join(registryRunDir, "summary.md"), "utf8");
assert.match(summaryMarkdown, /## Agent HWPOD Raw Steps/u);
assert.match(summaryMarkdown, /download/u);
assert.match(summaryMarkdown, /uvoptx file not found/u);
assert.match(summaryMarkdown, /uart-read/u);
assert.match(summaryMarkdown, /hwpod_uart_monitor_not_active/u);
assert.doesNotMatch(summaryMarkdown, /downloadSkipped/u);
const prompt = await readFile(path.join(cwd, ".state", "hwlab-cli", "caserun", "run-agent-flow", "agent-prompt.md"), "utf8");
assert.match(prompt, /without auto-grading/u);
@@ -683,7 +697,10 @@ function caseRunTraceEvents() {
const events: any[] = [
{ seq: 1, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "cat .agents/skills/hwpod-cli/SKILL.md", message: "use `hwpod build` for compile", exitCode: 0 },
{ seq: 248, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod-ctl spec validate --spec .hwlab/hwpod-spec.yaml'", message: "validated", exitCode: 0 },
{ seq: 323, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod inspect --spec .hwlab/hwpod-spec.yaml'", message: "inspected", exitCode: 0 }
{ seq: 323, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod inspect --spec .hwlab/hwpod-spec.yaml'", message: "inspected", exitCode: 0 },
{ seq: 360, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod workspace insert-after --spec .hwlab/hwpod-spec.yaml --path projects/01_baseline/User/main.c --anchor ready --line marker'", message: "inserted marker", exitCode: 0 },
{ seq: 480, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod download --spec .hwlab/hwpod-spec.yaml'", message: "uvoptx file not found; program accepted job 20260605_213000_aaaaaaaa", exitCode: 1 },
{ seq: 520, label: "item/commandExecution:completed", type: "tool_call", toolName: "commandExecution", status: "completed", command: "/bin/sh -lc 'hwpod uart read --spec .hwlab/hwpod-spec.yaml --port uart1 --max-bytes 4096'", message: "hwpod_uart_monitor_not_active on COM9", exitCode: 1 }
];
for (let index = 0; index < 31; index += 1) {
const seq = 400 + index * 2;
tools/src/hwlab-caserun-lib.ts
+60 -5
View File
@@ -511,7 +511,8 @@ export async function buildCaseRun(context: CaseContext, prepared?: PreparedCase
keilJob: job ? job.summary : null,
artifacts: job?.summary?.artifacts ?? [],
decisions: {
downloadSkipped: true,
autoEvaluation: false,
runnerPostAgentCompileCheck: "recorded",
reason: "flow-only run: CaseRun records agent, diff and compile evidence without auto-grading them"
}
});
@@ -1035,7 +1036,7 @@ function selectAgentTraceCommands(commands: AgentTraceCommand[]) {
}
function selectRepresentativeHwpodCommands(commands: AgentTraceCommand[]) {
return ["spec-validate", "inspect", "build"].flatMap((kind) => {
return ["spec-validate", "inspect", "workspace-edit", "build", "download", "uart-read"].flatMap((kind) => {
const candidates = commands.filter((command) => hwpodCommandKind(command) === kind);
return candidates.find((command) => text(command.status) === "completed" || command.exitCode === 0) ?? candidates[0] ?? [];
});
@@ -1045,7 +1046,10 @@ function hwpodCommandKind(command: AgentTraceCommand) {
const raw = normalizedHwpodCommand(command.normalizedCommand || command.command || "");
if (/^(?:hwpod-ctl|hwpod-cli)\s+spec\s+validate(?:\s|$)/iu.test(raw)) return "spec-validate";
if (/^hwpod\s+inspect(?:\s|$)/iu.test(raw)) return "inspect";
if (/^hwpod\s+workspace\s+(?:insert-after|apply-patch|write|replace|append)(?:\s|$)/iu.test(raw)) return "workspace-edit";
if (/^(?:hwpod|hwpod-cli)\s+build(?:\s|$)/iu.test(raw) || /^(?:bun\s+)?tools\/hwpod-cli\.ts\s+build(?:\s|$)/iu.test(raw)) return "build";
if (/^(?:hwpod|hwpod-cli)\s+download(?:\s|$)/iu.test(raw) || /^(?:bun\s+)?tools\/hwpod-cli\.ts\s+download(?:\s|$)/iu.test(raw)) return "download";
if (/^(?:hwpod|hwpod-cli)\s+(?:uart\s+read|io\s+uart\s+read)(?:\s|$)/iu.test(raw) || /^(?:bun\s+)?tools\/hwpod-cli\.ts\s+(?:uart\s+read|io\s+uart\s+read)(?:\s|$)/iu.test(raw)) return "uart-read";
return "";
}
@@ -1388,11 +1392,60 @@ function prepareSummary(run: PreparedCaseRun) {
}
function buildSummary(evidence: any) {
return { status: evidence.status, autoEvaluation: false, agentTraceCommandCount: evidence.agentTrace?.commandCount ?? 0, agentTraceHwpodCommandCount: evidence.agentTrace?.hwpodCommandCount ?? 0, hwpodExitCode: evidence.hwpod?.exitCode ?? null, hwpodSource: evidence.hwpod?.source ?? null, jobId: evidence.keilJob?.jobId ?? null, keilStatus: evidence.keilJob?.status ?? null, artifacts: evidence.artifacts ?? [] };
return { status: evidence.status, autoEvaluation: false, agentTraceCommandCount: evidence.agentTrace?.commandCount ?? 0, agentTraceHwpodCommandCount: evidence.agentTrace?.hwpodCommandCount ?? 0, agentStage: agentStageSummary(evidence), hwpodExitCode: evidence.hwpod?.exitCode ?? null, hwpodSource: evidence.hwpod?.source ?? null, jobId: evidence.keilJob?.jobId ?? null, keilStatus: evidence.keilJob?.status ?? null, artifacts: evidence.artifacts ?? [] };
}
function renderEvidenceSummary(evidence: any) {
return `# HWPOD CaseRun ${evidence.caseId}\n\n- runId: ${evidence.runId}\n- status: ${evidence.status}\n- autoEvaluation: false\n- compileOnly: ${evidence.compileOnly}\n- subjectRepoLocalPath: ${evidence.subject?.repoLocalPath ?? ""}\n- subjectCommitId: ${evidence.subject?.commitId ?? ""}\n- subjectWorktreePath: ${evidence.subject?.worktreePath ?? ""}\n- agentTraceId: ${evidence.agent?.traceId ?? ""}\n- agentSessionId: ${evidence.agent?.sessionId ?? ""}\n- agentTraceCommandCount: ${evidence.agentTrace?.commandCount ?? 0}\n- agentTraceHwpodCommandCount: ${evidence.agentTrace?.hwpodCommandCount ?? 0}\n- agentTraceHwpodBuildCommandCount: ${evidence.agentTrace?.hwpodBuildCommandCount ?? 0}\n- diffPatchPath: ${evidence.agentDiff?.diffPatchPath ?? ""}\n- runnerHwpodSource: ${evidence.hwpod?.source ?? ""}\n- hwpodExitCode: ${evidence.hwpod?.exitCode ?? ""}\n- jobId: ${evidence.keilJob?.jobId ?? ""}\n- downloadSkipped: true\n`;
const agentStage = agentStageSummary(evidence);
const stageTable = agentStage.commands.length > 0 ? [
``,
`## Agent HWPOD Raw Steps`,
``,
`| Step | Status | Exit | Command | Raw detail |`,
`|---|---:|---:|---|---|`,
...agentStage.commands.map((command: any) => `| ${command.kind} | ${command.status || ""} | ${command.exitCode ?? ""} | \`${markdownTableText(command.command, 160)}\` | ${markdownTableText(command.detail, 180)} |`)
].join("\n") : "";
return `# HWPOD CaseRun ${evidence.caseId}\n\n- runId: ${evidence.runId}\n- status: ${evidence.status}\n- autoEvaluation: false\n- compileOnly: ${evidence.compileOnly}\n- subjectRepoLocalPath: ${evidence.subject?.repoLocalPath ?? ""}\n- subjectCommitId: ${evidence.subject?.commitId ?? ""}\n- subjectWorktreePath: ${evidence.subject?.worktreePath ?? ""}\n- agentTraceId: ${evidence.agent?.traceId ?? ""}\n- agentSessionId: ${evidence.agent?.sessionId ?? ""}\n- agentTraceCommandCount: ${evidence.agentTrace?.commandCount ?? 0}\n- agentTraceHwpodCommandCount: ${evidence.agentTrace?.hwpodCommandCount ?? 0}\n- agentTraceHwpodBuildCommandCount: ${evidence.agentTrace?.hwpodBuildCommandCount ?? 0}\n- agentStageCommandCount: ${agentStage.commands.length}\n- agentStageKinds: ${agentStage.kinds.join(", ")}\n- diffPatchPath: ${evidence.agentDiff?.diffPatchPath ?? ""}\n- runnerHwpodSource: ${evidence.hwpod?.source ?? ""}\n- hwpodExitCode: ${evidence.hwpod?.exitCode ?? ""}\n- jobId: ${evidence.keilJob?.jobId ?? ""}\n- runnerPostAgentCompileCheck: recorded\n${stageTable}\n`;
}
function agentStageSummary(evidence: any) {
const commands = arrayOfObjects(evidence.agentTrace?.commands).filter(mentionsHwpod).map(agentStageCommand).filter((command) => text(command.kind));
const selected: Record<string, any> = {};
for (const command of commands) {
if (!selected[command.kind]) selected[command.kind] = command;
if (command.status === "completed" || command.exitCode === 0) selected[command.kind] = command;
}
const ordered = ["spec-validate", "inspect", "workspace-edit", "build", "download", "uart-read"]
.map((kind) => selected[kind])
.filter(Boolean);
const extras = commands.filter((command) => !ordered.some((item) => item.kind === command.kind && item.command === command.command));
return clean({
source: "agent-trace-commands",
autoEvaluation: false,
commands: [...ordered, ...extras].slice(0, MAX_AGENT_TRACE_COMMANDS),
kinds: uniqueStrings([...ordered, ...extras].map((command) => command.kind))
});
}
function agentStageCommand(command: AgentTraceCommand) {
return clean({
kind: hwpodCommandKind(command) || "hwpod-other",
seq: command.seq,
status: command.status,
exitCode: command.exitCode,
command: clipText(command.normalizedCommand || command.command, 1000),
detail: clipText(agentCommandDetail(command), 1000)
});
}
function agentCommandDetail(command: AgentTraceCommand) {
const body = text(command.bodyPreview);
if (!body) return "";
return body.split(/\r?\n/u).filter((line) => !line.includes(text(command.command))).join(" ").replace(/\s+/gu, " ").trim() || body;
}
function markdownTableText(value: unknown, maxBytes = 200) {
return clipText(value, maxBytes).replace(/\|/gu, "\\|").replace(/\r?\n/gu, " ").trim();
}
async function archiveCaseRunArtifacts(context: CaseContext, run: PreparedCaseRun, evidence: any): Promise<CaseRegistryArchive> {
@@ -1434,6 +1487,7 @@ function collectSummaryFromArchive(run: PreparedCaseRun, evidence: any, archive:
artifactCount: archive.files.length + 1,
skippedFiles: archive.skippedFiles,
trace: archive.manifest.trace ?? null,
agentStage: archive.manifest.agentStage ?? agentStageSummary(evidence),
status: evidence.status,
autoEvaluation: false
};
@@ -1666,11 +1720,12 @@ function caseRunArtifactManifest(context: CaseContext, run: PreparedCaseRun, evi
}),
prompt: clean({ path: "agent-prompt.md", sha256: evidence.agent?.promptSha256 ?? run.agent?.promptSha256 }),
diff: clean({ path: "agent-diff.patch", sha256: evidence.agentDiff?.diffPatchSha256 ?? run.agentDiff?.diffPatchSha256, statusShort: evidence.agentDiff?.statusShort ?? run.agentDiff?.statusShort, diffStat: evidence.agentDiff?.diffStat ?? run.agentDiff?.diffStat }),
agentStage: agentStageSummary(evidence),
keilJob: evidence.keilJob ?? null,
artifacts: evidence.artifacts ?? [],
decisions: {
autoEvaluation: false,
downloadSkipped: true,
runnerPostAgentCompileCheck: "recorded",
reason: "flow-only run: the manifest records artifacts and trace without auto-grading or gate decisions"
},
files,