From c2c016b20f76a3a12dc3f59e233c268403525308 Mon Sep 17 00:00:00 2001 From: Codex Agent Date: Sat, 6 Jun 2026 14:37:20 +0800 Subject: [PATCH] =?UTF-8?q?case-run:=20=E8=AE=B0=E5=BD=95=20agent=20compil?= =?UTF-8?q?e=20trace?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- tools/hwlab-cli/caserun.test.ts | 19 ++++ tools/src/hwlab-caserun-lib.ts | 196 ++++++++++++++++++++++++++++++-- 2 files changed, 205 insertions(+), 10 deletions(-) diff --git a/tools/hwlab-cli/caserun.test.ts b/tools/hwlab-cli/caserun.test.ts index ad0c6f77..663cd6c8 100644 --- a/tools/hwlab-cli/caserun.test.ts +++ b/tools/hwlab-cli/caserun.test.ts @@ -236,20 +236,32 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit assert.equal(result.payload.evidence.ok, undefined); assert.equal(result.payload.agent.stageStatus, "completed"); assert.equal(result.payload.agent.traceId.startsWith("trc_case_"), true); + assert.equal(result.payload.agentTrace.hwpodBuildCommandCount, 1); + assert.equal(result.payload.evidence.agentTrace.commandCount, 3); + assert.equal(result.payload.evidence.agentTrace.hwpodCommandCount, 3); + assert.equal(result.payload.evidence.agentTrace.keilJobCandidates[0], "20260605_203835_798515c0"); + assert.equal(result.payload.evidence.hwpod.source, "case-run-runner-post-agent-compile-check"); assert.match(result.payload.agentDiff.statusShort, /projects\/01_baseline\/main\.c/u); assert.equal(result.payload.build.autoEvaluation, false); + assert.equal(result.payload.build.agentTraceHwpodCommandCount, 3); assert.equal(result.payload.build.hwpodExitCode, 0); + assert.equal(result.payload.build.hwpodSource, "case-run-runner-post-agent-compile-check"); assert.equal(result.payload.build.keilStatus, "completed"); assert.equal(result.payload.collect.autoEvaluation, false); assert.equal(sawAgentRunningState, true); const prompt = await readFile(path.join(cwd, ".state", "hwlab-cli", "caserun", "run-agent-flow", "agent-prompt.md"), "utf8"); assert.match(prompt, /without auto-grading/u); + assert.match(prompt, /Run-local HWPOD spec/u); + assert.match(prompt, /hwpod-ctl spec validate --spec \.hwlab\/hwpod-spec\.yaml/u); + assert.match(prompt, /hwpod build --spec \.hwlab\/hwpod-spec\.yaml/u); + assert.match(prompt, /path: "F:\\\\Work\\\\HWLAB-CASE-F103\\\\\.worktree\\\\caserun-run-agent-flow"/u); const diff = await readFile(path.join(cwd, ".state", "hwlab-cli", "caserun", "run-agent-flow", "agent-diff.patch"), "utf8"); assert.match(diff, /printf\("hello"\)/u); assert.equal(requests.some((item) => item.url === "http://web.test/v1/agent/sessions"), true); assert.equal(requests.some((item) => item.url === "http://web.test/v1/agent/chat"), true); assert.equal(requests.some((item) => item.url === "http://web.test/v1/agent/chat/result/" + result.payload.agent.traceId), true); + assert.equal(requests.some((item) => item.url === "http://web.test/v1/agent/chat/trace/" + result.payload.agent.traceId), true); } finally { await rm(root, { recursive: true, force: true }); } @@ -300,6 +312,13 @@ function caseRunFlowFetch(requests: any[]) { resultPolls += 1; return new Response(JSON.stringify({ ok: true, status: resultPolls > 1 ? "completed" : "running", reply: { content: "done" } }), { status: 200, headers: { "content-type": "application/json" } }); } + if (String(url).startsWith("http://web.test/v1/agent/chat/trace/")) { + return new Response(JSON.stringify({ ok: true, status: "completed", sourceEventCount: 6, renderedRowCount: 3, traceSummary: { terminalStatus: "completed", agentRun: { runId: "run_agent_trace", commandId: "cmd_agent_trace" } }, rows: [ + { seq: 1, header: "ok commandExecution", body: "command: hwpod-ctl spec validate --spec .hwlab/hwpod-spec.yaml\nexitCode: 0" }, + { seq: 2, header: "ok commandExecution", body: "command: hwpod inspect --spec .hwlab/hwpod-spec.yaml\nexitCode: 0" }, + { seq: 3, header: "ok commandExecution", body: "command: hwpod build --spec .hwlab/hwpod-spec.yaml\nstdout: {\"job_id\":\"20260605_203835_798515c0\"}\nexitCode: 0" } + ] }), { status: 200, headers: { "content-type": "application/json" } }); + } if (body.ops?.[0]?.args?.argv?.includes("status") && body.ops?.length === 1) { return hwpodResponse([{ exitCode: 0, stdout: body.ops[0].args.argv.includes("--short") ? " M projects/01_baseline/main.c\n" : "" }]); } diff --git a/tools/src/hwlab-caserun-lib.ts b/tools/src/hwlab-caserun-lib.ts index 857730d8..a7fdb612 100644 --- a/tools/src/hwlab-caserun-lib.ts +++ b/tools/src/hwlab-caserun-lib.ts @@ -12,6 +12,7 @@ const DEFAULT_POLL_INTERVAL_MS = 1000; const DEFAULT_JOB_TIMEOUT_MS = 50000; const MAX_JOB_TIMEOUT_MS = 50000; const DEFAULT_AGENT_TIMEOUT_MS = 120000; +const MAX_AGENT_TRACE_COMMANDS = 30; const HWLAB_API_KEY_PREFIX = "hwl_live_"; const STANDARD_CASE_REPO_PATH = "/root/hwlab-case-registry"; const REMOVED_CASE_REPO_PATH = "/root/hwpod-cases"; @@ -52,6 +53,7 @@ type PreparedCaseRun = { subject: PreparedSubjectRun; agentTask?: PreparedAgentTask | null; agent?: AgentRunStage | null; + agentTrace?: AgentTraceStage | null; agentDiff?: AgentDiffStage | null; status?: string; stage?: string; @@ -108,6 +110,34 @@ type AgentDiffStage = { requests: Record[]; }; +type AgentTraceCommand = { + source: string; + seq?: number; + rowId?: string; + header?: string; + toolName?: string; + status?: string; + command?: string; + bodyPreview?: string; + exitCode?: number; +}; + +type AgentTraceStage = { + traceId: string; + status: string; + httpStatus: number; + sourceEventCount?: number; + renderedRowCount?: number; + commandCount: number; + hwpodCommandCount: number; + hwpodBuildCommandCount: number; + keilJobCandidates: string[]; + terminalStatus?: string; + agentRun?: Record | null; + commands: AgentTraceCommand[]; + error?: Record | null; +}; + export async function caseCommand(context: CaseContext) { const subcommand = context.rest[0] || "help"; if (["help", "--help", "-h"].includes(subcommand) || context.parsed.help === true) return caseHelp(); @@ -162,7 +192,8 @@ export function caseHelp() { export async function runCaseRun(context: CaseContext) { const prepared = await prepareCaseRun(context, "run"); const agentStage = await runAgentTaskStage(context, prepared.run); - const diffStage = await collectAgentDiff(context, agentStage.run); + const traceStage = await collectAgentTraceEvidence(context, agentStage.run); + const diffStage = await collectAgentDiff(context, traceStage.run); const built = await buildCaseRun(context, diffStage.run); const collected = await collectCaseRun(context, built.run, built.evidence); await writeRunControl(context, collected.run, { status: "completed", stage: "completed", completedAt: context.now(), resultPath: path.join(collected.run.runDir, "result.json") }); @@ -174,6 +205,7 @@ export async function runCaseRun(context: CaseContext) { compileOnly: true, prepare: prepared.summary, agent: agentSummary(agentStage.agent), + agentTrace: traceSummary(traceStage.trace), agentDiff: diffSummary(diffStage.diff), build: built.summary, collect: collected.summary, @@ -407,8 +439,10 @@ export async function buildCaseRun(context: CaseContext, prepared?: PreparedCase subject: run.subject, agentTask: run.agentTask ? agentTaskSummary(run.agentTask) : null, agent: run.agent ? agentSummary(run.agent) : null, + agentTrace: run.agentTrace ? traceSummary(run.agentTrace) : null, agentDiff: run.agentDiff ? diffSummary(run.agentDiff) : null, hwpod: clean({ + source: "case-run-runner-post-agent-compile-check", command: commandVisibility(command), exitCode: invoked.exitCode, stdoutJson: compactHwpodPayload(hwpodPayload), @@ -553,7 +587,7 @@ async function runAgentTaskStage(context: CaseContext, run: PreparedCaseRun) { run = await updateRun(context, run, { agentTask: defaultAgentTask(run.definition, run.caseId) }); } const promptPath = path.join(run.runDir, "agent-prompt.md"); - const prompt = renderAgentTaskPrompt(run, run.agentTask); + const prompt = await renderAgentTaskPrompt(run, run.agentTask); await writeFile(promptPath, prompt, "utf8"); const promptSha256 = sha256(prompt); const baseUrl = webUrlFrom(context, run.definition); @@ -626,6 +660,26 @@ async function runAgentTaskStage(context: CaseContext, run: PreparedCaseRun) { return { run: nextRun, agent }; } +async function collectAgentTraceEvidence(context: CaseContext, run: PreparedCaseRun) { + const agent = run.agent; + const traceId = text(agent?.traceId); + if (!agent || !traceId) { + const trace: AgentTraceStage = clean({ traceId, status: "skipped_no_trace_id", httpStatus: 0, commandCount: 0, hwpodCommandCount: 0, hwpodBuildCommandCount: 0, keilJobCandidates: [], commands: [] }); + const nextRun = await updateRun(context, run, { agentTrace: trace, stage: "agent-trace-collected" }); + return { run: nextRun, trace }; + } + const traceUrl = `${agent.baseUrl}/v1/agent/chat/trace/${encodeURIComponent(traceId)}`; + const response = await caseFetchJson(context, traceUrl, { method: "GET", headers: caseAgentHeaders(context) }, "agent_trace_fetch"); + if (!response.ok) { + const trace: AgentTraceStage = clean({ traceId, status: "fetch_failed", httpStatus: response.status, commandCount: 0, hwpodCommandCount: 0, hwpodBuildCommandCount: 0, keilJobCandidates: [], commands: [], error: response.error }); + const nextRun = await updateRun(context, run, { agentTrace: trace, stage: "agent-trace-collected" }); + return { run: nextRun, trace }; + } + const trace = summarizeAgentTrace(traceId, response.status, response.body); + const nextRun = await updateRun(context, run, { agentTrace: trace, stage: "agent-trace-collected" }); + return { run: nextRun, trace }; +} + async function tryCreateAgentSession(context: CaseContext, input: { baseUrl: string; projectId: string; providerProfile: string; conversationId: string }) { try { return await createAgentSession(context, input); @@ -780,12 +834,15 @@ async function updateRun(context: CaseContext, run: PreparedCaseRun, patch: Part return nextRun; } -function renderAgentTaskPrompt(run: PreparedCaseRun, agentTask: PreparedAgentTask) { +async function renderAgentTaskPrompt(run: PreparedCaseRun, agentTask: PreparedAgentTask) { + const specText = await readFile(run.specPath, "utf8"); const constraints = [ ...agentTask.constraints, "只能修改 isolated subject worktree，不得修改 case registry repo。", "不得修改原 subject repo checkout；所有源码修改必须落在 subjectWorktreePath。", - "完成后不要自行运行确定性 CaseRun 答案执行器；CaseRun 会读取 diff 并执行 compile-only 验收。" + "允许且必须在 AgentRun workspace 内写入/覆盖 .hwlab/hwpod-spec.yaml，用于本次 run-local HWPOD compile-only。", + "compile-only 阶段不要修改 subject 源码；如任务确需改源码，只能改 subjectWorktreePath。", + "不要运行 CaseRun 答案执行器；你本人必须通过 hwpod/hwpod-ctl 标准入口触发 compile-only。" ].filter(Boolean); return [ `# HWPOD CaseRun Code Agent Task`, @@ -798,6 +855,13 @@ function renderAgentTaskPrompt(run: PreparedCaseRun, agentTask: PreparedAgentTas `runLocalHwpodSpec: ${run.specPath}`, `compileOnly: true`, ``, + `## Run-local HWPOD spec`, + `Write exactly this YAML to \`.hwlab/hwpod-spec.yaml\` in your AgentRun workspace before running HWPOD commands. The Windows subjectWorktreePath is consumed by hwpod-node through this spec; do not try to cd into that Windows path from the Linux runner.`, + ``, + `\`\`\`yaml`, + specText.trimEnd(), + `\`\`\``, + ``, `## Task`, agentTask.prompt, ``, @@ -805,12 +869,106 @@ function renderAgentTaskPrompt(run: PreparedCaseRun, agentTask: PreparedAgentTas ...constraints.map((item) => `- ${item}`), ``, `## Flow`, - `- CaseRun will inspect git diff under subjectWorktreePath after your turn completes.`, - `- CaseRun will run HWPOD Keil compile-only smoke with the run-local hwpod-spec.yaml.`, - `- CaseRun records trace/session/conversation, workspace diff and Keil build evidence without auto-grading them.` + `- Create or overwrite \`.hwlab/hwpod-spec.yaml\` in your AgentRun workspace with the exact run-local spec above.`, + `- Run \`hwpod-ctl spec validate --spec .hwlab/hwpod-spec.yaml\`.`, + `- Run \`hwpod inspect --spec .hwlab/hwpod-spec.yaml\`.`, + `- Run \`hwpod build --spec .hwlab/hwpod-spec.yaml\` for compile-only verification and report the returned JSON/job/artifact summary.`, + `- CaseRun will inspect git diff under subjectWorktreePath after your turn completes and may run a runner post-check compile as separate evidence.`, + `- CaseRun records trace/session/conversation, agent commandExecution, workspace diff and Keil build evidence without auto-grading them.` ].join("\n"); } +function summarizeAgentTrace(traceId: string, httpStatus: number, body: any): AgentTraceStage { + const traceBody = body?.body ?? body; + const rows = arrayOfObjects(traceBody?.rows ?? traceBody?.renderedRows ?? traceBody?.traceRows); + const events = arrayOfObjects(traceBody?.events ?? traceBody?.trace?.events ?? traceBody?.sourceEvents); + const commands = [...commandsFromTraceRows(rows), ...commandsFromTraceEvents(events)].slice(0, MAX_AGENT_TRACE_COMMANDS); + const searchable = commands.map((command) => [command.command, command.bodyPreview, command.header, command.toolName].map((item) => text(item)).filter(Boolean).join("\n")).join("\n"); + const hwpodCommandCount = commands.filter((command) => mentionsHwpod(command)).length; + const hwpodBuildCommandCount = commands.filter((command) => mentionsHwpodBuild(command)).length; + return clean({ + traceId, + status: text(traceBody?.status ?? body?.status) || "recorded", + httpStatus, + sourceEventCount: numberOption(traceBody?.sourceEventCount ?? traceBody?.traceSummary?.sourceEventCount ?? body?.sourceEventCount), + renderedRowCount: rows.length || numberOption(traceBody?.renderedRowCount), + commandCount: commands.length, + hwpodCommandCount, + hwpodBuildCommandCount, + keilJobCandidates: uniqueStrings(Array.from(searchable.matchAll(/20\d{6}_\d{6}_[0-9a-f]{8}/giu)).map((match) => match[0])), + terminalStatus: text(traceBody?.terminalStatus ?? traceBody?.traceSummary?.terminalStatus ?? body?.terminalStatus), + agentRun: compactObject(traceBody?.agentRun ?? traceBody?.traceSummary?.agentRun ?? body?.agentRun ?? null), + commands + }); +} + +function commandsFromTraceRows(rows: Record[]) { + return rows.flatMap((row, index) => { + const header = text(row.header ?? row.title ?? row.label); + const body = text(row.body ?? row.content ?? row.text ?? row.message); + const command = text((row as any).command ?? commandFromText(`${header}\n${body}`)); + const looksLikeCommand = /commandExecution|tool|hwpod|hwpod-ctl|hwpod-cli|keil/iu.test(`${header}\n${body}\n${command}`); + if (!looksLikeCommand) return []; + return [clean({ + source: "trace-row", + seq: numberOption(row.seq) ?? index + 1, + rowId: text(row.id ?? row.rowId), + header: clipText(header, 500), + toolName: text(row.toolName ?? row.name), + status: text(row.status ?? row.level), + command: clipText(command, 1000), + bodyPreview: clipText(body, 1200), + exitCode: exitCodeFromText(body) + })]; + }); +} + +function commandsFromTraceEvents(events: Record[]) { + return events.flatMap((event, index) => { + const toolName = text(event.toolName ?? event.name ?? event.label); + const message = text(event.message ?? event.text ?? event.content ?? event.output ?? event.stdout); + const command = text(event.command ?? event.input ?? event.args ?? commandFromText(message)); + const marker = `${event.type ?? ""}\n${toolName}\n${message}\n${command}`; + if (!/commandExecution|tool|hwpod|hwpod-ctl|hwpod-cli|keil/iu.test(marker)) return []; + return [clean({ + source: "trace-event", + seq: numberOption(event.seq ?? event.index) ?? index + 1, + rowId: text(event.id ?? event.eventId), + toolName, + status: text(event.status ?? event.level), + command: clipText(command, 1000), + bodyPreview: clipText(message, 1200), + exitCode: numberOption(event.exitCode) ?? exitCodeFromText(message) + })]; + }); +} + +function arrayOfObjects(value: unknown): Record[] { + return Array.isArray(value) ? value.filter((item) => item && typeof item === "object" && !Array.isArray(item)) as Record[] : []; +} + +function commandFromText(value: string) { + const match = value.match(/(?:command|cmd)\s*[:=]\s*([^\n]+)/iu); + return text(match?.[1]); +} + +function exitCodeFromText(value: string) { + const match = value.match(/exit\s*code\s*[:=]?\s*(-?\d+)/iu) ?? value.match(/exitCode\s*[:=]?\s*(-?\d+)/u); + return numberOption(match?.[1]); +} + +function mentionsHwpod(command: AgentTraceCommand) { + return /\bhwpod(?:-ctl|-cli)?\b|tools\/hwpod-cli\.ts/iu.test([command.command, command.bodyPreview, command.header, command.toolName].map((item) => text(item)).join("\n")); +} + +function mentionsHwpodBuild(command: AgentTraceCommand) { + return /(?:\bhwpod(?:-cli)?\b|tools\/hwpod-cli\.ts)[^\n]*(?:\bbuild\b)/iu.test([command.command, command.bodyPreview, command.header, command.toolName].map((item) => text(item)).join("\n")); +} + +function uniqueStrings(values: string[]) { + return Array.from(new Set(values.map((value) => text(value)).filter(Boolean))); +} + async function caseFetchJson(context: CaseContext, url: string, init: RequestInit, errorCode: string) { try { const response = await context.fetchImpl(url, init); @@ -848,7 +1006,25 @@ function agentTaskSummary(agentTask: PreparedAgentTask) { } function agentSummary(agent: AgentRunStage) { - return clean({ stageStatus: agent.stageStatus, baseUrl: agent.baseUrl, projectId: agent.projectId, providerProfile: agent.providerProfile, conversationId: agent.conversationId, sessionId: agent.sessionId, threadId: agent.threadId, traceId: agent.traceId, promptPath: agent.promptPath, promptSha256: agent.promptSha256, polls: agent.polls, timedOut: agent.timedOut, error: agent.error }); + return clean({ stageStatus: agent.stageStatus, baseUrl: agent.baseUrl, projectId: agent.projectId, providerProfile: agent.providerProfile, conversationId: agent.conversationId, sessionId: agent.sessionId, threadId: agent.threadId, traceId: agent.traceId, promptPath: agent.promptPath, promptSha256: agent.promptSha256, polls: agent.polls, timedOut: agent.timedOut, lastHttpStatus: agent.lastHttpStatus, error: agent.error }); +} + +function traceSummary(trace: AgentTraceStage) { + return clean({ + traceId: trace.traceId, + status: trace.status, + httpStatus: trace.httpStatus, + sourceEventCount: trace.sourceEventCount, + renderedRowCount: trace.renderedRowCount, + commandCount: trace.commandCount, + hwpodCommandCount: trace.hwpodCommandCount, + hwpodBuildCommandCount: trace.hwpodBuildCommandCount, + keilJobCandidates: trace.keilJobCandidates, + terminalStatus: trace.terminalStatus, + agentRun: trace.agentRun, + commands: trace.commands, + error: trace.error + }); } function diffSummary(diff: AgentDiffStage) { @@ -1079,11 +1255,11 @@ function prepareSummary(run: PreparedCaseRun) { } function buildSummary(evidence: any) { - return { status: evidence.status, autoEvaluation: false, hwpodExitCode: evidence.hwpod?.exitCode ?? null, jobId: evidence.keilJob?.jobId ?? null, keilStatus: evidence.keilJob?.status ?? null, artifacts: evidence.artifacts ?? [] }; + return { status: evidence.status, autoEvaluation: false, agentTraceCommandCount: evidence.agentTrace?.commandCount ?? 0, agentTraceHwpodCommandCount: evidence.agentTrace?.hwpodCommandCount ?? 0, hwpodExitCode: evidence.hwpod?.exitCode ?? null, hwpodSource: evidence.hwpod?.source ?? null, jobId: evidence.keilJob?.jobId ?? null, keilStatus: evidence.keilJob?.status ?? null, artifacts: evidence.artifacts ?? [] }; } function renderEvidenceSummary(evidence: any) { - return `# HWPOD CaseRun ${evidence.caseId}\n\n- runId: ${evidence.runId}\n- status: ${evidence.status}\n- autoEvaluation: false\n- compileOnly: ${evidence.compileOnly}\n- subjectRepoLocalPath: ${evidence.subject?.repoLocalPath ?? ""}\n- subjectCommitId: ${evidence.subject?.commitId ?? ""}\n- subjectWorktreePath: ${evidence.subject?.worktreePath ?? ""}\n- agentTraceId: ${evidence.agent?.traceId ?? ""}\n- agentSessionId: ${evidence.agent?.sessionId ?? ""}\n- diffPatchPath: ${evidence.agentDiff?.diffPatchPath ?? ""}\n- hwpodExitCode: ${evidence.hwpod?.exitCode ?? ""}\n- jobId: ${evidence.keilJob?.jobId ?? ""}\n- downloadSkipped: true\n`; + return `# HWPOD CaseRun ${evidence.caseId}\n\n- runId: ${evidence.runId}\n- status: ${evidence.status}\n- autoEvaluation: false\n- compileOnly: ${evidence.compileOnly}\n- subjectRepoLocalPath: ${evidence.subject?.repoLocalPath ?? ""}\n- subjectCommitId: ${evidence.subject?.commitId ?? ""}\n- subjectWorktreePath: ${evidence.subject?.worktreePath ?? ""}\n- agentTraceId: ${evidence.agent?.traceId ?? ""}\n- agentSessionId: ${evidence.agent?.sessionId ?? ""}\n- agentTraceCommandCount: ${evidence.agentTrace?.commandCount ?? 0}\n- agentTraceHwpodCommandCount: ${evidence.agentTrace?.hwpodCommandCount ?? 0}\n- agentTraceHwpodBuildCommandCount: ${evidence.agentTrace?.hwpodBuildCommandCount ?? 0}\n- diffPatchPath: ${evidence.agentDiff?.diffPatchPath ?? ""}\n- runnerHwpodSource: ${evidence.hwpod?.source ?? ""}\n- hwpodExitCode: ${evidence.hwpod?.exitCode ?? ""}\n- jobId: ${evidence.keilJob?.jobId ?? ""}\n- downloadSkipped: true\n`; } async function runProcess(command: string[], cwd: string, env: Record) {