diff --git a/tools/hwlab-cli/caserun-offline-diagnostics.test.ts b/tools/hwlab-cli/caserun-offline-diagnostics.test.ts index 3126bedb..f6d76f0d 100644 --- a/tools/hwlab-cli/caserun-offline-diagnostics.test.ts +++ b/tools/hwlab-cli/caserun-offline-diagnostics.test.ts @@ -73,7 +73,70 @@ test("case run offline diagnostics inspect trace and skill usage without runtime assert.equal(skill.exitCode, 0, JSON.stringify(skill.payload, null, 2)); assert.equal(skill.payload.serviceRuntime, false); assert.equal(skill.payload.trace.readSkillFile, true); + assert.equal(skill.payload.trace.matchCount, 1); assert.equal(skill.payload.resourceBundle.materialized, true); + assert.equal(skill.payload.resourceBundle.evidence[0].source, "artifact-manifest.json"); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test("case run skill usage keeps prompt mentions separate from materialized bundles", async () => { + const fixture = await createRegistryFixture("prompt-only-skill"); + try { + await writeJson(path.join(fixture.runDir, "agent-messages.json"), { rows: [{ rowId: "tool:build", seq: 1, header: "工具调用", body: "hwpod build --spec .hwlab/hwpod-spec.yaml" }] }); + const manifest = JSON.parse(await readFile(path.join(fixture.runDir, "artifact-manifest.json"), "utf8")); + delete manifest.resourceBundle; + await writeJson(path.join(fixture.runDir, "artifact-manifest.json"), manifest); + + const result = await runHwlabCli(["case", "run", "skill-usage", fixture.runId, "--skill", "keil", "--case-repo", fixture.caseRepo], { cwd: fixture.root, runProcess: forbiddenRuntime }); + assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2)); + assert.equal(result.payload.prompt.mentionsSkill, true); + assert.equal(result.payload.trace.readSkillFile, false); + assert.equal(result.payload.resourceBundle.materialized, false); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test("case run inspect preserves missing authoritative agent final", async () => { + const fixture = await createRegistryFixture("missing-agent-final"); + try { + const reason = "finalResponse=null; provider returned no final assistant response"; + const evidence = JSON.parse(await readFile(path.join(fixture.runDir, "evidence.json"), "utf8")); + evidence.agentFinal = { present: false, reason, status: "blocked" }; + await writeJson(path.join(fixture.runDir, "evidence.json"), evidence); + const manifest = JSON.parse(await readFile(path.join(fixture.runDir, "artifact-manifest.json"), "utf8")); + manifest.agentFinal = { present: false, reason, status: "blocked" }; + await writeJson(path.join(fixture.runDir, "artifact-manifest.json"), manifest); + await writeFile(path.join(fixture.runDir, "final-response.md"), `# CaseRun Final Response\n\n${reason}\n`, "utf8"); + + const result = await runHwlabCli(["case", "run", "inspect", fixture.runId, "--case-repo", fixture.caseRepo], { cwd: fixture.root, runProcess: forbiddenRuntime }); + assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2)); + assert.equal(result.payload.agentFinal.present, false); + assert.equal(result.payload.finalResponse.present, false); + assert.equal(result.payload.finalResponse.reason, reason); + assert.match(result.payload.finalResponse.textPreview, /provider returned no final assistant response/u); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test("case run skill usage scans all rows before bounding returned matches", async () => { + const fixture = await createRegistryFixture("late-skill-read"); + try { + const rows = [ + ...Array.from({ length: 30 }, (_, index) => ({ rowId: `tool:other:${index + 1}`, seq: index + 1, header: "工具调用", body: "echo unrelated" })), + ...Array.from({ length: 35 }, (_, index) => ({ rowId: `tool:skill:${index + 31}`, seq: index + 31, header: "工具调用", body: "cat ~/.agents/skills/keil/SKILL.md" })), + ]; + await writeJson(path.join(fixture.runDir, "agent-messages.json"), { sourceEventCount: rows.length, renderedRowCount: rows.length, rows }); + + const result = await runHwlabCli(["case", "run", "skill-usage", fixture.runId, "--skill", "keil", "--case-repo", fixture.caseRepo], { cwd: fixture.root, runProcess: forbiddenRuntime }); + assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2)); + assert.equal(result.payload.trace.readSkillFile, true); + assert.equal(result.payload.trace.matchCount, 35); + assert.equal(result.payload.trace.matches.length, 30); + assert.equal(result.payload.trace.matches[0].seq, 31); } finally { await rm(fixture.root, { recursive: true, force: true }); } @@ -112,10 +175,10 @@ async function createRegistryFixture(suffix: string) { const agent = { stageStatus: "completed", baseUrl: "http://web.test", traceId: "trc_offline", sessionId: "ses_offline", conversationId: "cnv_offline", threadId: "thr_offline", providerProfile: "deepseek", requestedProviderProfile: "deepseek", resolvedBackendProfile: "deepseek-runtime", provider: "deepseek", model: "deepseek-chat", terminalStatus: "completed", commandStatus: "completed", agentRunStatus: "completed", promptSha256: "prompt-sha" }; const toolCallSummary = { source: "agentrun-result.toolCallSummary", count: 2, statusCounts: { completed: 2 }, exitCodeCounts: { "0": 2 }, terminalStatus: "completed", commandStatus: "completed", agentRunStatus: "completed", items: [{ kind: "build", toolName: "commandExecution", status: "completed", exitCode: 0, command: "hwpod build --spec .hwlab/hwpod-spec.yaml" }, { kind: "other", toolName: "commandExecution", status: "completed", exitCode: 0, command: "cat ~/.agents/skills/keil/SKILL.md" }] }; await writeJson(path.join(runDir, "run.json"), { caseId, runId, runDir, caseRepo, specPath: path.join(runDir, ".hwlab", "hwpod-spec.yaml"), compileOnly: true, createdAt: "2026-07-16T13:00:00.000Z", updatedAt: "2026-07-16T13:00:00.000Z", definition: {}, subject: { repoLocalPath: "C:/work/fw", commitId: "0".repeat(40), subdir: "", worktreePath: "C:/work/fw/.worktree/case", workspacePath: "C:/work/fw/.worktree/case", setup: {} }, agent }); - await writeJson(path.join(runDir, "evidence.json"), { contractVersion: "hwpod-case-run-evidence-v1", caseId, runId, compileOnly: true, autoEvaluation: false, status: "recorded", subject: { repoLocalPath: "C:/work/fw", commitId: "0".repeat(40), worktreePath: "C:/work/fw/.worktree/case" }, agent, agentTrace: { traceId: "trc_offline", terminalStatus: "completed", toolCallSummary, commandCount: 2, hwpodCommandCount: 1, hwpodBuildCommandCount: 1 }, agentFinal: { status: "completed", terminalStatus: "completed" }, postValidation: { source: "case-run-runner-post-agent-compile-check", status: "completed", returnCode: 0 }, decisions: { autoEvaluation: false, runnerPostAgentCompileCheck: "recorded" } }); + await writeJson(path.join(runDir, "evidence.json"), { contractVersion: "hwpod-case-run-evidence-v1", caseId, runId, compileOnly: true, autoEvaluation: false, status: "recorded", subject: { repoLocalPath: "C:/work/fw", commitId: "0".repeat(40), worktreePath: "C:/work/fw/.worktree/case" }, agent, agentTrace: { traceId: "trc_offline", terminalStatus: "completed", toolCallSummary, commandCount: 2, hwpodCommandCount: 1, hwpodBuildCommandCount: 1 }, agentFinal: { present: true, status: "completed", terminalStatus: "completed" }, postValidation: { source: "case-run-runner-post-agent-compile-check", status: "completed", returnCode: 0 }, decisions: { autoEvaluation: false, runnerPostAgentCompileCheck: "recorded" } }); await writeJson(path.join(runDir, "agent-trace.json"), { contractVersion: "hwpod-case-run-agent-trace-identity-v1", traceId: "trc_offline", status: "completed", terminalStatus: "completed", commandStatus: "completed", toolCallSummary }); await writeJson(path.join(runDir, "agent-messages.json"), { renderer: "tools/src/hwlab-cli/trace-renderer:traceDisplayRows", sourceEventCount: 2, renderedRowCount: 2, rows: [{ rowId: "tool:build", seq: 1, header: "工具调用", body: "hwpod build --spec .hwlab/hwpod-spec.yaml" }, { rowId: "tool:skill", seq: 2, header: "工具调用", body: "cat ~/.agents/skills/keil/SKILL.md" }] }); - await writeJson(path.join(runDir, "artifact-manifest.json"), { contractVersion: "hwpod-case-run-artifact-manifest-v1", caseId, runId, agent, trace: { traceId: "trc_offline", terminalStatus: "completed", toolCallSummary, commandCount: 2, hwpodCommandCount: 1, hwpodBuildCommandCount: 1 }, agentFinal: { status: "completed" }, postValidation: { status: "completed", returnCode: 0 }, files: [] }); + await writeJson(path.join(runDir, "artifact-manifest.json"), { contractVersion: "hwpod-case-run-artifact-manifest-v1", caseId, runId, agent, trace: { traceId: "trc_offline", terminalStatus: "completed", toolCallSummary, commandCount: 2, hwpodCommandCount: 1, hwpodBuildCommandCount: 1 }, resourceBundle: { materialized: true, skills: [{ name: "keil", manifest: ".agents/skills/keil/SKILL.md" }] }, agentFinal: { present: true, status: "completed" }, postValidation: { status: "completed", returnCode: 0 }, files: [] }); await writeFile(path.join(runDir, "agent-prompt.md"), "请读取 keil skill 后执行编译。\n", "utf8"); await writeFile(path.join(runDir, "final-response.md"), "# CaseRun Final Response\n\ncompleted\n", "utf8"); await writeFile(path.join(runDir, "agent-diff.patch"), "", "utf8"); diff --git a/tools/src/hwlab-caserun-diagnostics.ts b/tools/src/hwlab-caserun-diagnostics.ts index 43b72576..a951ba3d 100644 --- a/tools/src/hwlab-caserun-diagnostics.ts +++ b/tools/src/hwlab-caserun-diagnostics.ts @@ -73,15 +73,16 @@ export async function skillUsageCaseRegistryRun(context: CaseContext, caseRepo: if (!skill) throw cliError("missing_required_value", "skill is required", { field: "skill" }); const artifacts = await loadDiagnosticByRunId(caseRepo, text(context.parsed.runId ?? context.rest[2]), text(context.parsed.caseId ?? context.parsed.case)); const pattern = new RegExp(`(?:\\.agents[\\\\/]skills[\\\\/]${escapeRegExp(skill)}[\\\\/]SKILL\\.md|\\b${escapeRegExp(skill)}\\b)`, "iu"); - const sourceFiles = [ - ["agent-prompt.md", artifacts.promptText], - ["agent-trace.json", JSON.stringify(artifacts.agentTrace ?? {})], - ["agent-messages.json", JSON.stringify(artifacts.agentMessages ?? {})], - ["artifact-manifest.json", JSON.stringify(artifacts.manifest ?? {})], + const bundleSources = [ + ["artifact-manifest.json", artifacts.manifest.resourceBundle ?? artifacts.manifest.provenance?.resourceBundle], + ["evidence.json", artifacts.evidence.resourceBundle ?? artifacts.evidence.provenance?.resourceBundle], ] as const; - const evidence = sourceFiles.filter(([, value]) => pattern.test(value)).map(([source, value]) => ({ source, preview: clipText(value.replace(/\s+/gu, " "), 500) })); + const evidence = bundleSources + .filter(([, value]) => value && pattern.test(JSON.stringify(value))) + .map(([source, value]) => ({ source, materialized: value.materialized === true, preview: clipText(JSON.stringify(value).replace(/\s+/gu, " "), 500) })); const rows = traceRows(artifacts); - const matches = rows.filter((row) => pattern.test(`${row.header ?? ""}\n${row.body ?? ""}`)).map((row) => ({ rowId: row.rowId, seq: row.seq, header: row.header, preview: clipText(row.body, 500) })); + const allMatches = rows.filter((row) => pattern.test(`${row.header ?? ""}\n${row.body ?? ""}`)); + const matches = allMatches.slice(0, MAX_DIAGNOSTIC_ROWS).map((row) => ({ rowId: row.rowId, seq: row.seq, header: row.header, preview: clipText(row.body, 500) })); return ok("case.run.skill-usage", { caseId: artifacts.caseId, runId: artifacts.runId, @@ -89,8 +90,8 @@ export async function skillUsageCaseRegistryRun(context: CaseContext, caseRepo: caseRepoRunDir: artifacts.caseRepoRunDir, skill, prompt: { mentionsSkill: pattern.test(artifacts.promptText), path: path.join(artifacts.caseRepoRunDir, "agent-prompt.md") }, - trace: { readSkillFile: matches.length > 0, matchCount: matches.length, matches }, - resourceBundle: { materialized: evidence.length > 0, evidence }, + trace: { readSkillFile: allMatches.length > 0, matchCount: allMatches.length, matches }, + resourceBundle: { materialized: evidence.some((item) => item.materialized), evidence }, inspectionOnly: true, serviceRuntime: false, autoEvaluation: false, @@ -162,6 +163,7 @@ function diagnosticFromArtifacts(artifacts: any) { const trace = { ...(artifacts.run.agentTrace ?? {}), ...(artifacts.evidence.agentTrace ?? {}), ...(artifacts.manifest.trace ?? {}) }; const summary = boundedToolCallSummary(trace.toolCallSummary ?? agent.toolCallSummary ?? agentToolCallSummaryFromResult(artifacts.agentTrace?.body ?? artifacts.agentTrace) ?? null); const rows = traceRows(artifacts); + const agentFinal = clean({ ...(artifacts.evidence.agentFinal ?? {}), ...(artifacts.manifest.agentFinal ?? {}) }); const commands = summary?.items ?? rows.filter((row) => /commandExecution|tool_call|^tool:/iu.test(`${row.rowId ?? ""} ${row.header ?? ""}`)).map((row) => ({ rowId: row.rowId, seq: row.seq, command: row.command ?? row.body, status: row.status ?? row.tone })); return clean({ caseId: artifacts.caseId, @@ -191,10 +193,10 @@ function diagnosticFromArtifacts(artifacts: any) { commandCount: trace.commandCount ?? summary?.count ?? commands.length, hwpodCommandCount: trace.hwpodCommandCount, hwpodBuildCommandCount: trace.hwpodBuildCommandCount, - agentFinal: artifacts.manifest.agentFinal ?? artifacts.evidence.agentFinal, + agentFinal, postValidation: artifacts.manifest.postValidation ?? artifacts.evidence.postValidation, refresh: artifacts.manifest.refresh ?? artifacts.evidence.refresh, - finalResponse: { present: Boolean(artifacts.finalResponseText.trim()), path: path.join(artifacts.caseRepoRunDir, "final-response.md"), textPreview: clipText(artifacts.finalResponseText, 800) }, + finalResponse: { present: agentFinal.present === true, reason: agentFinal.reason, path: path.join(artifacts.caseRepoRunDir, "final-response.md"), textPreview: clipText(artifacts.finalResponseText, 800) }, traceCommands: commands.slice(0, MAX_DIAGNOSTIC_ROWS), paths: { runJson: path.join(artifacts.caseRepoRunDir, "run.json"), evidenceJson: path.join(artifacts.caseRepoRunDir, "evidence.json"), agentTraceJson: path.join(artifacts.caseRepoRunDir, "agent-trace.json"), agentMessagesJson: path.join(artifacts.caseRepoRunDir, "agent-messages.json"), artifactManifestJson: path.join(artifacts.caseRepoRunDir, "artifact-manifest.json"), aggregateMd: path.join(artifacts.caseRepoRunDir, "aggregate.md") }, }); @@ -242,7 +244,7 @@ function boundedToolCallSummary(summary: any) { function traceRows(artifacts: any) { const body = artifacts.agentTrace?.body && typeof artifacts.agentTrace.body === "object" ? artifacts.agentTrace.body : artifacts.agentTrace ?? {}; - return arrayOfObjects(artifacts.agentMessages?.rows ?? body.rows ?? body.renderedRows ?? body.traceRows).slice(0, MAX_DIAGNOSTIC_ROWS); + return arrayOfObjects(artifacts.agentMessages?.rows ?? body.rows ?? body.renderedRows ?? body.traceRows); } async function readJson(file: string) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } }