From 6af6711fc3a86086bdbb0ab7d8ebe87b24fd3a32 Mon Sep 17 00:00:00 2001 From: root Date: Thu, 16 Jul 2026 16:01:54 +0200 Subject: [PATCH] =?UTF-8?q?feat:=20=E5=AE=9E=E7=8E=B0=20CaseRun=20registry?= =?UTF-8?q?=20=E7=A6=BB=E7=BA=BF=E8=AF=8A=E6=96=AD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../caserun-offline-diagnostics.test.ts | 128 +++++++++ tools/src/hwlab-caserun-closeout.ts | 24 +- tools/src/hwlab-caserun-diagnostics.ts | 253 ++++++++++++++++++ tools/src/hwlab-caserun-runtime.ts | 17 +- 4 files changed, 413 insertions(+), 9 deletions(-) create mode 100644 tools/hwlab-cli/caserun-offline-diagnostics.test.ts create mode 100644 tools/src/hwlab-caserun-diagnostics.ts diff --git a/tools/hwlab-cli/caserun-offline-diagnostics.test.ts b/tools/hwlab-cli/caserun-offline-diagnostics.test.ts new file mode 100644 index 00000000..3126bedb --- /dev/null +++ b/tools/hwlab-cli/caserun-offline-diagnostics.test.ts @@ -0,0 +1,128 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import { test } from "bun:test"; + +import { runHwlabCli } from "../src/hwlab-cli-lib.ts"; + +test("case aggregate remains registry-only even when runtime credentials exist", async () => { + const fixture = await createRegistryFixture("aggregate-offline"); + try { + let runtimeCalls = 0; + const result = await runHwlabCli(["case", "aggregate", fixture.caseId, "--run-id", fixture.runId, "--case-repo", fixture.caseRepo, "--no-case-repo-git-sync"], { + cwd: fixture.root, + env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" }, + runProcess: async () => { + runtimeCalls += 1; + throw new Error("offline aggregate must not execute runtime commands"); + }, + }); + + assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2)); + assert.equal(runtimeCalls, 0); + assert.equal(result.payload.action, "case.aggregate"); + assert.match(await readFile(path.join(fixture.runDir, "aggregate.md"), "utf8"), /registry artifacts/u); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test("case refresh is explicit, registry-backed, and semantically idempotent", async () => { + const fixture = await createRegistryFixture("refresh-local"); + try { + const args = ["case", "refresh", fixture.caseId, "--run-id", fixture.runId, "--case-repo", fixture.caseRepo, "--source", "local", "--no-case-repo-git-sync"]; + const first = await runHwlabCli(args, { cwd: fixture.root, now: () => "2026-07-16T14:00:00.000Z", env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" }, runProcess: forbiddenRuntime }); + assert.equal(first.exitCode, 0, JSON.stringify(first.payload, null, 2)); + assert.deepEqual(first.payload.refreshed.provenance, { source: "local", authority: "registry", runtimeAccess: false }); + assert.equal(first.payload.refreshed.commandCount, 2); + assert.equal(first.payload.refreshed.hwpodCommandCount, 1); + assert.equal(first.payload.refreshed.hwpodBuildCommandCount, 1); + const before = await fileHashes(fixture.runDir, ["evidence.json", "agent-messages.json", "artifact-manifest.json", "aggregate.md"]); + + const second = await runHwlabCli(args, { cwd: fixture.root, now: () => "2026-07-16T15:00:00.000Z", env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" }, runProcess: forbiddenRuntime }); + assert.equal(second.exitCode, 0, JSON.stringify(second.payload, null, 2)); + assert.deepEqual(await fileHashes(fixture.runDir, Object.keys(before)), before); + const manifest = JSON.parse(await readFile(path.join(fixture.runDir, "artifact-manifest.json"), "utf8")); + assert.deepEqual(manifest.refresh, { source: "local", authority: "registry", runtimeAccess: false }); + assert.equal(manifest.trace.toolCallSummary.count, 2); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test("case run offline diagnostics inspect trace and skill usage without runtime access", async () => { + const fixture = await createRegistryFixture("inspect-offline"); + try { + const options = { cwd: fixture.root, env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" }, runProcess: forbiddenRuntime }; + const inspect = await runHwlabCli(["case", "run", "inspect", fixture.runId, "--case-repo", fixture.caseRepo], options); + assert.equal(inspect.exitCode, 0, JSON.stringify(inspect.payload, null, 2)); + assert.equal(inspect.payload.serviceRuntime, false); + assert.equal(inspect.payload.traceId, "trc_offline"); + assert.equal(inspect.payload.providerProfile, "deepseek"); + assert.equal(inspect.payload.terminalStatus, "completed"); + assert.equal(inspect.payload.toolCallSummary.count, 2); + + const trace = await runHwlabCli(["case", "run", "trace-summary", fixture.runId, "--case-repo", fixture.caseRepo], options); + assert.equal(trace.exitCode, 0, JSON.stringify(trace.payload, null, 2)); + assert.equal(trace.payload.commandCount, 2); + assert.equal(trace.payload.traceCommands.length, 2); + + const skill = await runHwlabCli(["case", "run", "skill-usage", fixture.runId, "--skill", "keil", "--case-repo", fixture.caseRepo], options); + assert.equal(skill.exitCode, 0, JSON.stringify(skill.payload, null, 2)); + assert.equal(skill.payload.serviceRuntime, false); + assert.equal(skill.payload.trace.readSkillFile, true); + assert.equal(skill.payload.resourceBundle.materialized, true); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +test("case refresh accesses runtime only with explicit runtime source", async () => { + const fixture = await createRegistryFixture("refresh-runtime"); + try { + let runtimeCalls = 0; + const result = await runHwlabCli(["case", "refresh", fixture.caseId, "--run-id", fixture.runId, "--case-repo", fixture.caseRepo, "--source", "runtime", "--no-case-repo-git-sync"], { + cwd: fixture.root, + env: { HWLAB_API_KEY: "hwl_live_runtime_test" }, + runProcess: async () => { + runtimeCalls += 1; + return { command: [], exitCode: 0, stderr: "", stdout: JSON.stringify({ body: { traceId: "trc_offline", source: "hwlab-cli.client.agent.trace", status: "completed", terminalStatus: "completed", rows: [{ rowId: "tool:runtime", seq: 1, header: "工具调用", body: "hwpod build --spec .hwlab/hwpod-spec.yaml" }, { rowId: "final", seq: 2, header: "助手最终消息", terminal: true, body: "runtime refreshed" }] } }) }; + }, + }); + + assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2)); + assert.equal(runtimeCalls, 1); + assert.deepEqual(result.payload.refreshed.provenance, { source: "runtime", authority: "registry", runtimeAccess: true }); + } finally { + await rm(fixture.root, { recursive: true, force: true }); + } +}); + +async function createRegistryFixture(suffix: string) { + const root = await mkdtemp(path.join(os.tmpdir(), `hwlab-caserun-${suffix}-`)); + const caseRepo = path.join(root, "hwlab-case-registry"); + const caseId = `case-${suffix}`; + const runId = `run-${suffix}`; + const runDir = path.join(caseRepo, "runs", caseId, runId); + await mkdir(path.join(caseRepo, ".git"), { recursive: true }); + await writeFile(path.join(caseRepo, ".git", "HEAD"), "ref: refs/heads/main\n", "utf8"); + await mkdir(path.join(runDir, ".hwlab"), { recursive: true }); + const agent = { stageStatus: "completed", baseUrl: "http://web.test", traceId: "trc_offline", sessionId: "ses_offline", conversationId: "cnv_offline", threadId: "thr_offline", providerProfile: "deepseek", requestedProviderProfile: "deepseek", resolvedBackendProfile: "deepseek-runtime", provider: "deepseek", model: "deepseek-chat", terminalStatus: "completed", commandStatus: "completed", agentRunStatus: "completed", promptSha256: "prompt-sha" }; + const toolCallSummary = { source: "agentrun-result.toolCallSummary", count: 2, statusCounts: { completed: 2 }, exitCodeCounts: { "0": 2 }, terminalStatus: "completed", commandStatus: "completed", agentRunStatus: "completed", items: [{ kind: "build", toolName: "commandExecution", status: "completed", exitCode: 0, command: "hwpod build --spec .hwlab/hwpod-spec.yaml" }, { kind: "other", toolName: "commandExecution", status: "completed", exitCode: 0, command: "cat ~/.agents/skills/keil/SKILL.md" }] }; + await writeJson(path.join(runDir, "run.json"), { caseId, runId, runDir, caseRepo, specPath: path.join(runDir, ".hwlab", "hwpod-spec.yaml"), compileOnly: true, createdAt: "2026-07-16T13:00:00.000Z", updatedAt: "2026-07-16T13:00:00.000Z", definition: {}, subject: { repoLocalPath: "C:/work/fw", commitId: "0".repeat(40), subdir: "", worktreePath: "C:/work/fw/.worktree/case", workspacePath: "C:/work/fw/.worktree/case", setup: {} }, agent }); + await writeJson(path.join(runDir, "evidence.json"), { contractVersion: "hwpod-case-run-evidence-v1", caseId, runId, compileOnly: true, autoEvaluation: false, status: "recorded", subject: { repoLocalPath: "C:/work/fw", commitId: "0".repeat(40), worktreePath: "C:/work/fw/.worktree/case" }, agent, agentTrace: { traceId: "trc_offline", terminalStatus: "completed", toolCallSummary, commandCount: 2, hwpodCommandCount: 1, hwpodBuildCommandCount: 1 }, agentFinal: { status: "completed", terminalStatus: "completed" }, postValidation: { source: "case-run-runner-post-agent-compile-check", status: "completed", returnCode: 0 }, decisions: { autoEvaluation: false, runnerPostAgentCompileCheck: "recorded" } }); + await writeJson(path.join(runDir, "agent-trace.json"), { contractVersion: "hwpod-case-run-agent-trace-identity-v1", traceId: "trc_offline", status: "completed", terminalStatus: "completed", commandStatus: "completed", toolCallSummary }); + await writeJson(path.join(runDir, "agent-messages.json"), { renderer: "tools/src/hwlab-cli/trace-renderer:traceDisplayRows", sourceEventCount: 2, renderedRowCount: 2, rows: [{ rowId: "tool:build", seq: 1, header: "工具调用", body: "hwpod build --spec .hwlab/hwpod-spec.yaml" }, { rowId: "tool:skill", seq: 2, header: "工具调用", body: "cat ~/.agents/skills/keil/SKILL.md" }] }); + await writeJson(path.join(runDir, "artifact-manifest.json"), { contractVersion: "hwpod-case-run-artifact-manifest-v1", caseId, runId, agent, trace: { traceId: "trc_offline", terminalStatus: "completed", toolCallSummary, commandCount: 2, hwpodCommandCount: 1, hwpodBuildCommandCount: 1 }, agentFinal: { status: "completed" }, postValidation: { status: "completed", returnCode: 0 }, files: [] }); + await writeFile(path.join(runDir, "agent-prompt.md"), "请读取 keil skill 后执行编译。\n", "utf8"); + await writeFile(path.join(runDir, "final-response.md"), "# CaseRun Final Response\n\ncompleted\n", "utf8"); + await writeFile(path.join(runDir, "agent-diff.patch"), "", "utf8"); + await writeFile(path.join(runDir, ".hwlab", "hwpod-spec.yaml"), "metadata:\n name: offline\n", "utf8"); + return { root, caseRepo, caseId, runId, runDir }; +} + +async function writeJson(file: string, value: unknown) { await writeFile(file, `${JSON.stringify(value, null, 2)}\n`, "utf8"); } +async function fileHashes(root: string, files: string[]) { return Object.fromEntries(await Promise.all(files.map(async (file) => [file, createHash("sha256").update(await readFile(path.join(root, file))).digest("hex")]))); } +async function forbiddenRuntime() { throw new Error("offline diagnostics must not execute runtime commands"); } diff --git a/tools/src/hwlab-caserun-closeout.ts b/tools/src/hwlab-caserun-closeout.ts index e214749b..6c109caa 100644 --- a/tools/src/hwlab-caserun-closeout.ts +++ b/tools/src/hwlab-caserun-closeout.ts @@ -204,7 +204,7 @@ export async function renderCaseAggregateMarkdown(context: CaseContext, input: { const diff = manifest.diff ?? evidence.agentDiff ?? run.agentDiff ?? {}; const keilJob = manifest.keilJob ?? evidence.keilJob ?? null; const agentStage = manifest.agentStage ?? agentStageSummary(evidence); - const messages = await aggregateMessagesWithFreshFullTrace(context, { archivedMessages, traceLookup, traceId: agent.traceId ?? trace.traceId ?? traceLookup?.traceId, run, evidence, manifest }); + const messages = archivedMessages; const traceBody = aggregateTraceBody(messages, traceMarkdown); return `${[ `# HWPOD CaseRun Aggregate: ${input.caseId}`, @@ -391,7 +391,7 @@ export function aggregateFence(language: string, body: string) { } export function aggregateArtifactIndex(manifest: any) { - const files = arrayOfObjects(manifest?.files); + const files = arrayOfObjects(manifest?.files).filter((file) => text(file.path) !== "aggregate.md"); if (files.length === 0) return `_artifact-manifest.json is missing or has no files list._`; return [`| Path | Bytes | SHA-256 |`, `|---|---:|---|`, ...files.map((file) => `| ${markdownTableText(file.path, 220)} | ${file.bytes ?? ""} | ${markdownTableText(file.sha256, 80)} |`)].join("\n"); } @@ -534,7 +534,7 @@ export async function archivedAgentTraceSnapshot(context: CaseContext, run: Prep const bodyForSummary = traceBody?.body && typeof traceBody.body === "object" ? traceBody.body : traceBody; const events = arrayOfObjects(bodyForSummary?.events ?? bodyForSummary?.trace?.events ?? bodyForSummary?.sourceEvents); const rows = traceRowsFromBody(bodyForSummary, events); - const hasRenderableTrace = rows.length > 0 || events.length > 0; + const hasRenderableTrace = rows.length > 0 || events.length > 0 || Boolean(bodyForSummary?.toolCallSummary); const summary = hasRenderableTrace ? summarizeAgentTrace(text(bodyForSummary?.traceId ?? traceBody?.traceId ?? evidence.agent?.traceId ?? run.agent?.traceId), numberOption(traceBody?.httpStatus) ?? 0, traceBody) : existingAgentTraceSummary(run, evidence, bodyForSummary); @@ -558,20 +558,28 @@ export async function archivedAgentTraceBody(run: PreparedCaseRun, evidence: any export async function materializeFullAgentTraceWithExistingCli(context: CaseContext, run: PreparedCaseRun, evidence: any, initialTrace: any) { const body = initialTrace?.body && typeof initialTrace.body === "object" ? initialTrace.body : initialTrace; const existingEvents = arrayOfObjects(body?.events ?? body?.trace?.events ?? body?.sourceEvents); - if (existingEvents.length > 0) return initialTrace; + const forceRuntimeRefresh = context.parsed.caseRefreshSource === "runtime"; + if (existingEvents.length > 0 && !forceRuntimeRefresh) return initialTrace; if (!run.agent) return initialTrace; const traceId = text(body?.traceId ?? initialTrace?.traceId ?? evidence.agent?.traceId ?? run.agent?.traceId); const lookup = agentTraceLookupForRun(run, evidence); const apiKey = text(context.env.HWLAB_API_KEY ?? process.env.HWLAB_API_KEY); - if (!traceId || !lookup || !apiKey.startsWith(HWLAB_API_KEY_PREFIX)) return initialTrace; + if (!traceId || !lookup || !apiKey.startsWith(HWLAB_API_KEY_PREFIX)) { + if (forceRuntimeRefresh) throw cliError("case_refresh_runtime_unavailable", "explicit runtime refresh requires trace identity, runtime base URL, and HWLAB_API_KEY", { traceId, hasLookup: Boolean(lookup), hasApiKey: apiKey.startsWith(HWLAB_API_KEY_PREFIX) }); + return initialTrace; + } const baseUrl = text((lookup as any).baseUrl) || webUrlFrom(context, run.definition); const command = [process.execPath, path.join(context.cwd, "tools/hwlab-cli/bin/hwlab-cli.mjs"), "client", "agent", "trace", traceId, "--render", "web", "--full"]; const result = await (context.runProcess ?? runProcess)(command, context.cwd, { ...process.env, ...context.env, HWLAB_RUNTIME_WEB_URL: baseUrl }); if (result.exitCode !== 0) { + if (forceRuntimeRefresh) throw cliError("case_refresh_trace_fetch_failed", "explicit runtime refresh failed through the existing trace CLI", { traceId, exitCode: result.exitCode, stderr: clipText(result.stderr, 1200) }); return clean({ ...body, traceFetch: { status: "failed", source: "existing-hwlab-cli", command: commandVisibility(command), exitCode: result.exitCode, stderr: clipText(result.stderr, 1200), stdout: clipText(result.stdout, 1200) } }); } const parsed = parseJsonMaybe(result.stdout); - if (!parsed) return clean({ ...body, traceFetch: { status: "failed", source: "existing-hwlab-cli", command: commandVisibility(command), exitCode: 0, error: "invalid_json_stdout", stdout: clipText(result.stdout, 1200) } }); + if (!parsed) { + if (forceRuntimeRefresh) throw cliError("case_refresh_trace_invalid_json", "explicit runtime refresh returned invalid trace JSON", { traceId, stdout: clipText(result.stdout, 1200) }); + return clean({ ...body, traceFetch: { status: "failed", source: "existing-hwlab-cli", command: commandVisibility(command), exitCode: 0, error: "invalid_json_stdout", stdout: clipText(result.stdout, 1200) } }); + } const parsedBody = parsed?.body && typeof parsed.body === "object" ? parsed.body : null; const preserved = clean({ toolCallSummary: body?.toolCallSummary, @@ -755,6 +763,9 @@ export function caseRunArtifactEntries(run: PreparedCaseRun) { } export async function copyCaseRunArtifact(source: string, destination: string) { + if (path.resolve(source) === path.resolve(destination)) { + try { return (await stat(source)).isFile(); } catch { return false; } + } try { const info = await stat(source); if (!info.isFile()) return false; @@ -842,6 +853,7 @@ export function caseRunArtifactManifest(context: CaseContext, run: PreparedCaseR runnerPostAgentCompileCheck: "recorded", reason: "flow-only run: the manifest records artifacts and trace without auto-grading or gate decisions" }, + refresh: evidence.refresh ?? existingManifest?.refresh, files, skippedFiles }); diff --git a/tools/src/hwlab-caserun-diagnostics.ts b/tools/src/hwlab-caserun-diagnostics.ts new file mode 100644 index 00000000..43b72576 --- /dev/null +++ b/tools/src/hwlab-caserun-diagnostics.ts @@ -0,0 +1,253 @@ +import { readFile, readdir, stat } from "node:fs/promises"; +import path from "node:path"; + +import { + archiveCaseRunArtifacts, + syncCaseRegistryRel, + writeCaseAggregateFromRegistry, +} from "./hwlab-caserun-closeout.ts"; +import { agentToolCallSummaryFromResult, arrayOfObjects, clean, clipText, text } from "./hwlab-caserun-evidence.ts"; +import type { CaseContext, PreparedCaseRun } from "./hwlab-caserun-shared.ts"; + +const MAX_DIAGNOSTIC_ROWS = 30; + +export async function refreshCaseRegistryArtifacts(context: CaseContext, caseRepo: string) { + const caseId = text(context.parsed.caseId ?? context.rest[1]); + const runId = text(context.parsed.runId ?? context.rest[2]); + if (!caseId) throw cliError("missing_required_value", "caseId is required", { field: "caseId" }); + if (!runId) throw cliError("missing_required_value", "runId is required", { field: "runId" }); + const caseRepoRunDir = path.join(caseRepo, "runs", caseId, runId); + const stored = await loadRegistryArtifacts(caseRepo, caseId, runId); + const source = text(context.parsed.source ?? context.parsed.from) || "local"; + if (source !== "local" && source !== "runtime") throw cliError("unsupported_case_refresh_source", "case refresh source must be local or runtime", { source }); + const run = registryPreparedRun(caseRepo, caseId, runId, caseRepoRunDir, stored.run, stored.evidence, stored.manifest, context.now()); + const evidence = { + ...stored.evidence, + caseId, + runId, + autoEvaluation: false, + refresh: { source, authority: "registry", runtimeAccess: source === "runtime" }, + }; + const offlineContext = { ...context, env: { ...context.env, HWLAB_API_KEY: undefined }, parsed: { ...context.parsed, noCaseRepoGitSync: true } }; + const archiveContext = source === "runtime" ? { ...context, parsed: { ...context.parsed, caseRefreshSource: "runtime" } } : offlineContext; + const first = await archiveCaseRunArtifacts(archiveContext, run, evidence, { sync: false }); + const aggregateContext = { ...offlineContext, parsed: { ...offlineContext.parsed, caseId, runId }, rest: ["aggregate", caseId, runId] }; + await writeCaseAggregateFromRegistry(aggregateContext, caseRepo, { sync: false }); + const final = await archiveCaseRunArtifacts(offlineContext, run, evidence, { sync: false }); + const sync = context.parsed.noCaseRepoGitSync === true + ? null + : await syncCaseRegistryRel(context, caseRepo, `runs/${caseId}/${runId}`, `data: refresh caserun artifacts ${runId}`); + const trace = final.manifest.trace ?? first.manifest.trace ?? {}; + return ok("case.refresh", { + caseId, + runId, + caseRepo, + caseRepoRunDir, + source, + refreshed: { + files: final.caseRepoFiles, + traceId: trace.traceId, + sourceEventCount: trace.sourceEventCount, + renderedRowCount: trace.renderedRowCount, + commandCount: trace.commandCount, + hwpodCommandCount: trace.hwpodCommandCount, + hwpodBuildCommandCount: trace.hwpodBuildCommandCount, + manifestPath: final.artifactManifestPath, + registrySync: sync, + provenance: evidence.refresh, + }, + aggregationOnly: false, + autoEvaluation: false, + }); +} + +export async function inspectCaseRegistryRun(context: CaseContext, caseRepo: string, view: "inspect" | "trace-summary") { + const artifacts = await loadDiagnosticByRunId(caseRepo, text(context.parsed.runId ?? context.rest[2]), text(context.parsed.caseId ?? context.parsed.case)); + const diagnostic = diagnosticFromArtifacts(artifacts); + const payload = view === "trace-summary" ? traceSummary(diagnostic) : diagnostic; + return ok(`case.run.${view}`, { ...payload, inspectionOnly: true, serviceRuntime: false, autoEvaluation: false }); +} + +export async function skillUsageCaseRegistryRun(context: CaseContext, caseRepo: string) { + const skill = text(context.parsed.skill ?? context.parsed.skillName ?? context.rest[3]); + if (!skill) throw cliError("missing_required_value", "skill is required", { field: "skill" }); + const artifacts = await loadDiagnosticByRunId(caseRepo, text(context.parsed.runId ?? context.rest[2]), text(context.parsed.caseId ?? context.parsed.case)); + const pattern = new RegExp(`(?:\\.agents[\\\\/]skills[\\\\/]${escapeRegExp(skill)}[\\\\/]SKILL\\.md|\\b${escapeRegExp(skill)}\\b)`, "iu"); + const sourceFiles = [ + ["agent-prompt.md", artifacts.promptText], + ["agent-trace.json", JSON.stringify(artifacts.agentTrace ?? {})], + ["agent-messages.json", JSON.stringify(artifacts.agentMessages ?? {})], + ["artifact-manifest.json", JSON.stringify(artifacts.manifest ?? {})], + ] as const; + const evidence = sourceFiles.filter(([, value]) => pattern.test(value)).map(([source, value]) => ({ source, preview: clipText(value.replace(/\s+/gu, " "), 500) })); + const rows = traceRows(artifacts); + const matches = rows.filter((row) => pattern.test(`${row.header ?? ""}\n${row.body ?? ""}`)).map((row) => ({ rowId: row.rowId, seq: row.seq, header: row.header, preview: clipText(row.body, 500) })); + return ok("case.run.skill-usage", { + caseId: artifacts.caseId, + runId: artifacts.runId, + caseRepo, + caseRepoRunDir: artifacts.caseRepoRunDir, + skill, + prompt: { mentionsSkill: pattern.test(artifacts.promptText), path: path.join(artifacts.caseRepoRunDir, "agent-prompt.md") }, + trace: { readSkillFile: matches.length > 0, matchCount: matches.length, matches }, + resourceBundle: { materialized: evidence.length > 0, evidence }, + inspectionOnly: true, + serviceRuntime: false, + autoEvaluation: false, + }); +} + +async function loadDiagnosticByRunId(caseRepo: string, runId: string, explicitCaseId: string) { + if (!runId) throw cliError("missing_required_value", "runId is required", { field: "runId" }); + const caseId = explicitCaseId || await resolveCaseId(caseRepo, runId); + return loadRegistryArtifacts(caseRepo, caseId, runId); +} + +async function loadRegistryArtifacts(caseRepo: string, caseId: string, runId: string) { + const caseRepoRunDir = path.join(caseRepo, "runs", caseId, runId); + const [run, evidence, manifest, agentTrace, agentMessages, finalResponseText, promptText] = await Promise.all([ + readJson(path.join(caseRepoRunDir, "run.json")), + readJson(path.join(caseRepoRunDir, "evidence.json")), + readJson(path.join(caseRepoRunDir, "artifact-manifest.json")), + readJson(path.join(caseRepoRunDir, "agent-trace.json")), + readJson(path.join(caseRepoRunDir, "agent-messages.json")), + readText(path.join(caseRepoRunDir, "final-response.md")), + readText(path.join(caseRepoRunDir, "agent-prompt.md")), + ]); + if (!run && !evidence && !manifest) throw cliError("case_registry_run_not_found", "case registry run artifacts were not found", { caseRepoRunDir, caseId, runId }); + return { caseRepo, caseId, runId, caseRepoRunDir, run: run ?? {}, evidence: evidence ?? {}, manifest: manifest ?? {}, agentTrace, agentMessages, finalResponseText, promptText }; +} + +async function resolveCaseId(caseRepo: string, runId: string) { + const root = path.join(caseRepo, "runs"); + const entries = await readdir(root).catch(() => []); + const matches: string[] = []; + for (const entry of entries.filter((item) => !item.startsWith("."))) { + if (await isDirectory(path.join(root, entry, runId))) matches.push(entry); + } + if (matches.length === 0) throw cliError("case_registry_run_not_found", "runId was not found in case registry runs", { caseRepo, runId }); + if (matches.length > 1) throw cliError("case_registry_run_ambiguous", "runId exists under multiple caseIds; pass --case-id", { caseRepo, runId, caseIds: matches }); + return matches[0]; +} + +function registryPreparedRun(caseRepo: string, caseId: string, runId: string, caseRepoRunDir: string, rawRun: any, rawEvidence: any, manifest: any, now: string): PreparedCaseRun { + const run = rawRun ?? {}; + const evidence = rawEvidence ?? {}; + const subject = { ...(run.subject ?? {}), ...(evidence.subject ?? {}), ...(manifest?.subject ?? {}) }; + const agent = { ...(run.agent ?? {}), ...(evidence.agent ?? {}), ...(manifest?.agent ?? {}) }; + return { + ...run, + caseId, + runId, + runDir: caseRepoRunDir, + caseRepo, + caseDir: run.caseDir ?? "", + caseFile: run.caseFile ?? "", + sourceSpecPath: run.sourceSpecPath ?? path.join(caseRepoRunDir, ".hwlab", "hwpod-spec.yaml"), + specPath: run.specPath ?? path.join(caseRepoRunDir, ".hwlab", "hwpod-spec.yaml"), + apiUrl: run.apiUrl ?? evidence.apiUrl ?? "", + compileOnly: run.compileOnly !== false, + createdAt: run.createdAt ?? now, + updatedAt: now, + definition: run.definition ?? {}, + subject: { repoLocalPath: subject.repoLocalPath ?? "", commitId: subject.commitId ?? "", subdir: subject.subdir ?? "", worktreePath: subject.worktreePath ?? "", workspacePath: subject.workspacePath ?? subject.worktreePath ?? "", setup: subject.setup ?? {} }, + agent: agent.traceId ? agent : null, + agentTrace: evidence.agentTrace ?? manifest?.trace ?? run.agentTrace, + agentDiff: evidence.agentDiff ?? manifest?.diff ?? run.agentDiff, + } as PreparedCaseRun; +} + +function diagnosticFromArtifacts(artifacts: any) { + const agent = { ...(artifacts.run.agent ?? {}), ...(artifacts.evidence.agent ?? {}), ...(artifacts.manifest.agent ?? {}) }; + const trace = { ...(artifacts.run.agentTrace ?? {}), ...(artifacts.evidence.agentTrace ?? {}), ...(artifacts.manifest.trace ?? {}) }; + const summary = boundedToolCallSummary(trace.toolCallSummary ?? agent.toolCallSummary ?? agentToolCallSummaryFromResult(artifacts.agentTrace?.body ?? artifacts.agentTrace) ?? null); + const rows = traceRows(artifacts); + const commands = summary?.items ?? rows.filter((row) => /commandExecution|tool_call|^tool:/iu.test(`${row.rowId ?? ""} ${row.header ?? ""}`)).map((row) => ({ rowId: row.rowId, seq: row.seq, command: row.command ?? row.body, status: row.status ?? row.tone })); + return clean({ + caseId: artifacts.caseId, + runId: artifacts.runId, + caseRepo: artifacts.caseRepo, + caseRepoRunDir: artifacts.caseRepoRunDir, + status: artifacts.run.status ?? artifacts.evidence.status ?? "recorded", + stage: artifacts.run.stage ?? artifacts.run._control?.stage, + compileOnly: artifacts.evidence.compileOnly ?? artifacts.run.compileOnly, + traceId: agent.traceId ?? trace.traceId, + sessionId: agent.sessionId ?? trace.sessionId, + conversationId: agent.conversationId ?? trace.conversationId, + threadId: agent.threadId ?? trace.threadId, + providerProfile: agent.providerProfile, + requestedProviderProfile: agent.requestedProviderProfile, + resolvedBackendProfile: agent.resolvedBackendProfile, + provider: agent.provider, + model: agent.model, + backend: agent.backend, + terminalStatus: trace.terminalStatus ?? agent.terminalStatus, + agentResultStatus: agent.result?.status ?? trace.status, + commandStatus: agent.commandStatus ?? summary?.commandStatus, + agentRunStatus: agent.agentRunStatus ?? summary?.agentRunStatus, + toolCallSummary: summary, + sourceEventCount: trace.sourceEventCount ?? artifacts.agentMessages?.sourceEventCount, + renderedRowCount: trace.renderedRowCount ?? artifacts.agentMessages?.renderedRowCount ?? rows.length, + commandCount: trace.commandCount ?? summary?.count ?? commands.length, + hwpodCommandCount: trace.hwpodCommandCount, + hwpodBuildCommandCount: trace.hwpodBuildCommandCount, + agentFinal: artifacts.manifest.agentFinal ?? artifacts.evidence.agentFinal, + postValidation: artifacts.manifest.postValidation ?? artifacts.evidence.postValidation, + refresh: artifacts.manifest.refresh ?? artifacts.evidence.refresh, + finalResponse: { present: Boolean(artifacts.finalResponseText.trim()), path: path.join(artifacts.caseRepoRunDir, "final-response.md"), textPreview: clipText(artifacts.finalResponseText, 800) }, + traceCommands: commands.slice(0, MAX_DIAGNOSTIC_ROWS), + paths: { runJson: path.join(artifacts.caseRepoRunDir, "run.json"), evidenceJson: path.join(artifacts.caseRepoRunDir, "evidence.json"), agentTraceJson: path.join(artifacts.caseRepoRunDir, "agent-trace.json"), agentMessagesJson: path.join(artifacts.caseRepoRunDir, "agent-messages.json"), artifactManifestJson: path.join(artifacts.caseRepoRunDir, "artifact-manifest.json"), aggregateMd: path.join(artifacts.caseRepoRunDir, "aggregate.md") }, + }); +} + +function traceSummary(diagnostic: any) { + return clean({ + caseId: diagnostic.caseId, + runId: diagnostic.runId, + caseRepo: diagnostic.caseRepo, + caseRepoRunDir: diagnostic.caseRepoRunDir, + traceId: diagnostic.traceId, + providerProfile: diagnostic.providerProfile, + requestedProviderProfile: diagnostic.requestedProviderProfile, + resolvedBackendProfile: diagnostic.resolvedBackendProfile, + provider: diagnostic.provider, + model: diagnostic.model, + backend: diagnostic.backend, + terminalStatus: diagnostic.terminalStatus, + agentResultStatus: diagnostic.agentResultStatus, + commandStatus: diagnostic.commandStatus, + agentRunStatus: diagnostic.agentRunStatus, + toolCallSummary: diagnostic.toolCallSummary, + sourceEventCount: diagnostic.sourceEventCount, + renderedRowCount: diagnostic.renderedRowCount, + commandCount: diagnostic.commandCount, + hwpodCommandCount: diagnostic.hwpodCommandCount, + hwpodBuildCommandCount: diagnostic.hwpodBuildCommandCount, + traceCommands: diagnostic.traceCommands, + paths: diagnostic.paths, + }); +} + +function boundedToolCallSummary(summary: any) { + if (!summary || typeof summary !== "object") return null; + return clean({ + ...summary, + items: arrayOfObjects(summary.items).slice(0, MAX_DIAGNOSTIC_ROWS).map((item) => clean({ + ...item, + command: clipText(item.command, 1000), + detail: clipText(item.detail, 1000), + })), + }); +} + +function traceRows(artifacts: any) { + const body = artifacts.agentTrace?.body && typeof artifacts.agentTrace.body === "object" ? artifacts.agentTrace.body : artifacts.agentTrace ?? {}; + return arrayOfObjects(artifacts.agentMessages?.rows ?? body.rows ?? body.renderedRows ?? body.traceRows).slice(0, MAX_DIAGNOSTIC_ROWS); +} + +async function readJson(file: string) { try { return JSON.parse(await readFile(file, "utf8")); } catch { return null; } } +async function readText(file: string) { try { return await readFile(file, "utf8"); } catch { return ""; } } +async function isDirectory(file: string) { try { return (await stat(file)).isDirectory(); } catch { return false; } } +function escapeRegExp(value: string) { return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); } +function cliError(code: string, message: string, details: Record = {}) { return Object.assign(new Error(message), { code, details }); } +function ok(action: string, extra: Record = {}) { return { ok: true, action, status: "completed", ...extra }; } diff --git a/tools/src/hwlab-caserun-runtime.ts b/tools/src/hwlab-caserun-runtime.ts index f5848113..5dadf83b 100644 --- a/tools/src/hwlab-caserun-runtime.ts +++ b/tools/src/hwlab-caserun-runtime.ts @@ -9,6 +9,7 @@ import path from "node:path"; import { renderTraceRowsMarkdown, traceDisplayRows, traceNoiseEventCount, type TraceEventRow } from "./hwlab-cli/trace-renderer.ts"; import { collectAgentWorkspaceDiff } from "./hwlab-caserun-diff.ts"; +import { inspectCaseRegistryRun, refreshCaseRegistryArtifacts, skillUsageCaseRegistryRun } from "./hwlab-caserun-diagnostics.ts"; import { agentTerminalEvidence, agentToolCallSummaryFromResult, agentWithResultEvidence, applyCaseRunEvidenceRelationships, commandsFromToolCallSummary, summarizeAgentTrace, firstNumberOption, firstTextOption, type AgentToolCallSummary } from "./hwlab-caserun-evidence.ts"; import { readHwpodSpec } from "./hwpod-harness-lib.ts"; import { prepareSubjectWorktree, caseRunForSubjectSnapshot, subjectFromDefinition, requestSubjectWorktree, subjectWorktreeStatus, keilSidecarCopyOp, subjectRelativeKeilProject, stripUvprojx, windowsJoin, keilSidecarCopyNodeScript, keilSidecarSummary, objectRecord, isAbsolutePathLike, subjectRelativeWorktreePath, normalizeLocalPath, subjectWorktreePath, looksLikeWindowsPath, rewriteHwpodSpecWorkspacePath, pollKeilJobStatus, requestKeilJobStatus, keilJobStatus, hwpodResultTexts, artifactsFrom } from "./hwlab-caserun-subject.ts"; @@ -34,6 +35,7 @@ export async function caseCommand(context: CaseContext) { if (["help", "--help", "-h"].includes(subcommand) || context.parsed.help === true) return caseHelp(); if (subcommand === "prompt") return promptCaseRun(context); if (subcommand === "aggregate") return aggregateCaseRun(context); + if (subcommand === "refresh" || subcommand === "backfill") return refreshCaseRegistryArtifacts(context, await resolveCaseRepo(context)); if (subcommand === "prepare") return prepareCaseRun(context); if (subcommand === "build") return buildCaseRun(context); if (subcommand === "collect") return collectCaseRun(context); @@ -47,6 +49,9 @@ async function caseRunCommand(context: CaseContext) { if (mode === "status") return statusCaseRun(context); if (mode === "result") return resultCaseRun(context); if (mode === "logs") return logsCaseRun(context); + if (mode === "inspect") return inspectCaseRegistryRun(context, await resolveCaseRepo(context), "inspect"); + if (mode === "trace-summary" || mode === "trace") return inspectCaseRegistryRun(context, await resolveCaseRepo(context), "trace-summary"); + if (mode === "skill-usage" || mode === "skills") return skillUsageCaseRegistryRun(context, await resolveCaseRepo(context)); if (mode === "worker") return runCaseRunWorker(context); return runCaseRun(context); } @@ -69,17 +74,23 @@ export function caseHelp() { `case build CASE_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--run-dir DIR]`, `case collect CASE_ID --case-repo ${STANDARD_CASE_REPO_PATH} --run-dir DIR`, `case aggregate CASE_ID --run-id RUN_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--output aggregate.md]`, + `case refresh CASE_ID --run-id RUN_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--source local|runtime]`, `case run CASE_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--provider-profile PROFILE]`, `case run start CASE_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--run-id RUN_ID]`, `case run status RUN_ID [--state-dir .state/hwlab-cli/caserun]`, `case run result RUN_ID [--state-dir .state/hwlab-cli/caserun]`, - `case run logs RUN_ID [--tail 8000]` + `case run logs RUN_ID [--tail 8000]`, + `case run inspect RUN_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--case-id CASE_ID]`, + `case run trace-summary RUN_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--case-id CASE_ID]`, + `case run skill-usage RUN_ID --skill SKILL_NAME --case-repo ${STANDARD_CASE_REPO_PATH} [--case-id CASE_ID]` ], notes: [ "case prompt 是无服务预览入口，只按真实 CaseRun 组装路径渲染输入 prompt，不准备 worktree、不提交 agent、不访问硬件。", "case build remains a compile-only HWPOD smoke stage.", "case run prepares the subject worktree, submits a prompt to HWLAB Code Agent, records agent provenance and workspace diff, then runs compile validation.", - "case aggregate is a service-free post-processing command: it reads an existing case registry run and writes one aggregate markdown entry without pass/fail grading.", + "case aggregate is registry-only and offline by default; it reads existing artifacts without querying runtime.", + "case refresh/backfill defaults to local registry artifacts; only explicit --source runtime may query the existing trace CLI.", + "case run inspect/trace-summary/skill-usage are bounded, service-free registry diagnostics.", "case run start/status/result/logs is the cli-spec async control surface for long CaseRun flows; start returns immediately and status/result/logs are short polling commands.", "This version records raw stage evidence only; it does not auto-grade agent output, diff contents, or Keil evidence." ] @@ -132,7 +143,7 @@ export async function promptCaseRun(context: CaseContext) { export async function aggregateCaseRun(context: CaseContext) { const caseRepo = await resolveCaseRepo(context); - const aggregate = await writeCaseAggregateFromRegistry(context, caseRepo, { sync: true }); + const aggregate = await writeCaseAggregateFromRegistry(context, caseRepo, { sync: context.parsed.noCaseRepoGitSync !== true }); return ok("case.aggregate", { caseId: aggregate.caseId, runId: aggregate.runId,