Merge remote-tracking branch 'origin/v0.3' into feat/caserun-static-audit
# Conflicts: # tools/src/hwlab-caserun-closeout.ts # tools/src/hwlab-caserun-runtime.ts
This commit is contained in:
@@ -0,0 +1,191 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { runHwlabCli } from "../src/hwlab-cli-lib.ts";
|
||||
|
||||
test("case aggregate remains registry-only even when runtime credentials exist", async () => {
|
||||
const fixture = await createRegistryFixture("aggregate-offline");
|
||||
try {
|
||||
let runtimeCalls = 0;
|
||||
const result = await runHwlabCli(["case", "aggregate", fixture.caseId, "--run-id", fixture.runId, "--case-repo", fixture.caseRepo, "--no-case-repo-git-sync"], {
|
||||
cwd: fixture.root,
|
||||
env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" },
|
||||
runProcess: async () => {
|
||||
runtimeCalls += 1;
|
||||
throw new Error("offline aggregate must not execute runtime commands");
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2));
|
||||
assert.equal(runtimeCalls, 0);
|
||||
assert.equal(result.payload.action, "case.aggregate");
|
||||
assert.match(await readFile(path.join(fixture.runDir, "aggregate.md"), "utf8"), /registry artifacts/u);
|
||||
} finally {
|
||||
await rm(fixture.root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("case refresh is explicit, registry-backed, and semantically idempotent", async () => {
|
||||
const fixture = await createRegistryFixture("refresh-local");
|
||||
try {
|
||||
const args = ["case", "refresh", fixture.caseId, "--run-id", fixture.runId, "--case-repo", fixture.caseRepo, "--source", "local", "--no-case-repo-git-sync"];
|
||||
const first = await runHwlabCli(args, { cwd: fixture.root, now: () => "2026-07-16T14:00:00.000Z", env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" }, runProcess: forbiddenRuntime });
|
||||
assert.equal(first.exitCode, 0, JSON.stringify(first.payload, null, 2));
|
||||
assert.deepEqual(first.payload.refreshed.provenance, { source: "local", authority: "registry", runtimeAccess: false });
|
||||
assert.equal(first.payload.refreshed.commandCount, 2);
|
||||
assert.equal(first.payload.refreshed.hwpodCommandCount, 1);
|
||||
assert.equal(first.payload.refreshed.hwpodBuildCommandCount, 1);
|
||||
const before = await fileHashes(fixture.runDir, ["evidence.json", "agent-messages.json", "artifact-manifest.json", "aggregate.md"]);
|
||||
|
||||
const second = await runHwlabCli(args, { cwd: fixture.root, now: () => "2026-07-16T15:00:00.000Z", env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" }, runProcess: forbiddenRuntime });
|
||||
assert.equal(second.exitCode, 0, JSON.stringify(second.payload, null, 2));
|
||||
assert.deepEqual(await fileHashes(fixture.runDir, Object.keys(before)), before);
|
||||
const manifest = JSON.parse(await readFile(path.join(fixture.runDir, "artifact-manifest.json"), "utf8"));
|
||||
assert.deepEqual(manifest.refresh, { source: "local", authority: "registry", runtimeAccess: false });
|
||||
assert.equal(manifest.trace.toolCallSummary.count, 2);
|
||||
} finally {
|
||||
await rm(fixture.root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("case run offline diagnostics inspect trace and skill usage without runtime access", async () => {
|
||||
const fixture = await createRegistryFixture("inspect-offline");
|
||||
try {
|
||||
const options = { cwd: fixture.root, env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" }, runProcess: forbiddenRuntime };
|
||||
const inspect = await runHwlabCli(["case", "run", "inspect", fixture.runId, "--case-repo", fixture.caseRepo], options);
|
||||
assert.equal(inspect.exitCode, 0, JSON.stringify(inspect.payload, null, 2));
|
||||
assert.equal(inspect.payload.serviceRuntime, false);
|
||||
assert.equal(inspect.payload.traceId, "trc_offline");
|
||||
assert.equal(inspect.payload.providerProfile, "deepseek");
|
||||
assert.equal(inspect.payload.terminalStatus, "completed");
|
||||
assert.equal(inspect.payload.toolCallSummary.count, 2);
|
||||
|
||||
const trace = await runHwlabCli(["case", "run", "trace-summary", fixture.runId, "--case-repo", fixture.caseRepo], options);
|
||||
assert.equal(trace.exitCode, 0, JSON.stringify(trace.payload, null, 2));
|
||||
assert.equal(trace.payload.commandCount, 2);
|
||||
assert.equal(trace.payload.traceCommands.length, 2);
|
||||
|
||||
const skill = await runHwlabCli(["case", "run", "skill-usage", fixture.runId, "--skill", "keil", "--case-repo", fixture.caseRepo], options);
|
||||
assert.equal(skill.exitCode, 0, JSON.stringify(skill.payload, null, 2));
|
||||
assert.equal(skill.payload.serviceRuntime, false);
|
||||
assert.equal(skill.payload.trace.readSkillFile, true);
|
||||
assert.equal(skill.payload.trace.matchCount, 1);
|
||||
assert.equal(skill.payload.resourceBundle.materialized, true);
|
||||
assert.equal(skill.payload.resourceBundle.evidence[0].source, "artifact-manifest.json");
|
||||
} finally {
|
||||
await rm(fixture.root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("case run skill usage keeps prompt mentions separate from materialized bundles", async () => {
|
||||
const fixture = await createRegistryFixture("prompt-only-skill");
|
||||
try {
|
||||
await writeJson(path.join(fixture.runDir, "agent-messages.json"), { rows: [{ rowId: "tool:build", seq: 1, header: "工具调用", body: "hwpod build --spec .hwlab/hwpod-spec.yaml" }] });
|
||||
const manifest = JSON.parse(await readFile(path.join(fixture.runDir, "artifact-manifest.json"), "utf8"));
|
||||
delete manifest.resourceBundle;
|
||||
await writeJson(path.join(fixture.runDir, "artifact-manifest.json"), manifest);
|
||||
|
||||
const result = await runHwlabCli(["case", "run", "skill-usage", fixture.runId, "--skill", "keil", "--case-repo", fixture.caseRepo], { cwd: fixture.root, runProcess: forbiddenRuntime });
|
||||
assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2));
|
||||
assert.equal(result.payload.prompt.mentionsSkill, true);
|
||||
assert.equal(result.payload.trace.readSkillFile, false);
|
||||
assert.equal(result.payload.resourceBundle.materialized, false);
|
||||
} finally {
|
||||
await rm(fixture.root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("case run inspect preserves missing authoritative agent final", async () => {
|
||||
const fixture = await createRegistryFixture("missing-agent-final");
|
||||
try {
|
||||
const reason = "finalResponse=null; provider returned no final assistant response";
|
||||
const evidence = JSON.parse(await readFile(path.join(fixture.runDir, "evidence.json"), "utf8"));
|
||||
evidence.agentFinal = { present: false, reason, status: "blocked" };
|
||||
await writeJson(path.join(fixture.runDir, "evidence.json"), evidence);
|
||||
const manifest = JSON.parse(await readFile(path.join(fixture.runDir, "artifact-manifest.json"), "utf8"));
|
||||
manifest.agentFinal = { present: false, reason, status: "blocked" };
|
||||
await writeJson(path.join(fixture.runDir, "artifact-manifest.json"), manifest);
|
||||
await writeFile(path.join(fixture.runDir, "final-response.md"), `# CaseRun Final Response\n\n${reason}\n`, "utf8");
|
||||
|
||||
const result = await runHwlabCli(["case", "run", "inspect", fixture.runId, "--case-repo", fixture.caseRepo], { cwd: fixture.root, runProcess: forbiddenRuntime });
|
||||
assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2));
|
||||
assert.equal(result.payload.agentFinal.present, false);
|
||||
assert.equal(result.payload.finalResponse.present, false);
|
||||
assert.equal(result.payload.finalResponse.reason, reason);
|
||||
assert.match(result.payload.finalResponse.textPreview, /provider returned no final assistant response/u);
|
||||
} finally {
|
||||
await rm(fixture.root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("case run skill usage scans all rows before bounding returned matches", async () => {
|
||||
const fixture = await createRegistryFixture("late-skill-read");
|
||||
try {
|
||||
const rows = [
|
||||
...Array.from({ length: 30 }, (_, index) => ({ rowId: `tool:other:${index + 1}`, seq: index + 1, header: "工具调用", body: "echo unrelated" })),
|
||||
...Array.from({ length: 35 }, (_, index) => ({ rowId: `tool:skill:${index + 31}`, seq: index + 31, header: "工具调用", body: "cat ~/.agents/skills/keil/SKILL.md" })),
|
||||
];
|
||||
await writeJson(path.join(fixture.runDir, "agent-messages.json"), { sourceEventCount: rows.length, renderedRowCount: rows.length, rows });
|
||||
|
||||
const result = await runHwlabCli(["case", "run", "skill-usage", fixture.runId, "--skill", "keil", "--case-repo", fixture.caseRepo], { cwd: fixture.root, runProcess: forbiddenRuntime });
|
||||
assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2));
|
||||
assert.equal(result.payload.trace.readSkillFile, true);
|
||||
assert.equal(result.payload.trace.matchCount, 35);
|
||||
assert.equal(result.payload.trace.matches.length, 30);
|
||||
assert.equal(result.payload.trace.matches[0].seq, 31);
|
||||
} finally {
|
||||
await rm(fixture.root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test("case refresh accesses runtime only with explicit runtime source", async () => {
|
||||
const fixture = await createRegistryFixture("refresh-runtime");
|
||||
try {
|
||||
let runtimeCalls = 0;
|
||||
const result = await runHwlabCli(["case", "refresh", fixture.caseId, "--run-id", fixture.runId, "--case-repo", fixture.caseRepo, "--source", "runtime", "--no-case-repo-git-sync"], {
|
||||
cwd: fixture.root,
|
||||
env: { HWLAB_API_KEY: "hwl_live_runtime_test" },
|
||||
runProcess: async () => {
|
||||
runtimeCalls += 1;
|
||||
return { command: [], exitCode: 0, stderr: "", stdout: JSON.stringify({ body: { traceId: "trc_offline", source: "hwlab-cli.client.agent.trace", status: "completed", terminalStatus: "completed", rows: [{ rowId: "tool:runtime", seq: 1, header: "工具调用", body: "hwpod build --spec .hwlab/hwpod-spec.yaml" }, { rowId: "final", seq: 2, header: "助手最终消息", terminal: true, body: "runtime refreshed" }] } }) };
|
||||
},
|
||||
});
|
||||
|
||||
assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2));
|
||||
assert.equal(runtimeCalls, 1);
|
||||
assert.deepEqual(result.payload.refreshed.provenance, { source: "runtime", authority: "registry", runtimeAccess: true });
|
||||
} finally {
|
||||
await rm(fixture.root, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
async function createRegistryFixture(suffix: string) {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), `hwlab-caserun-${suffix}-`));
|
||||
const caseRepo = path.join(root, "hwlab-case-registry");
|
||||
const caseId = `case-${suffix}`;
|
||||
const runId = `run-${suffix}`;
|
||||
const runDir = path.join(caseRepo, "runs", caseId, runId);
|
||||
await mkdir(path.join(caseRepo, ".git"), { recursive: true });
|
||||
await writeFile(path.join(caseRepo, ".git", "HEAD"), "ref: refs/heads/main\n", "utf8");
|
||||
await mkdir(path.join(runDir, ".hwlab"), { recursive: true });
|
||||
const agent = { stageStatus: "completed", baseUrl: "http://web.test", traceId: "trc_offline", sessionId: "ses_offline", conversationId: "cnv_offline", threadId: "thr_offline", providerProfile: "deepseek", requestedProviderProfile: "deepseek", resolvedBackendProfile: "deepseek-runtime", provider: "deepseek", model: "deepseek-chat", terminalStatus: "completed", commandStatus: "completed", agentRunStatus: "completed", promptSha256: "prompt-sha" };
|
||||
const toolCallSummary = { source: "agentrun-result.toolCallSummary", count: 2, statusCounts: { completed: 2 }, exitCodeCounts: { "0": 2 }, terminalStatus: "completed", commandStatus: "completed", agentRunStatus: "completed", items: [{ kind: "build", toolName: "commandExecution", status: "completed", exitCode: 0, command: "hwpod build --spec .hwlab/hwpod-spec.yaml" }, { kind: "other", toolName: "commandExecution", status: "completed", exitCode: 0, command: "cat ~/.agents/skills/keil/SKILL.md" }] };
|
||||
await writeJson(path.join(runDir, "run.json"), { caseId, runId, runDir, caseRepo, specPath: path.join(runDir, ".hwlab", "hwpod-spec.yaml"), compileOnly: true, createdAt: "2026-07-16T13:00:00.000Z", updatedAt: "2026-07-16T13:00:00.000Z", definition: {}, subject: { repoLocalPath: "C:/work/fw", commitId: "0".repeat(40), subdir: "", worktreePath: "C:/work/fw/.worktree/case", workspacePath: "C:/work/fw/.worktree/case", setup: {} }, agent });
|
||||
await writeJson(path.join(runDir, "evidence.json"), { contractVersion: "hwpod-case-run-evidence-v1", caseId, runId, compileOnly: true, autoEvaluation: false, status: "recorded", subject: { repoLocalPath: "C:/work/fw", commitId: "0".repeat(40), worktreePath: "C:/work/fw/.worktree/case" }, agent, agentTrace: { traceId: "trc_offline", terminalStatus: "completed", toolCallSummary, commandCount: 2, hwpodCommandCount: 1, hwpodBuildCommandCount: 1 }, agentFinal: { present: true, status: "completed", terminalStatus: "completed" }, postValidation: { source: "case-run-runner-post-agent-compile-check", status: "completed", returnCode: 0 }, decisions: { autoEvaluation: false, runnerPostAgentCompileCheck: "recorded" } });
|
||||
await writeJson(path.join(runDir, "agent-trace.json"), { contractVersion: "hwpod-case-run-agent-trace-identity-v1", traceId: "trc_offline", status: "completed", terminalStatus: "completed", commandStatus: "completed", toolCallSummary });
|
||||
await writeJson(path.join(runDir, "agent-messages.json"), { renderer: "tools/src/hwlab-cli/trace-renderer:traceDisplayRows", sourceEventCount: 2, renderedRowCount: 2, rows: [{ rowId: "tool:build", seq: 1, header: "工具调用", body: "hwpod build --spec .hwlab/hwpod-spec.yaml" }, { rowId: "tool:skill", seq: 2, header: "工具调用", body: "cat ~/.agents/skills/keil/SKILL.md" }] });
|
||||
await writeJson(path.join(runDir, "artifact-manifest.json"), { contractVersion: "hwpod-case-run-artifact-manifest-v1", caseId, runId, agent, trace: { traceId: "trc_offline", terminalStatus: "completed", toolCallSummary, commandCount: 2, hwpodCommandCount: 1, hwpodBuildCommandCount: 1 }, resourceBundle: { materialized: true, skills: [{ name: "keil", manifest: ".agents/skills/keil/SKILL.md" }] }, agentFinal: { present: true, status: "completed" }, postValidation: { status: "completed", returnCode: 0 }, files: [] });
|
||||
await writeFile(path.join(runDir, "agent-prompt.md"), "请读取 keil skill 后执行编译。\n", "utf8");
|
||||
await writeFile(path.join(runDir, "final-response.md"), "# CaseRun Final Response\n\ncompleted\n", "utf8");
|
||||
await writeFile(path.join(runDir, "agent-diff.patch"), "", "utf8");
|
||||
await writeFile(path.join(runDir, ".hwlab", "hwpod-spec.yaml"), "metadata:\n name: offline\n", "utf8");
|
||||
return { root, caseRepo, caseId, runId, runDir };
|
||||
}
|
||||
|
||||
async function writeJson(file: string, value: unknown) { await writeFile(file, `${JSON.stringify(value, null, 2)}\n`, "utf8"); }
|
||||
async function fileHashes(root: string, files: string[]) { return Object.fromEntries(await Promise.all(files.map(async (file) => [file, createHash("sha256").update(await readFile(path.join(root, file))).digest("hex")]))); }
|
||||
async function forbiddenRuntime() { throw new Error("offline diagnostics must not execute runtime commands"); }
|
||||
@@ -5,7 +5,7 @@ import path from "node:path";
|
||||
import { test } from "bun:test";
|
||||
|
||||
import { runHwlabCli } from "../src/hwlab-cli-lib.ts";
|
||||
import { artifactsFromKeilStatusForTest, extractKeilJobId, jobStatusCommandForTest } from "../src/hwlab-caserun-lib.ts";
|
||||
import { artifactsFromKeilStatusForTest, buildWaitBlockerForTest, extractKeilJobId, jobStatusCommandForTest, summarizeHwpodOperationForTest } from "../src/hwlab-caserun-lib.ts";
|
||||
|
||||
const SUBJECT_COMMIT_ID = "df7a4e6e551fa90d64bde5537cc000f89d63dd20";
|
||||
const SUBJECT_REPO_LOCAL_PATH = "F:\\Work\\HWLAB-CASE-F103";
|
||||
@@ -202,6 +202,28 @@ test("case run exposes Keil hex and axf artifacts in evidence summary", () => {
|
||||
assert.deepEqual(artifactsFromKeilStatusForTest({ result: { hex_file: "F:\\out.hex", axf_file: "F:\\out.axf" } }), ["F:\\out.hex", "F:\\out.axf"]);
|
||||
});
|
||||
|
||||
test("case run normalizes v0.3 HWPOD build operation topology and typed wait blockers", () => {
|
||||
const document = { metadata: { name: "d601-f103-v2" }, spec: { nodeBinding: { nodeId: "node-d601-f103-v2" }, workspace: { path: "F:\\Work\\case" } } };
|
||||
const operation = summarizeHwpodOperationForTest({ body: { hwpodId: "d601-f103-v2", nodeId: "node-d601-f103-v2", results: [{ opId: "op_build", op: "debug.build", ok: true, status: "completed", workspacePath: "F:\\Work\\case", output: { stdout: '{"job_id":"job-1"}' } }] } }, document, "debug.build");
|
||||
assert.equal(operation.topology.ok, true);
|
||||
assert.equal(operation.jobId, "job-1");
|
||||
assert.equal(buildWaitBlockerForTest(operation, { ok: true, status: "completed", warningCount: 7, artifacts: ["F:\\out.hex"] }, 0), null);
|
||||
assert.equal(buildWaitBlockerForTest({ ...operation, jobId: "" }, null, 0).code, "hwpod_build_job_id_missing");
|
||||
assert.equal(buildWaitBlockerForTest(operation, { ok: false, status: "timeout", timedOut: true, polls: 3 }, 0).code, "hwpod_build_wait_timeout");
|
||||
assert.equal(buildWaitBlockerForTest(operation, { ok: false, status: "failed", returnCode: 2 }, 0).code, "hwpod_build_failed");
|
||||
assert.equal(buildWaitBlockerForTest(operation, { ok: true, status: "completed", artifacts: ["F:\\out.axf"] }, 0).code, "hwpod_build_hex_missing");
|
||||
const mismatch = summarizeHwpodOperationForTest({ body: { nodeId: "wrong-node", results: [{ op: "debug.build", ok: true, output: { stdout: '{"job_id":"job-2"}' } }] } }, document, "debug.build");
|
||||
assert.equal(buildWaitBlockerForTest(mismatch, null, 0).code, "hwpod_topology_mismatch");
|
||||
const missingObserved = summarizeHwpodOperationForTest({ body: { results: [{ op: "debug.build", ok: true, output: { stdout: '{"job_id":"job-3"}' } }] } }, document, "debug.build");
|
||||
assert.deepEqual(missingObserved.topology.blocker.details, [
|
||||
{ field: "hwpodId", expected: "d601-f103-v2", observed: null },
|
||||
{ field: "nodeId", expected: "node-d601-f103-v2", observed: null },
|
||||
{ field: "workspacePath", expected: "F:\\Work\\case", observed: null }
|
||||
]);
|
||||
const yamlEmpty = summarizeHwpodOperationForTest({ body: { hwpodId: "response-only", nodeId: "response-node", results: [{ op: "debug.build", ok: true, workspacePath: "response-path", output: { stdout: '{"job_id":"job-4"}' } }] } }, { metadata: {}, spec: {} }, "debug.build");
|
||||
assert.equal(yamlEmpty.topology.ok, true);
|
||||
});
|
||||
|
||||
test("case run start returns immediately and status/result/logs are short polling commands", async () => {
|
||||
const root = await mkdtempCaseRoot();
|
||||
const caseRepo = path.join(root, "hwlab-case-registry");
|
||||
@@ -470,6 +492,7 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit
|
||||
assert.equal(result.payload.evidence.hwpod.source, "case-run-runner-post-agent-compile-check");
|
||||
assert.match(result.payload.agentDiff.statusShort, /projects\/01_baseline\/main\.c/u);
|
||||
assert.deepEqual(result.payload.agentDiff.diffCollection.untracked, { total: 2, included: 1, omitted: 1 });
|
||||
assert.deepEqual(result.payload.agentDiff.diffCollection.submodules, [{ path: "vendor/lib", commit: "0123456789abcdef", state: "modified" }]);
|
||||
assert.equal(result.payload.agentDiff.diffCollection.included[0].path, "projects/01_baseline/new.c");
|
||||
assert.equal(result.payload.agentDiff.diffCollection.omitted[0].path, "projects/01_baseline/build/out.bin");
|
||||
assert.equal(result.payload.agentDiff.diffCollection.omitted[0].reason, "generated output");
|
||||
@@ -1004,6 +1027,12 @@ function caseRunFlowFetch(requests: any[]) {
|
||||
JSON.stringify({ path: "projects/01_baseline/build/out.bin", bytes: 4096, sha256: "b".repeat(64) })
|
||||
].join("\n") }]);
|
||||
}
|
||||
if (body.ops?.[0]?.args?.argv?.includes("submodule")) {
|
||||
return hwpodResponse([{ exitCode: 0, stdout: "+0123456789abcdef vendor/lib (heads/main)\n" }]);
|
||||
}
|
||||
if (body.ops?.[0]?.args?.argv?.includes("submodule")) {
|
||||
return hwpodResponse([{ exitCode: 0, stdout: "+0123456789abcdef vendor/lib (heads/main)\n" }]);
|
||||
}
|
||||
if (body.ops?.[0]?.args?.argv?.includes("ls-files")) {
|
||||
return hwpodResponse([{ exitCode: 0, stdout: "projects/01_baseline/new.c\nprojects/01_baseline/build/out.bin\n" }]);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user