pikasTech-HWLAB/tools/hwlab-cli/caserun.test.ts

import assert from "node:assert/strict";
import { mkdir, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { test } from "bun:test";

import { runHwlabCli } from "../src/hwlab-cli-lib.ts";
import { artifactsFromKeilStatusForTest, extractKeilJobId, jobStatusCommandForTest } from "../src/hwlab-caserun-lib.ts";

const SUBJECT_COMMIT_ID = "df7a4e6e551fa90d64bde5537cc000f89d63dd20";
const SUBJECT_REPO_LOCAL_PATH = "F:\\Work\\HWLAB-CASE-F103";

test("hwlab-cli case prepare copies a case hwpod-spec into isolated run state", async () => {
  const root = await mkdtempCaseRoot();
  const caseRepo = path.join(root, "hwlab-case-registry");
  const cwd = path.join(root, "hwlab");
  const caseDir = path.join(caseRepo, "cases", "d601-f103-v2-compile");
  try {
    await mkdir(path.join(caseRepo, ".git"), { recursive: true });
    await writeFile(path.join(caseRepo, ".git", "HEAD"), "ref: refs/heads/main\n", "utf8");
    await mkdir(caseDir, { recursive: true });
    await mkdir(cwd, { recursive: true });
    await writeFile(path.join(caseDir, "case.json"), JSON.stringify({ contractVersion: "hwpod-case-v1", caseId: "d601-f103-v2-compile", hwpodSpec: "hwpod-spec.yaml", subject: { repoLocalPath: SUBJECT_REPO_LOCAL_PATH, commitId: SUBJECT_COMMIT_ID, subdir: "projects/01_baseline" } }, null, 2), "utf8");
    await writeFile(path.join(caseDir, "hwpod-spec.yaml"), hwpodSpecText(), "utf8");

    const requests: any[] = [];
    const result = await runHwlabCli(["case", "prepare", "d601-f103-v2-compile", "--case-repo", caseRepo, "--run-id", "run-test"], { cwd, now: () => "2026-06-05T00:00:00.000Z", fetchImpl: subjectWorktreeFetch(requests) });

    assert.equal(result.exitCode, 0);
    assert.equal(result.payload.action, "case.prepare.prepare");
    assert.equal(result.payload.runId, "run-test");
    assert.equal(result.payload.caseRepo, caseRepo);
    assert.equal(result.payload.compileOnly, true);
    assert.equal(result.payload.subject.repoLocalPath, SUBJECT_REPO_LOCAL_PATH);
    assert.equal(result.payload.subject.commitId, SUBJECT_COMMIT_ID);
    assert.equal(result.payload.subject.worktreePath, "F:\\Work\\HWLAB-CASE-F103\\.worktree\\caserun-run-test");
    const runLocalSpec = await readFile(path.join(cwd, ".state", "hwlab-cli", "caserun", "run-test", ".hwlab", "hwpod-spec.yaml"), "utf8");
    assert.match(runLocalSpec, /node-d601-f103-v2/u);
    assert.match(runLocalSpec, /path: "F:\\\\Work\\\\HWLAB-CASE-F103\\\\\.worktree\\\\caserun-run-test"/u);
    assert.equal(requests.length, 1);
    assert.equal(requests[0].body.ops.length, 3);
    assert.equal(requests[0].body.ops[0].args.workspacePath, SUBJECT_REPO_LOCAL_PATH);
    assert.deepEqual(requests[0].body.ops[0].args.argv, ["cat-file", "-e", SUBJECT_COMMIT_ID]);
    assert.deepEqual(requests[0].body.ops[1].args.argv, ["worktree", "add", "--detach", "--force", ".worktree\\caserun-run-test", SUBJECT_COMMIT_ID]);
    assert.deepEqual(requests[0].body.ops[2].args.argv, ["-C", ".worktree\\caserun-run-test", "rev-parse", "HEAD"]);
  } finally {
    await rm(root, { recursive: true, force: true });
  }
});

test("case prepare rejects legacy subject repo/ref without a local repo path and commit id", async () => {
  const root = await mkdtempCaseRoot();
  const caseRepo = path.join(root, "hwlab-case-registry");
  const cwd = path.join(root, "hwlab");
  const caseDir = path.join(caseRepo, "cases", "legacy-case");
  try {
    await mkdir(path.join(caseRepo, ".git"), { recursive: true });
    await writeFile(path.join(caseRepo, ".git", "HEAD"), "ref: refs/heads/main\n", "utf8");
    await mkdir(caseDir, { recursive: true });
    await mkdir(cwd, { recursive: true });
    await writeFile(path.join(caseDir, "case.json"), JSON.stringify({ contractVersion: "hwpod-case-v1", caseId: "legacy-case", hwpodSpec: "hwpod-spec.yaml", subject: { repo: "git@github.com:pikasTech/D601-HWLAB.git", ref: "main" } }, null, 2), "utf8");
    await writeFile(path.join(caseDir, "hwpod-spec.yaml"), hwpodSpecText(), "utf8");

    const result = await runHwlabCli(["case", "prepare", "legacy-case", "--case-repo", caseRepo], { cwd, now: () => "2026-06-05T00:00:00.000Z", fetchImpl: subjectWorktreeFetch([]) });

    assert.equal(result.exitCode, 1);
    assert.equal(result.payload.error.code, "subject_repo_local_path_required");
  } finally {
    await rm(root, { recursive: true, force: true });
  }
});

test("case prepare uses the new registry by default and refuses the removed hwpod-cases path", async () => {
  const root = await mkdtempCaseRoot();
  const cwd = path.join(root, "hwlab");
  const newRepo = path.join(root, "hwlab-case-registry");
  const removedRepo = path.join(root, "hwpod-cases");
  const caseId = "d601-f103-v2-compile";
  try {
    await createCaseRepo(newRepo, caseId);
    await createCaseRepo(removedRepo, caseId);
    await mkdir(cwd, { recursive: true });

    const defaultResult = await runHwlabCli(["case", "prepare", caseId, "--run-id", "run-default"], { cwd, now: () => "2026-06-05T00:00:00.000Z", fetchImpl: subjectWorktreeFetch([]) });
    assert.equal(defaultResult.exitCode, 0);
    assert.equal(defaultResult.payload.caseRepo, newRepo);

    const removedResult = await runHwlabCli(["case", "prepare", caseId, "--case-repo", removedRepo], { cwd, now: () => "2026-06-05T00:00:00.000Z" });
    assert.equal(removedResult.exitCode, 1);
    assert.equal(removedResult.payload.error.code, "removed_case_repo_path");
    assert.equal(removedResult.payload.error.details.standardCaseRepo, "/root/hwlab-case-registry");
  } finally {
    await rm(root, { recursive: true, force: true });
  }
});

test("case run extracts Keil async job ids from hwpod-node output", () => {
  const jobId = extractKeilJobId({ body: { results: [{ stdout: JSON.stringify({ ok: true, job_id: "20260605_203835_798515c0" }) }] } });
  assert.equal(jobId, "20260605_203835_798515c0");

  const nestedJobId = extractKeilJobId({ body: { results: [{ output: { stdout: JSON.stringify({ accepted: true, job_id: "20260605_210457_bdf3292a" }) } }] } });
  assert.equal(nestedJobId, "20260605_210457_bdf3292a");
});

test("case run uses cmd.run command plus argv for Keil job-status", () => {
  const command = jobStatusCommandForTest("py -3", "C:\\Users\\liang\\.agents\\skills\\keil\\keil-cli.py", "job-1");
  assert.equal(command.command, "cmd.exe");
  assert.deepEqual(command.argv.slice(0, 3), ["/d", "/s", "/c"]);
  assert.match(command.argv[3], /^py -3 C:\\Users\\liang\\\.agents\\skills\\keil\\keil-cli\.py job-status job-1$/u);
});

test("case run exposes Keil hex and axf artifacts in evidence summary", () => {
  assert.deepEqual(artifactsFromKeilStatusForTest({ result: { hex_file: "F:\\out.hex", axf_file: "F:\\out.axf" } }), ["F:\\out.hex", "F:\\out.axf"]);
});

test("case run start returns immediately and status/result/logs are short polling commands", async () => {
  const root = await mkdtempCaseRoot();
  const caseRepo = path.join(root, "hwlab-case-registry");
  const cwd = path.join(root, "hwlab");
  const caseId = "d601-f103-v2-compile";
  try {
    await createCaseRepo(caseRepo, caseId);
    await mkdir(cwd, { recursive: true });
    const started = await runHwlabCli([
      "case",
      "run",
      "start",
      caseId,
      "--case-repo",
      caseRepo,
      "--run-id",
      "run-async"
    ], {
      cwd,
      now: () => "2026-06-05T00:00:00.000Z",
      startProcess: async (input) => {
        await writeFile(input.stdoutPath, "worker accepted\n", "utf8");
        await writeFile(input.stderrPath, "", "utf8");
        return { pid: 4242 };
      }
    });

    assert.equal(started.exitCode, 0);
    assert.equal(started.payload.action, "case.run.start");
    assert.equal(started.payload.status, "submitted");
    assert.equal(started.payload.startReturned, true);
    assert.equal(started.payload.pid, 4242);
    assert.match(started.payload.statusCommand, /case run status run-async/u);
    assert.match(started.payload.resultCommand, /case run result run-async/u);
    assert.match(started.payload.logsCommand, /case run logs run-async/u);

    const status = await runHwlabCli(["case", "run", "status", "run-async"], { cwd, now: () => "2026-06-05T00:00:01.000Z" });
    assert.equal(status.exitCode, 0);
    assert.equal(status.payload.action, "case.run.status");
    assert.equal(status.payload.status, "running");
    assert.equal(status.payload.stage, "started");
    assert.equal(status.payload.pid, 4242);
    assert.equal(status.payload.stdoutBytes, Buffer.byteLength("worker accepted\n"));
    assert.equal(status.payload.stderrBytes, 0);
    assert.match(status.payload.nextPollCommand, /case run status run-async/u);

    const result = await runHwlabCli(["case", "run", "result", "run-async"], { cwd, now: () => "2026-06-05T00:00:02.000Z" });
    assert.equal(result.exitCode, 0);
    assert.equal(result.payload.action, "case.run.result");
    assert.equal(result.payload.ready, false);
    assert.equal(result.payload.status, "running");
    assert.match(result.payload.nextPollCommand, /case run status run-async/u);

    const logs = await runHwlabCli(["case", "run", "logs", "run-async", "--tail", "200"], { cwd, now: () => "2026-06-05T00:00:03.000Z" });
    assert.equal(logs.exitCode, 0);
    assert.equal(logs.payload.action, "case.run.logs");
    assert.equal(logs.payload.stdout, "worker accepted\n");
  } finally {
    await rm(root, { recursive: true, force: true });
  }
});

test("case run orchestrates agent prompt, diff capture, and compile evidence without auto evaluation", async () => {
  const root = await mkdtempCaseRoot();
  const caseRepo = path.join(root, "hwlab-case-registry");
  const cwd = path.join(root, "hwlab");
  const caseDir = path.join(caseRepo, "cases", "d601-f103-v2-compile");
  try {
    await mkdir(path.join(caseRepo, ".git"), { recursive: true });
    await writeFile(path.join(caseRepo, ".git", "HEAD"), "ref: refs/heads/main\n", "utf8");
    await mkdir(caseDir, { recursive: true });
    await mkdir(cwd, { recursive: true });
    await writeFile(path.join(caseDir, "case.json"), JSON.stringify({
      contractVersion: "hwpod-case-v1",
      caseId: "d601-f103-v2-compile",
      title: "D601-F103-V2 Keil compile-only CaseRun",
      hwpodSpec: "hwpod-spec.yaml",
      subject: { repoLocalPath: SUBJECT_REPO_LOCAL_PATH, commitId: SUBJECT_COMMIT_ID, subdir: "projects/01_baseline" },
      runtime: { apiUrl: "http://api.test", webUrl: "http://web.test" }
    }, null, 2), "utf8");
    await writeFile(path.join(caseDir, "hwpod-spec.yaml"), hwpodSpecText(), "utf8");

    const requests: any[] = [];
    let sawAgentRunningState = false;
    const result = await runHwlabCli([
      "case",
      "run",
      "d601-f103-v2-compile",
      "--case-repo",
      caseRepo,
      "--run-id",
      "run-agent-flow",
      "--poll-interval-ms",
      "10",
      "--agent-poll-interval-ms",
      "10"
    ], {
      cwd,
      now: () => "2026-06-05T00:00:00.000Z",
      sleep: async () => {
        const runState = JSON.parse(await readFile(path.join(cwd, ".state", "hwlab-cli", "caserun", "run-agent-flow", "run.json"), "utf8"));
        assert.equal(runState.stage, "agent-running");
        assert.equal(runState.agent.stageStatus, "running");
        assert.equal(runState.agent.sessionId, "ses_case_run");
        assert.equal(runState.agent.conversationId, "cnv_case_d601-f103-v2-compile_run-agent-flow");
        assert.equal(runState.agent.threadId, "thread-case");
        assert.match(runState.agent.traceId, /^trc_case_/u);
        assert.match(runState.agent.nextPollCommand, /client agent result trc_case_/u);
        sawAgentRunningState = true;
      },
      env: { HWLAB_API_KEY: "hwl_live_test_case_agent_key" },
      runProcess: async () => ({ command: [], exitCode: 0, stdout: JSON.stringify({ body: { results: [{ output: { stdout: JSON.stringify({ job_id: "job-compile-1" }) } }] } }), stderr: "" }),
      fetchImpl: caseRunFlowFetch(requests)
    });

    assert.equal(result.exitCode, 0);
    assert.equal(result.payload.action, "case.run");
    assert.equal(result.payload.status, "completed");
    assert.equal(result.payload.evidence.status, "recorded");
    assert.equal(result.payload.evidence.autoEvaluation, false);
    assert.equal(result.payload.evidence.ok, undefined);
    assert.equal(result.payload.agent.stageStatus, "completed");
    assert.equal(result.payload.agent.traceId.startsWith("trc_case_"), true);
    assert.equal(result.payload.agentTrace.hwpodBuildCommandCount, 1);
    assert.equal(result.payload.evidence.agentTrace.commandCount, 3);
    assert.equal(result.payload.evidence.agentTrace.hwpodCommandCount, 3);
    assert.equal(result.payload.evidence.agentTrace.keilJobCandidates[0], "20260605_203835_798515c0");
    assert.equal(result.payload.evidence.hwpod.source, "case-run-runner-post-agent-compile-check");
    assert.match(result.payload.agentDiff.statusShort, /projects\/01_baseline\/main\.c/u);
    assert.equal(result.payload.build.autoEvaluation, false);
    assert.equal(result.payload.build.agentTraceHwpodCommandCount, 3);
    assert.equal(result.payload.build.hwpodExitCode, 0);
    assert.equal(result.payload.build.hwpodSource, "case-run-runner-post-agent-compile-check");
    assert.equal(result.payload.build.keilStatus, "completed");
    assert.equal(result.payload.collect.autoEvaluation, false);
    assert.equal(sawAgentRunningState, true);

    const prompt = await readFile(path.join(cwd, ".state", "hwlab-cli", "caserun", "run-agent-flow", "agent-prompt.md"), "utf8");
    assert.match(prompt, /without auto-grading/u);
    assert.match(prompt, /Run-local HWPOD spec/u);
    assert.match(prompt, /hwpod-ctl spec validate --spec \.hwlab\/hwpod-spec\.yaml/u);
    assert.match(prompt, /hwpod build --spec \.hwlab\/hwpod-spec\.yaml/u);
    assert.match(prompt, /path: "F:\\\\Work\\\\HWLAB-CASE-F103\\\\\.worktree\\\\caserun-run-agent-flow"/u);
    const diff = await readFile(path.join(cwd, ".state", "hwlab-cli", "caserun", "run-agent-flow", "agent-diff.patch"), "utf8");
    assert.match(diff, /printf\("hello"\)/u);
    assert.equal(requests.some((item) => item.url === "http://web.test/v1/agent/sessions"), true);
    assert.equal(requests.some((item) => item.url === "http://web.test/v1/agent/chat"), true);
    assert.equal(requests.some((item) => item.url === "http://web.test/v1/agent/chat/result/" + result.payload.agent.traceId), true);
    assert.equal(requests.some((item) => item.url === "http://web.test/v1/agent/chat/trace/" + result.payload.agent.traceId), true);
  } finally {
    await rm(root, { recursive: true, force: true });
  }
});

async function mkdtempCaseRoot() {
  return await import("node:fs/promises").then(({ mkdtemp }) => mkdtemp(path.join(os.tmpdir(), "hwlab-caserun-")));
}

async function createCaseRepo(caseRepo: string, caseId: string) {
  const caseDir = path.join(caseRepo, "cases", caseId);
  await mkdir(path.join(caseRepo, ".git"), { recursive: true });
  await writeFile(path.join(caseRepo, ".git", "HEAD"), "ref: refs/heads/main\n", "utf8");
  await mkdir(caseDir, { recursive: true });
  await writeFile(path.join(caseDir, "case.json"), JSON.stringify({ contractVersion: "hwpod-case-v1", caseId, hwpodSpec: "hwpod-spec.yaml", subject: { repoLocalPath: SUBJECT_REPO_LOCAL_PATH, commitId: SUBJECT_COMMIT_ID, subdir: "projects/01_baseline" } }, null, 2), "utf8");
  await writeFile(path.join(caseDir, "hwpod-spec.yaml"), hwpodSpecText(), "utf8");
}

function hwpodSpecText() {
  return "kind: Hwpod\nmetadata:\n  name: d601-f103-v2\nspec:\n  nodeBinding:\n    nodeId: node-d601-f103-v2\n  workspace:\n    path: F:\\\\Work\\\\HWLAB-CASE-F103\n  targetDevice:\n    board: D601-F103-V2\n  debugProbe:\n    type: daplink\n  ioProbe:\n    uart:\n      port: COM9\n";
}

function subjectWorktreeFetch(requests: any[]) {
  return async (url: RequestInfo | URL, init?: RequestInit) => {
    const body = JSON.parse(String(init?.body ?? "{}"));
    requests.push({ url: String(url), body });
    return new Response(JSON.stringify({ body: { results: [
      { ok: true, output: { exitCode: 0, stdout: "" } },
      { ok: true, output: { exitCode: 0, stdout: "Preparing worktree\n" } },
      { ok: true, output: { exitCode: 0, stdout: `${SUBJECT_COMMIT_ID}\n` } }
    ] } }), { status: 200, headers: { "content-type": "application/json" } });
  };
}

function caseRunFlowFetch(requests: any[]) {
  let resultPolls = 0;
  return async (url: RequestInfo | URL, init?: RequestInit) => {
    const rawBody = String(init?.body ?? "{}");
    const body = rawBody ? JSON.parse(rawBody) : {};
    requests.push({ url: String(url), body });
    if (String(url) === "http://web.test/v1/agent/sessions") {
      return new Response(JSON.stringify({ ok: true, session: { sessionId: "ses_case_run", conversationId: body.conversationId, threadId: "thread-case", providerProfile: body.providerProfile } }), { status: 200, headers: { "content-type": "application/json" } });
    }
    if (String(url) === "http://web.test/v1/agent/chat") {
      return new Response(JSON.stringify({ ok: true, accepted: true, traceId: body.traceId, resultUrl: `/v1/agent/chat/result/${body.traceId}` }), { status: 202, headers: { "content-type": "application/json" } });
    }
    if (String(url).startsWith("http://web.test/v1/agent/chat/result/")) {
      resultPolls += 1;
      return new Response(JSON.stringify({ ok: true, status: resultPolls > 1 ? "completed" : "running", reply: { content: "done" } }), { status: 200, headers: { "content-type": "application/json" } });
    }
    if (String(url).startsWith("http://web.test/v1/agent/chat/trace/")) {
      return new Response(JSON.stringify({ ok: true, status: "completed", sourceEventCount: 6, renderedRowCount: 3, traceSummary: { terminalStatus: "completed", agentRun: { runId: "run_agent_trace", commandId: "cmd_agent_trace" } }, rows: [
        { seq: 1, header: "ok commandExecution", body: "command: hwpod-ctl spec validate --spec .hwlab/hwpod-spec.yaml\nexitCode: 0" },
        { seq: 2, header: "ok commandExecution", body: "command: hwpod inspect --spec .hwlab/hwpod-spec.yaml\nexitCode: 0" },
        { seq: 3, header: "ok commandExecution", body: "command: hwpod build --spec .hwlab/hwpod-spec.yaml\nstdout: {\"job_id\":\"20260605_203835_798515c0\"}\nexitCode: 0" }
      ] }), { status: 200, headers: { "content-type": "application/json" } });
    }
    if (body.ops?.[0]?.args?.argv?.includes("status") && body.ops?.length === 1) {
      return hwpodResponse([{ exitCode: 0, stdout: body.ops[0].args.argv.includes("--short") ? " M projects/01_baseline/main.c\n" : "" }]);
    }
    if (body.ops?.[0]?.args?.argv?.includes("--stat")) {
      return hwpodResponse([{ exitCode: 0, stdout: " projects/01_baseline/main.c | 1 +\n" }]);
    }
    if (body.ops?.[0]?.args?.argv?.includes("--binary")) {
      return hwpodResponse([{ exitCode: 0, stdout: "+printf(\"hello\");\n" }]);
    }
    if (body.ops?.[0]?.args?.argv?.includes("job-status")) {
      return hwpodResponse([{ exitCode: 0, stdout: JSON.stringify({ status: "completed", return_code: 0, result: { hex_file: "F:\\out.hex", axf_file: "F:\\out.axf" } }) }]);
    }
    return subjectWorktreeFetch(requests)(url, init);
  };
}

function hwpodResponse(outputs: Array<{ exitCode: number; stdout?: string; stderr?: string }>) {
  return new Response(JSON.stringify({ body: { results: outputs.map((output) => ({ ok: output.exitCode === 0, output })) } }), { status: 200, headers: { "content-type": "application/json" } });
}