pikasTech-HWLAB/tools/hwlab-cli/caserun-offline-diagnostics.test.ts

import assert from "node:assert/strict";
import { createHash } from "node:crypto";
import { mkdir, mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { test } from "bun:test";

import { runHwlabCli } from "../src/hwlab-cli-lib.ts";

test("case aggregate remains registry-only even when runtime credentials exist", async () => {
  const fixture = await createRegistryFixture("aggregate-offline");
  try {
    let runtimeCalls = 0;
    const result = await runHwlabCli(["case", "aggregate", fixture.caseId, "--run-id", fixture.runId, "--case-repo", fixture.caseRepo, "--no-case-repo-git-sync"], {
      cwd: fixture.root,
      env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" },
      runProcess: async () => {
        runtimeCalls += 1;
        throw new Error("offline aggregate must not execute runtime commands");
      },
    });

    assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2));
    assert.equal(runtimeCalls, 0);
    assert.equal(result.payload.action, "case.aggregate");
    assert.match(await readFile(path.join(fixture.runDir, "aggregate.md"), "utf8"), /registry artifacts/u);
  } finally {
    await rm(fixture.root, { recursive: true, force: true });
  }
});

test("case refresh is explicit, registry-backed, and semantically idempotent", async () => {
  const fixture = await createRegistryFixture("refresh-local");
  try {
    const args = ["case", "refresh", fixture.caseId, "--run-id", fixture.runId, "--case-repo", fixture.caseRepo, "--source", "local", "--no-case-repo-git-sync"];
    const first = await runHwlabCli(args, { cwd: fixture.root, now: () => "2026-07-16T14:00:00.000Z", env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" }, runProcess: forbiddenRuntime });
    assert.equal(first.exitCode, 0, JSON.stringify(first.payload, null, 2));
    assert.deepEqual(first.payload.refreshed.provenance, { source: "local", authority: "registry", runtimeAccess: false });
    assert.equal(first.payload.refreshed.commandCount, 2);
    assert.equal(first.payload.refreshed.hwpodCommandCount, 1);
    assert.equal(first.payload.refreshed.hwpodBuildCommandCount, 1);
    const before = await fileHashes(fixture.runDir, ["evidence.json", "agent-messages.json", "artifact-manifest.json", "aggregate.md"]);

    const second = await runHwlabCli(args, { cwd: fixture.root, now: () => "2026-07-16T15:00:00.000Z", env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" }, runProcess: forbiddenRuntime });
    assert.equal(second.exitCode, 0, JSON.stringify(second.payload, null, 2));
    assert.deepEqual(await fileHashes(fixture.runDir, Object.keys(before)), before);
    const manifest = JSON.parse(await readFile(path.join(fixture.runDir, "artifact-manifest.json"), "utf8"));
    assert.deepEqual(manifest.refresh, { source: "local", authority: "registry", runtimeAccess: false });
    assert.equal(manifest.trace.toolCallSummary.count, 2);
  } finally {
    await rm(fixture.root, { recursive: true, force: true });
  }
});

test("case run offline diagnostics inspect trace and skill usage without runtime access", async () => {
  const fixture = await createRegistryFixture("inspect-offline");
  try {
    const options = { cwd: fixture.root, env: { HWLAB_API_KEY: "hwl_live_should_not_be_used" }, runProcess: forbiddenRuntime };
    const inspect = await runHwlabCli(["case", "run", "inspect", fixture.runId, "--case-repo", fixture.caseRepo], options);
    assert.equal(inspect.exitCode, 0, JSON.stringify(inspect.payload, null, 2));
    assert.equal(inspect.payload.serviceRuntime, false);
    assert.equal(inspect.payload.traceId, "trc_offline");
    assert.equal(inspect.payload.providerProfile, "deepseek");
    assert.equal(inspect.payload.terminalStatus, "completed");
    assert.equal(inspect.payload.toolCallSummary.count, 2);

    const trace = await runHwlabCli(["case", "run", "trace-summary", fixture.runId, "--case-repo", fixture.caseRepo], options);
    assert.equal(trace.exitCode, 0, JSON.stringify(trace.payload, null, 2));
    assert.equal(trace.payload.commandCount, 2);
    assert.equal(trace.payload.traceCommands.length, 2);

    const skill = await runHwlabCli(["case", "run", "skill-usage", fixture.runId, "--skill", "keil", "--case-repo", fixture.caseRepo], options);
    assert.equal(skill.exitCode, 0, JSON.stringify(skill.payload, null, 2));
    assert.equal(skill.payload.serviceRuntime, false);
    assert.equal(skill.payload.trace.readSkillFile, true);
    assert.equal(skill.payload.trace.matchCount, 1);
    assert.equal(skill.payload.resourceBundle.materialized, true);
    assert.equal(skill.payload.resourceBundle.evidence[0].source, "artifact-manifest.json");
  } finally {
    await rm(fixture.root, { recursive: true, force: true });
  }
});

test("case run skill usage keeps prompt mentions separate from materialized bundles", async () => {
  const fixture = await createRegistryFixture("prompt-only-skill");
  try {
    await writeJson(path.join(fixture.runDir, "agent-messages.json"), { rows: [{ rowId: "tool:build", seq: 1, header: "工具调用", body: "hwpod build --spec .hwlab/hwpod-spec.yaml" }] });
    const manifest = JSON.parse(await readFile(path.join(fixture.runDir, "artifact-manifest.json"), "utf8"));
    delete manifest.resourceBundle;
    await writeJson(path.join(fixture.runDir, "artifact-manifest.json"), manifest);

    const result = await runHwlabCli(["case", "run", "skill-usage", fixture.runId, "--skill", "keil", "--case-repo", fixture.caseRepo], { cwd: fixture.root, runProcess: forbiddenRuntime });
    assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2));
    assert.equal(result.payload.prompt.mentionsSkill, true);
    assert.equal(result.payload.trace.readSkillFile, false);
    assert.equal(result.payload.resourceBundle.materialized, false);
  } finally {
    await rm(fixture.root, { recursive: true, force: true });
  }
});

test("case run inspect preserves missing authoritative agent final", async () => {
  const fixture = await createRegistryFixture("missing-agent-final");
  try {
    const reason = "finalResponse=null; provider returned no final assistant response";
    const evidence = JSON.parse(await readFile(path.join(fixture.runDir, "evidence.json"), "utf8"));
    evidence.agentFinal = { present: false, reason, status: "blocked" };
    await writeJson(path.join(fixture.runDir, "evidence.json"), evidence);
    const manifest = JSON.parse(await readFile(path.join(fixture.runDir, "artifact-manifest.json"), "utf8"));
    manifest.agentFinal = { present: false, reason, status: "blocked" };
    await writeJson(path.join(fixture.runDir, "artifact-manifest.json"), manifest);
    await writeFile(path.join(fixture.runDir, "final-response.md"), `# CaseRun Final Response\n\n${reason}\n`, "utf8");

    const result = await runHwlabCli(["case", "run", "inspect", fixture.runId, "--case-repo", fixture.caseRepo], { cwd: fixture.root, runProcess: forbiddenRuntime });
    assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2));
    assert.equal(result.payload.agentFinal.present, false);
    assert.equal(result.payload.finalResponse.present, false);
    assert.equal(result.payload.finalResponse.reason, reason);
    assert.match(result.payload.finalResponse.textPreview, /provider returned no final assistant response/u);
  } finally {
    await rm(fixture.root, { recursive: true, force: true });
  }
});

test("case run skill usage scans all rows before bounding returned matches", async () => {
  const fixture = await createRegistryFixture("late-skill-read");
  try {
    const rows = [
      ...Array.from({ length: 30 }, (_, index) => ({ rowId: `tool:other:${index + 1}`, seq: index + 1, header: "工具调用", body: "echo unrelated" })),
      ...Array.from({ length: 35 }, (_, index) => ({ rowId: `tool:skill:${index + 31}`, seq: index + 31, header: "工具调用", body: "cat ~/.agents/skills/keil/SKILL.md" })),
    ];
    await writeJson(path.join(fixture.runDir, "agent-messages.json"), { sourceEventCount: rows.length, renderedRowCount: rows.length, rows });

    const result = await runHwlabCli(["case", "run", "skill-usage", fixture.runId, "--skill", "keil", "--case-repo", fixture.caseRepo], { cwd: fixture.root, runProcess: forbiddenRuntime });
    assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2));
    assert.equal(result.payload.trace.readSkillFile, true);
    assert.equal(result.payload.trace.matchCount, 35);
    assert.equal(result.payload.trace.matches.length, 30);
    assert.equal(result.payload.trace.matches[0].seq, 31);
  } finally {
    await rm(fixture.root, { recursive: true, force: true });
  }
});

test("case refresh accesses runtime only with explicit runtime source", async () => {
  const fixture = await createRegistryFixture("refresh-runtime");
  try {
    let runtimeCalls = 0;
    const result = await runHwlabCli(["case", "refresh", fixture.caseId, "--run-id", fixture.runId, "--case-repo", fixture.caseRepo, "--source", "runtime", "--no-case-repo-git-sync"], {
      cwd: fixture.root,
      env: { HWLAB_API_KEY: "hwl_live_runtime_test" },
      runProcess: async () => {
        runtimeCalls += 1;
        return { command: [], exitCode: 0, stderr: "", stdout: JSON.stringify({ body: { traceId: "trc_offline", source: "hwlab-cli.client.agent.trace", status: "completed", terminalStatus: "completed", rows: [{ rowId: "tool:runtime", seq: 1, header: "工具调用", body: "hwpod build --spec .hwlab/hwpod-spec.yaml" }, { rowId: "final", seq: 2, header: "助手最终消息", terminal: true, body: "runtime refreshed" }] } }) };
      },
    });

    assert.equal(result.exitCode, 0, JSON.stringify(result.payload, null, 2));
    assert.equal(runtimeCalls, 1);
    assert.deepEqual(result.payload.refreshed.provenance, { source: "runtime", authority: "registry", runtimeAccess: true });
  } finally {
    await rm(fixture.root, { recursive: true, force: true });
  }
});

async function createRegistryFixture(suffix: string) {
  const root = await mkdtemp(path.join(os.tmpdir(), `hwlab-caserun-${suffix}-`));
  const caseRepo = path.join(root, "hwlab-case-registry");
  const caseId = `case-${suffix}`;
  const runId = `run-${suffix}`;
  const runDir = path.join(caseRepo, "runs", caseId, runId);
  await mkdir(path.join(caseRepo, ".git"), { recursive: true });
  await writeFile(path.join(caseRepo, ".git", "HEAD"), "ref: refs/heads/main\n", "utf8");
  await mkdir(path.join(runDir, ".hwlab"), { recursive: true });
  const agent = { stageStatus: "completed", baseUrl: "http://web.test", traceId: "trc_offline", sessionId: "ses_offline", conversationId: "cnv_offline", threadId: "thr_offline", providerProfile: "deepseek", requestedProviderProfile: "deepseek", resolvedBackendProfile: "deepseek-runtime", provider: "deepseek", model: "deepseek-chat", terminalStatus: "completed", commandStatus: "completed", agentRunStatus: "completed", promptSha256: "prompt-sha" };
  const toolCallSummary = { source: "agentrun-result.toolCallSummary", count: 2, statusCounts: { completed: 2 }, exitCodeCounts: { "0": 2 }, terminalStatus: "completed", commandStatus: "completed", agentRunStatus: "completed", items: [{ kind: "build", toolName: "commandExecution", status: "completed", exitCode: 0, command: "hwpod build --spec .hwlab/hwpod-spec.yaml" }, { kind: "other", toolName: "commandExecution", status: "completed", exitCode: 0, command: "cat ~/.agents/skills/keil/SKILL.md" }] };
  await writeJson(path.join(runDir, "run.json"), { caseId, runId, runDir, caseRepo, specPath: path.join(runDir, ".hwlab", "hwpod-spec.yaml"), compileOnly: true, createdAt: "2026-07-16T13:00:00.000Z", updatedAt: "2026-07-16T13:00:00.000Z", definition: {}, subject: { repoLocalPath: "C:/work/fw", commitId: "0".repeat(40), subdir: "", worktreePath: "C:/work/fw/.worktree/case", workspacePath: "C:/work/fw/.worktree/case", setup: {} }, agent });
  await writeJson(path.join(runDir, "evidence.json"), { contractVersion: "hwpod-case-run-evidence-v1", caseId, runId, compileOnly: true, autoEvaluation: false, status: "recorded", subject: { repoLocalPath: "C:/work/fw", commitId: "0".repeat(40), worktreePath: "C:/work/fw/.worktree/case" }, agent, agentTrace: { traceId: "trc_offline", terminalStatus: "completed", toolCallSummary, commandCount: 2, hwpodCommandCount: 1, hwpodBuildCommandCount: 1 }, agentFinal: { present: true, status: "completed", terminalStatus: "completed" }, postValidation: { source: "case-run-runner-post-agent-compile-check", status: "completed", returnCode: 0 }, decisions: { autoEvaluation: false, runnerPostAgentCompileCheck: "recorded" } });
  await writeJson(path.join(runDir, "agent-trace.json"), { contractVersion: "hwpod-case-run-agent-trace-identity-v1", traceId: "trc_offline", status: "completed", terminalStatus: "completed", commandStatus: "completed", toolCallSummary });
  await writeJson(path.join(runDir, "agent-messages.json"), { renderer: "tools/src/hwlab-cli/trace-renderer:traceDisplayRows", sourceEventCount: 2, renderedRowCount: 2, rows: [{ rowId: "tool:build", seq: 1, header: "工具调用", body: "hwpod build --spec .hwlab/hwpod-spec.yaml" }, { rowId: "tool:skill", seq: 2, header: "工具调用", body: "cat ~/.agents/skills/keil/SKILL.md" }] });
  await writeJson(path.join(runDir, "artifact-manifest.json"), { contractVersion: "hwpod-case-run-artifact-manifest-v1", caseId, runId, agent, trace: { traceId: "trc_offline", terminalStatus: "completed", toolCallSummary, commandCount: 2, hwpodCommandCount: 1, hwpodBuildCommandCount: 1 }, resourceBundle: { materialized: true, skills: [{ name: "keil", manifest: ".agents/skills/keil/SKILL.md" }] }, agentFinal: { present: true, status: "completed" }, postValidation: { status: "completed", returnCode: 0 }, files: [] });
  await writeFile(path.join(runDir, "agent-prompt.md"), "请读取 keil skill 后执行编译。\n", "utf8");
  await writeFile(path.join(runDir, "final-response.md"), "# CaseRun Final Response\n\ncompleted\n", "utf8");
  await writeFile(path.join(runDir, "agent-diff.patch"), "", "utf8");
  await writeFile(path.join(runDir, ".hwlab", "hwpod-spec.yaml"), "metadata:\n  name: offline\n", "utf8");
  return { root, caseRepo, caseId, runId, runDir };
}

async function writeJson(file: string, value: unknown) { await writeFile(file, `${JSON.stringify(value, null, 2)}\n`, "utf8"); }
async function fileHashes(root: string, files: string[]) { return Object.fromEntries(await Promise.all(files.map(async (file) => [file, createHash("sha256").update(await readFile(path.join(root, file))).digest("hex")]))); }
async function forbiddenRuntime() { throw new Error("offline diagnostics must not execute runtime commands"); }