pikasTech-HWLAB/tools/hwlab-cli/caserun-runtime-observation.test.ts

import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { test } from "bun:test";

import { compileHwpodNodeOpsPlan } from "../src/hwpod-harness-lib.ts";
import { executeHwpodNodeOpsPlan } from "../src/hwpod-node-lib.ts";
import { caseRunArtifactEntries, caseRunArtifactManifest, renderValidationObservationSummary } from "../src/hwlab-caserun-closeout.ts";
import { caseValidationPlanFromDefinition, normalizeValidationStepResult, validationReplayRelationship } from "../src/hwlab-caserun-validation.ts";

const DOCUMENT = {
  metadata: { name: "constart-71freq-c" },
  spec: {
    nodeBinding: { nodeId: "node-nc01-71freq" },
    workspace: { path: "C:\\work\\caserun" },
    targetDevice: { board: "71-FREQ" },
    debugProbe: { type: "daplink" },
    boardComm: { jsonrpcTcp: { host: "192.168.0.154", port: 8000, command: "py -3 board-comm-cli.py" } },
    ioProbe: {
      probes: [{ id: "main41.ai0.current", quantity: "current", unit: "mA", endpointRef: "main41", read: { method: "get", path: "ai/current", valuePath: "data.value" }, sample: { count: 3 } }],
      endpoints: { main41: { kind: "boardCommJsonRpcTcp", host: "192.168.0.154", port: 8000, command: "py -3 board-comm-cli.py" } }
    }
  }
};

function executePythonNodePlan(plan: unknown) {
  const pythonScript = `
import importlib.util, json, logging, sys
spec = importlib.util.spec_from_file_location("hwlab_node", "tools/hwlab-node.py")
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
plan = json.loads(sys.argv[1])
executor = module.NodeOpsExecutor({"nodeId": plan["nodeId"]}, logging.getLogger("test"))
print(json.dumps(executor.execute(plan)))
`;
  const python = spawnSync("python3", ["-c", pythonScript, JSON.stringify(plan)], { cwd: process.cwd(), encoding: "utf8" });
  assert.equal(python.status, 0, python.stderr);
  return JSON.parse(python.stdout);
}

test("CaseRun derives compile-only and runtime validation plans from case definition", () => {
  const compile = caseValidationPlanFromDefinition({});
  assert.equal(compile.mode, "compile-only");
  assert.equal(compile.compileOnly, true);
  assert.deepEqual(compile.steps.map((step) => step.kind), ["build"]);

  const ioProbe = caseValidationPlanFromDefinition({ validation: { mode: "io-probe", ioProbe: { probeId: "main41.ai0.current" } } });
  assert.equal(ioProbe.compileOnly, false);
  assert.deepEqual(ioProbe.steps.map((step) => step.kind), ["build", "download", "ioProbe"]);

  const boardComm = caseValidationPlanFromDefinition({ validation: { mode: "board-comm", boardComm: { method: "get", path: "system/status" } } });
  assert.deepEqual(boardComm.steps.map((step) => step.kind), ["build", "download", "boardComm"]);
});

test("HWPOD compiles ioProbe and board-comm validation into service cmd.run operations", () => {
  const ioProbe = compileHwpodNodeOpsPlan({ document: DOCUMENT, intent: "io.probe.read", args: { probeId: "main41.ai0.current", count: 3 } });
  assert.equal(ioProbe.ops[0].op, "cmd.run");
  assert.equal(ioProbe.ops[0].args.commandBinding.kind, "io-probe");
  assert.equal(ioProbe.ops[0].args.commandBinding.nativeHelper, "io-probe-observation");
  assert.equal(ioProbe.ops[0].args.commandBinding.probeId, "main41.ai0.current");
  assert.match(ioProbe.ops[0].args.argv[0], /hwpod-io-probe-observation\.mjs$/u);
  assert.equal(ioProbe.ops[0].args.argv.includes("-e"), false);

  const boardComm = compileHwpodNodeOpsPlan({ document: DOCUMENT, intent: "io.board-comm.jrpctcp", args: { method: "get", path: "system/status" } });
  assert.equal(boardComm.ops[0].op, "cmd.run");
  assert.equal(boardComm.ops[0].args.commandBinding.kind, "board-comm");
  assert.equal(boardComm.ops[0].args.commandBinding.path, "system/status");
});

test("HWPOD node owns ioProbe command output normalization before CaseRun projection", async () => {
  const workspacePath = await mkdtemp(path.join(os.tmpdir(), "hwlab-caserun-observation-authority-"));
  try {
    const fakeBoardComm = path.join(workspacePath, "fake-board-comm.mjs");
    await writeFile(fakeBoardComm, "console.log(JSON.stringify({ data: { value: 12 }, artifactRef: 'artifacts/io/raw.json' }));\n", "utf8");
    const document = structuredClone(DOCUMENT);
    document.spec.workspace.path = workspacePath;
    document.spec.ioProbe.endpoints.main41.command = `${process.execPath} ${fakeBoardComm}`;
    const plan = compileHwpodNodeOpsPlan({ document, intent: "io.probe.read", args: { probeId: "main41.ai0.current", count: 3 } });
    const nodeResult = await executeHwpodNodeOpsPlan(plan, { now: () => "2026-07-16T00:00:00.000Z" });
    const pythonResult = executePythonNodePlan(plan);

    assert.equal(nodeResult.ok, true);
    assert.equal(nodeResult.results[0].output.observation.probeId, "main41.ai0.current");
    assert.deepEqual(nodeResult.results[0].output.observation.samples, [12, 12, 12]);
    assert.equal(pythonResult.ok, true);
    assert.equal(pythonResult.results[0].output.observation.probeId, nodeResult.results[0].output.observation.probeId);
    assert.deepEqual(pythonResult.results[0].output.observation.samples, nodeResult.results[0].output.observation.samples);
    assert.deepEqual(pythonResult.results[0].output.observation.stats, nodeResult.results[0].output.observation.stats);
    assert.equal(pythonResult.results[0].output.observation.rawArtifactRef, nodeResult.results[0].output.observation.rawArtifactRef);

    const validationPlan = caseValidationPlanFromDefinition({ validation: { mode: "io-probe", ioProbe: { probeId: "main41.ai0.current", quantity: "current", unit: "mA" } } });
    const normalized = normalizeValidationStepResult({ step: validationPlan.steps[2], document, exitCode: 0, payload: { body: nodeResult } });
    assert.equal(normalized.blocker, undefined);
    assert.equal(normalized.observation.authority, "hwpod-service");
    assert.equal(normalized.observation.mean, 12);
  } finally {
    await rm(workspacePath, { recursive: true, force: true });
  }
});

test("HWPOD TS and Python nodes project identical UART observations before CaseRun normalization", async () => {
  const workspacePath = await mkdtemp(path.join(os.tmpdir(), "hwlab-caserun-uart-observation-"));
  try {
    const serialMonitor = path.join(workspacePath, "serial-monitor-fixture.mjs");
    await writeFile(serialMonitor, `
const args = process.argv.slice(2).join(" ");
if (args === "monitor status") console.log(JSON.stringify({ success: true, data: { isMonitoring: true, port: "COM9", baudRate: 115200 } }));
else if (args.startsWith("fetch")) console.log(JSON.stringify({ success: true, data: [{ timestamp: "2026-07-16T00:00:00Z", port: "COM9", data: "UART READY" }], count: 1, totalCount: 1, hasMore: false, truncated: false, sourceFile: "artifacts/uart/session.jsonl" }));
else { console.error("unsupported " + args); process.exit(2); }
`, "utf8");
    const document = structuredClone(DOCUMENT);
    document.spec.workspace.path = workspacePath;
    document.spec.ioProbe.uart = { id: "uart1", port: "COM9", baudrate: 115200 };
    document.spec.tooling = { serialMonitorDir: workspacePath, serialMonitorCommand: `${process.execPath} ${serialMonitor}` };
    const plan = compileHwpodNodeOpsPlan({ document, intent: "io.uart.read", args: { port: "uart1", maxBytes: 128 } });
    const nodeResult = await executeHwpodNodeOpsPlan(plan, { now: () => "2026-07-16T00:00:00.000Z" });
    const pythonResult = executePythonNodePlan(plan);

    assert.equal(nodeResult.ok, true);
    assert.equal(pythonResult.ok, true);
    assert.deepEqual(pythonResult.results[0].output.observation, nodeResult.results[0].output.observation);
    assert.equal(nodeResult.results[0].output.observation.text, "UART READY");
    assert.equal(nodeResult.results[0].output.observation.rawArtifactRef, "artifacts/uart/session.jsonl");

    const validationPlan = caseValidationPlanFromDefinition({ validation: { mode: "download-uart", uart: { port: "uart1" } } });
    const normalized = normalizeValidationStepResult({ step: validationPlan.steps[2], document, exitCode: 0, payload: { body: nodeResult } });
    assert.equal(normalized.blocker, undefined);
    assert.equal(normalized.observation.evidenceLevel, "serial-log");
    assert.equal(normalized.observation.rawArtifactRef, "artifacts/uart/session.jsonl");

    const stdoutOnly = structuredClone(nodeResult);
    stdoutOnly.results[0].output = { workspacePath, stdout: JSON.stringify(nodeResult.results[0].output.observation) };
    const rejected = normalizeValidationStepResult({ step: validationPlan.steps[2], document, exitCode: 0, payload: { body: stdoutOnly } });
    assert.equal(rejected.observation, undefined);
    assert.equal(rejected.blocker.code, "hwpod_validation_observation_missing");
  } finally {
    await rm(workspacePath, { recursive: true, force: true });
  }
});

test("CaseRun normalizes authoritative ioProbe statistics and replay relationships", () => {
  const plan = caseValidationPlanFromDefinition({ validation: { mode: "io-probe", ioProbe: { probeId: "main41.ai0.current", quantity: "current", unit: "mA" } } });
  const step = plan.steps[2];
  const normalized = normalizeValidationStepResult({
    step,
    document: DOCUMENT,
    exitCode: 0,
    payload: { body: { hwpodId: "constart-71freq-c", nodeId: "node-nc01-71freq", results: [{ opId: "op_probe", op: "cmd.run", ok: true, status: "completed", workspacePath: "C:\\work\\caserun", output: { observation: { observationId: "obs-1", probeId: "main41.ai0.current", quantity: "current", unit: "mA", samples: [11, 12, 13], stats: { count: 3, mean: 12, min: 11, max: 13 }, rawArtifactRef: "artifacts/io/main41-ai0-current.json" } } }] } }
  });
  assert.equal(normalized.blocker, undefined);
  assert.equal(normalized.observation.sampleCount, 3);
  assert.equal(normalized.observation.mean, 12);
  assert.equal(normalized.observation.min, 11);
  assert.equal(normalized.observation.max, 13);
  assert.equal(normalized.observation.evidenceLevel, "external-physical-reading");
  const replay = validationReplayRelationship(plan, [normalized]);
  assert.deepEqual(replay.observationIds, ["obs-1"]);
  assert.equal(replay.relationships[0].actionId, step.actionId);
  assert.equal(replay.relationships[0].operationId, "op_probe");
});

test("CaseRun records board response and returns typed blockers for missing observations", () => {
  const plan = caseValidationPlanFromDefinition({ validation: { mode: "board-comm", boardComm: { method: "get", path: "system/status" } } });
  const step = plan.steps[2];
  const observed = normalizeValidationStepResult({
    step,
    document: DOCUMENT,
    exitCode: 0,
    payload: { body: { hwpodId: "constart-71freq-c", nodeId: "node-nc01-71freq", results: [{ opId: "op_board", op: "cmd.run", ok: true, status: "completed", workspacePath: "C:\\work\\caserun", output: { observation: { observationId: "obs-board", response: { ok: true, firmware: "1.2.3" }, rawArtifactRef: "artifacts/board/system-status.json" } } }] } }
  });
  assert.equal(observed.blocker, undefined);
  assert.equal(observed.observation.response.firmware, "1.2.3");
  assert.equal(observed.observation.evidenceLevel, "board-internal-response");

  const stdoutOnly = normalizeValidationStepResult({ step, document: DOCUMENT, exitCode: 0, payload: { body: { hwpodId: "constart-71freq-c", nodeId: "node-nc01-71freq", results: [{ opId: "op_board", op: "cmd.run", ok: true, status: "completed", workspacePath: "C:\\work\\caserun", output: { stdout: JSON.stringify({ observationId: "not-authoritative", response: { ok: true } }) } }] } } });
  assert.equal(stdoutOnly.observation, undefined);
  assert.equal(stdoutOnly.blocker.code, "hwpod_validation_observation_missing");
});

test("manifest and aggregate project archived observation facts without grading Agent final", () => {
  const run: any = { caseId: "case-1", runId: "run-1", runDir: "/tmp/run-1", caseRepo: "/tmp/registry", specPath: "/tmp/run-1/.hwlab/hwpod-spec.yaml", compileOnly: false };
  assert.equal(caseRunArtifactEntries(run).some((entry) => entry.rel === "validation-observations.json"), true);
  const evidence = { validation: { plan: { mode: "io-probe", compileOnly: false }, results: [{ observation: { actionId: "validation_03_ioProbe", operationId: "op_probe", kind: "ioProbe", probeId: "main41.ai0.current", quantity: "current", unit: "mA", sampleCount: 3, mean: 12, min: 11, max: 13, evidenceLevel: "external-physical-reading", rawArtifactRef: "artifacts/io.json" } }] }, replay: { observationIds: ["obs-1"] }, agentFinal: { present: true, text: "agent answer" } };
  const manifest: any = caseRunArtifactManifest({ now: () => "2026-07-16T00:00:00.000Z" } as any, run, evidence, [], [], {} as any);
  assert.equal(manifest.validation.plan.mode, "io-probe");
  assert.deepEqual(manifest.replay.observationIds, ["obs-1"]);
  assert.equal(manifest.agentFinal.text, "agent answer");
  const markdown = renderValidationObservationSummary(evidence.validation);
  assert.match(markdown, /external-physical-reading/u);
  assert.match(markdown, /main41\.ai0\.current/u);
});