fix: 修正 CaseRun 观测权威边界

This commit is contained in:
root
2026-07-16 19:14:34 +02:00
parent 140a3a6b86
commit 166f45e485
6 changed files with 163 additions and 26 deletions
@@ -1,7 +1,11 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { test } from "bun:test"; import { test } from "bun:test";
import { compileHwpodNodeOpsPlan } from "../src/hwpod-harness-lib.ts"; import { compileHwpodNodeOpsPlan } from "../src/hwpod-harness-lib.ts";
import { executeHwpodNodeOpsPlan } from "../src/hwpod-node-lib.ts";
import { caseRunArtifactEntries, caseRunArtifactManifest, renderValidationObservationSummary } from "../src/hwlab-caserun-closeout.ts"; import { caseRunArtifactEntries, caseRunArtifactManifest, renderValidationObservationSummary } from "../src/hwlab-caserun-closeout.ts";
import { caseValidationPlanFromDefinition, normalizeValidationStepResult, validationReplayRelationship } from "../src/hwlab-caserun-validation.ts"; import { caseValidationPlanFromDefinition, normalizeValidationStepResult, validationReplayRelationship } from "../src/hwlab-caserun-validation.ts";
@@ -38,7 +42,10 @@ test("HWPOD compiles ioProbe and board-comm validation into service cmd.run oper
const ioProbe = compileHwpodNodeOpsPlan({ document: DOCUMENT, intent: "io.probe.read", args: { probeId: "main41.ai0.current", count: 3 } }); const ioProbe = compileHwpodNodeOpsPlan({ document: DOCUMENT, intent: "io.probe.read", args: { probeId: "main41.ai0.current", count: 3 } });
assert.equal(ioProbe.ops[0].op, "cmd.run"); assert.equal(ioProbe.ops[0].op, "cmd.run");
assert.equal(ioProbe.ops[0].args.commandBinding.kind, "io-probe"); assert.equal(ioProbe.ops[0].args.commandBinding.kind, "io-probe");
assert.equal(ioProbe.ops[0].args.commandBinding.nativeHelper, "io-probe-observation");
assert.equal(ioProbe.ops[0].args.commandBinding.probeId, "main41.ai0.current"); assert.equal(ioProbe.ops[0].args.commandBinding.probeId, "main41.ai0.current");
assert.match(ioProbe.ops[0].args.argv[0], /hwpod-io-probe-observation\.mjs$/u);
assert.equal(ioProbe.ops[0].args.argv.includes("-e"), false);
const boardComm = compileHwpodNodeOpsPlan({ document: DOCUMENT, intent: "io.board-comm.jrpctcp", args: { method: "get", path: "system/status" } }); const boardComm = compileHwpodNodeOpsPlan({ document: DOCUMENT, intent: "io.board-comm.jrpctcp", args: { method: "get", path: "system/status" } });
assert.equal(boardComm.ops[0].op, "cmd.run"); assert.equal(boardComm.ops[0].op, "cmd.run");
@@ -46,6 +53,31 @@ test("HWPOD compiles ioProbe and board-comm validation into service cmd.run oper
assert.equal(boardComm.ops[0].args.commandBinding.path, "system/status"); assert.equal(boardComm.ops[0].args.commandBinding.path, "system/status");
}); });
test("HWPOD node owns ioProbe command output normalization before CaseRun projection", async () => {
const workspacePath = await mkdtemp(path.join(os.tmpdir(), "hwlab-caserun-observation-authority-"));
try {
const fakeBoardComm = path.join(workspacePath, "fake-board-comm.mjs");
await writeFile(fakeBoardComm, "console.log(JSON.stringify({ data: { value: 12 }, artifactRef: 'artifacts/io/raw.json' }));\n", "utf8");
const document = structuredClone(DOCUMENT);
document.spec.workspace.path = workspacePath;
document.spec.ioProbe.endpoints.main41.command = `${process.execPath} ${fakeBoardComm}`;
const plan = compileHwpodNodeOpsPlan({ document, intent: "io.probe.read", args: { probeId: "main41.ai0.current", count: 3 } });
const nodeResult = await executeHwpodNodeOpsPlan(plan, { now: () => "2026-07-16T00:00:00.000Z" });
assert.equal(nodeResult.ok, true);
assert.equal(nodeResult.results[0].output.observation.probeId, "main41.ai0.current");
assert.deepEqual(nodeResult.results[0].output.observation.samples, [12, 12, 12]);
const validationPlan = caseValidationPlanFromDefinition({ validation: { mode: "io-probe", ioProbe: { probeId: "main41.ai0.current", quantity: "current", unit: "mA" } } });
const normalized = normalizeValidationStepResult({ step: validationPlan.steps[2], document, exitCode: 0, payload: { body: nodeResult } });
assert.equal(normalized.blocker, undefined);
assert.equal(normalized.observation.authority, "hwpod-service");
assert.equal(normalized.observation.mean, 12);
} finally {
await rm(workspacePath, { recursive: true, force: true });
}
});
test("CaseRun normalizes authoritative ioProbe statistics and replay relationships", () => { test("CaseRun normalizes authoritative ioProbe statistics and replay relationships", () => {
const plan = caseValidationPlanFromDefinition({ validation: { mode: "io-probe", ioProbe: { probeId: "main41.ai0.current", quantity: "current", unit: "mA" } } }); const plan = caseValidationPlanFromDefinition({ validation: { mode: "io-probe", ioProbe: { probeId: "main41.ai0.current", quantity: "current", unit: "mA" } } });
const step = plan.steps[2]; const step = plan.steps[2];
@@ -53,7 +85,7 @@ test("CaseRun normalizes authoritative ioProbe statistics and replay relationshi
step, step,
document: DOCUMENT, document: DOCUMENT,
exitCode: 0, exitCode: 0,
payload: { body: { hwpodId: "constart-71freq-c", nodeId: "node-nc01-71freq", results: [{ opId: "op_probe", op: "cmd.run", ok: true, status: "completed", workspacePath: "C:\\work\\caserun", output: { stdout: JSON.stringify({ observationId: "obs-1", probeId: "main41.ai0.current", quantity: "current", unit: "mA", samples: [11, 12, 13], stats: { count: 3, mean: 12, min: 11, max: 13 }, rawArtifactRef: "artifacts/io/main41-ai0-current.json" }) } }] } } payload: { body: { hwpodId: "constart-71freq-c", nodeId: "node-nc01-71freq", results: [{ opId: "op_probe", op: "cmd.run", ok: true, status: "completed", workspacePath: "C:\\work\\caserun", output: { observation: { observationId: "obs-1", probeId: "main41.ai0.current", quantity: "current", unit: "mA", samples: [11, 12, 13], stats: { count: 3, mean: 12, min: 11, max: 13 }, rawArtifactRef: "artifacts/io/main41-ai0-current.json" } } }] } }
}); });
assert.equal(normalized.blocker, undefined); assert.equal(normalized.blocker, undefined);
assert.equal(normalized.observation.sampleCount, 3); assert.equal(normalized.observation.sampleCount, 3);
@@ -74,14 +106,15 @@ test("CaseRun records board response and returns typed blockers for missing obse
step, step,
document: DOCUMENT, document: DOCUMENT,
exitCode: 0, exitCode: 0,
payload: { body: { hwpodId: "constart-71freq-c", nodeId: "node-nc01-71freq", results: [{ opId: "op_board", op: "cmd.run", ok: true, status: "completed", workspacePath: "C:\\work\\caserun", output: { stdout: JSON.stringify({ observationId: "obs-board", response: { ok: true, firmware: "1.2.3" }, rawArtifactRef: "artifacts/board/system-status.json" }) } }] } } payload: { body: { hwpodId: "constart-71freq-c", nodeId: "node-nc01-71freq", results: [{ opId: "op_board", op: "cmd.run", ok: true, status: "completed", workspacePath: "C:\\work\\caserun", output: { observation: { observationId: "obs-board", response: { ok: true, firmware: "1.2.3" }, rawArtifactRef: "artifacts/board/system-status.json" } } }] } }
}); });
assert.equal(observed.blocker, undefined); assert.equal(observed.blocker, undefined);
assert.equal(observed.observation.response.firmware, "1.2.3"); assert.equal(observed.observation.response.firmware, "1.2.3");
assert.equal(observed.observation.evidenceLevel, "board-internal-response"); assert.equal(observed.observation.evidenceLevel, "board-internal-response");
const missing = normalizeValidationStepResult({ step, document: DOCUMENT, exitCode: 0, payload: { body: { hwpodId: "constart-71freq-c", nodeId: "node-nc01-71freq", results: [{ opId: "op_board", op: "cmd.run", ok: true, status: "completed", workspacePath: "C:\\work\\caserun", output: {} }] } } }); const stdoutOnly = normalizeValidationStepResult({ step, document: DOCUMENT, exitCode: 0, payload: { body: { hwpodId: "constart-71freq-c", nodeId: "node-nc01-71freq", results: [{ opId: "op_board", op: "cmd.run", ok: true, status: "completed", workspacePath: "C:\\work\\caserun", output: { stdout: JSON.stringify({ observationId: "not-authoritative", response: { ok: true } }) } }] } } });
assert.equal(missing.blocker.code, "hwpod_validation_observation_inconsistent"); assert.equal(stdoutOnly.observation, undefined);
assert.equal(stdoutOnly.blocker.code, "hwpod_validation_observation_missing");
}); });
test("manifest and aggregate project archived observation facts without grading Agent final", () => { test("manifest and aggregate project archived observation facts without grading Agent final", () => {
+77
View File
@@ -0,0 +1,77 @@
import { spawnSync } from "node:child_process";
const input = JSON.parse(process.argv[2] || "{}");
function readPath(value, expression) {
return String(expression || "")
.replace(/^\$\.?/u, "")
.split(".")
.filter(Boolean)
.reduce((current, key) => current == null ? undefined : current[key], value);
}
function responseCandidates(value) {
return [value?.response?.response, value?.response, value].filter(Boolean);
}
function observationValue(value) {
for (const candidate of responseCandidates(value)) {
const observed = readPath(candidate, input.valuePath);
if (observed !== undefined) return Number(observed);
}
return Number.NaN;
}
function sleep(milliseconds) {
if (milliseconds > 0) Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
}
function boardCommand() {
return [
...input.commandBase,
"jrpctcp",
"--host",
input.host,
"--port",
String(input.port),
...(input.transport ? ["--transport", input.transport] : []),
...(input.targetNodeId ? ["--target-node-id", String(input.targetNodeId)] : []),
input.method,
input.path,
...input.params
];
}
sleep(input.settleMs);
const samples = [];
let response = null;
let exitCode = 0;
for (let index = 0; index < input.count; index += 1) {
const command = boardCommand();
const result = spawnSync(command[0], command.slice(1), { encoding: "utf8" });
try {
response = JSON.parse(String(result.stdout || "").trim());
} catch {
response = null;
}
const value = observationValue(response);
if (Number.isFinite(value)) samples.push(value);
if (result.status !== 0) exitCode = result.status || 1;
if (index < input.count - 1) sleep(input.intervalMs);
}
const mean = samples.length > 0 ? samples.reduce((sum, value) => sum + value, 0) / samples.length : undefined;
const observation = {
observationId: `obs_${input.probeId}_${Date.now()}`,
probeId: input.probeId,
quantity: input.quantity,
unit: input.unit,
samples,
stats: samples.length > 0 ? { count: samples.length, mean, min: Math.min(...samples), max: Math.max(...samples) } : {},
response,
rawArtifactRef: response?.log_path || response?.logPath || response?.artifactRef || null
};
console.log(JSON.stringify(observation));
process.exit(exitCode === 0 && samples.length > 0 ? 0 : exitCode || 1);
+28
View File
@@ -153,6 +153,34 @@ test("hwpod-node reports cmd.run non-zero exits as failed results", async () =>
} }
}); });
test("hwpod-node projects bound command JSON into authoritative observation", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-hwpod-node-observation-"));
try {
const result = await executeHwpodNodeOpsPlan({
contractVersion: "hwpod-node-ops-v1",
planId: "hwpod_plan_observation",
hwpodId: "hwpod-local",
nodeId: "pc-host-1",
ops: [{
opId: "op_observation",
op: "cmd.run",
args: {
workspacePath: root,
command: process.execPath,
argv: ["-e", "console.log(JSON.stringify({ observationId: 'obs-node', response: { ok: true } }))"],
commandBinding: { kind: "board-comm" }
}
}]
}, { now: () => "2026-07-16T00:00:00.000Z" });
assert.equal(result.ok, true);
assert.equal(result.results[0].output.observation.observationId, "obs-node");
assert.equal(result.results[0].output.observation.response.ok, true);
} finally {
await rm(root, { recursive: true, force: true });
}
});
test("hwpod-node refuses plans targeting another node id", async () => { test("hwpod-node refuses plans targeting another node id", async () => {
const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-hwpod-node-id-mismatch-")); const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-hwpod-node-id-mismatch-"));
try { try {
+2 -1
View File
@@ -58,7 +58,7 @@ export function normalizeValidationStepResult(input: { step: CaseValidationStep;
const results = Array.isArray(response.results) ? response.results : []; const results = Array.isArray(response.results) ? response.results : [];
const result = results.find((item: any) => text(item?.opId) === input.step.actionId || text(item?.op) === input.step.expectedOp) ?? results[0] ?? {}; const result = results.find((item: any) => text(item?.opId) === input.step.actionId || text(item?.op) === input.step.expectedOp) ?? results[0] ?? {};
const output = record(result?.output); const output = record(result?.output);
const outputJson = firstObject(result?.observation, output.observation, parseJsonMaybe(output.stdout), parseJsonMaybe(result?.stdout)); const outputJson = firstObject(result?.observation, output.observation);
const expected = { const expected = {
hwpodId: text(input.document?.metadata?.name ?? input.document?.metadata?.uid), hwpodId: text(input.document?.metadata?.name ?? input.document?.metadata?.uid),
nodeId: text(input.document?.spec?.nodeBinding?.nodeId), nodeId: text(input.document?.spec?.nodeBinding?.nodeId),
@@ -125,6 +125,7 @@ export function validationReplayRelationship(plan: CaseValidationPlan, results:
function normalizeObservation(step: CaseValidationStep, result: any, body: any, operation: any) { function normalizeObservation(step: CaseValidationStep, result: any, body: any, operation: any) {
if (step.kind !== "ioProbe" && step.kind !== "boardComm" && step.kind !== "uart") return undefined; if (step.kind !== "ioProbe" && step.kind !== "boardComm" && step.kind !== "uart") return undefined;
if (Object.keys(record(body)).length === 0) return undefined;
const samples = numberArray(body.samples ?? body.values ?? body.rawSamples?.map?.((item: any) => item?.value)); const samples = numberArray(body.samples ?? body.values ?? body.rawSamples?.map?.((item: any) => item?.value));
const stats = record(body.stats ?? body.statistics); const stats = record(body.stats ?? body.statistics);
const sampleCount = firstNumberOption(body.sampleCount, stats.count, samples.length) ?? 0; const sampleCount = firstNumberOption(body.sampleCount, stats.count, samples.length) ?? 0;
+2 -19
View File
@@ -602,38 +602,21 @@ function ioProbeObservationOps(common: any, args: any, document: any) {
intervalMs: Math.max(0, numberValue(args.intervalMs ?? sample.intervalMs) ?? 0), intervalMs: Math.max(0, numberValue(args.intervalMs ?? sample.intervalMs) ?? 0),
settleMs: Math.max(0, numberValue(args.settleMs ?? sample.settleMs) ?? 0) settleMs: Math.max(0, numberValue(args.settleMs ?? sample.settleMs) ?? 0)
}; };
const script = ioProbeObservationScript();
return [{ return [{
op: "cmd.run", op: "cmd.run",
args: clean({ args: clean({
...common, ...common,
workspacePath: text(endpoint.cwd ?? endpoint.boardCommDir) || common.workspacePath, workspacePath: text(endpoint.cwd ?? endpoint.boardCommDir) || common.workspacePath,
command: "node", command: "node",
argv: ["-e", script, JSON.stringify(input)], argv: ["hwpod-io-probe-observation.mjs", JSON.stringify(input)],
step: "io-probe-read", step: "io-probe-read",
commandBinding: { kind: "io-probe", probeId: probe.id, quantity: input.quantity, unit: input.unit, endpointKind: "boardCommJsonRpcTcp" }, commandBinding: { kind: "io-probe", nativeHelper: "io-probe-observation", probeId: probe.id, quantity: input.quantity, unit: input.unit, endpointKind: "boardCommJsonRpcTcp" },
timeoutMs: numberValue(args.timeoutMs ?? endpoint.timeoutMs), timeoutMs: numberValue(args.timeoutMs ?? endpoint.timeoutMs),
reason: text(args.reason) reason: text(args.reason)
}) })
}]; }];
} }
function ioProbeObservationScript() {
return [
"const {spawnSync}=require('child_process');",
"const i=JSON.parse(process.argv[1]||'{}');",
"const read=(o,p)=>String(p||'').replace(/^\\$\\.?/,'').split('.').filter(Boolean).reduce((a,k)=>a==null?undefined:a[k],o);",
"const unwrap=o=>[o?.response?.response,o?.response,o].filter(Boolean);",
"const value=o=>{for(const c of unwrap(o)){const v=read(c,i.valuePath);if(v!==undefined)return Number(v);}return NaN;};",
"const sleep=ms=>{if(ms>0)Atomics.wait(new Int32Array(new SharedArrayBuffer(4)),0,0,ms);};",
"sleep(i.settleMs);const samples=[];let last=null;let code=0;",
"for(let n=0;n<i.count;n++){const argv=[...i.commandBase.slice(1),'jrpctcp','--host',i.host,'--port',String(i.port),...(i.transport?['--transport',i.transport]:[]),...(i.targetNodeId?['--target-node-id',String(i.targetNodeId)]:[]),i.method,i.path,...i.params];const r=spawnSync(i.commandBase[0],argv,{encoding:'utf8'});try{last=JSON.parse(String(r.stdout||'').trim());}catch{last=null;}const v=value(last);if(Number.isFinite(v))samples.push(v);if(r.status!==0)code=r.status||1;if(n<i.count-1)sleep(i.intervalMs);}",
"const mean=samples.length?samples.reduce((a,b)=>a+b,0)/samples.length:undefined;",
"const body={observationId:`obs_${i.probeId}_${Date.now()}`,probeId:i.probeId,quantity:i.quantity,unit:i.unit,samples,stats:samples.length?{count:samples.length,mean,min:Math.min(...samples),max:Math.max(...samples)}:{},response:last,rawArtifactRef:last?.log_path||last?.logPath||last?.artifactRef||null};",
"console.log(JSON.stringify(body));process.exit(code===0&&samples.length>0?0:(code||1));"
].join("");
}
function boardCommEndpoint(document: any) { function boardCommEndpoint(document: any) {
const boardComm = objectValue(document.spec?.boardComm); const boardComm = objectValue(document.spec?.boardComm);
const ioProbe = objectValue(document.spec?.ioProbe); const ioProbe = objectValue(document.spec?.ioProbe);
+17 -2
View File
@@ -3,6 +3,7 @@ import { createServer } from "node:http";
import { mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises"; import { mkdir, readdir, readFile, rm, stat, writeFile } from "node:fs/promises";
import os from "node:os"; import os from "node:os";
import path from "node:path"; import path from "node:path";
import { fileURLToPath } from "node:url";
import { HWPOD_NODE_OPS_CONTRACT_VERSION } from "./hwpod-node-ops-contract.ts"; import { HWPOD_NODE_OPS_CONTRACT_VERSION } from "./hwpod-node-ops-contract.ts";
@@ -1318,10 +1319,24 @@ function resolvePatchFile(root: string, relativePath: string) {
async function cmdRun(args: any) { async function cmdRun(args: any) {
const command = requiredText(args.command, "command"); const command = requiredText(args.command, "command");
const argv = Array.isArray(args.argv) ? args.argv.map(String) : []; const commandBinding = objectValue(args.commandBinding);
const argv = nativeHelperArgv(Array.isArray(args.argv) ? args.argv.map(String) : [], commandBinding);
const cwd = workspaceRoot(args); const cwd = workspaceRoot(args);
const output = await spawnOutput([command, ...argv], { cwd, timeoutMs: numberValue(args.timeoutMs) ?? 30000 }); const output = await spawnOutput([command, ...argv], { cwd, timeoutMs: numberValue(args.timeoutMs) ?? 30000 });
return { cwd, command: [command, ...argv], ...output }; const observation = commandObservation(commandBinding, output);
return { cwd, command: [command, ...argv], ...output, ...(observation ? { observation } : {}) };
}
function nativeHelperArgv(argv: string[], commandBinding: Record<string, unknown>) {
if (text(commandBinding.nativeHelper) !== "io-probe-observation") return argv;
const helperPath = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../hwpod-io-probe-observation.mjs");
return [helperPath, ...argv.slice(1)];
}
function commandObservation(commandBinding: Record<string, unknown>, output: any) {
if (!["io-probe", "board-comm"].includes(text(commandBinding.kind))) return undefined;
const observation = parseJsonMaybe(output.stdout);
return observation && typeof observation === "object" && !Array.isArray(observation) ? observation : undefined;
} }
async function uartRead(args: any) { async function uartRead(args: any) {