Files
pikasTech-HWLAB/scripts/dev-m4-agent-loop-smoke.mjs
T
Lyon ae23972fd9 test: refresh DEV M4 agent smoke evidence (#139)
Commander review: direction is correct. This PR records DEV M4 agent-loop evidence as a bounded preflight, keeps M4 subordinate to the M3 trusted-loop mainline, and preserves blockers for DB live readiness and skill version injection. No PROD/service restart is included.
2026-05-22 20:45:23 +08:00

595 lines
22 KiB
JavaScript

#!/usr/bin/env node
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { mkdtemp, readFile, writeFile, mkdir, rm } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { fileURLToPath } from "node:url";
import {
AGENT_MGR_SERVICE_ID,
AGENT_SKILLS_SERVICE_ID,
AGENT_WORKER_SERVICE_ID
} from "../internal/agent/index.mjs";
import {
createLocalAgentSession,
getLocalAgentEvidence,
getLocalAgentStatus,
getLocalAgentTrace,
runLocalWorkerDryRun
} from "../internal/agent/runtime.mjs";
import { activeReportLifecycle } from "../internal/dev-report-lifecycle.mjs";
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs";
import {
collectD601K3sNativeEvidence,
collectPublicEntrypoints,
d601K3sReadonlyCommand,
d601Kubeconfig,
devNamespace,
requestJson,
workerServerDryRunCommand
} from "./src/dev-m4-agent-loop-smoke-lib.mjs";
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const fixedNow = () => "2026-05-22T00:00:00.000Z";
const requiredDocs = ["docs/dev-acceptance-matrix.md", "docs/dev-m4-agent-loop.md"];
const reportPath = "reports/dev-gate/dev-m4-agent-loop.json";
const preflightCommand = "node scripts/dev-m4-agent-loop-smoke.mjs --live --confirm-dev --confirmed-non-production";
const dryRunCommand = "node scripts/dev-m4-agent-loop-smoke.mjs --dry-run";
function parseArgs(argv) {
const flags = new Set();
for (const arg of argv) {
if (arg.startsWith("--")) {
flags.add(arg);
}
}
return {
live: flags.has("--live") || flags.has("--allow-live"),
dryRun: flags.has("--dry-run"),
confirmDev: flags.has("--confirm-dev"),
confirmedNonProduction: flags.has("--confirmed-non-production")
};
}
async function readJSON(relativePath) {
const raw = await readFile(path.join(repoRoot, relativePath), "utf8");
return JSON.parse(raw);
}
async function readText(relativePath) {
return readFile(path.join(repoRoot, relativePath), "utf8");
}
async function writeJSON(relativePath, value) {
const absolutePath = path.join(repoRoot, relativePath);
await mkdir(path.dirname(absolutePath), { recursive: true });
await writeFile(absolutePath, `${JSON.stringify(value, null, 2)}\n`, "utf8");
}
function currentOriginMainCommitId() {
return execFileSync("git", ["rev-parse", "--short=7", "origin/main"], {
cwd: repoRoot,
encoding: "utf8"
}).trim();
}
function statusForResult(result) {
return result.blocked || (result.additionalBlockers?.length ?? 0) > 0 ? "blocked" : "pass";
}
function blockerForResult(result) {
const blockers = [];
if (result.blocked && result.blockerType) {
blockers.push({
type: result.blockerType,
scope: result.blockerScope ?? "devPreconditions",
status: "open",
summary: result.blockerSummary,
classification: result.blockedClassification ?? "other"
});
}
for (const blocker of result.additionalBlockers ?? []) {
blockers.push({ status: "open", ...blocker });
}
const seen = new Set();
return blockers.filter((blocker) => {
const key = `${blocker.type}:${blocker.scope}`;
if (seen.has(key)) return false;
seen.add(key);
return true;
});
}
function classificationForResult(result) {
return result.blockedClassification ?? result.additionalBlockers?.[0]?.classification ?? "none";
}
function buildDryRunEvidence({ status = "not_run", details = [] } = {}) {
return {
status,
commands: [dryRunCommand],
evidence: details.length
? details
: ["No dry-run output is attached to this live preflight report."],
summary: status === "pass"
? "Local dry-run covered agent manager, worker, workspace isolation, skills commit, trace, evidence, and cleanup without live DEV mutation."
: "The live preflight report did not attach a dry-run agent run."
};
}
function buildComponentEvidence(fixture, dryRun, result) {
const dbLiveStatus = result.blockedClassification === "DB live"
? "blocked"
: result.blocked
? "not_run"
: "pass";
const m3DependencyStatus = result.blockedClassification === "hardware loop"
? "blocked"
: result.blocked
? "not_run"
: "pass";
return {
agentManagerWorker: {
level: dryRun.status === "pass" ? "DRY-RUN" : "LOCAL",
status: dryRun.status === "pass" ? "pass" : "not_run",
services: [AGENT_MGR_SERVICE_ID, AGENT_WORKER_SERVICE_ID],
summary: dryRun.status === "pass"
? "Manager created a scoped local session and worker completed the bounded dry-run task."
: "Manager/worker live scheduling was not attempted before preflight blockers were classified."
},
workspaceIsolation: {
level: dryRun.status === "pass" ? "DRY-RUN" : "LOCAL",
status: "pass",
evidence: [
`primary=${fixture.projectId}/${fixture.agentSessionId}`,
`isolation=${fixture.workspaceIsolation.projectId}/${fixture.workspaceIsolation.agentSessionId}`
],
summary: "Workspace identity is per agent session and fixture isolation remains source/local evidence only."
},
skillsCommit: {
level: dryRun.status === "pass" ? "DRY-RUN" : "LOCAL",
status: "pass",
serviceId: AGENT_SKILLS_SERVICE_ID,
commitId: fixture.skillCommitId,
version: fixture.skillVersion,
summary: "Agent skills are accepted only with explicit commitId and version wiring."
},
traceEvidenceCleanup: {
level: dryRun.status === "pass" ? "DRY-RUN" : "LOCAL",
status: dryRun.status === "pass" ? "pass" : "not_run",
summary: dryRun.status === "pass"
? "Dry-run produced trace/evidence records and removed the temporary worker workspace."
: "Trace/evidence/cleanup live closure was not attempted before preflight blockers were classified."
},
dbLive: {
level: "BLOCKED",
status: dbLiveStatus,
summary: dbLiveStatus === "blocked"
? "Live scheduling is stopped at cloud-api DB readiness; DB health is classified separately from local dry-run evidence."
: "Live cloud-api DB readiness did not block before the current M4 classification point."
},
m3HardwareLoopDependency: {
level: "BLOCKED",
status: m3DependencyStatus,
requiredPath: "DO1 -> hwlab-patch-panel -> DI1",
summary: m3DependencyStatus === "blocked"
? "M4 reached the hardware dispatch dependency, but the M3 patch-panel trusted loop is not accepted as live."
: "M4 can only support M3 through cloud-api/hardware API and the patch-panel trusted path; no direct box/gateway path is used."
}
};
}
function buildDependencyState(result) {
return {
dbLive: {
status: result.blockedClassification === "DB live" ? "blocked" : result.blocked ? "not_run" : "pass",
summary: result.blockedClassification === "DB live"
? result.blockerSummary
: "DB live readiness did not produce the active M4 blocker."
},
m3HardwareLoop: {
status: result.blockedClassification === "hardware loop" ? "blocked" : result.blocked ? "not_run" : "pass",
requiredEvidence: "Traceable DO1 -> hwlab-patch-panel -> DI1 operation with auditId and evidenceId.",
summary: result.blockedClassification === "hardware loop"
? result.blockerSummary
: "M3 hardware loop was not reached while an earlier M4 precondition remained blocked."
},
hardwareDispatchBoundary: {
status: "pass",
allowedPath: "cloud-api /rpc hardware.operation.request -> hwlab-patch-panel trusted route",
forbiddenPaths: [
"direct hwlab-box-simu access",
"direct hwlab-gateway-simu access",
"agent-side fixture promotion to DEV-LIVE"
]
}
};
}
function buildReport(
fixture,
liveEvidence,
result,
{ dryRun = buildDryRunEvidence(), d601K3s = null, publicEntrypoints = null } = {}
) {
const sourceContract = {
status: "pass",
documents: requiredDocs,
summary: "The DEV M4 report is anchored to the frozen acceptance matrix and the M4 loop doc."
};
const validationCommands = [
"node --check scripts/validate-dev-gate-report.mjs",
"node scripts/validate-dev-gate-report.mjs",
"node --check scripts/dev-m4-agent-loop-smoke.mjs",
"node --check scripts/src/dev-m4-agent-loop-smoke-lib.mjs",
dryRunCommand,
preflightCommand,
d601K3sReadonlyCommand,
workerServerDryRunCommand
];
const blockers = blockerForResult(result);
return {
$schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json",
$id: "https://hwlab.pikastech.local/reports/dev-gate/dev-m4-agent-loop.json",
reportVersion: "v1",
issue: "pikasTech/HWLAB#37",
taskId: "dev-m4-agent-loop",
commitId: currentOriginMainCommitId(),
acceptanceLevel: "dev_m4_agent_loop",
devOnly: true,
prodDisabled: true,
reportLifecycle: activeReportLifecycle("Current DEV M4 agent-loop preflight report; it does not substitute for M3 DEV-LIVE acceptance."),
sourceContract,
validationCommands,
localSmoke: {
status: "pass",
commands: ["node scripts/m4-agent-loop-smoke.mjs"],
evidence: [
"Local runtime skeleton smoke confirms create/start/trace/finish/cleanup coverage.",
"Workspace isolation and explicit skills commit wiring are covered by fixture assertions."
],
summary: "Local contract smoke passes on the repo fixture."
},
dryRun,
componentEvidence: buildComponentEvidence(fixture, dryRun, result),
dependencyState: buildDependencyState(result),
d601K3sNative: d601K3s,
publicEntrypoints,
safetyEvidence: {
devOnly: true,
namespace: devNamespace,
kubeconfig: d601Kubeconfig,
prodTouched: false,
prodNamespaceTouched: false,
secretMaterialRead: false,
secretResourcesRead: false,
servicesRestarted: false,
longRunningAgentStarted: false,
workerJobPersisted: false,
notes: [
"kubectl commands were scoped to hwlab-dev and /etc/rancher/k3s/k3s.yaml.",
"Secret RBAC was checked with kubectl auth can-i only; no Secret resource or secret value was read.",
"Worker createability used server-side dry-run with suspend=true and did not persist a Job."
]
},
devPreconditions: {
status: statusForResult(result),
classification: classificationForResult(result),
requirements: [
`DEV route remains frozen at ${DEV_ENDPOINT}`,
`DEV browser route remains frozen at ${DEV_FRONTEND_ENDPOINT}`,
`D601 native k3s access uses KUBECONFIG=${d601Kubeconfig} and namespace ${devNamespace}`,
"No secret reads, real deployment, or long-running agent task is attempted",
"Agent manager, worker, and skills are reachable only through the cloud API boundary",
"M4 hardware assistance must go through cloud-api/hardware API and patch-panel; direct box/gateway access is forbidden"
],
summary: result.summary
},
blockers,
livePreflight: {
status: blockers.length > 0 ? "blocked" : "pass",
classification: classificationForResult(result),
commands: [preflightCommand],
evidence: liveEvidence.length ? liveEvidence : ["No live DEV observation was recorded."],
summary: result.summary
},
notes: "DEV M4 agent loop report. SOURCE/LOCAL/DRY-RUN evidence is not DEV-LIVE, and no secret material is read."
};
}
async function loadFixture() {
const fixture = await readJSON("fixtures/mvp/m4-agent-loop/runtime.json");
const localEvidence = await readText(fixture.evidencePath);
assert.ok(localEvidence.includes("Boundary: local only"));
return fixture;
}
async function runDryRun(fixture) {
const stateDir = await mkdtemp(path.join(os.tmpdir(), "hwlab-dev-m4-agent-loop-"));
try {
const created = await createLocalAgentSession({
stateDir,
agentSessionId: fixture.agentSessionId,
projectId: fixture.projectId,
goal: fixture.goal,
prompt: fixture.prompt,
skillCommitId: fixture.skillCommitId,
skillVersion: fixture.skillVersion,
now: fixedNow
});
assert.equal(created.status, "ok");
assert.equal(created.environment, ENVIRONMENT_DEV);
const worker = await runLocalWorkerDryRun({
stateDir,
agentSessionId: fixture.agentSessionId,
skillCommitId: fixture.skillCommitId,
skillVersion: fixture.skillVersion,
now: fixedNow
});
assert.equal(worker.dryRun, true);
assert.equal(worker.health.status, "ok");
assert.equal(worker.agentSession.status, "cleanup");
assert.equal(worker.workerSession.status, "cleanup");
assert.equal(worker.workspace.cleaned, true);
assert.equal(worker.workspace.existsAfterCleanup, false);
const status = await getLocalAgentStatus({
stateDir,
agentSessionId: fixture.agentSessionId
});
const trace = await getLocalAgentTrace({
stateDir,
agentSessionId: fixture.agentSessionId
});
const evidence = await getLocalAgentEvidence({
stateDir,
agentSessionId: fixture.agentSessionId
});
assert.equal(status.evidenceCount, 1);
assert.equal(trace.traceEvents.length, 5);
assert.equal(evidence.evidenceRecords.length, 1);
return buildDryRunEvidence({
status: "pass",
details: [
`${AGENT_MGR_SERVICE_ID}:session=${fixture.agentSessionId}:created=${created.task.status}:final=${worker.agentSession.status}`,
`${AGENT_WORKER_SERVICE_ID}:session=${worker.workerSession.workerSessionId}:status=${worker.workerSession.status}`,
`workspace:${worker.workspace.workspaceVolumeId}:cleaned=${worker.workspace.cleaned}:existsAfterCleanup=${worker.workspace.existsAfterCleanup}`,
`${AGENT_SKILLS_SERVICE_ID}:commit=${fixture.skillCommitId}:version=${fixture.skillVersion}`,
`trace:${trace.traceId}:events=${trace.traceEvents.length}`,
`evidence:${evidence.evidenceRecords[0].evidenceId}:kind=${evidence.evidenceRecords[0].kind}`,
`cleanup:${worker.cleanup.cleanupId}:removed=${worker.cleanup.removed}`
]
});
} finally {
await rm(stateDir, { recursive: true, force: true });
}
}
async function runLivePreflight(fixture, dryRun) {
const liveEvidence = [];
const [d601K3s, publicEntrypoints] = await Promise.all([
collectD601K3sNativeEvidence(fixture),
collectPublicEntrypoints()
]);
liveEvidence.push(...d601K3s.evidence);
liveEvidence.push(publicEntrypoints.summary);
let result = {
blocked: true,
blockerType: "network_blocker",
blockerScope: "frp",
blockedClassification: "frp",
blockerSummary: "The fixed DEV endpoint did not accept a live connection from this runner.",
summary: `Blocked before agent scheduling because ${DEV_ENDPOINT} is not reachable from this D601 runner.`
};
try {
const health = await requestJson(`${DEV_ENDPOINT}/health`);
if (health.statusCode < 200 || health.statusCode >= 300) {
result = {
blocked: true,
blockerType: "network_blocker",
blockerScope: "frp",
blockedClassification: "frp",
blockerSummary: `DEV health returned HTTP ${health.statusCode}.`,
summary: `Blocked on DEV ingress health because ${DEV_ENDPOINT}/health did not return success.`
};
} else {
const body = health.body || {};
liveEvidence.push(`health:${body.serviceId}:${body.environment}:${body.status}`);
const live = await requestJson(`${DEV_ENDPOINT}/health/live`);
const liveBody = live.body || {};
if (live.statusCode < 200 || live.statusCode >= 300 || liveBody?.serviceId !== "hwlab-cloud-api") {
result = {
blocked: true,
blockerType: "network_blocker",
blockerScope: "frp",
blockedClassification: "frp",
blockerSummary: "DEV live probe did not present the expected HWLAB cloud API identity.",
summary: "Blocked because the live DEV boundary did not expose the expected cloud API identity."
};
} else if (isDbLiveBlocked(liveBody)) {
liveEvidence.push(`live:${liveBody.serviceId}:${liveBody.status}:db-blocked`);
result = {
blocked: true,
blockerType: "runtime_blocker",
blockerScope: "db-live",
blockedClassification: "DB live",
blockerSummary: summarizeDbBlocker(liveBody),
summary: "Blocked at DB live readiness before scheduling a DEV agent task."
};
} else {
liveEvidence.push(`live:${liveBody.serviceId}:${liveBody.status}`);
const rpcHealth = await requestJson(`${DEV_ENDPOINT}/rpc`, {
method: "POST",
headers: {
"content-type": "application/json"
},
body: JSON.stringify({
jsonrpc: "2.0",
id: "req_dev_m4_health",
method: "system.health",
params: {},
meta: {
traceId: "trc_dev_m4_health",
serviceId: "hwlab-cli",
environment: ENVIRONMENT_DEV
}
})
});
const rpcHealthBody = rpcHealth.body || {};
liveEvidence.push(`rpc-health:${rpcHealthBody.result?.serviceId ?? "unknown"}`);
if (rpcHealth.statusCode < 200 || rpcHealth.statusCode >= 300 || rpcHealthBody.error) {
result = {
blocked: true,
blockerType: "agent_blocker",
blockerScope: "agent-runtime",
blockedClassification: "agent runtime",
blockerSummary: rpcHealthBody.error?.message ?? `system.health returned HTTP ${rpcHealth.statusCode}.`,
summary: "Blocked at agent runtime health before attempting hardware dispatch."
};
} else {
const rpcHardware = await requestJson(`${DEV_ENDPOINT}/rpc`, {
method: "POST",
headers: {
"content-type": "application/json"
},
body: JSON.stringify({
jsonrpc: "2.0",
id: "req_dev_m4_hw",
method: "hardware.operation.request",
params: {
projectId: fixture.projectId,
input: {
requestedPath: "DO1 -> hwlab-patch-panel -> DI1",
directBoxGatewayAccess: false,
source: "dev-m4-agent-loop-preflight"
}
},
meta: {
traceId: "trc_dev_m4_hw",
serviceId: "hwlab-cli",
environment: ENVIRONMENT_DEV
}
})
});
const rpcHardwareBody = rpcHardware.body || {};
const accepted = rpcHardwareBody.result?.accepted === true;
const status = rpcHardwareBody.result?.status || rpcHardwareBody.error?.message || "unknown";
liveEvidence.push(`rpc-hardware:${status}`);
if (!accepted) {
result = {
blocked: true,
blockerType: "agent_blocker",
blockerScope: "hardware-loop",
blockedClassification: "hardware loop",
blockerSummary: "hardware.operation.request did not accept the patch-panel-bounded M3 request.",
summary: "Blocked by M3 readiness because M4 did not observe an accepted DO1 -> patch-panel -> DI1 hardware request."
};
} else if (!rpcHardwareBody.result?.auditId || !rpcHardwareBody.result?.evidenceId) {
result = {
blocked: true,
blockerType: "observability_blocker",
blockerScope: "evidence-audit",
blockedClassification: "evidence/audit",
blockerSummary: "hardware.operation.request accepted but did not return audit/evidence identifiers.",
summary: "Blocked because accepted DEV agent hardware dispatch lacks audit/evidence closure."
};
} else {
result = {
blocked: false,
blockerType: null,
blockerScope: null,
blockedClassification: "none",
blockerSummary: "",
summary: "Live DEV preflight observed a non-skeleton agent hardware path."
};
}
}
}
}
} catch (error) {
result = {
blocked: true,
blockerType: "network_blocker",
blockerScope: "frp",
blockedClassification: "frp",
blockerSummary: error instanceof Error ? error.message : String(error),
summary: `Blocked before agent scheduling because ${DEV_ENDPOINT} is unreachable from this runner.`
};
}
result.additionalBlockers = d601K3s.blockers;
if (!result.blocked && result.additionalBlockers.length > 0) {
result.summary = "D601 native k3s agent evidence is blocked even though the public preflight path did not produce an earlier blocker.";
}
const report = buildReport(fixture, liveEvidence, result, { dryRun, d601K3s, publicEntrypoints });
await writeJSON(reportPath, report);
process.stdout.write(JSON.stringify({
reportPath,
status: report.devPreconditions.status,
blocker: report.blockers[0] ?? null,
classification: report.devPreconditions.classification,
evidence: report.livePreflight.evidence,
summary: report.livePreflight.summary,
observedAt: fixedNow()
}, null, 2));
process.stdout.write("\n");
}
async function main() {
const args = parseArgs(process.argv.slice(2));
const fixture = await loadFixture();
if (args.dryRun) {
assert.equal(args.live, false, "--dry-run cannot be combined with --live");
const dryRun = await runDryRun(fixture);
process.stdout.write(JSON.stringify({
reportPath,
status: dryRun.status,
evidence: dryRun.evidence,
summary: dryRun.summary,
observedAt: fixedNow()
}, null, 2));
process.stdout.write("\n");
return;
}
assert.equal(args.live, true, "live preflight requires --live");
assert.equal(args.confirmDev, true, "live preflight requires --confirm-dev");
assert.equal(args.confirmedNonProduction, true, "live preflight requires --confirmed-non-production");
await runLivePreflight(fixture, await runDryRun(fixture));
}
function isDbLiveBlocked(body) {
const db = body?.db;
if (!db || typeof db !== "object") {
return false;
}
return db.connected === false || db.ready === false || ["blocked", "degraded"].includes(db.status);
}
function summarizeDbBlocker(body) {
const db = body?.db ?? {};
const missing = Array.isArray(db.missingEnv) && db.missingEnv.length > 0
? ` missing ${db.missingEnv.join(", ")}`
: "";
return `cloud-api /health/live reports DB ${db.status ?? "not ready"}; connected=${db.connected ?? "unknown"}; ready=${db.ready ?? "unknown"}.${missing}`;
}
await main();