From ae30b1a2ba5c01b50c1d65c43bfb92f2672d4e60 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Fri, 22 May 2026 18:09:06 +0800 Subject: [PATCH] test: refresh M4 DEV agent loop evidence Refs HWLAB#37, HWLAB#64, HWLAB#78. Reviewed by commander. This keeps M4 as blocked at DB live readiness, does not promote dry-run/source evidence to DEV-LIVE, and preserves the patch-panel boundary. --- docs/dev-m4-agent-loop.md | 14 ++ reports/dev-gate/dev-m4-agent-loop.json | 99 +++++++-- scripts/dev-m4-agent-loop-smoke.mjs | 267 +++++++++++++++++++++--- 3 files changed, 334 insertions(+), 46 deletions(-) diff --git a/docs/dev-m4-agent-loop.md b/docs/dev-m4-agent-loop.md index 1785bb69..68fbcdd5 100644 --- a/docs/dev-m4-agent-loop.md +++ b/docs/dev-m4-agent-loop.md @@ -23,6 +23,16 @@ node scripts/m4-agent-loop-smoke.mjs That smoke stays local and checks the runtime skeleton, workspace isolation, and explicit skills commit handling. +Run the bounded M4 dry-run before live preflight: + +```sh +node scripts/dev-m4-agent-loop-smoke.mjs --dry-run +``` + +The dry-run uses only temporary local state. It records manager, worker, +workspace, skills commit, trace, evidence, and cleanup evidence as `DRY-RUN`; +it is not `DEV-LIVE`. + ## Live DEV Preflight Run the live preflight only if the DEV route is reachable and the operator has @@ -36,6 +46,10 @@ The live path must only observe existing DEV state. It must not read tokens or secrets, must not try to run a real agent workload, and must stop if the M3 runtime preconditions are not ready. +Any hardware assistance from M4 must go through the cloud API hardware surface +and the patch-panel trusted path for `DO1 -> patch-panel -> DI1`. The agent +must not connect directly to box or gateway simulators. + ## Expected Evidence - `agent-mgr` creates a scoped worker session. diff --git a/reports/dev-gate/dev-m4-agent-loop.json b/reports/dev-gate/dev-m4-agent-loop.json index dc65ba0f..e8535642 100644 --- a/reports/dev-gate/dev-m4-agent-loop.json +++ b/reports/dev-gate/dev-m4-agent-loop.json @@ -4,10 +4,18 @@ "reportVersion": "v1", "issue": "pikasTech/HWLAB#37", "taskId": "dev-m4-agent-loop", - "commitId": "a1c3cb5", + "commitId": "8e89409", "acceptanceLevel": "dev_m4_agent_loop", "devOnly": true, "prodDisabled": true, + "reportLifecycle": { + "version": "v1", + "state": "active", + "activeEndpoint": "http://74.48.78.17:16667", + "activeBrowserEndpoint": "http://74.48.78.17:16666", + "deprecatedEndpoint": null, + "summary": "Current DEV M4 agent-loop preflight report; it does not substitute for M3 DEV-LIVE acceptance." + }, "sourceContract": { "status": "pass", "documents": [ @@ -35,14 +43,84 @@ "summary": "Local contract smoke passes on the repo fixture." }, "dryRun": { - "status": "not_run", + "status": "pass", "commands": [ "node scripts/dev-m4-agent-loop-smoke.mjs --dry-run" ], "evidence": [ - "No dry-run output is attached to the live preflight report." + "hwlab-agent-mgr:session=agt_m4_agent_loop_01:created=queued:final=cleanup", + "hwlab-agent-worker:session=wkr_agt_m4_agent_loop_01:status=cleanup", + "workspace:vol_agt_m4_agent_loop_01:cleaned=true:existsAfterCleanup=false", + "hwlab-agent-skills:commit=6509a35:version=v1", + "trace:trc_agt_m4_agent_loop_01:events=5", + "evidence:evi_agt_m4_agent_loop_01:kind=report", + "cleanup:cln_agt_m4_agent_loop_01:removed=true" ], - "summary": "The live preflight report does not attempt a dry-run agent run." + "summary": "Local dry-run covered agent manager, worker, workspace isolation, skills commit, trace, evidence, and cleanup without live DEV mutation." + }, + "componentEvidence": { + "agentManagerWorker": { + "level": "DRY-RUN", + "status": "pass", + "services": [ + "hwlab-agent-mgr", + "hwlab-agent-worker" + ], + "summary": "Manager created a scoped local session and worker completed the bounded dry-run task." + }, + "workspaceIsolation": { + "level": "DRY-RUN", + "status": "pass", + "evidence": [ + "primary=prj_m4_agent_loop/agt_m4_agent_loop_01", + "isolation=prj_m4_agent_loop_alt/agt_m4_agent_loop_02" + ], + "summary": "Workspace identity is per agent session and fixture isolation remains source/local evidence only." + }, + "skillsCommit": { + "level": "DRY-RUN", + "status": "pass", + "serviceId": "hwlab-agent-skills", + "commitId": "6509a35", + "version": "v1", + "summary": "Agent skills are accepted only with explicit commitId and version wiring." + }, + "traceEvidenceCleanup": { + "level": "DRY-RUN", + "status": "pass", + "summary": "Dry-run produced trace/evidence records and removed the temporary worker workspace." + }, + "dbLive": { + "level": "BLOCKED", + "status": "blocked", + "summary": "Live scheduling is stopped at cloud-api DB readiness; DB health is classified separately from local dry-run evidence." + }, + "m3HardwareLoopDependency": { + "level": "BLOCKED", + "status": "not_run", + "requiredPath": "DO1 -> hwlab-patch-panel -> DI1", + "summary": "M4 can only support M3 through cloud-api/hardware API and the patch-panel trusted path; no direct box/gateway path is used." + } + }, + "dependencyState": { + "dbLive": { + "status": "blocked", + "summary": "cloud-api /health/live reports DB degraded; connected=false; ready=false." + }, + "m3HardwareLoop": { + "status": "not_run", + "requiredEvidence": "Traceable DO1 -> hwlab-patch-panel -> DI1 operation with auditId and evidenceId.", + "summary": "M3 hardware loop was not reached while an earlier M4 precondition remained blocked." + }, + "hardwareDispatchBoundary": { + "status": "pass", + "allowedPath": "cloud-api /rpc hardware.operation.request -> hwlab-patch-panel trusted route", + "forbiddenPaths": [ + "direct hwlab-box-simu access", + "direct hwlab-gateway-simu access", + "agent-side fixture promotion to DEV-LIVE" + ] + } }, "devPreconditions": { "status": "blocked", @@ -50,7 +128,8 @@ "requirements": [ "DEV route remains frozen at http://74.48.78.17:16667", "No secret reads, real deployment, or long-running agent task is attempted", - "Agent manager, worker, and skills are reachable only through the cloud API boundary" + "Agent manager, worker, and skills are reachable only through the cloud API boundary", + "M4 hardware assistance must go through cloud-api/hardware API and patch-panel; direct box/gateway access is forbidden" ], "summary": "Blocked at DB live readiness before scheduling a DEV agent task." }, @@ -75,13 +154,5 @@ ], "summary": "Blocked at DB live readiness before scheduling a DEV agent task." }, - "notes": "DEV M4 agent loop live preflight report. No secret material is read.", - "reportLifecycle": { - "version": "v1", - "state": "active", - "activeEndpoint": "http://74.48.78.17:16667", - "activeBrowserEndpoint": "http://74.48.78.17:16666", - "deprecatedEndpoint": null, - "summary": "Current DEV M4 agent-loop preflight report; it does not substitute for M3 DEV-LIVE acceptance." - } + "notes": "DEV M4 agent loop report. SOURCE/LOCAL/DRY-RUN evidence is not DEV-LIVE, and no secret material is read." } diff --git a/scripts/dev-m4-agent-loop-smoke.mjs b/scripts/dev-m4-agent-loop-smoke.mjs index 84437891..2a86e06e 100644 --- a/scripts/dev-m4-agent-loop-smoke.mjs +++ b/scripts/dev-m4-agent-loop-smoke.mjs @@ -3,15 +3,28 @@ import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; import { request as httpRequest } from "node:http"; import { request as httpsRequest } from "node:https"; -import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { mkdtemp, readFile, writeFile, mkdir, rm } from "node:fs/promises"; +import os from "node:os"; import path from "node:path"; import { fileURLToPath } from "node:url"; +import { + AGENT_MGR_SERVICE_ID, + AGENT_SKILLS_SERVICE_ID, + AGENT_WORKER_SERVICE_ID +} from "../internal/agent/index.mjs"; +import { + createLocalAgentSession, + getLocalAgentEvidence, + getLocalAgentStatus, + getLocalAgentTrace, + runLocalWorkerDryRun +} from "../internal/agent/runtime.mjs"; import { activeReportLifecycle } from "../internal/dev-report-lifecycle.mjs"; import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); -const fixedNow = () => "2026-05-21T00:00:00.000Z"; +const fixedNow = () => "2026-05-22T00:00:00.000Z"; const requiredDocs = ["docs/dev-acceptance-matrix.md", "docs/dev-m4-agent-loop.md"]; const reportPath = "reports/dev-gate/dev-m4-agent-loop.json"; const preflightCommand = "node scripts/dev-m4-agent-loop-smoke.mjs --live --confirm-dev --confirmed-non-production"; @@ -127,7 +140,110 @@ function blockerForResult(result) { : []; } -function buildReport(fixture, evidence, result) { +function buildDryRunEvidence({ status = "not_run", details = [] } = {}) { + return { + status, + commands: [dryRunCommand], + evidence: details.length + ? details + : ["No dry-run output is attached to this live preflight report."], + summary: status === "pass" + ? "Local dry-run covered agent manager, worker, workspace isolation, skills commit, trace, evidence, and cleanup without live DEV mutation." + : "The live preflight report did not attach a dry-run agent run." + }; +} + +function buildComponentEvidence(fixture, dryRun, result) { + const dbLiveStatus = result.blockedClassification === "DB live" + ? "blocked" + : result.blocked + ? "not_run" + : "pass"; + const m3DependencyStatus = result.blockedClassification === "hardware loop" + ? "blocked" + : result.blocked + ? "not_run" + : "pass"; + + return { + agentManagerWorker: { + level: dryRun.status === "pass" ? "DRY-RUN" : "LOCAL", + status: dryRun.status === "pass" ? "pass" : "not_run", + services: [AGENT_MGR_SERVICE_ID, AGENT_WORKER_SERVICE_ID], + summary: dryRun.status === "pass" + ? "Manager created a scoped local session and worker completed the bounded dry-run task." + : "Manager/worker live scheduling was not attempted before preflight blockers were classified." + }, + workspaceIsolation: { + level: dryRun.status === "pass" ? "DRY-RUN" : "LOCAL", + status: "pass", + evidence: [ + `primary=${fixture.projectId}/${fixture.agentSessionId}`, + `isolation=${fixture.workspaceIsolation.projectId}/${fixture.workspaceIsolation.agentSessionId}` + ], + summary: "Workspace identity is per agent session and fixture isolation remains source/local evidence only." + }, + skillsCommit: { + level: dryRun.status === "pass" ? "DRY-RUN" : "LOCAL", + status: "pass", + serviceId: AGENT_SKILLS_SERVICE_ID, + commitId: fixture.skillCommitId, + version: fixture.skillVersion, + summary: "Agent skills are accepted only with explicit commitId and version wiring." + }, + traceEvidenceCleanup: { + level: dryRun.status === "pass" ? "DRY-RUN" : "LOCAL", + status: dryRun.status === "pass" ? "pass" : "not_run", + summary: dryRun.status === "pass" + ? "Dry-run produced trace/evidence records and removed the temporary worker workspace." + : "Trace/evidence/cleanup live closure was not attempted before preflight blockers were classified." + }, + dbLive: { + level: "BLOCKED", + status: dbLiveStatus, + summary: dbLiveStatus === "blocked" + ? "Live scheduling is stopped at cloud-api DB readiness; DB health is classified separately from local dry-run evidence." + : "Live cloud-api DB readiness did not block before the current M4 classification point." + }, + m3HardwareLoopDependency: { + level: "BLOCKED", + status: m3DependencyStatus, + requiredPath: "DO1 -> hwlab-patch-panel -> DI1", + summary: m3DependencyStatus === "blocked" + ? "M4 reached the hardware dispatch dependency, but the M3 patch-panel trusted loop is not accepted as live." + : "M4 can only support M3 through cloud-api/hardware API and the patch-panel trusted path; no direct box/gateway path is used." + } + }; +} + +function buildDependencyState(result) { + return { + dbLive: { + status: result.blockedClassification === "DB live" ? "blocked" : result.blocked ? "not_run" : "pass", + summary: result.blockedClassification === "DB live" + ? result.blockerSummary + : "DB live readiness did not produce the active M4 blocker." + }, + m3HardwareLoop: { + status: result.blockedClassification === "hardware loop" ? "blocked" : result.blocked ? "not_run" : "pass", + requiredEvidence: "Traceable DO1 -> hwlab-patch-panel -> DI1 operation with auditId and evidenceId.", + summary: result.blockedClassification === "hardware loop" + ? result.blockerSummary + : "M3 hardware loop was not reached while an earlier M4 precondition remained blocked." + }, + hardwareDispatchBoundary: { + status: "pass", + allowedPath: "cloud-api /rpc hardware.operation.request -> hwlab-patch-panel trusted route", + forbiddenPaths: [ + "direct hwlab-box-simu access", + "direct hwlab-gateway-simu access", + "agent-side fixture promotion to DEV-LIVE" + ] + } + }; +} + +function buildReport(fixture, liveEvidence, result, { dryRun = buildDryRunEvidence() } = {}) { const sourceContract = { status: "pass", documents: requiredDocs, @@ -164,21 +280,17 @@ function buildReport(fixture, evidence, result) { ], summary: "Local contract smoke passes on the repo fixture." }, - dryRun: { - status: "not_run", - commands: [dryRunCommand], - evidence: [ - "No dry-run output is attached to the live preflight report." - ], - summary: "The live preflight report does not attempt a dry-run agent run." - }, + dryRun, + componentEvidence: buildComponentEvidence(fixture, dryRun, result), + dependencyState: buildDependencyState(result), devPreconditions: { status: statusForResult(result), classification: result.blockedClassification ?? "none", requirements: [ `DEV route remains frozen at ${DEV_ENDPOINT}`, "No secret reads, real deployment, or long-running agent task is attempted", - "Agent manager, worker, and skills are reachable only through the cloud API boundary" + "Agent manager, worker, and skills are reachable only through the cloud API boundary", + "M4 hardware assistance must go through cloud-api/hardware API and patch-panel; direct box/gateway access is forbidden" ], summary: result.summary }, @@ -187,25 +299,86 @@ function buildReport(fixture, evidence, result) { status: result.blocked ? "blocked" : "pass", classification: result.blockedClassification ?? "none", commands: [preflightCommand], - evidence: evidence.length ? evidence : ["No live DEV observation was recorded."], + evidence: liveEvidence.length ? liveEvidence : ["No live DEV observation was recorded."], summary: result.summary }, - notes: "DEV M4 agent loop live preflight report. No secret material is read." + notes: "DEV M4 agent loop report. SOURCE/LOCAL/DRY-RUN evidence is not DEV-LIVE, and no secret material is read." }; } -async function main() { - const args = parseArgs(process.argv.slice(2)); - assert.equal(args.dryRun, false, "--dry-run is not supported for this live preflight"); - assert.equal(args.live, true, "live preflight requires --live"); - assert.equal(args.confirmDev, true, "live preflight requires --confirm-dev"); - assert.equal(args.confirmedNonProduction, true, "live preflight requires --confirmed-non-production"); - +async function loadFixture() { const fixture = await readJSON("fixtures/mvp/m4-agent-loop/runtime.json"); - const evidence = []; - const localEvidence = await readText(fixture.evidencePath); assert.ok(localEvidence.includes("Boundary: local only")); + return fixture; +} + +async function runDryRun(fixture) { + const stateDir = await mkdtemp(path.join(os.tmpdir(), "hwlab-dev-m4-agent-loop-")); + try { + const created = await createLocalAgentSession({ + stateDir, + agentSessionId: fixture.agentSessionId, + projectId: fixture.projectId, + goal: fixture.goal, + prompt: fixture.prompt, + skillCommitId: fixture.skillCommitId, + skillVersion: fixture.skillVersion, + now: fixedNow + }); + assert.equal(created.status, "ok"); + assert.equal(created.environment, ENVIRONMENT_DEV); + + const worker = await runLocalWorkerDryRun({ + stateDir, + agentSessionId: fixture.agentSessionId, + skillCommitId: fixture.skillCommitId, + skillVersion: fixture.skillVersion, + now: fixedNow + }); + assert.equal(worker.dryRun, true); + assert.equal(worker.health.status, "ok"); + assert.equal(worker.agentSession.status, "cleanup"); + assert.equal(worker.workerSession.status, "cleanup"); + assert.equal(worker.workspace.cleaned, true); + assert.equal(worker.workspace.existsAfterCleanup, false); + + const status = await getLocalAgentStatus({ + stateDir, + agentSessionId: fixture.agentSessionId + }); + const trace = await getLocalAgentTrace({ + stateDir, + agentSessionId: fixture.agentSessionId + }); + const evidence = await getLocalAgentEvidence({ + stateDir, + agentSessionId: fixture.agentSessionId + }); + + assert.equal(status.evidenceCount, 1); + assert.equal(trace.traceEvents.length, 5); + assert.equal(evidence.evidenceRecords.length, 1); + + return buildDryRunEvidence({ + status: "pass", + details: [ + `${AGENT_MGR_SERVICE_ID}:session=${fixture.agentSessionId}:created=${created.task.status}:final=${worker.agentSession.status}`, + `${AGENT_WORKER_SERVICE_ID}:session=${worker.workerSession.workerSessionId}:status=${worker.workerSession.status}`, + `workspace:${worker.workspace.workspaceVolumeId}:cleaned=${worker.workspace.cleaned}:existsAfterCleanup=${worker.workspace.existsAfterCleanup}`, + `${AGENT_SKILLS_SERVICE_ID}:commit=${fixture.skillCommitId}:version=${fixture.skillVersion}`, + `trace:${trace.traceId}:events=${trace.traceEvents.length}`, + `evidence:${evidence.evidenceRecords[0].evidenceId}:kind=${evidence.evidenceRecords[0].kind}`, + `cleanup:${worker.cleanup.cleanupId}:removed=${worker.cleanup.removed}` + ] + }); + } finally { + await rm(stateDir, { recursive: true, force: true }); + } +} + +async function runLivePreflight(fixture, dryRun) { + const liveEvidence = []; let result = { blocked: true, @@ -229,7 +402,7 @@ async function main() { }; } else { const body = health.body || {}; - evidence.push(`health:${body.serviceId}:${body.environment}:${body.status}`); + liveEvidence.push(`health:${body.serviceId}:${body.environment}:${body.status}`); const live = await requestJson(`${DEV_ENDPOINT}/health/live`); const liveBody = live.body || {}; @@ -243,7 +416,7 @@ async function main() { summary: "Blocked because the live DEV boundary did not expose the expected cloud API identity." }; } else if (isDbLiveBlocked(liveBody)) { - evidence.push(`live:${liveBody.serviceId}:${liveBody.status}:db-blocked`); + liveEvidence.push(`live:${liveBody.serviceId}:${liveBody.status}:db-blocked`); result = { blocked: true, blockerType: "runtime_blocker", @@ -253,7 +426,7 @@ async function main() { summary: "Blocked at DB live readiness before scheduling a DEV agent task." }; } else { - evidence.push(`live:${liveBody.serviceId}:${liveBody.status}`); + liveEvidence.push(`live:${liveBody.serviceId}:${liveBody.status}`); const rpcHealth = await requestJson(`${DEV_ENDPOINT}/rpc`, { method: "POST", @@ -273,7 +446,7 @@ async function main() { }) }); const rpcHealthBody = rpcHealth.body || {}; - evidence.push(`rpc-health:${rpcHealthBody.result?.serviceId ?? "unknown"}`); + liveEvidence.push(`rpc-health:${rpcHealthBody.result?.serviceId ?? "unknown"}`); if (rpcHealth.statusCode < 200 || rpcHealth.statusCode >= 300 || rpcHealthBody.error) { result = { blocked: true, @@ -295,7 +468,12 @@ async function main() { id: "req_dev_m4_hw", method: "hardware.operation.request", params: { - projectId: fixture.projectId + projectId: fixture.projectId, + input: { + requestedPath: "DO1 -> hwlab-patch-panel -> DI1", + directBoxGatewayAccess: false, + source: "dev-m4-agent-loop-preflight" + } }, meta: { traceId: "trc_dev_m4_hw", @@ -307,7 +485,7 @@ async function main() { const rpcHardwareBody = rpcHardware.body || {}; const accepted = rpcHardwareBody.result?.accepted === true; const status = rpcHardwareBody.result?.status || rpcHardwareBody.error?.message || "unknown"; - evidence.push(`rpc-hardware:${status}`); + liveEvidence.push(`rpc-hardware:${status}`); if (!accepted) { result = { @@ -315,8 +493,8 @@ async function main() { blockerType: "agent_blocker", blockerScope: "hardware-loop", blockedClassification: "hardware loop", - blockerSummary: "hardware.operation.request remains a skeleton response in the current DEV runtime.", - summary: "Blocked by M3 readiness because the agent hardware dispatch path is still skeleton-only." + blockerSummary: "hardware.operation.request did not accept the patch-panel-bounded M3 request.", + summary: "Blocked by M3 readiness because M4 did not observe an accepted DO1 -> patch-panel -> DI1 hardware request." }; } else if (!rpcHardwareBody.result?.auditId || !rpcHardwareBody.result?.evidenceId) { result = { @@ -351,7 +529,7 @@ async function main() { }; } - const report = buildReport(fixture, evidence, result); + const report = buildReport(fixture, liveEvidence, result, { dryRun }); await writeJSON(reportPath, report); process.stdout.write(JSON.stringify({ @@ -366,6 +544,31 @@ async function main() { process.stdout.write("\n"); } +async function main() { + const args = parseArgs(process.argv.slice(2)); + const fixture = await loadFixture(); + + if (args.dryRun) { + assert.equal(args.live, false, "--dry-run cannot be combined with --live"); + const dryRun = await runDryRun(fixture); + process.stdout.write(JSON.stringify({ + reportPath, + status: dryRun.status, + evidence: dryRun.evidence, + summary: dryRun.summary, + observedAt: fixedNow() + }, null, 2)); + process.stdout.write("\n"); + return; + } + + assert.equal(args.live, true, "live preflight requires --live"); + assert.equal(args.confirmDev, true, "live preflight requires --confirm-dev"); + assert.equal(args.confirmedNonProduction, true, "live preflight requires --confirmed-non-production"); + + await runLivePreflight(fixture, await runDryRun(fixture)); +} + function isDbLiveBlocked(body) { const db = body?.db; if (!db || typeof db !== "object") {