diff --git a/docs/dev-gate-report.md b/docs/dev-gate-report.md index fa1d242c..f1da5215 100644 --- a/docs/dev-gate-report.md +++ b/docs/dev-gate-report.md @@ -39,6 +39,11 @@ Each committed DEV gate report must carry `reportLifecycle`: M4 agent loop, and artifact/desired-state source status as separate layers. If the only read-only evidence is `16666/16667` reachability, M5 must remain blocked/non-green. +- Gate/report aggregators must cross-check DB, M3, M4, and M5 acceptance + against the corresponding source reports before showing `DEV-LIVE` or green + status. Frontend/browser revision facts, public route reachability, source + manifests, local smoke, dry-run output, and fixtures are non-promoting + support evidence for those layers. ## Files @@ -114,6 +119,16 @@ The validator scans `reports/dev-gate/*.json` by default. It only checks the contract shape and the frozen report metadata; it does not run any DEV or PROD deployment. +For M5 aggregator reports, the validator also verifies non-promotion +boundaries by reading the declared source report paths. DB acceptance must come +from redacted live DB readiness (`ready=true`, `connected=true`, and +`liveDbEvidence=true`); M3 must come from a real live operation with operation, +trace, audit, and evidence identifiers; M4 must come from the live preflight; +and M5 must remain blocked until DB, M3, M4, and the M5 live source report +each pass their own source checks. Frontend route, source, dry-run, and +fixture evidence may remain in the report, but it cannot promote those layers +to DEV-LIVE acceptance. + ## Invalidate/Rebuild Workflow When the public DEV endpoint mapping changes, do not rewrite historical diff --git a/docs/reference/cloud-workbench.md b/docs/reference/cloud-workbench.md index d7b8383b..443e8ffe 100644 --- a/docs/reference/cloud-workbench.md +++ b/docs/reference/cloud-workbench.md @@ -10,6 +10,9 @@ Cloud Workbench is the default user-facing frontend at - `/` must show the Cloud Workbench, not Gate, status, help, or diagnostics. - The default first screen must not show Gate, `BLOCKED`, `M0-M5`, or acceptance-review copy. Those terms may remain in hidden/internal views. +- Checked-in Gate reports and aggregator summaries are read-only support data. + They must not be rendered as the default homepage or used to rename the + default route away from Cloud Workbench. - Gate and diagnostics are secondary routes or right-panel support views. - `/gate` and `/diagnostics/gate` are internal diagnostic aliases served by the same Cloud Web app router. They may carry Gate, blocker, and M0-M5 evidence diff --git a/scripts/validate-dev-gate-report-guard.test.mjs b/scripts/validate-dev-gate-report-guard.test.mjs index c637923e..68464e36 100644 --- a/scripts/validate-dev-gate-report-guard.test.mjs +++ b/scripts/validate-dev-gate-report-guard.test.mjs @@ -9,12 +9,18 @@ import { fileURLToPath } from "node:url"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const validator = path.join(repoRoot, "scripts/validate-dev-gate-report.mjs"); const baseReportPath = path.join(repoRoot, "reports/dev-gate/dev-cloud-workbench-live.json"); +const baseAggregatorReportPath = path.join(repoRoot, "reports/dev-gate/dev-m5-gate-aggregator-v2.json"); const baseReport = JSON.parse(await readFile(baseReportPath, "utf8")); +const baseAggregatorReport = JSON.parse(await readFile(baseAggregatorReportPath, "utf8")); function cloneBaseReport() { return JSON.parse(JSON.stringify(baseReport)); } +function cloneBaseAggregatorReport() { + return JSON.parse(JSON.stringify(baseAggregatorReport)); +} + async function withReport(report, fn) { const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-")); const reportPath = path.join(directory, "report.json"); @@ -108,3 +114,101 @@ test("rejects accepted workbench evidence that uses the synthetic gpt-5 placehol journey.observations.networkEvents[0].body.model = "gpt-5"; await assertRejected(report, /synthetic placeholder model/); }); + +test("accepts the current blocked M5 aggregator non-promotion contract", async () => { + await withReport(cloneBaseAggregatorReport(), async (reportPath) => { + const result = runValidator(reportPath); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /validated 1 dev-gate report JSON file/); + }); +}); + +test("rejects aggregator DB acceptance promoted from frontend route evidence", async () => { + const report = cloneBaseAggregatorReport(); + const check = report.dod.checks.find((item) => item.id === "cloud-api-db-ready"); + Object.assign(check, { + status: "pass", + evidenceLevel: "DEV-LIVE", + summary: "Frontend route HTTP 200 proves DB ready." + }); + Object.assign(report.currentDevLayering.dbLive, { + status: "pass", + evidenceLevel: "DEV-LIVE", + summary: "Promoted from active frontend/API route evidence." + }); + await assertRejected(report, /DB DEV-LIVE acceptance source reports/); +}); + +test("rejects aggregator M3 acceptance promoted from source evidence", async () => { + const report = cloneBaseAggregatorReport(); + const sourceEvidence = report.evidence.find((item) => + item.milestone === "M3" && item.category === "hardware-loop-cardinality" + ); + sourceEvidence.level = "DEV-LIVE"; + report.levels["DEV-LIVE"].push({ + milestone: "M3", + issue: sourceEvidence.issue, + taskId: sourceEvidence.taskId, + lifecycleState: "active", + status: sourceEvidence.status, + category: sourceEvidence.category, + reportPath: sourceEvidence.reportPath, + summary: sourceEvidence.summary + }); + await assertRejected(report, /M3 DEV-LIVE evidence must come from hardware-loop-live/); +}); + +test("rejects aggregator M4 acceptance promoted from dry-run evidence", async () => { + const report = cloneBaseAggregatorReport(); + const dryRunEvidence = report.evidence.find((item) => + item.milestone === "M4" && item.category === "agent-loop-dry-run" + ); + dryRunEvidence.level = "DEV-LIVE"; + report.levels["DEV-LIVE"].push({ + milestone: "M4", + issue: dryRunEvidence.issue, + taskId: dryRunEvidence.taskId, + lifecycleState: "active", + status: dryRunEvidence.status, + category: dryRunEvidence.category, + reportPath: dryRunEvidence.reportPath, + summary: dryRunEvidence.summary + }); + await assertRejected(report, /M4 DEV-LIVE evidence must come from agent-loop-live-preflight/); +}); + +test("rejects aggregator M5 acceptance promoted from fixture evidence", async () => { + const report = cloneBaseAggregatorReport(); + const fixtureEvidence = { + milestone: "M5", + issue: "pikasTech/HWLAB#39", + taskId: "dev-mvp-gate-report", + reportPath: "reports/dev-gate/dev-mvp-gate-report.json", + commitId: "fixture", + lifecycleState: "active", + level: "DEV-LIVE", + status: "pass", + category: "fixture-mvp-e2e", + evidence: ["fixture transcript marked green"], + commands: ["node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run"], + summary: "Fixture evidence was incorrectly promoted to M5 DEV-LIVE." + }; + report.evidence.push(fixtureEvidence); + report.levels["DEV-LIVE"].push({ + milestone: fixtureEvidence.milestone, + issue: fixtureEvidence.issue, + taskId: fixtureEvidence.taskId, + lifecycleState: fixtureEvidence.lifecycleState, + status: fixtureEvidence.status, + category: fixtureEvidence.category, + reportPath: fixtureEvidence.reportPath, + summary: fixtureEvidence.summary + }); + await assertRejected(report, /M5 DEV-LIVE evidence must come from mvp-e2e-live/); +}); + +test("rejects aggregator frontend fact that claims promotion authority", async () => { + const report = cloneBaseAggregatorReport(); + report.latestFrontendDevFact.promotesM3M4M5 = true; + await assertRejected(report, /latestFrontendDevFact\.promotesM3M4M5 must be false/); +}); diff --git a/scripts/validate-dev-gate-report.mjs b/scripts/validate-dev-gate-report.mjs index eb69734a..51645a45 100644 --- a/scripts/validate-dev-gate-report.mjs +++ b/scripts/validate-dev-gate-report.mjs @@ -273,6 +273,12 @@ const requiredPreflightSupports = [ "pikasTech/HWLAB#31", "pikasTech/HWLAB#66" ]; +const protectedAggregatorLiveCategories = Object.freeze({ + M3: "hardware-loop-live", + M4: "agent-loop-live-preflight", + M5: "mvp-e2e-live" +}); +const protectedAggregatorMilestones = new Set(Object.keys(protectedAggregatorLiveCategories)); function assertObject(value, label) { assert.ok(value && typeof value === "object" && !Array.isArray(value), `${label} must be an object`); @@ -402,6 +408,10 @@ function primaryReportStatus(report) { (Array.isArray(report.blockers) && report.blockers.length > 0 ? "blocked" : "not_run"); } +function statusIsPass(value) { + return value === "pass"; +} + function validateHistoricalReport(report, label) { assert.equal(report.devOnly, true, `${label}.devOnly`); assert.equal(report.prodDisabled, true, `${label}.prodDisabled`); @@ -2177,6 +2187,241 @@ function assertBoolean(value, label) { assert.equal(typeof value, "boolean", `${label} must be a boolean`); } +async function readAggregatorSourceReports(report, label) { + const sources = {}; + for (const key of ["devEdgeHealth", "devM3Hardware", "devM4Agent", "devM5Gate"]) { + const sourceReport = report.sourceReports[key]; + const absolutePath = assertRepoRelativePath( + sourceReport.path, + `${label}.sourceReports.${key}.path` + ); + sources[key] = JSON.parse(await readFile(absolutePath, "utf8")); + } + return sources; +} + +function observedLiveIdentifier(value) { + return typeof value === "string" && + value.trim().length > 0 && + !["not_observed", "not_run", "unknown", "blocked", "missing"].includes(value); +} + +function aggregatorLiveStateFromSources(sources) { + const cloudApiDb = sources.devEdgeHealth.edgeHealth?.contracts?.deploy?.cloudApiDb ?? {}; + const dbLive = cloudApiDb.ready === true && + (cloudApiDb.connected === true || cloudApiDb.liveConnected === true) && + cloudApiDb.liveDbEvidence === true; + const m3Operation = sources.devM3Hardware.liveOperation ?? {}; + const m3Live = statusIsPass(m3Operation.status) && + ["operationId", "traceId", "auditId", "evidenceId"].every((field) => + observedLiveIdentifier(m3Operation[field]) + ); + const m4Live = statusIsPass(sources.devM4Agent.livePreflight?.status); + const m5Live = dbLive && + m3Live && + m4Live && + statusIsPass(sources.devM5Gate.devPreconditions?.status); + + return { + dbLive, + m3Live, + m4Live, + m5Live + }; +} + +function findById(items, id) { + return items.find((item) => item.id === id); +} + +function findByMilestone(items, milestone) { + return items.find((item) => item.milestone === milestone || item.id === milestone); +} + +function assertAggregatorBlockedDod(check, label, reason) { + assertObject(check, label); + assert.equal(check.status, "blocked", `${label}.status must stay blocked until ${reason}`); + assert.equal(check.evidenceLevel, "BLOCKED", `${label}.evidenceLevel must stay BLOCKED until ${reason}`); +} + +function assertAggregatorBlockedLayer(layer, label, reason) { + assertObject(layer, label); + assert.equal(layer.status, "blocked", `${label}.status must stay blocked until ${reason}`); + assert.equal(layer.evidenceLevel, "BLOCKED", `${label}.evidenceLevel must stay BLOCKED until ${reason}`); +} + +function assertAggregatorBlockedMilestone(milestone, label, reason) { + assertObject(milestone, label); + assert.equal(milestone.status, "blocked", `${label}.status must stay blocked until ${reason}`); + assert.equal(milestone.liveEvidence, "missing_or_blocked", `${label}.liveEvidence must stay missing_or_blocked until ${reason}`); + assert.notEqual(milestone.highestVisibleLevel, "DEV-LIVE", `${label}.highestVisibleLevel must not promote non-live evidence`); +} + +function assertAggregatorBlockedClassification(classification, label, reason) { + assertObject(classification, label); + assert.equal(classification.status, "blocked", `${label}.status must stay blocked until ${reason}`); + assert.equal(classification.currentLevel, "BLOCKED", `${label}.currentLevel must stay BLOCKED until ${reason}`); +} + +function assertProtectedAggregatorDevLiveEvidence(item, label, liveState, assumedLevel = item.level) { + if (!protectedAggregatorMilestones.has(item.milestone) || assumedLevel !== "DEV-LIVE") { + return; + } + + const requiredCategory = protectedAggregatorLiveCategories[item.milestone]; + assert.equal( + item.category, + requiredCategory, + `${label} ${item.milestone} DEV-LIVE evidence must come from ${requiredCategory}, not frontend, route, source, dry-run, or fixture evidence` + ); + + const sourceLive = { + M3: liveState.m3Live, + M4: liveState.m4Live, + M5: liveState.m5Live + }[item.milestone]; + assert.equal( + sourceLive, + true, + `${label} ${item.milestone} DEV-LIVE evidence must be backed by its live source report` + ); +} + +async function assertAggregatorNonPromotionBoundaries(report, label) { + const sources = await readAggregatorSourceReports(report, label); + const liveState = aggregatorLiveStateFromSources(sources); + const anyProtectedLiveBlocked = !liveState.dbLive || !liveState.m3Live || !liveState.m4Live || !liveState.m5Live; + + assertObject(report.latestFrontendDevFact, `${label}.latestFrontendDevFact`); + assert.equal( + report.latestFrontendDevFact.promotesM3M4M5, + false, + `${label}.latestFrontendDevFact.promotesM3M4M5 must be false` + ); + + if (anyProtectedLiveBlocked) { + assert.equal(report.overall.status, "blocked", `${label}.overall.status must remain blocked while DB/M3/M4/M5 live evidence is incomplete`); + assert.equal(report.overall.green, false, `${label}.overall.green must remain false while DB/M3/M4/M5 live evidence is incomplete`); + } + + if (!liveState.dbLive) { + assertAggregatorBlockedDod( + findById(report.dod.checks, "cloud-api-db-ready"), + `${label}.dod.checks.cloud-api-db-ready`, + "DB DEV-LIVE acceptance source reports prove ready=true, connected=true, and liveDbEvidence=true" + ); + if (report.currentDevLayering?.dbLive) { + assertAggregatorBlockedLayer( + report.currentDevLayering.dbLive, + `${label}.currentDevLayering.dbLive`, + "DB DEV-LIVE acceptance source reports prove ready=true, connected=true, and liveDbEvidence=true" + ); + } + } + + if (!liveState.m3Live) { + assertAggregatorBlockedDod( + findById(report.dod.checks, "m3-hardware-trusted-loop"), + `${label}.dod.checks.m3-hardware-trusted-loop`, + "M3 liveOperation proves operationId, traceId, auditId, and evidenceId" + ); + assertAggregatorBlockedMilestone( + findByMilestone(report.milestones, "M3"), + `${label}.milestones.M3`, + "M3 liveOperation proves operationId, traceId, auditId, and evidenceId" + ); + if (report.currentDevLayering?.m3HardwareTrustedLoop) { + assertAggregatorBlockedLayer( + report.currentDevLayering.m3HardwareTrustedLoop, + `${label}.currentDevLayering.m3HardwareTrustedLoop`, + "M3 liveOperation proves operationId, traceId, auditId, and evidenceId" + ); + } + if (Array.isArray(report.milestoneLevelClassification)) { + assertAggregatorBlockedClassification( + findByMilestone(report.milestoneLevelClassification, "M3"), + `${label}.milestoneLevelClassification.M3`, + "M3 liveOperation proves operationId, traceId, auditId, and evidenceId" + ); + } + if (Array.isArray(report.milestoneBlockerClassification)) { + assertAggregatorBlockedClassification( + findByMilestone(report.milestoneBlockerClassification, "M3"), + `${label}.milestoneBlockerClassification.M3`, + "M3 liveOperation proves operationId, traceId, auditId, and evidenceId" + ); + } + } + + if (!liveState.m4Live) { + assertAggregatorBlockedDod( + findById(report.dod.checks, "m4-agent-loop-live"), + `${label}.dod.checks.m4-agent-loop-live`, + "M4 livePreflight source report passes" + ); + assertAggregatorBlockedMilestone( + findByMilestone(report.milestones, "M4"), + `${label}.milestones.M4`, + "M4 livePreflight source report passes" + ); + if (report.currentDevLayering?.m4AgentLoop) { + assertAggregatorBlockedLayer( + report.currentDevLayering.m4AgentLoop, + `${label}.currentDevLayering.m4AgentLoop`, + "M4 livePreflight source report passes" + ); + } + if (Array.isArray(report.milestoneLevelClassification)) { + assertAggregatorBlockedClassification( + findByMilestone(report.milestoneLevelClassification, "M4"), + `${label}.milestoneLevelClassification.M4`, + "M4 livePreflight source report passes" + ); + } + if (Array.isArray(report.milestoneBlockerClassification)) { + assertAggregatorBlockedClassification( + findByMilestone(report.milestoneBlockerClassification, "M4"), + `${label}.milestoneBlockerClassification.M4`, + "M4 livePreflight source report passes" + ); + } + } + + if (!liveState.m5Live) { + assertAggregatorBlockedDod( + findById(report.dod.checks, "m5-mvp-dev-live"), + `${label}.dod.checks.m5-mvp-dev-live`, + "M5 source report passes with DB, M3, and M4 live evidence" + ); + assertAggregatorBlockedMilestone( + findByMilestone(report.milestones, "M5"), + `${label}.milestones.M5`, + "M5 source report passes with DB, M3, and M4 live evidence" + ); + if (Array.isArray(report.milestoneLevelClassification)) { + assertAggregatorBlockedClassification( + findByMilestone(report.milestoneLevelClassification, "M5"), + `${label}.milestoneLevelClassification.M5`, + "M5 source report passes with DB, M3, and M4 live evidence" + ); + } + if (Array.isArray(report.milestoneBlockerClassification)) { + assertAggregatorBlockedClassification( + findByMilestone(report.milestoneBlockerClassification, "M5"), + `${label}.milestoneBlockerClassification.M5`, + "M5 source report passes with DB, M3, and M4 live evidence" + ); + } + } + + for (const [index, evidence] of report.evidence.entries()) { + assertProtectedAggregatorDevLiveEvidence(evidence, `${label}.evidence[${index}]`, liveState); + } + for (const [index, evidence] of (report.levels["DEV-LIVE"] ?? []).entries()) { + assertProtectedAggregatorDevLiveEvidence(evidence, `${label}.levels.DEV-LIVE[${index}]`, liveState, "DEV-LIVE"); + } +} + async function validateAggregatorV2Report(relativePath, report) { const label = relativePath; @@ -2412,6 +2657,8 @@ async function validateAggregatorV2Report(relativePath, report) { `${label}.validationCommands missing ${requiredCommand}` ); } + + await assertAggregatorNonPromotionBoundaries(report, label); } async function main() {