test: consolidate dev gate non-promotion guards
This commit is contained in:
unidesk-code-queue-runner
@@ -9,12 +9,18 @@ import { fileURLToPath } from "node:url";
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const validator = path.join(repoRoot, "scripts/validate-dev-gate-report.mjs");
|
||||
const baseReportPath = path.join(repoRoot, "reports/dev-gate/dev-cloud-workbench-live.json");
|
||||
const baseAggregatorReportPath = path.join(repoRoot, "reports/dev-gate/dev-m5-gate-aggregator-v2.json");
|
||||
const baseReport = JSON.parse(await readFile(baseReportPath, "utf8"));
|
||||
const baseAggregatorReport = JSON.parse(await readFile(baseAggregatorReportPath, "utf8"));
|
||||
|
||||
function cloneBaseReport() {
|
||||
return JSON.parse(JSON.stringify(baseReport));
|
||||
}
|
||||
|
||||
function cloneBaseAggregatorReport() {
|
||||
return JSON.parse(JSON.stringify(baseAggregatorReport));
|
||||
}
|
||||
|
||||
async function withReport(report, fn) {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-"));
|
||||
const reportPath = path.join(directory, "report.json");
|
||||
@@ -108,3 +114,101 @@ test("rejects accepted workbench evidence that uses the synthetic gpt-5 placehol
|
||||
journey.observations.networkEvents[0].body.model = "gpt-5";
|
||||
await assertRejected(report, /synthetic placeholder model/);
|
||||
});
|
||||
|
||||
test("accepts the current blocked M5 aggregator non-promotion contract", async () => {
|
||||
await withReport(cloneBaseAggregatorReport(), async (reportPath) => {
|
||||
const result = runValidator(reportPath);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
||||
});
|
||||
});
|
||||
|
||||
test("rejects aggregator DB acceptance promoted from frontend route evidence", async () => {
|
||||
const report = cloneBaseAggregatorReport();
|
||||
const check = report.dod.checks.find((item) => item.id === "cloud-api-db-ready");
|
||||
Object.assign(check, {
|
||||
status: "pass",
|
||||
evidenceLevel: "DEV-LIVE",
|
||||
summary: "Frontend route HTTP 200 proves DB ready."
|
||||
});
|
||||
Object.assign(report.currentDevLayering.dbLive, {
|
||||
status: "pass",
|
||||
evidenceLevel: "DEV-LIVE",
|
||||
summary: "Promoted from active frontend/API route evidence."
|
||||
});
|
||||
await assertRejected(report, /DB DEV-LIVE acceptance source reports/);
|
||||
});
|
||||
|
||||
test("rejects aggregator M3 acceptance promoted from source evidence", async () => {
|
||||
const report = cloneBaseAggregatorReport();
|
||||
const sourceEvidence = report.evidence.find((item) =>
|
||||
item.milestone === "M3" && item.category === "hardware-loop-cardinality"
|
||||
);
|
||||
sourceEvidence.level = "DEV-LIVE";
|
||||
report.levels["DEV-LIVE"].push({
|
||||
milestone: "M3",
|
||||
issue: sourceEvidence.issue,
|
||||
taskId: sourceEvidence.taskId,
|
||||
lifecycleState: "active",
|
||||
status: sourceEvidence.status,
|
||||
category: sourceEvidence.category,
|
||||
reportPath: sourceEvidence.reportPath,
|
||||
summary: sourceEvidence.summary
|
||||
});
|
||||
await assertRejected(report, /M3 DEV-LIVE evidence must come from hardware-loop-live/);
|
||||
});
|
||||
|
||||
test("rejects aggregator M4 acceptance promoted from dry-run evidence", async () => {
|
||||
const report = cloneBaseAggregatorReport();
|
||||
const dryRunEvidence = report.evidence.find((item) =>
|
||||
item.milestone === "M4" && item.category === "agent-loop-dry-run"
|
||||
);
|
||||
dryRunEvidence.level = "DEV-LIVE";
|
||||
report.levels["DEV-LIVE"].push({
|
||||
milestone: "M4",
|
||||
issue: dryRunEvidence.issue,
|
||||
taskId: dryRunEvidence.taskId,
|
||||
lifecycleState: "active",
|
||||
status: dryRunEvidence.status,
|
||||
category: dryRunEvidence.category,
|
||||
reportPath: dryRunEvidence.reportPath,
|
||||
summary: dryRunEvidence.summary
|
||||
});
|
||||
await assertRejected(report, /M4 DEV-LIVE evidence must come from agent-loop-live-preflight/);
|
||||
});
|
||||
|
||||
test("rejects aggregator M5 acceptance promoted from fixture evidence", async () => {
|
||||
const report = cloneBaseAggregatorReport();
|
||||
const fixtureEvidence = {
|
||||
milestone: "M5",
|
||||
issue: "pikasTech/HWLAB#39",
|
||||
taskId: "dev-mvp-gate-report",
|
||||
reportPath: "reports/dev-gate/dev-mvp-gate-report.json",
|
||||
commitId: "fixture",
|
||||
lifecycleState: "active",
|
||||
level: "DEV-LIVE",
|
||||
status: "pass",
|
||||
category: "fixture-mvp-e2e",
|
||||
evidence: ["fixture transcript marked green"],
|
||||
commands: ["node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run"],
|
||||
summary: "Fixture evidence was incorrectly promoted to M5 DEV-LIVE."
|
||||
};
|
||||
report.evidence.push(fixtureEvidence);
|
||||
report.levels["DEV-LIVE"].push({
|
||||
milestone: fixtureEvidence.milestone,
|
||||
issue: fixtureEvidence.issue,
|
||||
taskId: fixtureEvidence.taskId,
|
||||
lifecycleState: fixtureEvidence.lifecycleState,
|
||||
status: fixtureEvidence.status,
|
||||
category: fixtureEvidence.category,
|
||||
reportPath: fixtureEvidence.reportPath,
|
||||
summary: fixtureEvidence.summary
|
||||
});
|
||||
await assertRejected(report, /M5 DEV-LIVE evidence must come from mvp-e2e-live/);
|
||||
});
|
||||
|
||||
test("rejects aggregator frontend fact that claims promotion authority", async () => {
|
||||
const report = cloneBaseAggregatorReport();
|
||||
report.latestFrontendDevFact.promotesM3M4M5 = true;
|
||||
await assertRejected(report, /latestFrontendDevFact\.promotesM3M4M5 must be false/);
|
||||
});
|
||||
|
||||
@@ -273,6 +273,12 @@ const requiredPreflightSupports = [
|
||||
"pikasTech/HWLAB#31",
|
||||
"pikasTech/HWLAB#66"
|
||||
];
|
||||
const protectedAggregatorLiveCategories = Object.freeze({
|
||||
M3: "hardware-loop-live",
|
||||
M4: "agent-loop-live-preflight",
|
||||
M5: "mvp-e2e-live"
|
||||
});
|
||||
const protectedAggregatorMilestones = new Set(Object.keys(protectedAggregatorLiveCategories));
|
||||
|
||||
function assertObject(value, label) {
|
||||
assert.ok(value && typeof value === "object" && !Array.isArray(value), `${label} must be an object`);
|
||||
@@ -402,6 +408,10 @@ function primaryReportStatus(report) {
|
||||
(Array.isArray(report.blockers) && report.blockers.length > 0 ? "blocked" : "not_run");
|
||||
}
|
||||
|
||||
function statusIsPass(value) {
|
||||
return value === "pass";
|
||||
}
|
||||
|
||||
function validateHistoricalReport(report, label) {
|
||||
assert.equal(report.devOnly, true, `${label}.devOnly`);
|
||||
assert.equal(report.prodDisabled, true, `${label}.prodDisabled`);
|
||||
@@ -2177,6 +2187,241 @@ function assertBoolean(value, label) {
|
||||
assert.equal(typeof value, "boolean", `${label} must be a boolean`);
|
||||
}
|
||||
|
||||
async function readAggregatorSourceReports(report, label) {
|
||||
const sources = {};
|
||||
for (const key of ["devEdgeHealth", "devM3Hardware", "devM4Agent", "devM5Gate"]) {
|
||||
const sourceReport = report.sourceReports[key];
|
||||
const absolutePath = assertRepoRelativePath(
|
||||
sourceReport.path,
|
||||
`${label}.sourceReports.${key}.path`
|
||||
);
|
||||
sources[key] = JSON.parse(await readFile(absolutePath, "utf8"));
|
||||
}
|
||||
return sources;
|
||||
}
|
||||
|
||||
function observedLiveIdentifier(value) {
|
||||
return typeof value === "string" &&
|
||||
value.trim().length > 0 &&
|
||||
!["not_observed", "not_run", "unknown", "blocked", "missing"].includes(value);
|
||||
}
|
||||
|
||||
function aggregatorLiveStateFromSources(sources) {
|
||||
const cloudApiDb = sources.devEdgeHealth.edgeHealth?.contracts?.deploy?.cloudApiDb ?? {};
|
||||
const dbLive = cloudApiDb.ready === true &&
|
||||
(cloudApiDb.connected === true || cloudApiDb.liveConnected === true) &&
|
||||
cloudApiDb.liveDbEvidence === true;
|
||||
const m3Operation = sources.devM3Hardware.liveOperation ?? {};
|
||||
const m3Live = statusIsPass(m3Operation.status) &&
|
||||
["operationId", "traceId", "auditId", "evidenceId"].every((field) =>
|
||||
observedLiveIdentifier(m3Operation[field])
|
||||
);
|
||||
const m4Live = statusIsPass(sources.devM4Agent.livePreflight?.status);
|
||||
const m5Live = dbLive &&
|
||||
m3Live &&
|
||||
m4Live &&
|
||||
statusIsPass(sources.devM5Gate.devPreconditions?.status);
|
||||
|
||||
return {
|
||||
dbLive,
|
||||
m3Live,
|
||||
m4Live,
|
||||
m5Live
|
||||
};
|
||||
}
|
||||
|
||||
function findById(items, id) {
|
||||
return items.find((item) => item.id === id);
|
||||
}
|
||||
|
||||
function findByMilestone(items, milestone) {
|
||||
return items.find((item) => item.milestone === milestone || item.id === milestone);
|
||||
}
|
||||
|
||||
function assertAggregatorBlockedDod(check, label, reason) {
|
||||
assertObject(check, label);
|
||||
assert.equal(check.status, "blocked", `${label}.status must stay blocked until ${reason}`);
|
||||
assert.equal(check.evidenceLevel, "BLOCKED", `${label}.evidenceLevel must stay BLOCKED until ${reason}`);
|
||||
}
|
||||
|
||||
function assertAggregatorBlockedLayer(layer, label, reason) {
|
||||
assertObject(layer, label);
|
||||
assert.equal(layer.status, "blocked", `${label}.status must stay blocked until ${reason}`);
|
||||
assert.equal(layer.evidenceLevel, "BLOCKED", `${label}.evidenceLevel must stay BLOCKED until ${reason}`);
|
||||
}
|
||||
|
||||
function assertAggregatorBlockedMilestone(milestone, label, reason) {
|
||||
assertObject(milestone, label);
|
||||
assert.equal(milestone.status, "blocked", `${label}.status must stay blocked until ${reason}`);
|
||||
assert.equal(milestone.liveEvidence, "missing_or_blocked", `${label}.liveEvidence must stay missing_or_blocked until ${reason}`);
|
||||
assert.notEqual(milestone.highestVisibleLevel, "DEV-LIVE", `${label}.highestVisibleLevel must not promote non-live evidence`);
|
||||
}
|
||||
|
||||
function assertAggregatorBlockedClassification(classification, label, reason) {
|
||||
assertObject(classification, label);
|
||||
assert.equal(classification.status, "blocked", `${label}.status must stay blocked until ${reason}`);
|
||||
assert.equal(classification.currentLevel, "BLOCKED", `${label}.currentLevel must stay BLOCKED until ${reason}`);
|
||||
}
|
||||
|
||||
function assertProtectedAggregatorDevLiveEvidence(item, label, liveState, assumedLevel = item.level) {
|
||||
if (!protectedAggregatorMilestones.has(item.milestone) || assumedLevel !== "DEV-LIVE") {
|
||||
return;
|
||||
}
|
||||
|
||||
const requiredCategory = protectedAggregatorLiveCategories[item.milestone];
|
||||
assert.equal(
|
||||
item.category,
|
||||
requiredCategory,
|
||||
`${label} ${item.milestone} DEV-LIVE evidence must come from ${requiredCategory}, not frontend, route, source, dry-run, or fixture evidence`
|
||||
);
|
||||
|
||||
const sourceLive = {
|
||||
M3: liveState.m3Live,
|
||||
M4: liveState.m4Live,
|
||||
M5: liveState.m5Live
|
||||
}[item.milestone];
|
||||
assert.equal(
|
||||
sourceLive,
|
||||
true,
|
||||
`${label} ${item.milestone} DEV-LIVE evidence must be backed by its live source report`
|
||||
);
|
||||
}
|
||||
|
||||
async function assertAggregatorNonPromotionBoundaries(report, label) {
|
||||
const sources = await readAggregatorSourceReports(report, label);
|
||||
const liveState = aggregatorLiveStateFromSources(sources);
|
||||
const anyProtectedLiveBlocked = !liveState.dbLive || !liveState.m3Live || !liveState.m4Live || !liveState.m5Live;
|
||||
|
||||
assertObject(report.latestFrontendDevFact, `${label}.latestFrontendDevFact`);
|
||||
assert.equal(
|
||||
report.latestFrontendDevFact.promotesM3M4M5,
|
||||
false,
|
||||
`${label}.latestFrontendDevFact.promotesM3M4M5 must be false`
|
||||
);
|
||||
|
||||
if (anyProtectedLiveBlocked) {
|
||||
assert.equal(report.overall.status, "blocked", `${label}.overall.status must remain blocked while DB/M3/M4/M5 live evidence is incomplete`);
|
||||
assert.equal(report.overall.green, false, `${label}.overall.green must remain false while DB/M3/M4/M5 live evidence is incomplete`);
|
||||
}
|
||||
|
||||
if (!liveState.dbLive) {
|
||||
assertAggregatorBlockedDod(
|
||||
findById(report.dod.checks, "cloud-api-db-ready"),
|
||||
`${label}.dod.checks.cloud-api-db-ready`,
|
||||
"DB DEV-LIVE acceptance source reports prove ready=true, connected=true, and liveDbEvidence=true"
|
||||
);
|
||||
if (report.currentDevLayering?.dbLive) {
|
||||
assertAggregatorBlockedLayer(
|
||||
report.currentDevLayering.dbLive,
|
||||
`${label}.currentDevLayering.dbLive`,
|
||||
"DB DEV-LIVE acceptance source reports prove ready=true, connected=true, and liveDbEvidence=true"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (!liveState.m3Live) {
|
||||
assertAggregatorBlockedDod(
|
||||
findById(report.dod.checks, "m3-hardware-trusted-loop"),
|
||||
`${label}.dod.checks.m3-hardware-trusted-loop`,
|
||||
"M3 liveOperation proves operationId, traceId, auditId, and evidenceId"
|
||||
);
|
||||
assertAggregatorBlockedMilestone(
|
||||
findByMilestone(report.milestones, "M3"),
|
||||
`${label}.milestones.M3`,
|
||||
"M3 liveOperation proves operationId, traceId, auditId, and evidenceId"
|
||||
);
|
||||
if (report.currentDevLayering?.m3HardwareTrustedLoop) {
|
||||
assertAggregatorBlockedLayer(
|
||||
report.currentDevLayering.m3HardwareTrustedLoop,
|
||||
`${label}.currentDevLayering.m3HardwareTrustedLoop`,
|
||||
"M3 liveOperation proves operationId, traceId, auditId, and evidenceId"
|
||||
);
|
||||
}
|
||||
if (Array.isArray(report.milestoneLevelClassification)) {
|
||||
assertAggregatorBlockedClassification(
|
||||
findByMilestone(report.milestoneLevelClassification, "M3"),
|
||||
`${label}.milestoneLevelClassification.M3`,
|
||||
"M3 liveOperation proves operationId, traceId, auditId, and evidenceId"
|
||||
);
|
||||
}
|
||||
if (Array.isArray(report.milestoneBlockerClassification)) {
|
||||
assertAggregatorBlockedClassification(
|
||||
findByMilestone(report.milestoneBlockerClassification, "M3"),
|
||||
`${label}.milestoneBlockerClassification.M3`,
|
||||
"M3 liveOperation proves operationId, traceId, auditId, and evidenceId"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (!liveState.m4Live) {
|
||||
assertAggregatorBlockedDod(
|
||||
findById(report.dod.checks, "m4-agent-loop-live"),
|
||||
`${label}.dod.checks.m4-agent-loop-live`,
|
||||
"M4 livePreflight source report passes"
|
||||
);
|
||||
assertAggregatorBlockedMilestone(
|
||||
findByMilestone(report.milestones, "M4"),
|
||||
`${label}.milestones.M4`,
|
||||
"M4 livePreflight source report passes"
|
||||
);
|
||||
if (report.currentDevLayering?.m4AgentLoop) {
|
||||
assertAggregatorBlockedLayer(
|
||||
report.currentDevLayering.m4AgentLoop,
|
||||
`${label}.currentDevLayering.m4AgentLoop`,
|
||||
"M4 livePreflight source report passes"
|
||||
);
|
||||
}
|
||||
if (Array.isArray(report.milestoneLevelClassification)) {
|
||||
assertAggregatorBlockedClassification(
|
||||
findByMilestone(report.milestoneLevelClassification, "M4"),
|
||||
`${label}.milestoneLevelClassification.M4`,
|
||||
"M4 livePreflight source report passes"
|
||||
);
|
||||
}
|
||||
if (Array.isArray(report.milestoneBlockerClassification)) {
|
||||
assertAggregatorBlockedClassification(
|
||||
findByMilestone(report.milestoneBlockerClassification, "M4"),
|
||||
`${label}.milestoneBlockerClassification.M4`,
|
||||
"M4 livePreflight source report passes"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (!liveState.m5Live) {
|
||||
assertAggregatorBlockedDod(
|
||||
findById(report.dod.checks, "m5-mvp-dev-live"),
|
||||
`${label}.dod.checks.m5-mvp-dev-live`,
|
||||
"M5 source report passes with DB, M3, and M4 live evidence"
|
||||
);
|
||||
assertAggregatorBlockedMilestone(
|
||||
findByMilestone(report.milestones, "M5"),
|
||||
`${label}.milestones.M5`,
|
||||
"M5 source report passes with DB, M3, and M4 live evidence"
|
||||
);
|
||||
if (Array.isArray(report.milestoneLevelClassification)) {
|
||||
assertAggregatorBlockedClassification(
|
||||
findByMilestone(report.milestoneLevelClassification, "M5"),
|
||||
`${label}.milestoneLevelClassification.M5`,
|
||||
"M5 source report passes with DB, M3, and M4 live evidence"
|
||||
);
|
||||
}
|
||||
if (Array.isArray(report.milestoneBlockerClassification)) {
|
||||
assertAggregatorBlockedClassification(
|
||||
findByMilestone(report.milestoneBlockerClassification, "M5"),
|
||||
`${label}.milestoneBlockerClassification.M5`,
|
||||
"M5 source report passes with DB, M3, and M4 live evidence"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [index, evidence] of report.evidence.entries()) {
|
||||
assertProtectedAggregatorDevLiveEvidence(evidence, `${label}.evidence[${index}]`, liveState);
|
||||
}
|
||||
for (const [index, evidence] of (report.levels["DEV-LIVE"] ?? []).entries()) {
|
||||
assertProtectedAggregatorDevLiveEvidence(evidence, `${label}.levels.DEV-LIVE[${index}]`, liveState, "DEV-LIVE");
|
||||
}
|
||||
}
|
||||
|
||||
async function validateAggregatorV2Report(relativePath, report) {
|
||||
const label = relativePath;
|
||||
|
||||
@@ -2412,6 +2657,8 @@ async function validateAggregatorV2Report(relativePath, report) {
|
||||
`${label}.validationCommands missing ${requiredCommand}`
|
||||
);
|
||||
}
|
||||
|
||||
await assertAggregatorNonPromotionBoundaries(report, label);
|
||||
}
|
||||
|
||||
async function main() {
|
||||
|
||||
Reference in New Issue
Block a user