fix: harden dev cd runtime postflight
Summary: - shared runtime durable readiness classifier for live-status and DEV postflight - structured postflight blockers propagated through dev-cd apply - contract coverage for ready queryResult, auth/schema/migration blockers, evidence persistence blockers, and bounded stdout Verification: - git diff --check - node --check changed scripts/tests/live-status modules - node --test scripts/src/dev-runtime-postflight.test.mjs scripts/src/dev-cd-apply.test.mjs web/hwlab-cloud-web/scripts/live-status-contract.test.mjs - node scripts/dev-runtime-postflight.mjs --check - node web/hwlab-cloud-web/scripts/check.mjs - npm run check No DEV apply/rollout/live health verification was run.
This commit is contained in:
@@ -1815,11 +1815,55 @@ function parseLastJsonObject(text) {
|
||||
}
|
||||
|
||||
function stepBlocker(stepResult) {
|
||||
return {
|
||||
const stdoutBlockers = Array.isArray(stepResult.stdoutJson?.blockers) ? stepResult.stdoutJson.blockers : [];
|
||||
const primary = stdoutBlockers.find((item) => item?.status === "open") ?? stdoutBlockers[0] ?? null;
|
||||
const reason = primary?.reason ?? primary?.summary ?? `${stepResult.command} exited ${stepResult.code}.`;
|
||||
const impact = primary?.impact ?? stepBlockerImpact(stepResult);
|
||||
const safeNextAction = primary?.safeNextAction ?? stepBlockerSafeNextAction(stepResult);
|
||||
return structuredBlocker({
|
||||
type: "runtime_blocker",
|
||||
scope: stepResult.id,
|
||||
reason,
|
||||
impact,
|
||||
safeNextAction,
|
||||
retryable: primary?.retryable ?? true,
|
||||
evidence: {
|
||||
command: stepResult.command,
|
||||
code: stepResult.code,
|
||||
reportPath: stepResult.reportPath ?? null,
|
||||
blockerScope: primary?.scope ?? null,
|
||||
k8sJob: stepResult.k8sJob ?? null
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function stepBlockerImpact(stepResult) {
|
||||
if (stepResult.id === "runtime-db-provisioning") return "DEV CD cannot prove DB role/database provisioning before workload apply.";
|
||||
if (stepResult.id === "runtime-db-migration") return "DEV CD cannot prove runtime schema migration before workload apply.";
|
||||
if (stepResult.id === "runtime-durable-postflight") return "DEV CD cannot prove /health/live, /v1, and M3 durable evidence persistence after apply.";
|
||||
if (stepResult.id === "dev-deploy-apply") return "DEV desired-state apply did not complete, so live readiness cannot be trusted.";
|
||||
return "DEV CD transaction cannot safely continue.";
|
||||
}
|
||||
|
||||
function stepBlockerSafeNextAction(stepResult) {
|
||||
if (stepResult.id === "runtime-db-provisioning") return "Repair repo-owned runtime provisioning inputs/SecretRefs without printing Secret values, then rerun DEV CD.";
|
||||
if (stepResult.id === "runtime-db-migration") return "Repair repo-owned runtime migration/schema state, then rerun DEV CD.";
|
||||
if (stepResult.id === "runtime-durable-postflight") return "Inspect the runtime postflight report, repair durable readiness or M3 evidence persistence, then rerun DEV CD after host-controlled rollout if needed.";
|
||||
if (stepResult.id === "dev-deploy-apply") return "Inspect reports/dev-gate/dev-deploy-report.json and rerun the repo-owned DEV CD path after the apply blocker is fixed.";
|
||||
return "Fix the reported step blocker and rerun the repo-owned DEV CD transaction.";
|
||||
}
|
||||
|
||||
function structuredBlocker({ type = "runtime_blocker", scope, reason, impact, safeNextAction, retryable = true, evidence = null }) {
|
||||
return {
|
||||
type,
|
||||
scope,
|
||||
status: "open",
|
||||
summary: `${stepResult.command} exited ${stepResult.code}.`
|
||||
reason: oneLine(reason),
|
||||
impact: oneLine(impact),
|
||||
safeNextAction: oneLine(safeNextAction),
|
||||
retryable: Boolean(retryable),
|
||||
summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`),
|
||||
...(evidence ? { evidence } : {})
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -215,6 +215,7 @@ function makeRunCommand({
|
||||
],
|
||||
desiredStatePromotionStatus = "pass",
|
||||
runtimeJobLogSuffix = "",
|
||||
runtimePostflightReport = null,
|
||||
kubeContext = "d601",
|
||||
kubeServer = "https://d601:6443",
|
||||
kubeNodes = ["d601"],
|
||||
@@ -390,6 +391,24 @@ function makeRunCommand({
|
||||
};
|
||||
}
|
||||
if (command === process.execPath || command.endsWith("/node")) {
|
||||
if (args.includes("scripts/dev-runtime-postflight.mjs")) {
|
||||
const report = runtimePostflightReport ?? {
|
||||
conclusion: { status: "pass" },
|
||||
blockers: [],
|
||||
safety: { secretValuesPrinted: false },
|
||||
apiHealth: { runtimeDurableReadiness: { ready: true } },
|
||||
apiV1: { runtimeDurableReadiness: { ready: true } },
|
||||
m3: {
|
||||
persistenceContract: { ready: true },
|
||||
operationCount: 4
|
||||
}
|
||||
};
|
||||
return {
|
||||
code: report.conclusion?.status === "pass" ? 0 : 1,
|
||||
stdout: `${JSON.stringify(report, null, 2)}\n`,
|
||||
stderr: ""
|
||||
};
|
||||
}
|
||||
return { code: 0, stdout: JSON.stringify({ ok: true, command: args.join(" ") }), stderr: "" };
|
||||
}
|
||||
return { code: 0, stdout: "", stderr: "" };
|
||||
@@ -1294,6 +1313,93 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a
|
||||
assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2);
|
||||
});
|
||||
|
||||
test("transaction blocks with structured runtime postflight blocker when M3 evidence is not persisted", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const runtimePostflightReport = {
|
||||
conclusion: {
|
||||
status: "blocked",
|
||||
blockerCount: 1
|
||||
},
|
||||
apiHealth: {
|
||||
runtimeDurableReadiness: {
|
||||
ready: true,
|
||||
classification: "runtime_durable_ready"
|
||||
}
|
||||
},
|
||||
apiV1: {
|
||||
runtimeDurableReadiness: {
|
||||
ready: true,
|
||||
classification: "runtime_durable_ready"
|
||||
}
|
||||
},
|
||||
m3: {
|
||||
persistenceContract: {
|
||||
ready: false,
|
||||
sequence: [
|
||||
{ id: "write-do1-true", persisted: true, idsPresent: true },
|
||||
{ id: "read-di1-true", persisted: false, idsPresent: true }
|
||||
]
|
||||
},
|
||||
operationCount: 4
|
||||
},
|
||||
blockers: [
|
||||
{
|
||||
type: "runtime_blocker",
|
||||
scope: "m3-durable-postflight",
|
||||
status: "open",
|
||||
reason: "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.",
|
||||
impact: "A later rollout could appear ready while losing evidence persistence.",
|
||||
safeNextAction: "Repair durable audit/evidence persistence, then rerun DEV runtime postflight.",
|
||||
retryable: true
|
||||
}
|
||||
],
|
||||
safety: {
|
||||
secretValuesPrinted: false,
|
||||
readsKubernetesSecrets: false
|
||||
}
|
||||
};
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-postflight-blocked",
|
||||
"--kubeconfig",
|
||||
"/etc/rancher/k3s/k3s.yaml",
|
||||
"--write-report"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({ commandLog, runtimePostflightReport }),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const summary = JSON.parse(output);
|
||||
const report = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
|
||||
const blocker = report.blockers.find((item) => item.scope === "runtime-durable-postflight");
|
||||
assert.equal(code, 2);
|
||||
assert.equal(summary.status, "blocked");
|
||||
assert.equal(summary.devCdApply, undefined);
|
||||
assert.equal(output.length < 12000, true);
|
||||
assert.equal(report.status, "blocked");
|
||||
assert.equal(report.devCdApply.steps.at(-1).id, "runtime-durable-postflight");
|
||||
assert.equal(report.devCdApply.steps.at(-1).status, "blocked");
|
||||
assert.ok(blocker);
|
||||
assert.equal(blocker.reason, runtimePostflightReport.blockers[0].reason);
|
||||
assert.equal(blocker.impact, runtimePostflightReport.blockers[0].impact);
|
||||
assert.equal(blocker.safeNextAction, runtimePostflightReport.blockers[0].safeNextAction);
|
||||
assert.equal(blocker.retryable, true);
|
||||
assert.equal(blocker.evidence.blockerScope, "m3-durable-postflight");
|
||||
assert.equal(report.devCdApply.liveVerify.status, "not_run");
|
||||
assert.equal(JSON.stringify(report).includes("postgresql://"), false);
|
||||
assert.equal(JSON.stringify(report).includes("SECRET"), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-runtime-postflight.mjs")), true);
|
||||
});
|
||||
|
||||
test("runtime maintenance job reports are parsed after redacting secret-like log output", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
|
||||
@@ -4,6 +4,7 @@ import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
||||
import { classifyRuntimeDurableReadiness } from "../../web/hwlab-cloud-web/live-status.mjs";
|
||||
import {
|
||||
buildM3IoControlSourceReport,
|
||||
runM3IoControlLiveReport
|
||||
@@ -78,8 +79,16 @@ export async function buildDevRuntimePostflightReport(args = {}, options = {}) {
|
||||
blockerCount: sourceReport.summary?.status === "pass" ? 0 : 1
|
||||
};
|
||||
if (sourceReport.summary?.status !== "pass") {
|
||||
addBlocker(report, "contract_blocker", "m3-source-contract", "M3 postflight source contract is blocked.", {
|
||||
classification: sourceReport.summary?.classification ?? "unknown"
|
||||
addBlocker(report, {
|
||||
type: "contract_blocker",
|
||||
scope: "m3-source-contract",
|
||||
reason: "M3 postflight source contract is blocked.",
|
||||
impact: "DEV CD cannot rely on the M3 durable postflight harness contract.",
|
||||
safeNextAction: "Fix the source M3 IO control contract tests before running live DEV postflight.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
classification: sourceReport.summary?.classification ?? "unknown"
|
||||
}
|
||||
});
|
||||
}
|
||||
return report;
|
||||
@@ -227,6 +236,7 @@ async function defaultHttpGetJson(url, timeoutMs) {
|
||||
function summarizeApiPayload(check, response) {
|
||||
const json = response.json ?? {};
|
||||
const runtime = summarizeRuntime(json.runtime);
|
||||
const runtimeDurableReadiness = classifyRuntimeDurableReadiness(json, { requirePostgresAdapter: true });
|
||||
return {
|
||||
check,
|
||||
httpStatus: response.status,
|
||||
@@ -236,6 +246,7 @@ function summarizeApiPayload(check, response) {
|
||||
status: json.status ?? null,
|
||||
ready: json.ready === true,
|
||||
runtime,
|
||||
runtimeDurableReadiness,
|
||||
db: summarizeDb(json.db),
|
||||
readiness: {
|
||||
ready: json.readiness?.ready === true,
|
||||
@@ -372,23 +383,48 @@ function summarizeM3PersistenceContract(operations = []) {
|
||||
function addApiReadinessBlockers(report) {
|
||||
for (const [scope, payload] of [["api-health-live", report.apiHealth], ["api-v1", report.apiV1]]) {
|
||||
if (!payload.reachable) {
|
||||
addBlocker(report, "runtime_blocker", scope, `${payload.check} was not reachable.`, {
|
||||
httpStatus: payload.httpStatus,
|
||||
error: payload.error
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope,
|
||||
reason: `${payload.check} was not reachable.`,
|
||||
impact: "DEV CD cannot prove cloud-api durable readiness, so M3 evidence persistence postflight cannot safely run.",
|
||||
safeNextAction: "Restore DEV cloud-api reachability on the repo-owned public endpoint, then rerun postflight.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
httpStatus: payload.httpStatus,
|
||||
error: payload.error
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (!runtimeDurableReady(payload.runtime)) {
|
||||
addBlocker(report, "runtime_blocker", scope, `${payload.check} did not prove durable runtime readiness.`, {
|
||||
runtime: payload.runtime,
|
||||
blockerCodes: payload.blockerCodes
|
||||
if (payload.runtimeDurableReadiness?.ready !== true) {
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope,
|
||||
reason: `${payload.check} did not prove durable runtime readiness: ${payload.runtimeDurableReadiness?.reason ?? "not ready"}.`,
|
||||
impact: payload.runtimeDurableReadiness?.impact ?? "M3 durable postflight cannot prove evidence persistence.",
|
||||
safeNextAction: payload.runtimeDurableReadiness?.safeNextAction ?? "Repair runtime durability through repo-owned DEV CD steps, then rerun postflight.",
|
||||
retryable: payload.runtimeDurableReadiness?.retryable !== false,
|
||||
evidence: {
|
||||
runtime: payload.runtime,
|
||||
runtimeDurableReadiness: payload.runtimeDurableReadiness,
|
||||
blockerCodes: payload.blockerCodes
|
||||
}
|
||||
});
|
||||
}
|
||||
if (scope === "api-v1" && !apiV1M3ContractReady(payload)) {
|
||||
addBlocker(report, "runtime_blocker", scope, "GET /v1 did not expose the controlled M3 IO route required by durable postflight.", {
|
||||
m3IoControl: payload.m3IoControl,
|
||||
requiredRoute: "/v1/m3/io",
|
||||
requiredContractVersion: "m3-io-control-v1"
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope,
|
||||
reason: "GET /v1 did not expose the controlled M3 IO route required by durable postflight.",
|
||||
impact: "DEV CD cannot run the M3 true/false IO persistence smoke through the cloud-api controlled boundary.",
|
||||
safeNextAction: "Restore /v1 m3IoControl route metadata for /v1/m3/io without exposing direct frontend simulator access.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
m3IoControl: payload.m3IoControl,
|
||||
requiredRoute: "/v1/m3/io",
|
||||
requiredContractVersion: "m3-io-control-v1"
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -406,58 +442,96 @@ function apiV1M3ContractReady(payload) {
|
||||
|
||||
function addM3Blockers(report) {
|
||||
if (report.m3?.persistenceContract?.ready !== true) {
|
||||
addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.", {
|
||||
persistenceContract: report.m3?.persistenceContract ?? null,
|
||||
trustedGreen: report.m3?.trustedGreen === true,
|
||||
operationCount: report.m3?.operationCount ?? 0
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope: "m3-durable-postflight",
|
||||
reason: "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.",
|
||||
impact: "A later rollout could appear ready while losing operation, audit, or evidence persistence for M3 hardware IO.",
|
||||
safeNextAction: "Inspect the failed M3 operation sequence, repair durable audit/evidence persistence, then rerun DEV runtime postflight.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
persistenceContract: report.m3?.persistenceContract ?? null,
|
||||
trustedGreen: report.m3?.trustedGreen === true,
|
||||
operationCount: report.m3?.operationCount ?? 0
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (report.m3?.trustedGreen !== true || report.m3?.status !== "pass") {
|
||||
addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 true/false durable postflight did not produce trusted green persisted evidence.", {
|
||||
classification: report.m3?.classification ?? "unknown",
|
||||
trustedGreen: report.m3?.trustedGreen === true,
|
||||
operationCount: report.m3?.operationCount ?? 0
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope: "m3-durable-postflight",
|
||||
reason: "M3 true/false durable postflight did not produce trusted green persisted evidence.",
|
||||
impact: "M3 durable readiness cannot be accepted because trusted evidence is not green.",
|
||||
safeNextAction: "Fix the M3 control path or trusted-record persistence and rerun DEV runtime postflight.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
classification: report.m3?.classification ?? "unknown",
|
||||
trustedGreen: report.m3?.trustedGreen === true,
|
||||
operationCount: report.m3?.operationCount ?? 0
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
for (const operation of report.m3.operations ?? []) {
|
||||
if (!operation.operationId || !operation.auditId || !operation.evidenceId || operation.evidenceState?.status !== "green" || operation.evidenceState?.durable !== true) {
|
||||
addBlocker(report, "runtime_blocker", "m3-durable-evidence", "M3 operation/audit/evidence durable fields were incomplete.", {
|
||||
operationIdPresent: Boolean(operation.operationId),
|
||||
auditIdPresent: Boolean(operation.auditId),
|
||||
evidenceIdPresent: Boolean(operation.evidenceId),
|
||||
evidenceState: operation.evidenceState
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope: "m3-durable-evidence",
|
||||
reason: "M3 operation/audit/evidence durable fields were incomplete.",
|
||||
impact: "The M3 postflight cannot prove every IO operation is bound to persisted audit and evidence records.",
|
||||
safeNextAction: "Repair operation, audit, and evidence ID propagation before accepting DEV readiness.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
operationIdPresent: Boolean(operation.operationId),
|
||||
traceIdPresent: Boolean(operation.traceId),
|
||||
auditIdPresent: Boolean(operation.auditId),
|
||||
evidenceIdPresent: Boolean(operation.evidenceId),
|
||||
evidenceState: operation.evidenceState
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runtimeDurableReady(runtime) {
|
||||
return runtime.adapter === "postgres" &&
|
||||
runtime.durable === true &&
|
||||
runtime.ready === true &&
|
||||
runtime.liveRuntimeEvidence === true;
|
||||
}
|
||||
|
||||
function addSafetyRefusal(report, summary) {
|
||||
report.safetyRefusal = true;
|
||||
addBlocker(report, "safety_refusal", "runtime-postflight-live-boundary", summary, {
|
||||
devOnly: true,
|
||||
prodAllowed: false,
|
||||
secretValuesPrinted: false
|
||||
addBlocker(report, {
|
||||
type: "safety_refusal",
|
||||
scope: "runtime-postflight-live-boundary",
|
||||
reason: summary,
|
||||
impact: "Live DEV reads and M3 writes are not allowed without explicit non-production confirmation.",
|
||||
safeNextAction: "Rerun with --live --confirm-dev --confirmed-non-production for DEV only, or use --check for source-only validation.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
devOnly: true,
|
||||
prodAllowed: false,
|
||||
secretValuesPrinted: false
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function addBlocker(report, type, scope, summary, evidence = {}) {
|
||||
function addBlocker(report, {
|
||||
type,
|
||||
scope,
|
||||
reason,
|
||||
impact,
|
||||
safeNextAction,
|
||||
retryable,
|
||||
evidence = {}
|
||||
}) {
|
||||
report.blockers.push({
|
||||
type,
|
||||
scope,
|
||||
status: "open",
|
||||
summary,
|
||||
reason: oneLine(reason),
|
||||
impact: oneLine(impact),
|
||||
safeNextAction: oneLine(safeNextAction),
|
||||
retryable: Boolean(retryable),
|
||||
summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`),
|
||||
sourceIssue: issue,
|
||||
evidence
|
||||
});
|
||||
@@ -475,6 +549,10 @@ function finalizeReport(report) {
|
||||
return report;
|
||||
}
|
||||
|
||||
function oneLine(value) {
|
||||
return String(value ?? "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function buildSafety(args) {
|
||||
return {
|
||||
devOnly: true,
|
||||
|
||||
@@ -56,6 +56,8 @@ test("live mode skips M3 writes when /health/live durable runtime is blocked", a
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
assert.equal(report.m3.status, "not_run");
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-health-live"), true);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
assert.match(report.blockers.find((blocker) => blocker.scope === "api-health-live").reason, /auth_blocked/u);
|
||||
assert.equal(JSON.stringify(report).includes("postgresql://"), false);
|
||||
});
|
||||
|
||||
@@ -74,6 +76,7 @@ test("live mode requires /v1 durable readiness before M3 writes", async () => {
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-v1"), true);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
});
|
||||
|
||||
test("live mode requires /v1 to expose the controlled M3 IO route before M3 writes", async () => {
|
||||
@@ -100,6 +103,7 @@ test("live mode requires /v1 to expose the controlled M3 IO route before M3 writ
|
||||
blocker.scope === "api-v1" &&
|
||||
blocker.summary.includes("controlled M3 IO route")
|
||||
), true);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
});
|
||||
|
||||
test("live mode does not mistake DB live evidence for durable runtime readiness", async () => {
|
||||
@@ -121,6 +125,51 @@ test("live mode does not mistake DB live evidence for durable runtime readiness"
|
||||
assert.equal(report.apiHealth.readiness.durabilityReady, false);
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-health-live"), true);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
});
|
||||
|
||||
test("live mode blocks auth, schema, and migration durable readiness before M3 writes", async () => {
|
||||
for (const [blocker, queryResult, blockedLayer] of [
|
||||
["runtime_durable_adapter_auth_blocked", "auth_blocked", "auth"],
|
||||
["runtime_durable_adapter_schema_blocked", "schema_blocked", "schema"],
|
||||
["runtime_durable_adapter_migration_blocked", "migration_blocked", "migration"]
|
||||
]) {
|
||||
const report = await buildDevRuntimePostflightReport(
|
||||
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
|
||||
{
|
||||
httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: false, blocker, queryResult, blockedLayer }) }),
|
||||
m3Runner: async () => {
|
||||
throw new Error("M3 runner should not run when durable readiness is blocked");
|
||||
},
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
assert.equal(report.m3.status, "not_run");
|
||||
assert.equal(report.apiHealth.runtimeDurableReadiness.classification, blocker);
|
||||
assert.equal(report.apiHealth.runtimeDurableReadiness.evidence.queryResult, queryResult);
|
||||
assert.equal(report.apiV1.runtimeDurableReadiness.classification, blocker);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
}
|
||||
});
|
||||
|
||||
test("live mode treats durable_readiness_ready queryResult as ready, not blocked", async () => {
|
||||
const report = await buildDevRuntimePostflightReport(
|
||||
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
|
||||
{
|
||||
httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: true, queryResult: "durable_readiness_ready" }) }),
|
||||
m3Runner: async () => m3GreenReport(),
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "pass");
|
||||
assert.equal(report.actions.m3MutationAttempted, true);
|
||||
assert.equal(report.apiHealth.runtimeDurableReadiness.ready, true);
|
||||
assert.equal(report.apiHealth.runtimeDurableReadiness.evidence.queryResult, "durable_readiness_ready");
|
||||
assert.equal(report.blockers.length, 0);
|
||||
});
|
||||
|
||||
test("live mode records M3 true/false durable green evidence", async () => {
|
||||
@@ -165,6 +214,7 @@ test("live mode blocks when M3 operation evidence is not durable green", async (
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-postflight"), true);
|
||||
assert.equal(report.m3.persistenceContract.ready, false);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
});
|
||||
|
||||
test("live mode blocks when M3 true/false persisted sequence is incomplete", async () => {
|
||||
@@ -183,9 +233,29 @@ test("live mode blocks when M3 true/false persisted sequence is incomplete", asy
|
||||
assert.equal(report.m3.persistenceContract.ready, false);
|
||||
assert.equal(report.m3.persistenceContract.sequence.at(-1).status, "missing");
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-postflight"), true);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
});
|
||||
|
||||
function apiPayload({ ready }) {
|
||||
function assertStructuredBlockers(blockers) {
|
||||
assert.ok(blockers.length > 0);
|
||||
for (const blocker of blockers) {
|
||||
assert.equal(typeof blocker.scope, "string");
|
||||
assert.equal(typeof blocker.reason, "string");
|
||||
assert.equal(typeof blocker.impact, "string");
|
||||
assert.equal(typeof blocker.safeNextAction, "string");
|
||||
assert.equal(typeof blocker.retryable, "boolean");
|
||||
assert.notEqual(blocker.reason, "");
|
||||
assert.notEqual(blocker.impact, "");
|
||||
assert.notEqual(blocker.safeNextAction, "");
|
||||
}
|
||||
}
|
||||
|
||||
function apiPayload({
|
||||
ready,
|
||||
blocker = "runtime_durable_adapter_auth_blocked",
|
||||
queryResult = ready ? "durable_readiness_ready" : "auth_blocked",
|
||||
blockedLayer = "auth"
|
||||
}) {
|
||||
return {
|
||||
serviceId: "hwlab-cloud-api",
|
||||
environment: "dev",
|
||||
@@ -200,8 +270,8 @@ function apiPayload({ ready }) {
|
||||
runtimeReadiness: {
|
||||
ready,
|
||||
status: ready ? "ready" : "blocked",
|
||||
blocker: ready ? null : "runtime_durable_adapter_auth_blocked",
|
||||
queryResult: ready ? "durable_readiness_ready" : "auth_blocked",
|
||||
blocker: ready ? null : blocker,
|
||||
queryResult,
|
||||
requiredEvidence: "runtime_adapter_schema_migration_read_query"
|
||||
}
|
||||
},
|
||||
@@ -211,18 +281,21 @@ function apiPayload({ ready }) {
|
||||
durableRequested: true,
|
||||
ready,
|
||||
status: ready ? "ready" : "blocked",
|
||||
blocker: ready ? null : "runtime_durable_adapter_auth_blocked",
|
||||
blocker: ready ? null : blocker,
|
||||
liveRuntimeEvidence: ready,
|
||||
connection: {
|
||||
queryResult
|
||||
},
|
||||
durabilityContract: {
|
||||
requiredEvidence: "runtime_adapter_schema_migration_read_query",
|
||||
blockedLayer: ready ? null : "auth"
|
||||
blockedLayer: ready ? null : blockedLayer
|
||||
},
|
||||
gates: {
|
||||
ssl: gate(true),
|
||||
auth: gate(ready, "runtime_durable_adapter_auth_blocked"),
|
||||
schema: gate(ready),
|
||||
migration: gate(ready),
|
||||
durability: gate(ready)
|
||||
auth: gate(ready || !["auth"].includes(blockedLayer), blockedLayer === "auth" ? blocker : null),
|
||||
schema: gate(ready || !["schema"].includes(blockedLayer), blockedLayer === "schema" ? blocker : null),
|
||||
migration: gate(ready || !["migration"].includes(blockedLayer), blockedLayer === "migration" ? blocker : null),
|
||||
durability: gate(ready, blocker)
|
||||
}
|
||||
},
|
||||
readiness: {
|
||||
@@ -230,11 +303,14 @@ function apiPayload({ ready }) {
|
||||
status: ready ? "ready" : "blocked",
|
||||
durability: {
|
||||
ready,
|
||||
blockedLayer: ready ? null : "auth"
|
||||
status: ready ? "ready" : "blocked",
|
||||
blocker: ready ? null : blocker,
|
||||
blockedLayer: ready ? null : blockedLayer,
|
||||
queryResult
|
||||
}
|
||||
},
|
||||
blockerCodes: ready ? []
|
||||
: ["runtime_durable_adapter_auth_blocked"],
|
||||
: [blocker],
|
||||
m3IoControl: {
|
||||
route: "/v1/m3/io",
|
||||
status: "available",
|
||||
|
||||
Reference in New Issue
Block a user