fix: harden dev cd runtime postflight

Summary:
- shared runtime durable readiness classifier for live-status and DEV postflight
- structured postflight blockers propagated through dev-cd apply
- contract coverage for ready queryResult, auth/schema/migration blockers, evidence persistence blockers, and bounded stdout

Verification:
- git diff --check
- node --check changed scripts/tests/live-status modules
- node --test scripts/src/dev-runtime-postflight.test.mjs scripts/src/dev-cd-apply.test.mjs web/hwlab-cloud-web/scripts/live-status-contract.test.mjs
- node scripts/dev-runtime-postflight.mjs --check
- node web/hwlab-cloud-web/scripts/check.mjs
- npm run check

No DEV apply/rollout/live health verification was run.
This commit is contained in:
Lyon
2026-05-23 22:15:04 +08:00
committed by GitHub
parent 8a04c27d31
commit 83e5f52c98
6 changed files with 812 additions and 101 deletions
+46 -2
View File
@@ -1815,11 +1815,55 @@ function parseLastJsonObject(text) {
}
function stepBlocker(stepResult) {
return {
const stdoutBlockers = Array.isArray(stepResult.stdoutJson?.blockers) ? stepResult.stdoutJson.blockers : [];
const primary = stdoutBlockers.find((item) => item?.status === "open") ?? stdoutBlockers[0] ?? null;
const reason = primary?.reason ?? primary?.summary ?? `${stepResult.command} exited ${stepResult.code}.`;
const impact = primary?.impact ?? stepBlockerImpact(stepResult);
const safeNextAction = primary?.safeNextAction ?? stepBlockerSafeNextAction(stepResult);
return structuredBlocker({
type: "runtime_blocker",
scope: stepResult.id,
reason,
impact,
safeNextAction,
retryable: primary?.retryable ?? true,
evidence: {
command: stepResult.command,
code: stepResult.code,
reportPath: stepResult.reportPath ?? null,
blockerScope: primary?.scope ?? null,
k8sJob: stepResult.k8sJob ?? null
}
});
}
function stepBlockerImpact(stepResult) {
if (stepResult.id === "runtime-db-provisioning") return "DEV CD cannot prove DB role/database provisioning before workload apply.";
if (stepResult.id === "runtime-db-migration") return "DEV CD cannot prove runtime schema migration before workload apply.";
if (stepResult.id === "runtime-durable-postflight") return "DEV CD cannot prove /health/live, /v1, and M3 durable evidence persistence after apply.";
if (stepResult.id === "dev-deploy-apply") return "DEV desired-state apply did not complete, so live readiness cannot be trusted.";
return "DEV CD transaction cannot safely continue.";
}
function stepBlockerSafeNextAction(stepResult) {
if (stepResult.id === "runtime-db-provisioning") return "Repair repo-owned runtime provisioning inputs/SecretRefs without printing Secret values, then rerun DEV CD.";
if (stepResult.id === "runtime-db-migration") return "Repair repo-owned runtime migration/schema state, then rerun DEV CD.";
if (stepResult.id === "runtime-durable-postflight") return "Inspect the runtime postflight report, repair durable readiness or M3 evidence persistence, then rerun DEV CD after host-controlled rollout if needed.";
if (stepResult.id === "dev-deploy-apply") return "Inspect reports/dev-gate/dev-deploy-report.json and rerun the repo-owned DEV CD path after the apply blocker is fixed.";
return "Fix the reported step blocker and rerun the repo-owned DEV CD transaction.";
}
function structuredBlocker({ type = "runtime_blocker", scope, reason, impact, safeNextAction, retryable = true, evidence = null }) {
return {
type,
scope,
status: "open",
summary: `${stepResult.command} exited ${stepResult.code}.`
reason: oneLine(reason),
impact: oneLine(impact),
safeNextAction: oneLine(safeNextAction),
retryable: Boolean(retryable),
summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`),
...(evidence ? { evidence } : {})
};
}
+106
View File
@@ -215,6 +215,7 @@ function makeRunCommand({
],
desiredStatePromotionStatus = "pass",
runtimeJobLogSuffix = "",
runtimePostflightReport = null,
kubeContext = "d601",
kubeServer = "https://d601:6443",
kubeNodes = ["d601"],
@@ -390,6 +391,24 @@ function makeRunCommand({
};
}
if (command === process.execPath || command.endsWith("/node")) {
if (args.includes("scripts/dev-runtime-postflight.mjs")) {
const report = runtimePostflightReport ?? {
conclusion: { status: "pass" },
blockers: [],
safety: { secretValuesPrinted: false },
apiHealth: { runtimeDurableReadiness: { ready: true } },
apiV1: { runtimeDurableReadiness: { ready: true } },
m3: {
persistenceContract: { ready: true },
operationCount: 4
}
};
return {
code: report.conclusion?.status === "pass" ? 0 : 1,
stdout: `${JSON.stringify(report, null, 2)}\n`,
stderr: ""
};
}
return { code: 0, stdout: JSON.stringify({ ok: true, command: args.join(" ") }), stderr: "" };
}
return { code: 0, stdout: "", stderr: "" };
@@ -1294,6 +1313,93 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a
assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2);
});
test("transaction blocks with structured runtime postflight blocker when M3 evidence is not persisted", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const runtimePostflightReport = {
conclusion: {
status: "blocked",
blockerCount: 1
},
apiHealth: {
runtimeDurableReadiness: {
ready: true,
classification: "runtime_durable_ready"
}
},
apiV1: {
runtimeDurableReadiness: {
ready: true,
classification: "runtime_durable_ready"
}
},
m3: {
persistenceContract: {
ready: false,
sequence: [
{ id: "write-do1-true", persisted: true, idsPresent: true },
{ id: "read-di1-true", persisted: false, idsPresent: true }
]
},
operationCount: 4
},
blockers: [
{
type: "runtime_blocker",
scope: "m3-durable-postflight",
status: "open",
reason: "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.",
impact: "A later rollout could appear ready while losing evidence persistence.",
safeNextAction: "Repair durable audit/evidence persistence, then rerun DEV runtime postflight.",
retryable: true
}
],
safety: {
secretValuesPrinted: false,
readsKubernetesSecrets: false
}
};
const code = await runDevCdApply([
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--owner-task-id",
"task-postflight-blocked",
"--kubeconfig",
"/etc/rancher/k3s/k3s.yaml",
"--write-report"
], {
repoRoot,
env: {},
runCommand: makeRunCommand({ commandLog, runtimePostflightReport }),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const summary = JSON.parse(output);
const report = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
const blocker = report.blockers.find((item) => item.scope === "runtime-durable-postflight");
assert.equal(code, 2);
assert.equal(summary.status, "blocked");
assert.equal(summary.devCdApply, undefined);
assert.equal(output.length < 12000, true);
assert.equal(report.status, "blocked");
assert.equal(report.devCdApply.steps.at(-1).id, "runtime-durable-postflight");
assert.equal(report.devCdApply.steps.at(-1).status, "blocked");
assert.ok(blocker);
assert.equal(blocker.reason, runtimePostflightReport.blockers[0].reason);
assert.equal(blocker.impact, runtimePostflightReport.blockers[0].impact);
assert.equal(blocker.safeNextAction, runtimePostflightReport.blockers[0].safeNextAction);
assert.equal(blocker.retryable, true);
assert.equal(blocker.evidence.blockerScope, "m3-durable-postflight");
assert.equal(report.devCdApply.liveVerify.status, "not_run");
assert.equal(JSON.stringify(report).includes("postgresql://"), false);
assert.equal(JSON.stringify(report).includes("SECRET"), false);
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-runtime-postflight.mjs")), true);
});
test("runtime maintenance job reports are parsed after redacting secret-like log output", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
+117 -39
View File
@@ -4,6 +4,7 @@ import path from "node:path";
import { fileURLToPath } from "node:url";
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
import { classifyRuntimeDurableReadiness } from "../../web/hwlab-cloud-web/live-status.mjs";
import {
buildM3IoControlSourceReport,
runM3IoControlLiveReport
@@ -78,8 +79,16 @@ export async function buildDevRuntimePostflightReport(args = {}, options = {}) {
blockerCount: sourceReport.summary?.status === "pass" ? 0 : 1
};
if (sourceReport.summary?.status !== "pass") {
addBlocker(report, "contract_blocker", "m3-source-contract", "M3 postflight source contract is blocked.", {
classification: sourceReport.summary?.classification ?? "unknown"
addBlocker(report, {
type: "contract_blocker",
scope: "m3-source-contract",
reason: "M3 postflight source contract is blocked.",
impact: "DEV CD cannot rely on the M3 durable postflight harness contract.",
safeNextAction: "Fix the source M3 IO control contract tests before running live DEV postflight.",
retryable: true,
evidence: {
classification: sourceReport.summary?.classification ?? "unknown"
}
});
}
return report;
@@ -227,6 +236,7 @@ async function defaultHttpGetJson(url, timeoutMs) {
function summarizeApiPayload(check, response) {
const json = response.json ?? {};
const runtime = summarizeRuntime(json.runtime);
const runtimeDurableReadiness = classifyRuntimeDurableReadiness(json, { requirePostgresAdapter: true });
return {
check,
httpStatus: response.status,
@@ -236,6 +246,7 @@ function summarizeApiPayload(check, response) {
status: json.status ?? null,
ready: json.ready === true,
runtime,
runtimeDurableReadiness,
db: summarizeDb(json.db),
readiness: {
ready: json.readiness?.ready === true,
@@ -372,23 +383,48 @@ function summarizeM3PersistenceContract(operations = []) {
function addApiReadinessBlockers(report) {
for (const [scope, payload] of [["api-health-live", report.apiHealth], ["api-v1", report.apiV1]]) {
if (!payload.reachable) {
addBlocker(report, "runtime_blocker", scope, `${payload.check} was not reachable.`, {
httpStatus: payload.httpStatus,
error: payload.error
addBlocker(report, {
type: "runtime_blocker",
scope,
reason: `${payload.check} was not reachable.`,
impact: "DEV CD cannot prove cloud-api durable readiness, so M3 evidence persistence postflight cannot safely run.",
safeNextAction: "Restore DEV cloud-api reachability on the repo-owned public endpoint, then rerun postflight.",
retryable: true,
evidence: {
httpStatus: payload.httpStatus,
error: payload.error
}
});
continue;
}
if (!runtimeDurableReady(payload.runtime)) {
addBlocker(report, "runtime_blocker", scope, `${payload.check} did not prove durable runtime readiness.`, {
runtime: payload.runtime,
blockerCodes: payload.blockerCodes
if (payload.runtimeDurableReadiness?.ready !== true) {
addBlocker(report, {
type: "runtime_blocker",
scope,
reason: `${payload.check} did not prove durable runtime readiness: ${payload.runtimeDurableReadiness?.reason ?? "not ready"}.`,
impact: payload.runtimeDurableReadiness?.impact ?? "M3 durable postflight cannot prove evidence persistence.",
safeNextAction: payload.runtimeDurableReadiness?.safeNextAction ?? "Repair runtime durability through repo-owned DEV CD steps, then rerun postflight.",
retryable: payload.runtimeDurableReadiness?.retryable !== false,
evidence: {
runtime: payload.runtime,
runtimeDurableReadiness: payload.runtimeDurableReadiness,
blockerCodes: payload.blockerCodes
}
});
}
if (scope === "api-v1" && !apiV1M3ContractReady(payload)) {
addBlocker(report, "runtime_blocker", scope, "GET /v1 did not expose the controlled M3 IO route required by durable postflight.", {
m3IoControl: payload.m3IoControl,
requiredRoute: "/v1/m3/io",
requiredContractVersion: "m3-io-control-v1"
addBlocker(report, {
type: "runtime_blocker",
scope,
reason: "GET /v1 did not expose the controlled M3 IO route required by durable postflight.",
impact: "DEV CD cannot run the M3 true/false IO persistence smoke through the cloud-api controlled boundary.",
safeNextAction: "Restore /v1 m3IoControl route metadata for /v1/m3/io without exposing direct frontend simulator access.",
retryable: true,
evidence: {
m3IoControl: payload.m3IoControl,
requiredRoute: "/v1/m3/io",
requiredContractVersion: "m3-io-control-v1"
}
});
}
}
@@ -406,58 +442,96 @@ function apiV1M3ContractReady(payload) {
function addM3Blockers(report) {
if (report.m3?.persistenceContract?.ready !== true) {
addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.", {
persistenceContract: report.m3?.persistenceContract ?? null,
trustedGreen: report.m3?.trustedGreen === true,
operationCount: report.m3?.operationCount ?? 0
addBlocker(report, {
type: "runtime_blocker",
scope: "m3-durable-postflight",
reason: "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.",
impact: "A later rollout could appear ready while losing operation, audit, or evidence persistence for M3 hardware IO.",
safeNextAction: "Inspect the failed M3 operation sequence, repair durable audit/evidence persistence, then rerun DEV runtime postflight.",
retryable: true,
evidence: {
persistenceContract: report.m3?.persistenceContract ?? null,
trustedGreen: report.m3?.trustedGreen === true,
operationCount: report.m3?.operationCount ?? 0
}
});
return;
}
if (report.m3?.trustedGreen !== true || report.m3?.status !== "pass") {
addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 true/false durable postflight did not produce trusted green persisted evidence.", {
classification: report.m3?.classification ?? "unknown",
trustedGreen: report.m3?.trustedGreen === true,
operationCount: report.m3?.operationCount ?? 0
addBlocker(report, {
type: "runtime_blocker",
scope: "m3-durable-postflight",
reason: "M3 true/false durable postflight did not produce trusted green persisted evidence.",
impact: "M3 durable readiness cannot be accepted because trusted evidence is not green.",
safeNextAction: "Fix the M3 control path or trusted-record persistence and rerun DEV runtime postflight.",
retryable: true,
evidence: {
classification: report.m3?.classification ?? "unknown",
trustedGreen: report.m3?.trustedGreen === true,
operationCount: report.m3?.operationCount ?? 0
}
});
return;
}
for (const operation of report.m3.operations ?? []) {
if (!operation.operationId || !operation.auditId || !operation.evidenceId || operation.evidenceState?.status !== "green" || operation.evidenceState?.durable !== true) {
addBlocker(report, "runtime_blocker", "m3-durable-evidence", "M3 operation/audit/evidence durable fields were incomplete.", {
operationIdPresent: Boolean(operation.operationId),
auditIdPresent: Boolean(operation.auditId),
evidenceIdPresent: Boolean(operation.evidenceId),
evidenceState: operation.evidenceState
addBlocker(report, {
type: "runtime_blocker",
scope: "m3-durable-evidence",
reason: "M3 operation/audit/evidence durable fields were incomplete.",
impact: "The M3 postflight cannot prove every IO operation is bound to persisted audit and evidence records.",
safeNextAction: "Repair operation, audit, and evidence ID propagation before accepting DEV readiness.",
retryable: true,
evidence: {
operationIdPresent: Boolean(operation.operationId),
traceIdPresent: Boolean(operation.traceId),
auditIdPresent: Boolean(operation.auditId),
evidenceIdPresent: Boolean(operation.evidenceId),
evidenceState: operation.evidenceState
}
});
return;
}
}
}
function runtimeDurableReady(runtime) {
return runtime.adapter === "postgres" &&
runtime.durable === true &&
runtime.ready === true &&
runtime.liveRuntimeEvidence === true;
}
function addSafetyRefusal(report, summary) {
report.safetyRefusal = true;
addBlocker(report, "safety_refusal", "runtime-postflight-live-boundary", summary, {
devOnly: true,
prodAllowed: false,
secretValuesPrinted: false
addBlocker(report, {
type: "safety_refusal",
scope: "runtime-postflight-live-boundary",
reason: summary,
impact: "Live DEV reads and M3 writes are not allowed without explicit non-production confirmation.",
safeNextAction: "Rerun with --live --confirm-dev --confirmed-non-production for DEV only, or use --check for source-only validation.",
retryable: true,
evidence: {
devOnly: true,
prodAllowed: false,
secretValuesPrinted: false
}
});
}
function addBlocker(report, type, scope, summary, evidence = {}) {
function addBlocker(report, {
type,
scope,
reason,
impact,
safeNextAction,
retryable,
evidence = {}
}) {
report.blockers.push({
type,
scope,
status: "open",
summary,
reason: oneLine(reason),
impact: oneLine(impact),
safeNextAction: oneLine(safeNextAction),
retryable: Boolean(retryable),
summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`),
sourceIssue: issue,
evidence
});
@@ -475,6 +549,10 @@ function finalizeReport(report) {
return report;
}
function oneLine(value) {
return String(value ?? "").replace(/\s+/g, " ").trim();
}
function buildSafety(args) {
return {
devOnly: true,
+87 -11
View File
@@ -56,6 +56,8 @@ test("live mode skips M3 writes when /health/live durable runtime is blocked", a
assert.equal(report.actions.m3MutationAttempted, false);
assert.equal(report.m3.status, "not_run");
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-health-live"), true);
assertStructuredBlockers(report.blockers);
assert.match(report.blockers.find((blocker) => blocker.scope === "api-health-live").reason, /auth_blocked/u);
assert.equal(JSON.stringify(report).includes("postgresql://"), false);
});
@@ -74,6 +76,7 @@ test("live mode requires /v1 durable readiness before M3 writes", async () => {
assert.equal(report.conclusion.status, "blocked");
assert.equal(report.actions.m3MutationAttempted, false);
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-v1"), true);
assertStructuredBlockers(report.blockers);
});
test("live mode requires /v1 to expose the controlled M3 IO route before M3 writes", async () => {
@@ -100,6 +103,7 @@ test("live mode requires /v1 to expose the controlled M3 IO route before M3 writ
blocker.scope === "api-v1" &&
blocker.summary.includes("controlled M3 IO route")
), true);
assertStructuredBlockers(report.blockers);
});
test("live mode does not mistake DB live evidence for durable runtime readiness", async () => {
@@ -121,6 +125,51 @@ test("live mode does not mistake DB live evidence for durable runtime readiness"
assert.equal(report.apiHealth.readiness.durabilityReady, false);
assert.equal(report.actions.m3MutationAttempted, false);
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-health-live"), true);
assertStructuredBlockers(report.blockers);
});
test("live mode blocks auth, schema, and migration durable readiness before M3 writes", async () => {
for (const [blocker, queryResult, blockedLayer] of [
["runtime_durable_adapter_auth_blocked", "auth_blocked", "auth"],
["runtime_durable_adapter_schema_blocked", "schema_blocked", "schema"],
["runtime_durable_adapter_migration_blocked", "migration_blocked", "migration"]
]) {
const report = await buildDevRuntimePostflightReport(
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
{
httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: false, blocker, queryResult, blockedLayer }) }),
m3Runner: async () => {
throw new Error("M3 runner should not run when durable readiness is blocked");
},
now: () => "2026-05-23T00:00:00.000Z"
}
);
assert.equal(report.conclusion.status, "blocked");
assert.equal(report.actions.m3MutationAttempted, false);
assert.equal(report.m3.status, "not_run");
assert.equal(report.apiHealth.runtimeDurableReadiness.classification, blocker);
assert.equal(report.apiHealth.runtimeDurableReadiness.evidence.queryResult, queryResult);
assert.equal(report.apiV1.runtimeDurableReadiness.classification, blocker);
assertStructuredBlockers(report.blockers);
}
});
test("live mode treats durable_readiness_ready queryResult as ready, not blocked", async () => {
const report = await buildDevRuntimePostflightReport(
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
{
httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: true, queryResult: "durable_readiness_ready" }) }),
m3Runner: async () => m3GreenReport(),
now: () => "2026-05-23T00:00:00.000Z"
}
);
assert.equal(report.conclusion.status, "pass");
assert.equal(report.actions.m3MutationAttempted, true);
assert.equal(report.apiHealth.runtimeDurableReadiness.ready, true);
assert.equal(report.apiHealth.runtimeDurableReadiness.evidence.queryResult, "durable_readiness_ready");
assert.equal(report.blockers.length, 0);
});
test("live mode records M3 true/false durable green evidence", async () => {
@@ -165,6 +214,7 @@ test("live mode blocks when M3 operation evidence is not durable green", async (
assert.equal(report.conclusion.status, "blocked");
assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-postflight"), true);
assert.equal(report.m3.persistenceContract.ready, false);
assertStructuredBlockers(report.blockers);
});
test("live mode blocks when M3 true/false persisted sequence is incomplete", async () => {
@@ -183,9 +233,29 @@ test("live mode blocks when M3 true/false persisted sequence is incomplete", asy
assert.equal(report.m3.persistenceContract.ready, false);
assert.equal(report.m3.persistenceContract.sequence.at(-1).status, "missing");
assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-postflight"), true);
assertStructuredBlockers(report.blockers);
});
function apiPayload({ ready }) {
function assertStructuredBlockers(blockers) {
assert.ok(blockers.length > 0);
for (const blocker of blockers) {
assert.equal(typeof blocker.scope, "string");
assert.equal(typeof blocker.reason, "string");
assert.equal(typeof blocker.impact, "string");
assert.equal(typeof blocker.safeNextAction, "string");
assert.equal(typeof blocker.retryable, "boolean");
assert.notEqual(blocker.reason, "");
assert.notEqual(blocker.impact, "");
assert.notEqual(blocker.safeNextAction, "");
}
}
function apiPayload({
ready,
blocker = "runtime_durable_adapter_auth_blocked",
queryResult = ready ? "durable_readiness_ready" : "auth_blocked",
blockedLayer = "auth"
}) {
return {
serviceId: "hwlab-cloud-api",
environment: "dev",
@@ -200,8 +270,8 @@ function apiPayload({ ready }) {
runtimeReadiness: {
ready,
status: ready ? "ready" : "blocked",
blocker: ready ? null : "runtime_durable_adapter_auth_blocked",
queryResult: ready ? "durable_readiness_ready" : "auth_blocked",
blocker: ready ? null : blocker,
queryResult,
requiredEvidence: "runtime_adapter_schema_migration_read_query"
}
},
@@ -211,18 +281,21 @@ function apiPayload({ ready }) {
durableRequested: true,
ready,
status: ready ? "ready" : "blocked",
blocker: ready ? null : "runtime_durable_adapter_auth_blocked",
blocker: ready ? null : blocker,
liveRuntimeEvidence: ready,
connection: {
queryResult
},
durabilityContract: {
requiredEvidence: "runtime_adapter_schema_migration_read_query",
blockedLayer: ready ? null : "auth"
blockedLayer: ready ? null : blockedLayer
},
gates: {
ssl: gate(true),
auth: gate(ready, "runtime_durable_adapter_auth_blocked"),
schema: gate(ready),
migration: gate(ready),
durability: gate(ready)
auth: gate(ready || !["auth"].includes(blockedLayer), blockedLayer === "auth" ? blocker : null),
schema: gate(ready || !["schema"].includes(blockedLayer), blockedLayer === "schema" ? blocker : null),
migration: gate(ready || !["migration"].includes(blockedLayer), blockedLayer === "migration" ? blocker : null),
durability: gate(ready, blocker)
}
},
readiness: {
@@ -230,11 +303,14 @@ function apiPayload({ ready }) {
status: ready ? "ready" : "blocked",
durability: {
ready,
blockedLayer: ready ? null : "auth"
status: ready ? "ready" : "blocked",
blocker: ready ? null : blocker,
blockedLayer: ready ? null : blockedLayer,
queryResult
}
},
blockerCodes: ready ? []
: ["runtime_durable_adapter_auth_blocked"],
: [blocker],
m3IoControl: {
route: "/v1/m3/io",
status: "available",
+294 -49
View File
@@ -3,6 +3,26 @@ const ERROR_STATUSES = Object.freeze(["error", "failed", "failure"]);
const RAW_DEGRADED_STATUSES = Object.freeze(["degraded"]);
const BLOCKED_STATUSES = Object.freeze(["blocked", "unavailable", "rejected"]);
const UNVERIFIED_STATUSES = Object.freeze(["", "unknown", "pending", "loading", "probing", "unverified", "not_observed"]);
const RUNTIME_DURABLE_READY_QUERY_RESULTS = Object.freeze([
"durable_readiness_ready",
"runtime_durable_ready",
"query_ready",
"readiness_ready",
"ready",
"ok",
"pass",
"passed"
]);
const RUNTIME_DURABLE_BLOCKED_QUERY_RESULTS = Object.freeze({
driver_missing: "runtime_durable_adapter_driver_missing",
ssl_negotiation_blocked: "runtime_durable_adapter_ssl_blocked",
auth_blocked: "runtime_durable_adapter_auth_blocked",
schema_blocked: "runtime_durable_adapter_schema_blocked",
migration_blocked: "runtime_durable_adapter_migration_blocked",
query_blocked: "runtime_durable_adapter_query_blocked",
not_ready: "runtime_durable_adapter_query_blocked"
});
const RUNTIME_DURABLE_GATE_LAYERS = Object.freeze(["ssl", "auth", "schema", "migration", "durability"]);
const READ_ONLY_REASON_CODES = Object.freeze([
"read_only",
"read-only",
@@ -480,70 +500,286 @@ function classifyM3StatusProbe(live = {}) {
}
function durableRuntimeReason(payload) {
const runtime = payload?.runtime;
const durability = payload?.readiness?.durability;
const runtimeDurable = runtime?.durable === true || durability?.durable === true;
const runtimeExplicitlyNonDurable = runtime?.durable === false || durability?.durable === false;
const runtimeReady = runtime?.ready === true || durability?.ready === true;
const durabilityReady = durability?.ready === true || normalizedStatus(durability?.status) === "ready";
const liveRuntimeEvidence = runtime?.liveRuntimeEvidence === true || durability?.liveRuntimeEvidence === true;
if (runtimeDurable && runtimeReady && durabilityReady && !hasDurableRuntimeBlockedSignal(payload)) return null;
const reasonCode = firstNonEmpty(
runtime?.blocker,
durability?.blocker,
runtime?.durabilityContract?.blockedLayer,
durability?.blockedLayer,
blockedDurabilityQueryResult(runtime?.connection?.queryResult),
blockedDurabilityQueryResult(durability?.queryResult),
blockerCodeStartingWith(payload, "runtime_durable_adapter_")
);
const blockedLayer = firstNonEmpty(runtime?.durabilityContract?.blockedLayer, durability?.blockedLayer);
if (!reasonCode && runtimeDurable && runtimeReady && liveRuntimeEvidence) return null;
if (runtimeExplicitlyNonDurable) {
return {
reasonCode: "runtime_durable_false",
reason: `runtime durable=false; adapter=${firstNonEmpty(runtime?.adapter, durability?.adapter, "unknown")}`,
layer: "runtime-durable"
};
}
if (!hasRuntimeDurableSignal(payload)) return null;
const classification = classifyRuntimeDurableReadiness(payload);
if (classification.ready) return null;
if (
reasonCode &&
classification.status === "blocked" &&
(
isReadOnlyReason(reasonCode) ||
String(reasonCode).endsWith("_blocked") ||
normalizedStatus(blockedLayer).includes("durability")
isReadOnlyReason(classification.reasonCode) ||
String(classification.reasonCode).endsWith("_blocked") ||
normalizedStatus(classification.evidence?.blockedLayer).includes("durability")
)
) {
return {
reasonCode,
reason: `runtime durable blocked at ${blockedLayer || "durability"}; queryResult=${firstNonEmpty(runtime?.connection?.queryResult, durability?.queryResult, "unknown")}`,
layer: blockedLayer || "runtime-durable"
reasonCode: classification.reasonCode,
reason: classification.reason,
layer: classification.evidence?.blockedLayer || "runtime-durable",
classification
};
}
return null;
}
function hasDurableRuntimeBlockedSignal(payload) {
function hasRuntimeDurableSignal(payload = {}) {
const runtime = payload?.runtime;
const durability = payload?.readiness?.durability;
return [
runtime?.blocker,
durability?.blocker,
runtime?.durabilityContract?.blockedLayer,
durability?.blockedLayer,
blockedDurabilityQueryResult(runtime?.connection?.queryResult),
blockedDurabilityQueryResult(durability?.queryResult),
blockerCodeStartingWith(payload, "runtime_durable_adapter_")
].some(Boolean);
const dbRuntimeReadiness = payload?.db?.runtimeReadiness;
return Boolean(
blockerCodeStartingWith(payload, "runtime_durable_adapter_") ||
hasOwnNonEmpty(runtime, ["adapter", "durable", "durableRequested", "ready", "blocker", "liveRuntimeEvidence", "connection", "durabilityContract", "gates"]) ||
hasOwnNonEmpty(durability, ["adapter", "durable", "ready", "blocker", "blockedLayer", "queryResult", "liveRuntimeEvidence", "gates"]) ||
hasOwnNonEmpty(dbRuntimeReadiness, ["adapter", "durable", "ready", "blocker", "blockedLayer", "queryResult", "liveRuntimeEvidence"])
);
}
function blockedDurabilityQueryResult(value) {
const text = normalizedStatus(value);
if (!text || text === "durable_readiness_ready" || text === "ready") return "";
if (text.endsWith("_blocked") || text.includes("blocked")) return String(value).trim();
function hasOwnNonEmpty(object, fields) {
if (!object || typeof object !== "object") return false;
return fields.some((field) => Object.hasOwn(object, field) && object[field] !== undefined && object[field] !== null && object[field] !== "");
}
export function classifyRuntimeDurableReadiness(payload = {}, options = {}) {
const runtime = payload?.runtime ?? {};
const durability = payload?.readiness?.durability ?? {};
const dbRuntimeReadiness = payload?.db?.runtimeReadiness ?? {};
const gates = runtime?.gates ?? durability?.gates ?? {};
const gateBlocker = firstBlockedRuntimeGate(gates);
const queryResult = firstNonEmpty(
runtime?.connection?.queryResult,
durability?.queryResult,
dbRuntimeReadiness?.queryResult
);
const directBlocker = firstNonEmpty(
runtime?.blocker,
durability?.blocker,
dbRuntimeReadiness?.blocker,
blockerCodeStartingWith(payload, "runtime_durable_adapter_")
);
const queryBlocker = runtimeDurableBlockerFromQueryResult(queryResult);
const reasonCode = firstNonEmpty(directBlocker, gateBlocker?.blocker, queryBlocker);
const blockedLayer = firstNonEmpty(
runtime?.durabilityContract?.blockedLayer,
durability?.blockedLayer,
dbRuntimeReadiness?.blockedLayer,
gateBlocker?.layer,
runtimeDurableLayerFromBlocker(reasonCode),
runtimeDurableLayerFromQueryResult(queryResult)
);
const adapter = firstNonEmpty(runtime?.adapter, durability?.adapter, dbRuntimeReadiness?.adapter, "unknown");
const runtimeDurable = runtime?.durable === true || durability?.durable === true || dbRuntimeReadiness?.durable === true;
const runtimeExplicitlyNonDurable = runtime?.durable === false || durability?.durable === false || dbRuntimeReadiness?.durable === false;
const runtimeReady = runtime?.ready === true || durability?.runtimeReady === true || dbRuntimeReadiness?.ready === true;
const durabilityReady = durability?.ready === true || dbRuntimeReadiness?.ready === true || runtimeReady;
const liveRuntimeEvidence =
runtime?.liveRuntimeEvidence === true ||
durability?.liveRuntimeEvidence === true ||
dbRuntimeReadiness?.liveRuntimeEvidence === true;
const rawStatus = rawStatusFrom(payload);
const rawStatusBlocked = [...ERROR_STATUSES, ...BLOCKED_STATUSES, ...RAW_DEGRADED_STATUSES].includes(normalizedStatus(rawStatus));
const readyQueryResult = isRuntimeDurableReadyQueryResult(queryResult);
const postgresRequired = options.requirePostgresAdapter === true;
const adapterReady = !postgresRequired || adapter === "postgres";
const ready = adapterReady &&
runtimeDurable &&
runtimeReady &&
durabilityReady &&
liveRuntimeEvidence &&
!reasonCode &&
!rawStatusBlocked;
if (ready) {
return {
status: "ready",
ready: true,
classification: "runtime_durable_ready",
reasonCode: "runtime_durable_ready",
reason: `runtime durable ready; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`,
impact: "M3 durable postflight may run because runtime persistence readiness is green.",
safeNextAction: "Continue with repo-owned DEV CD postflight; do not read or print Secret values.",
retryable: true,
readonly: false,
evidence: runtimeDurableEvidence({
adapter,
runtimeDurable,
runtimeReady,
durabilityReady,
liveRuntimeEvidence,
blockedLayer: null,
queryResult,
readyQueryResult,
reasonCode: null,
gates,
rawStatus,
postgresRequired
})
};
}
const nonReadyReasonCode = firstNonEmpty(
reasonCode,
postgresRequired && adapter !== "postgres" ? "runtime_durable_adapter_not_postgres" : "",
runtimeExplicitlyNonDurable ? "runtime_durable_false" : "",
runtimeDurable ? "" : "runtime_durable_adapter_missing",
runtimeReady ? "" : "runtime_durable_runtime_not_ready",
liveRuntimeEvidence ? "" : "runtime_durable_live_evidence_missing",
rawStatusBlocked ? rawStatus : "",
"runtime_durable_adapter_not_ready"
);
const layer = firstNonEmpty(
blockedLayer,
runtimeDurableLayerFromBlocker(nonReadyReasonCode),
runtimeExplicitlyNonDurable ? "runtime-durable" : "",
"runtime-durable"
);
return {
status: "blocked",
ready: false,
classification: nonReadyReasonCode,
reasonCode: nonReadyReasonCode,
reason: runtimeDurableBlockedReason({ reasonCode: nonReadyReasonCode, adapter, layer, queryResult }),
impact: "M3 true/false IO durable postflight is skipped or blocked so DEV CD cannot prove evidence persistence.",
safeNextAction: runtimeDurableSafeNextAction(nonReadyReasonCode, layer),
retryable: true,
readonly: true,
evidence: runtimeDurableEvidence({
adapter,
runtimeDurable,
runtimeReady,
durabilityReady,
liveRuntimeEvidence,
blockedLayer: layer,
queryResult,
readyQueryResult,
reasonCode: nonReadyReasonCode,
gates,
rawStatus,
postgresRequired
})
};
}
function firstBlockedRuntimeGate(gates = {}) {
for (const layer of RUNTIME_DURABLE_GATE_LAYERS) {
const gate = gates?.[layer];
const status = normalizedStatus(gate?.status);
if (gate?.ready === false || status === "blocked" || status === "failed") {
return {
layer,
blocker: firstNonEmpty(gate?.blocker, runtimeDurableBlockerForLayer(layer))
};
}
}
return null;
}
function runtimeDurableBlockerForLayer(layer) {
const text = normalizedStatus(layer);
if (text === "ssl") return "runtime_durable_adapter_ssl_blocked";
if (text === "auth") return "runtime_durable_adapter_auth_blocked";
if (text === "schema") return "runtime_durable_adapter_schema_blocked";
if (text === "migration") return "runtime_durable_adapter_migration_blocked";
if (text === "durability" || text === "durability_query") return "runtime_durable_adapter_query_blocked";
return "runtime_durable_adapter_blocked";
}
function runtimeDurableLayerFromBlocker(blocker) {
const text = normalizedStatus(blocker);
if (!text) return "";
if (text.includes("ssl")) return "ssl";
if (text.includes("auth")) return "auth";
if (text.includes("schema")) return "schema";
if (text.includes("migration")) return "migration";
if (text.includes("query")) return "durability_query";
if (text.includes("missing") || text.includes("unconfigured") || text.includes("driver")) return "adapter";
return "";
}
function runtimeDurableLayerFromQueryResult(queryResult) {
const text = normalizedStatus(queryResult);
if (!text || isRuntimeDurableReadyQueryResult(text)) return "";
if (text.includes("ssl")) return "ssl";
if (text.includes("auth")) return "auth";
if (text.includes("schema")) return "schema";
if (text.includes("migration")) return "migration";
if (text.includes("query")) return "durability_query";
if (text.includes("driver")) return "adapter";
return "";
}
function runtimeDurableBlockerFromQueryResult(queryResult) {
const text = normalizedStatus(queryResult);
if (!text || isRuntimeDurableReadyQueryResult(text)) return "";
return RUNTIME_DURABLE_BLOCKED_QUERY_RESULTS[text] ?? "";
}
function isRuntimeDurableReadyQueryResult(queryResult) {
const text = normalizedStatus(queryResult);
return RUNTIME_DURABLE_READY_QUERY_RESULTS.includes(text);
}
function runtimeDurableEvidence({
adapter,
runtimeDurable,
runtimeReady,
durabilityReady,
liveRuntimeEvidence,
blockedLayer,
queryResult,
readyQueryResult,
reasonCode,
gates,
rawStatus,
postgresRequired
}) {
return {
adapter,
durable: runtimeDurable,
runtimeReady,
durabilityReady,
liveRuntimeEvidence,
blocker: reasonCode,
blockedLayer,
queryResult: queryResult || null,
readyQueryResult,
rawStatus: rawStatus || null,
postgresRequired,
gates: summarizeRuntimeGateStatuses(gates),
secretMaterialRead: false
};
}
function summarizeRuntimeGateStatuses(gates = {}) {
return Object.fromEntries(RUNTIME_DURABLE_GATE_LAYERS.map((layer) => [
layer,
{
checked: gates?.[layer]?.checked === true,
ready: gates?.[layer]?.ready === true,
status: gates?.[layer]?.status ?? "unknown",
blocker: gates?.[layer]?.blocker ?? null
}
]));
}
function runtimeDurableBlockedReason({ reasonCode, adapter, layer, queryResult }) {
if (reasonCode === "runtime_durable_false") {
return `runtime durable=false; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`;
}
if (reasonCode === "runtime_durable_adapter_not_postgres") {
return `runtime durable adapter is not postgres; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`;
}
return `runtime durable blocked at ${layer || "runtime-durable"}; blocker=${reasonCode}; queryResult=${queryResult || "unknown"}`;
}
function runtimeDurableSafeNextAction(reasonCode, layer) {
const text = normalizedStatus(`${reasonCode} ${layer}`);
if (text.includes("auth")) return "Repair DEV DB auth/SecretRef metadata through repo-owned provisioning, then rerun postflight without printing Secret values.";
if (text.includes("schema") || text.includes("migration")) return "Run repo-owned DEV runtime provisioning and migration, then rerun postflight.";
if (text.includes("ssl")) return "Align DEV DB SSL mode with the repo-owned DEV contract, then rerun postflight.";
if (text.includes("query")) return "Inspect durable runtime query readiness, preserve evidence persistence, and rerun postflight.";
if (text.includes("postgres")) return "Run DEV with HWLAB_CLOUD_RUNTIME_ADAPTER=postgres and DB URLs injected only through SecretRefs.";
return "Restore durable runtime readiness through repo-owned DEV CD steps, then rerun postflight.";
}
function firstBlocker(payload) {
const blockers = [
...(Array.isArray(payload?.blockers) ? payload.blockers : []),
@@ -626,13 +862,22 @@ function probeAttribution(probe) {
return [
`${probe.serviceId} ${probe.apiPath}`,
probe.httpStatus ? `HTTP ${probe.httpStatus}` : null,
probe.reasonCode,
probe.reason && probe.reason !== probe.reasonCode ? probe.reason : null,
displayReasonCode(probe.reasonCode),
probe.reason && probe.reason !== probe.reasonCode ? displayReasonText(probe.reason) : null,
probe.traceId ? `trace=${probe.traceId}` : null,
probe.evidenceSummary
].filter(Boolean).join(" / ");
}
function displayReasonCode(reasonCode) {
const text = String(reasonCode ?? "");
return text.startsWith("runtime_durable_adapter_") ? "runtime durable" : text;
}
function displayReasonText(reason) {
return String(reason ?? "").replace(/\bruntime_durable_adapter_[a-z0-9_]+\b/giu, "runtime durable");
}
function normalizedStatus(value) {
return String(value ?? "").trim().toLowerCase();
}
@@ -34,6 +34,25 @@ const okRest = Object.freeze({
serviceId: "hwlab-cloud-api",
status: "ok",
ready: true,
readiness: {
status: "ok",
ready: true,
durability: {
status: "ready",
ready: true,
queryResult: "durable_readiness_ready"
}
},
runtime: {
adapter: "postgres",
status: "ready",
durable: true,
ready: true,
liveRuntimeEvidence: true,
connection: {
queryResult: "durable_readiness_ready"
}
},
codeAgent: {
endpoint: "POST /v1/agent/chat",
status: "available",
@@ -52,6 +71,25 @@ const codexStdioRest = Object.freeze({
serviceId: "hwlab-cloud-api",
status: "ok",
ready: true,
readiness: {
status: "ok",
ready: true,
durability: {
status: "ready",
ready: true,
queryResult: "durable_readiness_ready"
}
},
runtime: {
adapter: "postgres",
status: "ready",
durable: true,
ready: true,
liveRuntimeEvidence: true,
connection: {
queryResult: "durable_readiness_ready"
}
},
codeAgent: {
endpoint: "POST /v1/agent/chat",
status: "codex-stdio-feasible",
@@ -78,6 +116,25 @@ const readonlyOnlyRest = Object.freeze({
serviceId: "hwlab-cloud-api",
status: "degraded",
ready: false,
readiness: {
status: "ok",
ready: true,
durability: {
status: "ready",
ready: true,
queryResult: "durable_readiness_ready"
}
},
runtime: {
adapter: "postgres",
status: "ready",
durable: true,
ready: true,
liveRuntimeEvidence: true,
connection: {
queryResult: "durable_readiness_ready"
}
},
codeAgent: {
endpoint: "POST /v1/agent/chat",
status: "partial",
@@ -232,6 +289,111 @@ test("classifies durable runtime degraded as explicit read-only mode", () => {
assert.deepEqual(status.internalRawStatuses.filter((item) => item === "degraded"), ["degraded"]);
});
test("does not classify ready queryResult durable runtime as blocked", () => {
const status = classifyWorkbenchLiveStatus({
healthLive: {
ok: true,
status: 200,
data: {
serviceId: "hwlab-cloud-api",
status: "ready",
ready: true,
runtime: {
adapter: "postgres",
durable: true,
ready: true,
liveRuntimeEvidence: true,
status: "ready",
blocker: null,
connection: {
queryResult: "durable_readiness_ready"
}
},
readiness: {
status: "ready",
ready: true,
durability: {
status: "ready",
ready: true,
blocker: null,
blockedLayer: null,
queryResult: "durable_readiness_ready"
}
},
blockerCodes: []
}
},
restIndex: okRest,
health: okApi,
adapter: okApi,
m3Control: okM3Control,
m3Status: okM3Status
});
assert.equal(status.kind, "pass");
assert.equal(status.reasonCode, "ready");
assert.equal(status.probes.find((probe) => probe.apiPath === "/health/live")?.reasonCode, "ready");
});
test("classifies auth and migration durable blockers with shared runtime reason", () => {
for (const [reasonCode, queryResult, blockedLayer] of [
["runtime_durable_adapter_auth_blocked", "auth_blocked", "auth"],
["runtime_durable_adapter_migration_blocked", "migration_blocked", "migration"]
]) {
const status = classifyWorkbenchLiveStatus({
healthLive: {
ok: true,
status: 200,
data: {
serviceId: "hwlab-cloud-api",
status: "degraded",
ready: false,
runtime: {
adapter: "postgres",
durable: false,
durableRequested: true,
ready: false,
liveRuntimeEvidence: false,
status: "blocked",
blocker: reasonCode,
connection: { queryResult },
durabilityContract: { blockedLayer },
gates: {
[blockedLayer]: {
checked: true,
ready: false,
status: "blocked",
blocker: reasonCode
}
}
},
readiness: {
ready: false,
status: "blocked",
durability: {
ready: false,
status: "blocked",
blocker: reasonCode,
blockedLayer,
queryResult
}
},
blockerCodes: [reasonCode]
}
},
restIndex: okRest,
health: okApi,
adapter: okApi,
m3Control: okM3Control,
m3Status: okM3Status
});
assert.equal(status.kind, "readonly");
assert.equal(status.reasonCode, reasonCode);
assert.match(status.detail, new RegExp(queryResult, "u"));
}
});
test("classifies missing probes as unverified instead of degraded", () => {
const status = classifyWorkbenchLiveStatus({});