fix: harden dev cd runtime postflight
Summary: - shared runtime durable readiness classifier for live-status and DEV postflight - structured postflight blockers propagated through dev-cd apply - contract coverage for ready queryResult, auth/schema/migration blockers, evidence persistence blockers, and bounded stdout Verification: - git diff --check - node --check changed scripts/tests/live-status modules - node --test scripts/src/dev-runtime-postflight.test.mjs scripts/src/dev-cd-apply.test.mjs web/hwlab-cloud-web/scripts/live-status-contract.test.mjs - node scripts/dev-runtime-postflight.mjs --check - node web/hwlab-cloud-web/scripts/check.mjs - npm run check No DEV apply/rollout/live health verification was run.
This commit is contained in:
@@ -1815,11 +1815,55 @@ function parseLastJsonObject(text) {
|
||||
}
|
||||
|
||||
function stepBlocker(stepResult) {
|
||||
return {
|
||||
const stdoutBlockers = Array.isArray(stepResult.stdoutJson?.blockers) ? stepResult.stdoutJson.blockers : [];
|
||||
const primary = stdoutBlockers.find((item) => item?.status === "open") ?? stdoutBlockers[0] ?? null;
|
||||
const reason = primary?.reason ?? primary?.summary ?? `${stepResult.command} exited ${stepResult.code}.`;
|
||||
const impact = primary?.impact ?? stepBlockerImpact(stepResult);
|
||||
const safeNextAction = primary?.safeNextAction ?? stepBlockerSafeNextAction(stepResult);
|
||||
return structuredBlocker({
|
||||
type: "runtime_blocker",
|
||||
scope: stepResult.id,
|
||||
reason,
|
||||
impact,
|
||||
safeNextAction,
|
||||
retryable: primary?.retryable ?? true,
|
||||
evidence: {
|
||||
command: stepResult.command,
|
||||
code: stepResult.code,
|
||||
reportPath: stepResult.reportPath ?? null,
|
||||
blockerScope: primary?.scope ?? null,
|
||||
k8sJob: stepResult.k8sJob ?? null
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function stepBlockerImpact(stepResult) {
|
||||
if (stepResult.id === "runtime-db-provisioning") return "DEV CD cannot prove DB role/database provisioning before workload apply.";
|
||||
if (stepResult.id === "runtime-db-migration") return "DEV CD cannot prove runtime schema migration before workload apply.";
|
||||
if (stepResult.id === "runtime-durable-postflight") return "DEV CD cannot prove /health/live, /v1, and M3 durable evidence persistence after apply.";
|
||||
if (stepResult.id === "dev-deploy-apply") return "DEV desired-state apply did not complete, so live readiness cannot be trusted.";
|
||||
return "DEV CD transaction cannot safely continue.";
|
||||
}
|
||||
|
||||
function stepBlockerSafeNextAction(stepResult) {
|
||||
if (stepResult.id === "runtime-db-provisioning") return "Repair repo-owned runtime provisioning inputs/SecretRefs without printing Secret values, then rerun DEV CD.";
|
||||
if (stepResult.id === "runtime-db-migration") return "Repair repo-owned runtime migration/schema state, then rerun DEV CD.";
|
||||
if (stepResult.id === "runtime-durable-postflight") return "Inspect the runtime postflight report, repair durable readiness or M3 evidence persistence, then rerun DEV CD after host-controlled rollout if needed.";
|
||||
if (stepResult.id === "dev-deploy-apply") return "Inspect reports/dev-gate/dev-deploy-report.json and rerun the repo-owned DEV CD path after the apply blocker is fixed.";
|
||||
return "Fix the reported step blocker and rerun the repo-owned DEV CD transaction.";
|
||||
}
|
||||
|
||||
function structuredBlocker({ type = "runtime_blocker", scope, reason, impact, safeNextAction, retryable = true, evidence = null }) {
|
||||
return {
|
||||
type,
|
||||
scope,
|
||||
status: "open",
|
||||
summary: `${stepResult.command} exited ${stepResult.code}.`
|
||||
reason: oneLine(reason),
|
||||
impact: oneLine(impact),
|
||||
safeNextAction: oneLine(safeNextAction),
|
||||
retryable: Boolean(retryable),
|
||||
summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`),
|
||||
...(evidence ? { evidence } : {})
|
||||
};
|
||||
}
|
||||
|
||||
|
||||
@@ -215,6 +215,7 @@ function makeRunCommand({
|
||||
],
|
||||
desiredStatePromotionStatus = "pass",
|
||||
runtimeJobLogSuffix = "",
|
||||
runtimePostflightReport = null,
|
||||
kubeContext = "d601",
|
||||
kubeServer = "https://d601:6443",
|
||||
kubeNodes = ["d601"],
|
||||
@@ -390,6 +391,24 @@ function makeRunCommand({
|
||||
};
|
||||
}
|
||||
if (command === process.execPath || command.endsWith("/node")) {
|
||||
if (args.includes("scripts/dev-runtime-postflight.mjs")) {
|
||||
const report = runtimePostflightReport ?? {
|
||||
conclusion: { status: "pass" },
|
||||
blockers: [],
|
||||
safety: { secretValuesPrinted: false },
|
||||
apiHealth: { runtimeDurableReadiness: { ready: true } },
|
||||
apiV1: { runtimeDurableReadiness: { ready: true } },
|
||||
m3: {
|
||||
persistenceContract: { ready: true },
|
||||
operationCount: 4
|
||||
}
|
||||
};
|
||||
return {
|
||||
code: report.conclusion?.status === "pass" ? 0 : 1,
|
||||
stdout: `${JSON.stringify(report, null, 2)}\n`,
|
||||
stderr: ""
|
||||
};
|
||||
}
|
||||
return { code: 0, stdout: JSON.stringify({ ok: true, command: args.join(" ") }), stderr: "" };
|
||||
}
|
||||
return { code: 0, stdout: "", stderr: "" };
|
||||
@@ -1294,6 +1313,93 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a
|
||||
assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2);
|
||||
});
|
||||
|
||||
test("transaction blocks with structured runtime postflight blocker when M3 evidence is not persisted", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const runtimePostflightReport = {
|
||||
conclusion: {
|
||||
status: "blocked",
|
||||
blockerCount: 1
|
||||
},
|
||||
apiHealth: {
|
||||
runtimeDurableReadiness: {
|
||||
ready: true,
|
||||
classification: "runtime_durable_ready"
|
||||
}
|
||||
},
|
||||
apiV1: {
|
||||
runtimeDurableReadiness: {
|
||||
ready: true,
|
||||
classification: "runtime_durable_ready"
|
||||
}
|
||||
},
|
||||
m3: {
|
||||
persistenceContract: {
|
||||
ready: false,
|
||||
sequence: [
|
||||
{ id: "write-do1-true", persisted: true, idsPresent: true },
|
||||
{ id: "read-di1-true", persisted: false, idsPresent: true }
|
||||
]
|
||||
},
|
||||
operationCount: 4
|
||||
},
|
||||
blockers: [
|
||||
{
|
||||
type: "runtime_blocker",
|
||||
scope: "m3-durable-postflight",
|
||||
status: "open",
|
||||
reason: "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.",
|
||||
impact: "A later rollout could appear ready while losing evidence persistence.",
|
||||
safeNextAction: "Repair durable audit/evidence persistence, then rerun DEV runtime postflight.",
|
||||
retryable: true
|
||||
}
|
||||
],
|
||||
safety: {
|
||||
secretValuesPrinted: false,
|
||||
readsKubernetesSecrets: false
|
||||
}
|
||||
};
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-postflight-blocked",
|
||||
"--kubeconfig",
|
||||
"/etc/rancher/k3s/k3s.yaml",
|
||||
"--write-report"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({ commandLog, runtimePostflightReport }),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const summary = JSON.parse(output);
|
||||
const report = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
|
||||
const blocker = report.blockers.find((item) => item.scope === "runtime-durable-postflight");
|
||||
assert.equal(code, 2);
|
||||
assert.equal(summary.status, "blocked");
|
||||
assert.equal(summary.devCdApply, undefined);
|
||||
assert.equal(output.length < 12000, true);
|
||||
assert.equal(report.status, "blocked");
|
||||
assert.equal(report.devCdApply.steps.at(-1).id, "runtime-durable-postflight");
|
||||
assert.equal(report.devCdApply.steps.at(-1).status, "blocked");
|
||||
assert.ok(blocker);
|
||||
assert.equal(blocker.reason, runtimePostflightReport.blockers[0].reason);
|
||||
assert.equal(blocker.impact, runtimePostflightReport.blockers[0].impact);
|
||||
assert.equal(blocker.safeNextAction, runtimePostflightReport.blockers[0].safeNextAction);
|
||||
assert.equal(blocker.retryable, true);
|
||||
assert.equal(blocker.evidence.blockerScope, "m3-durable-postflight");
|
||||
assert.equal(report.devCdApply.liveVerify.status, "not_run");
|
||||
assert.equal(JSON.stringify(report).includes("postgresql://"), false);
|
||||
assert.equal(JSON.stringify(report).includes("SECRET"), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-runtime-postflight.mjs")), true);
|
||||
});
|
||||
|
||||
test("runtime maintenance job reports are parsed after redacting secret-like log output", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
|
||||
@@ -4,6 +4,7 @@ import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
||||
import { classifyRuntimeDurableReadiness } from "../../web/hwlab-cloud-web/live-status.mjs";
|
||||
import {
|
||||
buildM3IoControlSourceReport,
|
||||
runM3IoControlLiveReport
|
||||
@@ -78,8 +79,16 @@ export async function buildDevRuntimePostflightReport(args = {}, options = {}) {
|
||||
blockerCount: sourceReport.summary?.status === "pass" ? 0 : 1
|
||||
};
|
||||
if (sourceReport.summary?.status !== "pass") {
|
||||
addBlocker(report, "contract_blocker", "m3-source-contract", "M3 postflight source contract is blocked.", {
|
||||
classification: sourceReport.summary?.classification ?? "unknown"
|
||||
addBlocker(report, {
|
||||
type: "contract_blocker",
|
||||
scope: "m3-source-contract",
|
||||
reason: "M3 postflight source contract is blocked.",
|
||||
impact: "DEV CD cannot rely on the M3 durable postflight harness contract.",
|
||||
safeNextAction: "Fix the source M3 IO control contract tests before running live DEV postflight.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
classification: sourceReport.summary?.classification ?? "unknown"
|
||||
}
|
||||
});
|
||||
}
|
||||
return report;
|
||||
@@ -227,6 +236,7 @@ async function defaultHttpGetJson(url, timeoutMs) {
|
||||
function summarizeApiPayload(check, response) {
|
||||
const json = response.json ?? {};
|
||||
const runtime = summarizeRuntime(json.runtime);
|
||||
const runtimeDurableReadiness = classifyRuntimeDurableReadiness(json, { requirePostgresAdapter: true });
|
||||
return {
|
||||
check,
|
||||
httpStatus: response.status,
|
||||
@@ -236,6 +246,7 @@ function summarizeApiPayload(check, response) {
|
||||
status: json.status ?? null,
|
||||
ready: json.ready === true,
|
||||
runtime,
|
||||
runtimeDurableReadiness,
|
||||
db: summarizeDb(json.db),
|
||||
readiness: {
|
||||
ready: json.readiness?.ready === true,
|
||||
@@ -372,23 +383,48 @@ function summarizeM3PersistenceContract(operations = []) {
|
||||
function addApiReadinessBlockers(report) {
|
||||
for (const [scope, payload] of [["api-health-live", report.apiHealth], ["api-v1", report.apiV1]]) {
|
||||
if (!payload.reachable) {
|
||||
addBlocker(report, "runtime_blocker", scope, `${payload.check} was not reachable.`, {
|
||||
httpStatus: payload.httpStatus,
|
||||
error: payload.error
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope,
|
||||
reason: `${payload.check} was not reachable.`,
|
||||
impact: "DEV CD cannot prove cloud-api durable readiness, so M3 evidence persistence postflight cannot safely run.",
|
||||
safeNextAction: "Restore DEV cloud-api reachability on the repo-owned public endpoint, then rerun postflight.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
httpStatus: payload.httpStatus,
|
||||
error: payload.error
|
||||
}
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (!runtimeDurableReady(payload.runtime)) {
|
||||
addBlocker(report, "runtime_blocker", scope, `${payload.check} did not prove durable runtime readiness.`, {
|
||||
runtime: payload.runtime,
|
||||
blockerCodes: payload.blockerCodes
|
||||
if (payload.runtimeDurableReadiness?.ready !== true) {
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope,
|
||||
reason: `${payload.check} did not prove durable runtime readiness: ${payload.runtimeDurableReadiness?.reason ?? "not ready"}.`,
|
||||
impact: payload.runtimeDurableReadiness?.impact ?? "M3 durable postflight cannot prove evidence persistence.",
|
||||
safeNextAction: payload.runtimeDurableReadiness?.safeNextAction ?? "Repair runtime durability through repo-owned DEV CD steps, then rerun postflight.",
|
||||
retryable: payload.runtimeDurableReadiness?.retryable !== false,
|
||||
evidence: {
|
||||
runtime: payload.runtime,
|
||||
runtimeDurableReadiness: payload.runtimeDurableReadiness,
|
||||
blockerCodes: payload.blockerCodes
|
||||
}
|
||||
});
|
||||
}
|
||||
if (scope === "api-v1" && !apiV1M3ContractReady(payload)) {
|
||||
addBlocker(report, "runtime_blocker", scope, "GET /v1 did not expose the controlled M3 IO route required by durable postflight.", {
|
||||
m3IoControl: payload.m3IoControl,
|
||||
requiredRoute: "/v1/m3/io",
|
||||
requiredContractVersion: "m3-io-control-v1"
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope,
|
||||
reason: "GET /v1 did not expose the controlled M3 IO route required by durable postflight.",
|
||||
impact: "DEV CD cannot run the M3 true/false IO persistence smoke through the cloud-api controlled boundary.",
|
||||
safeNextAction: "Restore /v1 m3IoControl route metadata for /v1/m3/io without exposing direct frontend simulator access.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
m3IoControl: payload.m3IoControl,
|
||||
requiredRoute: "/v1/m3/io",
|
||||
requiredContractVersion: "m3-io-control-v1"
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -406,58 +442,96 @@ function apiV1M3ContractReady(payload) {
|
||||
|
||||
function addM3Blockers(report) {
|
||||
if (report.m3?.persistenceContract?.ready !== true) {
|
||||
addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.", {
|
||||
persistenceContract: report.m3?.persistenceContract ?? null,
|
||||
trustedGreen: report.m3?.trustedGreen === true,
|
||||
operationCount: report.m3?.operationCount ?? 0
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope: "m3-durable-postflight",
|
||||
reason: "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.",
|
||||
impact: "A later rollout could appear ready while losing operation, audit, or evidence persistence for M3 hardware IO.",
|
||||
safeNextAction: "Inspect the failed M3 operation sequence, repair durable audit/evidence persistence, then rerun DEV runtime postflight.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
persistenceContract: report.m3?.persistenceContract ?? null,
|
||||
trustedGreen: report.m3?.trustedGreen === true,
|
||||
operationCount: report.m3?.operationCount ?? 0
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (report.m3?.trustedGreen !== true || report.m3?.status !== "pass") {
|
||||
addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 true/false durable postflight did not produce trusted green persisted evidence.", {
|
||||
classification: report.m3?.classification ?? "unknown",
|
||||
trustedGreen: report.m3?.trustedGreen === true,
|
||||
operationCount: report.m3?.operationCount ?? 0
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope: "m3-durable-postflight",
|
||||
reason: "M3 true/false durable postflight did not produce trusted green persisted evidence.",
|
||||
impact: "M3 durable readiness cannot be accepted because trusted evidence is not green.",
|
||||
safeNextAction: "Fix the M3 control path or trusted-record persistence and rerun DEV runtime postflight.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
classification: report.m3?.classification ?? "unknown",
|
||||
trustedGreen: report.m3?.trustedGreen === true,
|
||||
operationCount: report.m3?.operationCount ?? 0
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
for (const operation of report.m3.operations ?? []) {
|
||||
if (!operation.operationId || !operation.auditId || !operation.evidenceId || operation.evidenceState?.status !== "green" || operation.evidenceState?.durable !== true) {
|
||||
addBlocker(report, "runtime_blocker", "m3-durable-evidence", "M3 operation/audit/evidence durable fields were incomplete.", {
|
||||
operationIdPresent: Boolean(operation.operationId),
|
||||
auditIdPresent: Boolean(operation.auditId),
|
||||
evidenceIdPresent: Boolean(operation.evidenceId),
|
||||
evidenceState: operation.evidenceState
|
||||
addBlocker(report, {
|
||||
type: "runtime_blocker",
|
||||
scope: "m3-durable-evidence",
|
||||
reason: "M3 operation/audit/evidence durable fields were incomplete.",
|
||||
impact: "The M3 postflight cannot prove every IO operation is bound to persisted audit and evidence records.",
|
||||
safeNextAction: "Repair operation, audit, and evidence ID propagation before accepting DEV readiness.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
operationIdPresent: Boolean(operation.operationId),
|
||||
traceIdPresent: Boolean(operation.traceId),
|
||||
auditIdPresent: Boolean(operation.auditId),
|
||||
evidenceIdPresent: Boolean(operation.evidenceId),
|
||||
evidenceState: operation.evidenceState
|
||||
}
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runtimeDurableReady(runtime) {
|
||||
return runtime.adapter === "postgres" &&
|
||||
runtime.durable === true &&
|
||||
runtime.ready === true &&
|
||||
runtime.liveRuntimeEvidence === true;
|
||||
}
|
||||
|
||||
function addSafetyRefusal(report, summary) {
|
||||
report.safetyRefusal = true;
|
||||
addBlocker(report, "safety_refusal", "runtime-postflight-live-boundary", summary, {
|
||||
devOnly: true,
|
||||
prodAllowed: false,
|
||||
secretValuesPrinted: false
|
||||
addBlocker(report, {
|
||||
type: "safety_refusal",
|
||||
scope: "runtime-postflight-live-boundary",
|
||||
reason: summary,
|
||||
impact: "Live DEV reads and M3 writes are not allowed without explicit non-production confirmation.",
|
||||
safeNextAction: "Rerun with --live --confirm-dev --confirmed-non-production for DEV only, or use --check for source-only validation.",
|
||||
retryable: true,
|
||||
evidence: {
|
||||
devOnly: true,
|
||||
prodAllowed: false,
|
||||
secretValuesPrinted: false
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
function addBlocker(report, type, scope, summary, evidence = {}) {
|
||||
function addBlocker(report, {
|
||||
type,
|
||||
scope,
|
||||
reason,
|
||||
impact,
|
||||
safeNextAction,
|
||||
retryable,
|
||||
evidence = {}
|
||||
}) {
|
||||
report.blockers.push({
|
||||
type,
|
||||
scope,
|
||||
status: "open",
|
||||
summary,
|
||||
reason: oneLine(reason),
|
||||
impact: oneLine(impact),
|
||||
safeNextAction: oneLine(safeNextAction),
|
||||
retryable: Boolean(retryable),
|
||||
summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`),
|
||||
sourceIssue: issue,
|
||||
evidence
|
||||
});
|
||||
@@ -475,6 +549,10 @@ function finalizeReport(report) {
|
||||
return report;
|
||||
}
|
||||
|
||||
function oneLine(value) {
|
||||
return String(value ?? "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function buildSafety(args) {
|
||||
return {
|
||||
devOnly: true,
|
||||
|
||||
@@ -56,6 +56,8 @@ test("live mode skips M3 writes when /health/live durable runtime is blocked", a
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
assert.equal(report.m3.status, "not_run");
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-health-live"), true);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
assert.match(report.blockers.find((blocker) => blocker.scope === "api-health-live").reason, /auth_blocked/u);
|
||||
assert.equal(JSON.stringify(report).includes("postgresql://"), false);
|
||||
});
|
||||
|
||||
@@ -74,6 +76,7 @@ test("live mode requires /v1 durable readiness before M3 writes", async () => {
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-v1"), true);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
});
|
||||
|
||||
test("live mode requires /v1 to expose the controlled M3 IO route before M3 writes", async () => {
|
||||
@@ -100,6 +103,7 @@ test("live mode requires /v1 to expose the controlled M3 IO route before M3 writ
|
||||
blocker.scope === "api-v1" &&
|
||||
blocker.summary.includes("controlled M3 IO route")
|
||||
), true);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
});
|
||||
|
||||
test("live mode does not mistake DB live evidence for durable runtime readiness", async () => {
|
||||
@@ -121,6 +125,51 @@ test("live mode does not mistake DB live evidence for durable runtime readiness"
|
||||
assert.equal(report.apiHealth.readiness.durabilityReady, false);
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-health-live"), true);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
});
|
||||
|
||||
test("live mode blocks auth, schema, and migration durable readiness before M3 writes", async () => {
|
||||
for (const [blocker, queryResult, blockedLayer] of [
|
||||
["runtime_durable_adapter_auth_blocked", "auth_blocked", "auth"],
|
||||
["runtime_durable_adapter_schema_blocked", "schema_blocked", "schema"],
|
||||
["runtime_durable_adapter_migration_blocked", "migration_blocked", "migration"]
|
||||
]) {
|
||||
const report = await buildDevRuntimePostflightReport(
|
||||
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
|
||||
{
|
||||
httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: false, blocker, queryResult, blockedLayer }) }),
|
||||
m3Runner: async () => {
|
||||
throw new Error("M3 runner should not run when durable readiness is blocked");
|
||||
},
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
assert.equal(report.m3.status, "not_run");
|
||||
assert.equal(report.apiHealth.runtimeDurableReadiness.classification, blocker);
|
||||
assert.equal(report.apiHealth.runtimeDurableReadiness.evidence.queryResult, queryResult);
|
||||
assert.equal(report.apiV1.runtimeDurableReadiness.classification, blocker);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
}
|
||||
});
|
||||
|
||||
test("live mode treats durable_readiness_ready queryResult as ready, not blocked", async () => {
|
||||
const report = await buildDevRuntimePostflightReport(
|
||||
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
|
||||
{
|
||||
httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: true, queryResult: "durable_readiness_ready" }) }),
|
||||
m3Runner: async () => m3GreenReport(),
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "pass");
|
||||
assert.equal(report.actions.m3MutationAttempted, true);
|
||||
assert.equal(report.apiHealth.runtimeDurableReadiness.ready, true);
|
||||
assert.equal(report.apiHealth.runtimeDurableReadiness.evidence.queryResult, "durable_readiness_ready");
|
||||
assert.equal(report.blockers.length, 0);
|
||||
});
|
||||
|
||||
test("live mode records M3 true/false durable green evidence", async () => {
|
||||
@@ -165,6 +214,7 @@ test("live mode blocks when M3 operation evidence is not durable green", async (
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-postflight"), true);
|
||||
assert.equal(report.m3.persistenceContract.ready, false);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
});
|
||||
|
||||
test("live mode blocks when M3 true/false persisted sequence is incomplete", async () => {
|
||||
@@ -183,9 +233,29 @@ test("live mode blocks when M3 true/false persisted sequence is incomplete", asy
|
||||
assert.equal(report.m3.persistenceContract.ready, false);
|
||||
assert.equal(report.m3.persistenceContract.sequence.at(-1).status, "missing");
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-postflight"), true);
|
||||
assertStructuredBlockers(report.blockers);
|
||||
});
|
||||
|
||||
function apiPayload({ ready }) {
|
||||
function assertStructuredBlockers(blockers) {
|
||||
assert.ok(blockers.length > 0);
|
||||
for (const blocker of blockers) {
|
||||
assert.equal(typeof blocker.scope, "string");
|
||||
assert.equal(typeof blocker.reason, "string");
|
||||
assert.equal(typeof blocker.impact, "string");
|
||||
assert.equal(typeof blocker.safeNextAction, "string");
|
||||
assert.equal(typeof blocker.retryable, "boolean");
|
||||
assert.notEqual(blocker.reason, "");
|
||||
assert.notEqual(blocker.impact, "");
|
||||
assert.notEqual(blocker.safeNextAction, "");
|
||||
}
|
||||
}
|
||||
|
||||
function apiPayload({
|
||||
ready,
|
||||
blocker = "runtime_durable_adapter_auth_blocked",
|
||||
queryResult = ready ? "durable_readiness_ready" : "auth_blocked",
|
||||
blockedLayer = "auth"
|
||||
}) {
|
||||
return {
|
||||
serviceId: "hwlab-cloud-api",
|
||||
environment: "dev",
|
||||
@@ -200,8 +270,8 @@ function apiPayload({ ready }) {
|
||||
runtimeReadiness: {
|
||||
ready,
|
||||
status: ready ? "ready" : "blocked",
|
||||
blocker: ready ? null : "runtime_durable_adapter_auth_blocked",
|
||||
queryResult: ready ? "durable_readiness_ready" : "auth_blocked",
|
||||
blocker: ready ? null : blocker,
|
||||
queryResult,
|
||||
requiredEvidence: "runtime_adapter_schema_migration_read_query"
|
||||
}
|
||||
},
|
||||
@@ -211,18 +281,21 @@ function apiPayload({ ready }) {
|
||||
durableRequested: true,
|
||||
ready,
|
||||
status: ready ? "ready" : "blocked",
|
||||
blocker: ready ? null : "runtime_durable_adapter_auth_blocked",
|
||||
blocker: ready ? null : blocker,
|
||||
liveRuntimeEvidence: ready,
|
||||
connection: {
|
||||
queryResult
|
||||
},
|
||||
durabilityContract: {
|
||||
requiredEvidence: "runtime_adapter_schema_migration_read_query",
|
||||
blockedLayer: ready ? null : "auth"
|
||||
blockedLayer: ready ? null : blockedLayer
|
||||
},
|
||||
gates: {
|
||||
ssl: gate(true),
|
||||
auth: gate(ready, "runtime_durable_adapter_auth_blocked"),
|
||||
schema: gate(ready),
|
||||
migration: gate(ready),
|
||||
durability: gate(ready)
|
||||
auth: gate(ready || !["auth"].includes(blockedLayer), blockedLayer === "auth" ? blocker : null),
|
||||
schema: gate(ready || !["schema"].includes(blockedLayer), blockedLayer === "schema" ? blocker : null),
|
||||
migration: gate(ready || !["migration"].includes(blockedLayer), blockedLayer === "migration" ? blocker : null),
|
||||
durability: gate(ready, blocker)
|
||||
}
|
||||
},
|
||||
readiness: {
|
||||
@@ -230,11 +303,14 @@ function apiPayload({ ready }) {
|
||||
status: ready ? "ready" : "blocked",
|
||||
durability: {
|
||||
ready,
|
||||
blockedLayer: ready ? null : "auth"
|
||||
status: ready ? "ready" : "blocked",
|
||||
blocker: ready ? null : blocker,
|
||||
blockedLayer: ready ? null : blockedLayer,
|
||||
queryResult
|
||||
}
|
||||
},
|
||||
blockerCodes: ready ? []
|
||||
: ["runtime_durable_adapter_auth_blocked"],
|
||||
: [blocker],
|
||||
m3IoControl: {
|
||||
route: "/v1/m3/io",
|
||||
status: "available",
|
||||
|
||||
@@ -3,6 +3,26 @@ const ERROR_STATUSES = Object.freeze(["error", "failed", "failure"]);
|
||||
const RAW_DEGRADED_STATUSES = Object.freeze(["degraded"]);
|
||||
const BLOCKED_STATUSES = Object.freeze(["blocked", "unavailable", "rejected"]);
|
||||
const UNVERIFIED_STATUSES = Object.freeze(["", "unknown", "pending", "loading", "probing", "unverified", "not_observed"]);
|
||||
const RUNTIME_DURABLE_READY_QUERY_RESULTS = Object.freeze([
|
||||
"durable_readiness_ready",
|
||||
"runtime_durable_ready",
|
||||
"query_ready",
|
||||
"readiness_ready",
|
||||
"ready",
|
||||
"ok",
|
||||
"pass",
|
||||
"passed"
|
||||
]);
|
||||
const RUNTIME_DURABLE_BLOCKED_QUERY_RESULTS = Object.freeze({
|
||||
driver_missing: "runtime_durable_adapter_driver_missing",
|
||||
ssl_negotiation_blocked: "runtime_durable_adapter_ssl_blocked",
|
||||
auth_blocked: "runtime_durable_adapter_auth_blocked",
|
||||
schema_blocked: "runtime_durable_adapter_schema_blocked",
|
||||
migration_blocked: "runtime_durable_adapter_migration_blocked",
|
||||
query_blocked: "runtime_durable_adapter_query_blocked",
|
||||
not_ready: "runtime_durable_adapter_query_blocked"
|
||||
});
|
||||
const RUNTIME_DURABLE_GATE_LAYERS = Object.freeze(["ssl", "auth", "schema", "migration", "durability"]);
|
||||
const READ_ONLY_REASON_CODES = Object.freeze([
|
||||
"read_only",
|
||||
"read-only",
|
||||
@@ -480,70 +500,286 @@ function classifyM3StatusProbe(live = {}) {
|
||||
}
|
||||
|
||||
function durableRuntimeReason(payload) {
|
||||
const runtime = payload?.runtime;
|
||||
const durability = payload?.readiness?.durability;
|
||||
const runtimeDurable = runtime?.durable === true || durability?.durable === true;
|
||||
const runtimeExplicitlyNonDurable = runtime?.durable === false || durability?.durable === false;
|
||||
const runtimeReady = runtime?.ready === true || durability?.ready === true;
|
||||
const durabilityReady = durability?.ready === true || normalizedStatus(durability?.status) === "ready";
|
||||
const liveRuntimeEvidence = runtime?.liveRuntimeEvidence === true || durability?.liveRuntimeEvidence === true;
|
||||
if (runtimeDurable && runtimeReady && durabilityReady && !hasDurableRuntimeBlockedSignal(payload)) return null;
|
||||
const reasonCode = firstNonEmpty(
|
||||
runtime?.blocker,
|
||||
durability?.blocker,
|
||||
runtime?.durabilityContract?.blockedLayer,
|
||||
durability?.blockedLayer,
|
||||
blockedDurabilityQueryResult(runtime?.connection?.queryResult),
|
||||
blockedDurabilityQueryResult(durability?.queryResult),
|
||||
blockerCodeStartingWith(payload, "runtime_durable_adapter_")
|
||||
);
|
||||
const blockedLayer = firstNonEmpty(runtime?.durabilityContract?.blockedLayer, durability?.blockedLayer);
|
||||
if (!reasonCode && runtimeDurable && runtimeReady && liveRuntimeEvidence) return null;
|
||||
if (runtimeExplicitlyNonDurable) {
|
||||
return {
|
||||
reasonCode: "runtime_durable_false",
|
||||
reason: `runtime durable=false; adapter=${firstNonEmpty(runtime?.adapter, durability?.adapter, "unknown")}`,
|
||||
layer: "runtime-durable"
|
||||
};
|
||||
}
|
||||
if (!hasRuntimeDurableSignal(payload)) return null;
|
||||
const classification = classifyRuntimeDurableReadiness(payload);
|
||||
if (classification.ready) return null;
|
||||
if (
|
||||
reasonCode &&
|
||||
classification.status === "blocked" &&
|
||||
(
|
||||
isReadOnlyReason(reasonCode) ||
|
||||
String(reasonCode).endsWith("_blocked") ||
|
||||
normalizedStatus(blockedLayer).includes("durability")
|
||||
isReadOnlyReason(classification.reasonCode) ||
|
||||
String(classification.reasonCode).endsWith("_blocked") ||
|
||||
normalizedStatus(classification.evidence?.blockedLayer).includes("durability")
|
||||
)
|
||||
) {
|
||||
return {
|
||||
reasonCode,
|
||||
reason: `runtime durable blocked at ${blockedLayer || "durability"}; queryResult=${firstNonEmpty(runtime?.connection?.queryResult, durability?.queryResult, "unknown")}`,
|
||||
layer: blockedLayer || "runtime-durable"
|
||||
reasonCode: classification.reasonCode,
|
||||
reason: classification.reason,
|
||||
layer: classification.evidence?.blockedLayer || "runtime-durable",
|
||||
classification
|
||||
};
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function hasDurableRuntimeBlockedSignal(payload) {
|
||||
function hasRuntimeDurableSignal(payload = {}) {
|
||||
const runtime = payload?.runtime;
|
||||
const durability = payload?.readiness?.durability;
|
||||
return [
|
||||
runtime?.blocker,
|
||||
durability?.blocker,
|
||||
runtime?.durabilityContract?.blockedLayer,
|
||||
durability?.blockedLayer,
|
||||
blockedDurabilityQueryResult(runtime?.connection?.queryResult),
|
||||
blockedDurabilityQueryResult(durability?.queryResult),
|
||||
blockerCodeStartingWith(payload, "runtime_durable_adapter_")
|
||||
].some(Boolean);
|
||||
const dbRuntimeReadiness = payload?.db?.runtimeReadiness;
|
||||
return Boolean(
|
||||
blockerCodeStartingWith(payload, "runtime_durable_adapter_") ||
|
||||
hasOwnNonEmpty(runtime, ["adapter", "durable", "durableRequested", "ready", "blocker", "liveRuntimeEvidence", "connection", "durabilityContract", "gates"]) ||
|
||||
hasOwnNonEmpty(durability, ["adapter", "durable", "ready", "blocker", "blockedLayer", "queryResult", "liveRuntimeEvidence", "gates"]) ||
|
||||
hasOwnNonEmpty(dbRuntimeReadiness, ["adapter", "durable", "ready", "blocker", "blockedLayer", "queryResult", "liveRuntimeEvidence"])
|
||||
);
|
||||
}
|
||||
|
||||
function blockedDurabilityQueryResult(value) {
|
||||
const text = normalizedStatus(value);
|
||||
if (!text || text === "durable_readiness_ready" || text === "ready") return "";
|
||||
if (text.endsWith("_blocked") || text.includes("blocked")) return String(value).trim();
|
||||
function hasOwnNonEmpty(object, fields) {
|
||||
if (!object || typeof object !== "object") return false;
|
||||
return fields.some((field) => Object.hasOwn(object, field) && object[field] !== undefined && object[field] !== null && object[field] !== "");
|
||||
}
|
||||
|
||||
export function classifyRuntimeDurableReadiness(payload = {}, options = {}) {
|
||||
const runtime = payload?.runtime ?? {};
|
||||
const durability = payload?.readiness?.durability ?? {};
|
||||
const dbRuntimeReadiness = payload?.db?.runtimeReadiness ?? {};
|
||||
const gates = runtime?.gates ?? durability?.gates ?? {};
|
||||
const gateBlocker = firstBlockedRuntimeGate(gates);
|
||||
const queryResult = firstNonEmpty(
|
||||
runtime?.connection?.queryResult,
|
||||
durability?.queryResult,
|
||||
dbRuntimeReadiness?.queryResult
|
||||
);
|
||||
const directBlocker = firstNonEmpty(
|
||||
runtime?.blocker,
|
||||
durability?.blocker,
|
||||
dbRuntimeReadiness?.blocker,
|
||||
blockerCodeStartingWith(payload, "runtime_durable_adapter_")
|
||||
);
|
||||
const queryBlocker = runtimeDurableBlockerFromQueryResult(queryResult);
|
||||
const reasonCode = firstNonEmpty(directBlocker, gateBlocker?.blocker, queryBlocker);
|
||||
const blockedLayer = firstNonEmpty(
|
||||
runtime?.durabilityContract?.blockedLayer,
|
||||
durability?.blockedLayer,
|
||||
dbRuntimeReadiness?.blockedLayer,
|
||||
gateBlocker?.layer,
|
||||
runtimeDurableLayerFromBlocker(reasonCode),
|
||||
runtimeDurableLayerFromQueryResult(queryResult)
|
||||
);
|
||||
const adapter = firstNonEmpty(runtime?.adapter, durability?.adapter, dbRuntimeReadiness?.adapter, "unknown");
|
||||
const runtimeDurable = runtime?.durable === true || durability?.durable === true || dbRuntimeReadiness?.durable === true;
|
||||
const runtimeExplicitlyNonDurable = runtime?.durable === false || durability?.durable === false || dbRuntimeReadiness?.durable === false;
|
||||
const runtimeReady = runtime?.ready === true || durability?.runtimeReady === true || dbRuntimeReadiness?.ready === true;
|
||||
const durabilityReady = durability?.ready === true || dbRuntimeReadiness?.ready === true || runtimeReady;
|
||||
const liveRuntimeEvidence =
|
||||
runtime?.liveRuntimeEvidence === true ||
|
||||
durability?.liveRuntimeEvidence === true ||
|
||||
dbRuntimeReadiness?.liveRuntimeEvidence === true;
|
||||
const rawStatus = rawStatusFrom(payload);
|
||||
const rawStatusBlocked = [...ERROR_STATUSES, ...BLOCKED_STATUSES, ...RAW_DEGRADED_STATUSES].includes(normalizedStatus(rawStatus));
|
||||
const readyQueryResult = isRuntimeDurableReadyQueryResult(queryResult);
|
||||
const postgresRequired = options.requirePostgresAdapter === true;
|
||||
const adapterReady = !postgresRequired || adapter === "postgres";
|
||||
const ready = adapterReady &&
|
||||
runtimeDurable &&
|
||||
runtimeReady &&
|
||||
durabilityReady &&
|
||||
liveRuntimeEvidence &&
|
||||
!reasonCode &&
|
||||
!rawStatusBlocked;
|
||||
|
||||
if (ready) {
|
||||
return {
|
||||
status: "ready",
|
||||
ready: true,
|
||||
classification: "runtime_durable_ready",
|
||||
reasonCode: "runtime_durable_ready",
|
||||
reason: `runtime durable ready; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`,
|
||||
impact: "M3 durable postflight may run because runtime persistence readiness is green.",
|
||||
safeNextAction: "Continue with repo-owned DEV CD postflight; do not read or print Secret values.",
|
||||
retryable: true,
|
||||
readonly: false,
|
||||
evidence: runtimeDurableEvidence({
|
||||
adapter,
|
||||
runtimeDurable,
|
||||
runtimeReady,
|
||||
durabilityReady,
|
||||
liveRuntimeEvidence,
|
||||
blockedLayer: null,
|
||||
queryResult,
|
||||
readyQueryResult,
|
||||
reasonCode: null,
|
||||
gates,
|
||||
rawStatus,
|
||||
postgresRequired
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
const nonReadyReasonCode = firstNonEmpty(
|
||||
reasonCode,
|
||||
postgresRequired && adapter !== "postgres" ? "runtime_durable_adapter_not_postgres" : "",
|
||||
runtimeExplicitlyNonDurable ? "runtime_durable_false" : "",
|
||||
runtimeDurable ? "" : "runtime_durable_adapter_missing",
|
||||
runtimeReady ? "" : "runtime_durable_runtime_not_ready",
|
||||
liveRuntimeEvidence ? "" : "runtime_durable_live_evidence_missing",
|
||||
rawStatusBlocked ? rawStatus : "",
|
||||
"runtime_durable_adapter_not_ready"
|
||||
);
|
||||
const layer = firstNonEmpty(
|
||||
blockedLayer,
|
||||
runtimeDurableLayerFromBlocker(nonReadyReasonCode),
|
||||
runtimeExplicitlyNonDurable ? "runtime-durable" : "",
|
||||
"runtime-durable"
|
||||
);
|
||||
return {
|
||||
status: "blocked",
|
||||
ready: false,
|
||||
classification: nonReadyReasonCode,
|
||||
reasonCode: nonReadyReasonCode,
|
||||
reason: runtimeDurableBlockedReason({ reasonCode: nonReadyReasonCode, adapter, layer, queryResult }),
|
||||
impact: "M3 true/false IO durable postflight is skipped or blocked so DEV CD cannot prove evidence persistence.",
|
||||
safeNextAction: runtimeDurableSafeNextAction(nonReadyReasonCode, layer),
|
||||
retryable: true,
|
||||
readonly: true,
|
||||
evidence: runtimeDurableEvidence({
|
||||
adapter,
|
||||
runtimeDurable,
|
||||
runtimeReady,
|
||||
durabilityReady,
|
||||
liveRuntimeEvidence,
|
||||
blockedLayer: layer,
|
||||
queryResult,
|
||||
readyQueryResult,
|
||||
reasonCode: nonReadyReasonCode,
|
||||
gates,
|
||||
rawStatus,
|
||||
postgresRequired
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
function firstBlockedRuntimeGate(gates = {}) {
|
||||
for (const layer of RUNTIME_DURABLE_GATE_LAYERS) {
|
||||
const gate = gates?.[layer];
|
||||
const status = normalizedStatus(gate?.status);
|
||||
if (gate?.ready === false || status === "blocked" || status === "failed") {
|
||||
return {
|
||||
layer,
|
||||
blocker: firstNonEmpty(gate?.blocker, runtimeDurableBlockerForLayer(layer))
|
||||
};
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function runtimeDurableBlockerForLayer(layer) {
|
||||
const text = normalizedStatus(layer);
|
||||
if (text === "ssl") return "runtime_durable_adapter_ssl_blocked";
|
||||
if (text === "auth") return "runtime_durable_adapter_auth_blocked";
|
||||
if (text === "schema") return "runtime_durable_adapter_schema_blocked";
|
||||
if (text === "migration") return "runtime_durable_adapter_migration_blocked";
|
||||
if (text === "durability" || text === "durability_query") return "runtime_durable_adapter_query_blocked";
|
||||
return "runtime_durable_adapter_blocked";
|
||||
}
|
||||
|
||||
function runtimeDurableLayerFromBlocker(blocker) {
|
||||
const text = normalizedStatus(blocker);
|
||||
if (!text) return "";
|
||||
if (text.includes("ssl")) return "ssl";
|
||||
if (text.includes("auth")) return "auth";
|
||||
if (text.includes("schema")) return "schema";
|
||||
if (text.includes("migration")) return "migration";
|
||||
if (text.includes("query")) return "durability_query";
|
||||
if (text.includes("missing") || text.includes("unconfigured") || text.includes("driver")) return "adapter";
|
||||
return "";
|
||||
}
|
||||
|
||||
function runtimeDurableLayerFromQueryResult(queryResult) {
|
||||
const text = normalizedStatus(queryResult);
|
||||
if (!text || isRuntimeDurableReadyQueryResult(text)) return "";
|
||||
if (text.includes("ssl")) return "ssl";
|
||||
if (text.includes("auth")) return "auth";
|
||||
if (text.includes("schema")) return "schema";
|
||||
if (text.includes("migration")) return "migration";
|
||||
if (text.includes("query")) return "durability_query";
|
||||
if (text.includes("driver")) return "adapter";
|
||||
return "";
|
||||
}
|
||||
|
||||
function runtimeDurableBlockerFromQueryResult(queryResult) {
|
||||
const text = normalizedStatus(queryResult);
|
||||
if (!text || isRuntimeDurableReadyQueryResult(text)) return "";
|
||||
return RUNTIME_DURABLE_BLOCKED_QUERY_RESULTS[text] ?? "";
|
||||
}
|
||||
|
||||
function isRuntimeDurableReadyQueryResult(queryResult) {
|
||||
const text = normalizedStatus(queryResult);
|
||||
return RUNTIME_DURABLE_READY_QUERY_RESULTS.includes(text);
|
||||
}
|
||||
|
||||
function runtimeDurableEvidence({
|
||||
adapter,
|
||||
runtimeDurable,
|
||||
runtimeReady,
|
||||
durabilityReady,
|
||||
liveRuntimeEvidence,
|
||||
blockedLayer,
|
||||
queryResult,
|
||||
readyQueryResult,
|
||||
reasonCode,
|
||||
gates,
|
||||
rawStatus,
|
||||
postgresRequired
|
||||
}) {
|
||||
return {
|
||||
adapter,
|
||||
durable: runtimeDurable,
|
||||
runtimeReady,
|
||||
durabilityReady,
|
||||
liveRuntimeEvidence,
|
||||
blocker: reasonCode,
|
||||
blockedLayer,
|
||||
queryResult: queryResult || null,
|
||||
readyQueryResult,
|
||||
rawStatus: rawStatus || null,
|
||||
postgresRequired,
|
||||
gates: summarizeRuntimeGateStatuses(gates),
|
||||
secretMaterialRead: false
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeRuntimeGateStatuses(gates = {}) {
|
||||
return Object.fromEntries(RUNTIME_DURABLE_GATE_LAYERS.map((layer) => [
|
||||
layer,
|
||||
{
|
||||
checked: gates?.[layer]?.checked === true,
|
||||
ready: gates?.[layer]?.ready === true,
|
||||
status: gates?.[layer]?.status ?? "unknown",
|
||||
blocker: gates?.[layer]?.blocker ?? null
|
||||
}
|
||||
]));
|
||||
}
|
||||
|
||||
function runtimeDurableBlockedReason({ reasonCode, adapter, layer, queryResult }) {
|
||||
if (reasonCode === "runtime_durable_false") {
|
||||
return `runtime durable=false; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`;
|
||||
}
|
||||
if (reasonCode === "runtime_durable_adapter_not_postgres") {
|
||||
return `runtime durable adapter is not postgres; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`;
|
||||
}
|
||||
return `runtime durable blocked at ${layer || "runtime-durable"}; blocker=${reasonCode}; queryResult=${queryResult || "unknown"}`;
|
||||
}
|
||||
|
||||
function runtimeDurableSafeNextAction(reasonCode, layer) {
|
||||
const text = normalizedStatus(`${reasonCode} ${layer}`);
|
||||
if (text.includes("auth")) return "Repair DEV DB auth/SecretRef metadata through repo-owned provisioning, then rerun postflight without printing Secret values.";
|
||||
if (text.includes("schema") || text.includes("migration")) return "Run repo-owned DEV runtime provisioning and migration, then rerun postflight.";
|
||||
if (text.includes("ssl")) return "Align DEV DB SSL mode with the repo-owned DEV contract, then rerun postflight.";
|
||||
if (text.includes("query")) return "Inspect durable runtime query readiness, preserve evidence persistence, and rerun postflight.";
|
||||
if (text.includes("postgres")) return "Run DEV with HWLAB_CLOUD_RUNTIME_ADAPTER=postgres and DB URLs injected only through SecretRefs.";
|
||||
return "Restore durable runtime readiness through repo-owned DEV CD steps, then rerun postflight.";
|
||||
}
|
||||
|
||||
function firstBlocker(payload) {
|
||||
const blockers = [
|
||||
...(Array.isArray(payload?.blockers) ? payload.blockers : []),
|
||||
@@ -626,13 +862,22 @@ function probeAttribution(probe) {
|
||||
return [
|
||||
`${probe.serviceId} ${probe.apiPath}`,
|
||||
probe.httpStatus ? `HTTP ${probe.httpStatus}` : null,
|
||||
probe.reasonCode,
|
||||
probe.reason && probe.reason !== probe.reasonCode ? probe.reason : null,
|
||||
displayReasonCode(probe.reasonCode),
|
||||
probe.reason && probe.reason !== probe.reasonCode ? displayReasonText(probe.reason) : null,
|
||||
probe.traceId ? `trace=${probe.traceId}` : null,
|
||||
probe.evidenceSummary
|
||||
].filter(Boolean).join(" / ");
|
||||
}
|
||||
|
||||
function displayReasonCode(reasonCode) {
|
||||
const text = String(reasonCode ?? "");
|
||||
return text.startsWith("runtime_durable_adapter_") ? "runtime durable" : text;
|
||||
}
|
||||
|
||||
function displayReasonText(reason) {
|
||||
return String(reason ?? "").replace(/\bruntime_durable_adapter_[a-z0-9_]+\b/giu, "runtime durable");
|
||||
}
|
||||
|
||||
function normalizedStatus(value) {
|
||||
return String(value ?? "").trim().toLowerCase();
|
||||
}
|
||||
|
||||
@@ -34,6 +34,25 @@ const okRest = Object.freeze({
|
||||
serviceId: "hwlab-cloud-api",
|
||||
status: "ok",
|
||||
ready: true,
|
||||
readiness: {
|
||||
status: "ok",
|
||||
ready: true,
|
||||
durability: {
|
||||
status: "ready",
|
||||
ready: true,
|
||||
queryResult: "durable_readiness_ready"
|
||||
}
|
||||
},
|
||||
runtime: {
|
||||
adapter: "postgres",
|
||||
status: "ready",
|
||||
durable: true,
|
||||
ready: true,
|
||||
liveRuntimeEvidence: true,
|
||||
connection: {
|
||||
queryResult: "durable_readiness_ready"
|
||||
}
|
||||
},
|
||||
codeAgent: {
|
||||
endpoint: "POST /v1/agent/chat",
|
||||
status: "available",
|
||||
@@ -52,6 +71,25 @@ const codexStdioRest = Object.freeze({
|
||||
serviceId: "hwlab-cloud-api",
|
||||
status: "ok",
|
||||
ready: true,
|
||||
readiness: {
|
||||
status: "ok",
|
||||
ready: true,
|
||||
durability: {
|
||||
status: "ready",
|
||||
ready: true,
|
||||
queryResult: "durable_readiness_ready"
|
||||
}
|
||||
},
|
||||
runtime: {
|
||||
adapter: "postgres",
|
||||
status: "ready",
|
||||
durable: true,
|
||||
ready: true,
|
||||
liveRuntimeEvidence: true,
|
||||
connection: {
|
||||
queryResult: "durable_readiness_ready"
|
||||
}
|
||||
},
|
||||
codeAgent: {
|
||||
endpoint: "POST /v1/agent/chat",
|
||||
status: "codex-stdio-feasible",
|
||||
@@ -78,6 +116,25 @@ const readonlyOnlyRest = Object.freeze({
|
||||
serviceId: "hwlab-cloud-api",
|
||||
status: "degraded",
|
||||
ready: false,
|
||||
readiness: {
|
||||
status: "ok",
|
||||
ready: true,
|
||||
durability: {
|
||||
status: "ready",
|
||||
ready: true,
|
||||
queryResult: "durable_readiness_ready"
|
||||
}
|
||||
},
|
||||
runtime: {
|
||||
adapter: "postgres",
|
||||
status: "ready",
|
||||
durable: true,
|
||||
ready: true,
|
||||
liveRuntimeEvidence: true,
|
||||
connection: {
|
||||
queryResult: "durable_readiness_ready"
|
||||
}
|
||||
},
|
||||
codeAgent: {
|
||||
endpoint: "POST /v1/agent/chat",
|
||||
status: "partial",
|
||||
@@ -232,6 +289,111 @@ test("classifies durable runtime degraded as explicit read-only mode", () => {
|
||||
assert.deepEqual(status.internalRawStatuses.filter((item) => item === "degraded"), ["degraded"]);
|
||||
});
|
||||
|
||||
test("does not classify ready queryResult durable runtime as blocked", () => {
|
||||
const status = classifyWorkbenchLiveStatus({
|
||||
healthLive: {
|
||||
ok: true,
|
||||
status: 200,
|
||||
data: {
|
||||
serviceId: "hwlab-cloud-api",
|
||||
status: "ready",
|
||||
ready: true,
|
||||
runtime: {
|
||||
adapter: "postgres",
|
||||
durable: true,
|
||||
ready: true,
|
||||
liveRuntimeEvidence: true,
|
||||
status: "ready",
|
||||
blocker: null,
|
||||
connection: {
|
||||
queryResult: "durable_readiness_ready"
|
||||
}
|
||||
},
|
||||
readiness: {
|
||||
status: "ready",
|
||||
ready: true,
|
||||
durability: {
|
||||
status: "ready",
|
||||
ready: true,
|
||||
blocker: null,
|
||||
blockedLayer: null,
|
||||
queryResult: "durable_readiness_ready"
|
||||
}
|
||||
},
|
||||
blockerCodes: []
|
||||
}
|
||||
},
|
||||
restIndex: okRest,
|
||||
health: okApi,
|
||||
adapter: okApi,
|
||||
m3Control: okM3Control,
|
||||
m3Status: okM3Status
|
||||
});
|
||||
|
||||
assert.equal(status.kind, "pass");
|
||||
assert.equal(status.reasonCode, "ready");
|
||||
assert.equal(status.probes.find((probe) => probe.apiPath === "/health/live")?.reasonCode, "ready");
|
||||
});
|
||||
|
||||
test("classifies auth and migration durable blockers with shared runtime reason", () => {
|
||||
for (const [reasonCode, queryResult, blockedLayer] of [
|
||||
["runtime_durable_adapter_auth_blocked", "auth_blocked", "auth"],
|
||||
["runtime_durable_adapter_migration_blocked", "migration_blocked", "migration"]
|
||||
]) {
|
||||
const status = classifyWorkbenchLiveStatus({
|
||||
healthLive: {
|
||||
ok: true,
|
||||
status: 200,
|
||||
data: {
|
||||
serviceId: "hwlab-cloud-api",
|
||||
status: "degraded",
|
||||
ready: false,
|
||||
runtime: {
|
||||
adapter: "postgres",
|
||||
durable: false,
|
||||
durableRequested: true,
|
||||
ready: false,
|
||||
liveRuntimeEvidence: false,
|
||||
status: "blocked",
|
||||
blocker: reasonCode,
|
||||
connection: { queryResult },
|
||||
durabilityContract: { blockedLayer },
|
||||
gates: {
|
||||
[blockedLayer]: {
|
||||
checked: true,
|
||||
ready: false,
|
||||
status: "blocked",
|
||||
blocker: reasonCode
|
||||
}
|
||||
}
|
||||
},
|
||||
readiness: {
|
||||
ready: false,
|
||||
status: "blocked",
|
||||
durability: {
|
||||
ready: false,
|
||||
status: "blocked",
|
||||
blocker: reasonCode,
|
||||
blockedLayer,
|
||||
queryResult
|
||||
}
|
||||
},
|
||||
blockerCodes: [reasonCode]
|
||||
}
|
||||
},
|
||||
restIndex: okRest,
|
||||
health: okApi,
|
||||
adapter: okApi,
|
||||
m3Control: okM3Control,
|
||||
m3Status: okM3Status
|
||||
});
|
||||
|
||||
assert.equal(status.kind, "readonly");
|
||||
assert.equal(status.reasonCode, reasonCode);
|
||||
assert.match(status.detail, new RegExp(queryResult, "u"));
|
||||
}
|
||||
});
|
||||
|
||||
test("classifies missing probes as unverified instead of degraded", () => {
|
||||
const status = classifyWorkbenchLiveStatus({});
|
||||
|
||||
|
||||
Reference in New Issue
Block a user