From 83e5f52c98ed061e1707c8ae078a095eb0de7358 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Sat, 23 May 2026 22:15:04 +0800 Subject: [PATCH] fix: harden dev cd runtime postflight Summary: - shared runtime durable readiness classifier for live-status and DEV postflight - structured postflight blockers propagated through dev-cd apply - contract coverage for ready queryResult, auth/schema/migration blockers, evidence persistence blockers, and bounded stdout Verification: - git diff --check - node --check changed scripts/tests/live-status modules - node --test scripts/src/dev-runtime-postflight.test.mjs scripts/src/dev-cd-apply.test.mjs web/hwlab-cloud-web/scripts/live-status-contract.test.mjs - node scripts/dev-runtime-postflight.mjs --check - node web/hwlab-cloud-web/scripts/check.mjs - npm run check No DEV apply/rollout/live health verification was run. --- scripts/src/dev-cd-apply.mjs | 48 ++- scripts/src/dev-cd-apply.test.mjs | 106 ++++++ scripts/src/dev-runtime-postflight.mjs | 156 ++++++-- scripts/src/dev-runtime-postflight.test.mjs | 98 ++++- web/hwlab-cloud-web/live-status.mjs | 343 +++++++++++++++--- .../scripts/live-status-contract.test.mjs | 162 +++++++++ 6 files changed, 812 insertions(+), 101 deletions(-) diff --git a/scripts/src/dev-cd-apply.mjs b/scripts/src/dev-cd-apply.mjs index 1a39df30..25cb14e7 100644 --- a/scripts/src/dev-cd-apply.mjs +++ b/scripts/src/dev-cd-apply.mjs @@ -1815,11 +1815,55 @@ function parseLastJsonObject(text) { } function stepBlocker(stepResult) { - return { + const stdoutBlockers = Array.isArray(stepResult.stdoutJson?.blockers) ? stepResult.stdoutJson.blockers : []; + const primary = stdoutBlockers.find((item) => item?.status === "open") ?? stdoutBlockers[0] ?? null; + const reason = primary?.reason ?? primary?.summary ?? `${stepResult.command} exited ${stepResult.code}.`; + const impact = primary?.impact ?? stepBlockerImpact(stepResult); + const safeNextAction = primary?.safeNextAction ?? stepBlockerSafeNextAction(stepResult); + return structuredBlocker({ type: "runtime_blocker", scope: stepResult.id, + reason, + impact, + safeNextAction, + retryable: primary?.retryable ?? true, + evidence: { + command: stepResult.command, + code: stepResult.code, + reportPath: stepResult.reportPath ?? null, + blockerScope: primary?.scope ?? null, + k8sJob: stepResult.k8sJob ?? null + } + }); +} + +function stepBlockerImpact(stepResult) { + if (stepResult.id === "runtime-db-provisioning") return "DEV CD cannot prove DB role/database provisioning before workload apply."; + if (stepResult.id === "runtime-db-migration") return "DEV CD cannot prove runtime schema migration before workload apply."; + if (stepResult.id === "runtime-durable-postflight") return "DEV CD cannot prove /health/live, /v1, and M3 durable evidence persistence after apply."; + if (stepResult.id === "dev-deploy-apply") return "DEV desired-state apply did not complete, so live readiness cannot be trusted."; + return "DEV CD transaction cannot safely continue."; +} + +function stepBlockerSafeNextAction(stepResult) { + if (stepResult.id === "runtime-db-provisioning") return "Repair repo-owned runtime provisioning inputs/SecretRefs without printing Secret values, then rerun DEV CD."; + if (stepResult.id === "runtime-db-migration") return "Repair repo-owned runtime migration/schema state, then rerun DEV CD."; + if (stepResult.id === "runtime-durable-postflight") return "Inspect the runtime postflight report, repair durable readiness or M3 evidence persistence, then rerun DEV CD after host-controlled rollout if needed."; + if (stepResult.id === "dev-deploy-apply") return "Inspect reports/dev-gate/dev-deploy-report.json and rerun the repo-owned DEV CD path after the apply blocker is fixed."; + return "Fix the reported step blocker and rerun the repo-owned DEV CD transaction."; +} + +function structuredBlocker({ type = "runtime_blocker", scope, reason, impact, safeNextAction, retryable = true, evidence = null }) { + return { + type, + scope, status: "open", - summary: `${stepResult.command} exited ${stepResult.code}.` + reason: oneLine(reason), + impact: oneLine(impact), + safeNextAction: oneLine(safeNextAction), + retryable: Boolean(retryable), + summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`), + ...(evidence ? { evidence } : {}) }; } diff --git a/scripts/src/dev-cd-apply.test.mjs b/scripts/src/dev-cd-apply.test.mjs index 58d02b00..7f44a0dc 100644 --- a/scripts/src/dev-cd-apply.test.mjs +++ b/scripts/src/dev-cd-apply.test.mjs @@ -215,6 +215,7 @@ function makeRunCommand({ ], desiredStatePromotionStatus = "pass", runtimeJobLogSuffix = "", + runtimePostflightReport = null, kubeContext = "d601", kubeServer = "https://d601:6443", kubeNodes = ["d601"], @@ -390,6 +391,24 @@ function makeRunCommand({ }; } if (command === process.execPath || command.endsWith("/node")) { + if (args.includes("scripts/dev-runtime-postflight.mjs")) { + const report = runtimePostflightReport ?? { + conclusion: { status: "pass" }, + blockers: [], + safety: { secretValuesPrinted: false }, + apiHealth: { runtimeDurableReadiness: { ready: true } }, + apiV1: { runtimeDurableReadiness: { ready: true } }, + m3: { + persistenceContract: { ready: true }, + operationCount: 4 + } + }; + return { + code: report.conclusion?.status === "pass" ? 0 : 1, + stdout: `${JSON.stringify(report, null, 2)}\n`, + stderr: "" + }; + } return { code: 0, stdout: JSON.stringify({ ok: true, command: args.join(" ") }), stderr: "" }; } return { code: 0, stdout: "", stderr: "" }; @@ -1294,6 +1313,93 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2); }); +test("transaction blocks with structured runtime postflight blocker when M3 evidence is not persisted", async () => { + const repoRoot = await makeRepo(); + const commandLog = []; + let output = ""; + const runtimePostflightReport = { + conclusion: { + status: "blocked", + blockerCount: 1 + }, + apiHealth: { + runtimeDurableReadiness: { + ready: true, + classification: "runtime_durable_ready" + } + }, + apiV1: { + runtimeDurableReadiness: { + ready: true, + classification: "runtime_durable_ready" + } + }, + m3: { + persistenceContract: { + ready: false, + sequence: [ + { id: "write-do1-true", persisted: true, idsPresent: true }, + { id: "read-di1-true", persisted: false, idsPresent: true } + ] + }, + operationCount: 4 + }, + blockers: [ + { + type: "runtime_blocker", + scope: "m3-durable-postflight", + status: "open", + reason: "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.", + impact: "A later rollout could appear ready while losing evidence persistence.", + safeNextAction: "Repair durable audit/evidence persistence, then rerun DEV runtime postflight.", + retryable: true + } + ], + safety: { + secretValuesPrinted: false, + readsKubernetesSecrets: false + } + }; + const code = await runDevCdApply([ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--owner-task-id", + "task-postflight-blocked", + "--kubeconfig", + "/etc/rancher/k3s/k3s.yaml", + "--write-report" + ], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ commandLog, runtimePostflightReport }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const summary = JSON.parse(output); + const report = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8")); + const blocker = report.blockers.find((item) => item.scope === "runtime-durable-postflight"); + assert.equal(code, 2); + assert.equal(summary.status, "blocked"); + assert.equal(summary.devCdApply, undefined); + assert.equal(output.length < 12000, true); + assert.equal(report.status, "blocked"); + assert.equal(report.devCdApply.steps.at(-1).id, "runtime-durable-postflight"); + assert.equal(report.devCdApply.steps.at(-1).status, "blocked"); + assert.ok(blocker); + assert.equal(blocker.reason, runtimePostflightReport.blockers[0].reason); + assert.equal(blocker.impact, runtimePostflightReport.blockers[0].impact); + assert.equal(blocker.safeNextAction, runtimePostflightReport.blockers[0].safeNextAction); + assert.equal(blocker.retryable, true); + assert.equal(blocker.evidence.blockerScope, "m3-durable-postflight"); + assert.equal(report.devCdApply.liveVerify.status, "not_run"); + assert.equal(JSON.stringify(report).includes("postgresql://"), false); + assert.equal(JSON.stringify(report).includes("SECRET"), false); + assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-runtime-postflight.mjs")), true); +}); + test("runtime maintenance job reports are parsed after redacting secret-like log output", async () => { const repoRoot = await makeRepo(); const commandLog = []; diff --git a/scripts/src/dev-runtime-postflight.mjs b/scripts/src/dev-runtime-postflight.mjs index f0fe3b03..bc15c827 100644 --- a/scripts/src/dev-runtime-postflight.mjs +++ b/scripts/src/dev-runtime-postflight.mjs @@ -4,6 +4,7 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; +import { classifyRuntimeDurableReadiness } from "../../web/hwlab-cloud-web/live-status.mjs"; import { buildM3IoControlSourceReport, runM3IoControlLiveReport @@ -78,8 +79,16 @@ export async function buildDevRuntimePostflightReport(args = {}, options = {}) { blockerCount: sourceReport.summary?.status === "pass" ? 0 : 1 }; if (sourceReport.summary?.status !== "pass") { - addBlocker(report, "contract_blocker", "m3-source-contract", "M3 postflight source contract is blocked.", { - classification: sourceReport.summary?.classification ?? "unknown" + addBlocker(report, { + type: "contract_blocker", + scope: "m3-source-contract", + reason: "M3 postflight source contract is blocked.", + impact: "DEV CD cannot rely on the M3 durable postflight harness contract.", + safeNextAction: "Fix the source M3 IO control contract tests before running live DEV postflight.", + retryable: true, + evidence: { + classification: sourceReport.summary?.classification ?? "unknown" + } }); } return report; @@ -227,6 +236,7 @@ async function defaultHttpGetJson(url, timeoutMs) { function summarizeApiPayload(check, response) { const json = response.json ?? {}; const runtime = summarizeRuntime(json.runtime); + const runtimeDurableReadiness = classifyRuntimeDurableReadiness(json, { requirePostgresAdapter: true }); return { check, httpStatus: response.status, @@ -236,6 +246,7 @@ function summarizeApiPayload(check, response) { status: json.status ?? null, ready: json.ready === true, runtime, + runtimeDurableReadiness, db: summarizeDb(json.db), readiness: { ready: json.readiness?.ready === true, @@ -372,23 +383,48 @@ function summarizeM3PersistenceContract(operations = []) { function addApiReadinessBlockers(report) { for (const [scope, payload] of [["api-health-live", report.apiHealth], ["api-v1", report.apiV1]]) { if (!payload.reachable) { - addBlocker(report, "runtime_blocker", scope, `${payload.check} was not reachable.`, { - httpStatus: payload.httpStatus, - error: payload.error + addBlocker(report, { + type: "runtime_blocker", + scope, + reason: `${payload.check} was not reachable.`, + impact: "DEV CD cannot prove cloud-api durable readiness, so M3 evidence persistence postflight cannot safely run.", + safeNextAction: "Restore DEV cloud-api reachability on the repo-owned public endpoint, then rerun postflight.", + retryable: true, + evidence: { + httpStatus: payload.httpStatus, + error: payload.error + } }); continue; } - if (!runtimeDurableReady(payload.runtime)) { - addBlocker(report, "runtime_blocker", scope, `${payload.check} did not prove durable runtime readiness.`, { - runtime: payload.runtime, - blockerCodes: payload.blockerCodes + if (payload.runtimeDurableReadiness?.ready !== true) { + addBlocker(report, { + type: "runtime_blocker", + scope, + reason: `${payload.check} did not prove durable runtime readiness: ${payload.runtimeDurableReadiness?.reason ?? "not ready"}.`, + impact: payload.runtimeDurableReadiness?.impact ?? "M3 durable postflight cannot prove evidence persistence.", + safeNextAction: payload.runtimeDurableReadiness?.safeNextAction ?? "Repair runtime durability through repo-owned DEV CD steps, then rerun postflight.", + retryable: payload.runtimeDurableReadiness?.retryable !== false, + evidence: { + runtime: payload.runtime, + runtimeDurableReadiness: payload.runtimeDurableReadiness, + blockerCodes: payload.blockerCodes + } }); } if (scope === "api-v1" && !apiV1M3ContractReady(payload)) { - addBlocker(report, "runtime_blocker", scope, "GET /v1 did not expose the controlled M3 IO route required by durable postflight.", { - m3IoControl: payload.m3IoControl, - requiredRoute: "/v1/m3/io", - requiredContractVersion: "m3-io-control-v1" + addBlocker(report, { + type: "runtime_blocker", + scope, + reason: "GET /v1 did not expose the controlled M3 IO route required by durable postflight.", + impact: "DEV CD cannot run the M3 true/false IO persistence smoke through the cloud-api controlled boundary.", + safeNextAction: "Restore /v1 m3IoControl route metadata for /v1/m3/io without exposing direct frontend simulator access.", + retryable: true, + evidence: { + m3IoControl: payload.m3IoControl, + requiredRoute: "/v1/m3/io", + requiredContractVersion: "m3-io-control-v1" + } }); } } @@ -406,58 +442,96 @@ function apiV1M3ContractReady(payload) { function addM3Blockers(report) { if (report.m3?.persistenceContract?.ready !== true) { - addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.", { - persistenceContract: report.m3?.persistenceContract ?? null, - trustedGreen: report.m3?.trustedGreen === true, - operationCount: report.m3?.operationCount ?? 0 + addBlocker(report, { + type: "runtime_blocker", + scope: "m3-durable-postflight", + reason: "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.", + impact: "A later rollout could appear ready while losing operation, audit, or evidence persistence for M3 hardware IO.", + safeNextAction: "Inspect the failed M3 operation sequence, repair durable audit/evidence persistence, then rerun DEV runtime postflight.", + retryable: true, + evidence: { + persistenceContract: report.m3?.persistenceContract ?? null, + trustedGreen: report.m3?.trustedGreen === true, + operationCount: report.m3?.operationCount ?? 0 + } }); return; } if (report.m3?.trustedGreen !== true || report.m3?.status !== "pass") { - addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 true/false durable postflight did not produce trusted green persisted evidence.", { - classification: report.m3?.classification ?? "unknown", - trustedGreen: report.m3?.trustedGreen === true, - operationCount: report.m3?.operationCount ?? 0 + addBlocker(report, { + type: "runtime_blocker", + scope: "m3-durable-postflight", + reason: "M3 true/false durable postflight did not produce trusted green persisted evidence.", + impact: "M3 durable readiness cannot be accepted because trusted evidence is not green.", + safeNextAction: "Fix the M3 control path or trusted-record persistence and rerun DEV runtime postflight.", + retryable: true, + evidence: { + classification: report.m3?.classification ?? "unknown", + trustedGreen: report.m3?.trustedGreen === true, + operationCount: report.m3?.operationCount ?? 0 + } }); return; } for (const operation of report.m3.operations ?? []) { if (!operation.operationId || !operation.auditId || !operation.evidenceId || operation.evidenceState?.status !== "green" || operation.evidenceState?.durable !== true) { - addBlocker(report, "runtime_blocker", "m3-durable-evidence", "M3 operation/audit/evidence durable fields were incomplete.", { - operationIdPresent: Boolean(operation.operationId), - auditIdPresent: Boolean(operation.auditId), - evidenceIdPresent: Boolean(operation.evidenceId), - evidenceState: operation.evidenceState + addBlocker(report, { + type: "runtime_blocker", + scope: "m3-durable-evidence", + reason: "M3 operation/audit/evidence durable fields were incomplete.", + impact: "The M3 postflight cannot prove every IO operation is bound to persisted audit and evidence records.", + safeNextAction: "Repair operation, audit, and evidence ID propagation before accepting DEV readiness.", + retryable: true, + evidence: { + operationIdPresent: Boolean(operation.operationId), + traceIdPresent: Boolean(operation.traceId), + auditIdPresent: Boolean(operation.auditId), + evidenceIdPresent: Boolean(operation.evidenceId), + evidenceState: operation.evidenceState + } }); return; } } } -function runtimeDurableReady(runtime) { - return runtime.adapter === "postgres" && - runtime.durable === true && - runtime.ready === true && - runtime.liveRuntimeEvidence === true; -} - function addSafetyRefusal(report, summary) { report.safetyRefusal = true; - addBlocker(report, "safety_refusal", "runtime-postflight-live-boundary", summary, { - devOnly: true, - prodAllowed: false, - secretValuesPrinted: false + addBlocker(report, { + type: "safety_refusal", + scope: "runtime-postflight-live-boundary", + reason: summary, + impact: "Live DEV reads and M3 writes are not allowed without explicit non-production confirmation.", + safeNextAction: "Rerun with --live --confirm-dev --confirmed-non-production for DEV only, or use --check for source-only validation.", + retryable: true, + evidence: { + devOnly: true, + prodAllowed: false, + secretValuesPrinted: false + } }); } -function addBlocker(report, type, scope, summary, evidence = {}) { +function addBlocker(report, { + type, + scope, + reason, + impact, + safeNextAction, + retryable, + evidence = {} +}) { report.blockers.push({ type, scope, status: "open", - summary, + reason: oneLine(reason), + impact: oneLine(impact), + safeNextAction: oneLine(safeNextAction), + retryable: Boolean(retryable), + summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`), sourceIssue: issue, evidence }); @@ -475,6 +549,10 @@ function finalizeReport(report) { return report; } +function oneLine(value) { + return String(value ?? "").replace(/\s+/g, " ").trim(); +} + function buildSafety(args) { return { devOnly: true, diff --git a/scripts/src/dev-runtime-postflight.test.mjs b/scripts/src/dev-runtime-postflight.test.mjs index a677aaff..3ce0bf3d 100644 --- a/scripts/src/dev-runtime-postflight.test.mjs +++ b/scripts/src/dev-runtime-postflight.test.mjs @@ -56,6 +56,8 @@ test("live mode skips M3 writes when /health/live durable runtime is blocked", a assert.equal(report.actions.m3MutationAttempted, false); assert.equal(report.m3.status, "not_run"); assert.equal(report.blockers.some((blocker) => blocker.scope === "api-health-live"), true); + assertStructuredBlockers(report.blockers); + assert.match(report.blockers.find((blocker) => blocker.scope === "api-health-live").reason, /auth_blocked/u); assert.equal(JSON.stringify(report).includes("postgresql://"), false); }); @@ -74,6 +76,7 @@ test("live mode requires /v1 durable readiness before M3 writes", async () => { assert.equal(report.conclusion.status, "blocked"); assert.equal(report.actions.m3MutationAttempted, false); assert.equal(report.blockers.some((blocker) => blocker.scope === "api-v1"), true); + assertStructuredBlockers(report.blockers); }); test("live mode requires /v1 to expose the controlled M3 IO route before M3 writes", async () => { @@ -100,6 +103,7 @@ test("live mode requires /v1 to expose the controlled M3 IO route before M3 writ blocker.scope === "api-v1" && blocker.summary.includes("controlled M3 IO route") ), true); + assertStructuredBlockers(report.blockers); }); test("live mode does not mistake DB live evidence for durable runtime readiness", async () => { @@ -121,6 +125,51 @@ test("live mode does not mistake DB live evidence for durable runtime readiness" assert.equal(report.apiHealth.readiness.durabilityReady, false); assert.equal(report.actions.m3MutationAttempted, false); assert.equal(report.blockers.some((blocker) => blocker.scope === "api-health-live"), true); + assertStructuredBlockers(report.blockers); +}); + +test("live mode blocks auth, schema, and migration durable readiness before M3 writes", async () => { + for (const [blocker, queryResult, blockedLayer] of [ + ["runtime_durable_adapter_auth_blocked", "auth_blocked", "auth"], + ["runtime_durable_adapter_schema_blocked", "schema_blocked", "schema"], + ["runtime_durable_adapter_migration_blocked", "migration_blocked", "migration"] + ]) { + const report = await buildDevRuntimePostflightReport( + parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]), + { + httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: false, blocker, queryResult, blockedLayer }) }), + m3Runner: async () => { + throw new Error("M3 runner should not run when durable readiness is blocked"); + }, + now: () => "2026-05-23T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "blocked"); + assert.equal(report.actions.m3MutationAttempted, false); + assert.equal(report.m3.status, "not_run"); + assert.equal(report.apiHealth.runtimeDurableReadiness.classification, blocker); + assert.equal(report.apiHealth.runtimeDurableReadiness.evidence.queryResult, queryResult); + assert.equal(report.apiV1.runtimeDurableReadiness.classification, blocker); + assertStructuredBlockers(report.blockers); + } +}); + +test("live mode treats durable_readiness_ready queryResult as ready, not blocked", async () => { + const report = await buildDevRuntimePostflightReport( + parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]), + { + httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: true, queryResult: "durable_readiness_ready" }) }), + m3Runner: async () => m3GreenReport(), + now: () => "2026-05-23T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "pass"); + assert.equal(report.actions.m3MutationAttempted, true); + assert.equal(report.apiHealth.runtimeDurableReadiness.ready, true); + assert.equal(report.apiHealth.runtimeDurableReadiness.evidence.queryResult, "durable_readiness_ready"); + assert.equal(report.blockers.length, 0); }); test("live mode records M3 true/false durable green evidence", async () => { @@ -165,6 +214,7 @@ test("live mode blocks when M3 operation evidence is not durable green", async ( assert.equal(report.conclusion.status, "blocked"); assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-postflight"), true); assert.equal(report.m3.persistenceContract.ready, false); + assertStructuredBlockers(report.blockers); }); test("live mode blocks when M3 true/false persisted sequence is incomplete", async () => { @@ -183,9 +233,29 @@ test("live mode blocks when M3 true/false persisted sequence is incomplete", asy assert.equal(report.m3.persistenceContract.ready, false); assert.equal(report.m3.persistenceContract.sequence.at(-1).status, "missing"); assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-postflight"), true); + assertStructuredBlockers(report.blockers); }); -function apiPayload({ ready }) { +function assertStructuredBlockers(blockers) { + assert.ok(blockers.length > 0); + for (const blocker of blockers) { + assert.equal(typeof blocker.scope, "string"); + assert.equal(typeof blocker.reason, "string"); + assert.equal(typeof blocker.impact, "string"); + assert.equal(typeof blocker.safeNextAction, "string"); + assert.equal(typeof blocker.retryable, "boolean"); + assert.notEqual(blocker.reason, ""); + assert.notEqual(blocker.impact, ""); + assert.notEqual(blocker.safeNextAction, ""); + } +} + +function apiPayload({ + ready, + blocker = "runtime_durable_adapter_auth_blocked", + queryResult = ready ? "durable_readiness_ready" : "auth_blocked", + blockedLayer = "auth" +}) { return { serviceId: "hwlab-cloud-api", environment: "dev", @@ -200,8 +270,8 @@ function apiPayload({ ready }) { runtimeReadiness: { ready, status: ready ? "ready" : "blocked", - blocker: ready ? null : "runtime_durable_adapter_auth_blocked", - queryResult: ready ? "durable_readiness_ready" : "auth_blocked", + blocker: ready ? null : blocker, + queryResult, requiredEvidence: "runtime_adapter_schema_migration_read_query" } }, @@ -211,18 +281,21 @@ function apiPayload({ ready }) { durableRequested: true, ready, status: ready ? "ready" : "blocked", - blocker: ready ? null : "runtime_durable_adapter_auth_blocked", + blocker: ready ? null : blocker, liveRuntimeEvidence: ready, + connection: { + queryResult + }, durabilityContract: { requiredEvidence: "runtime_adapter_schema_migration_read_query", - blockedLayer: ready ? null : "auth" + blockedLayer: ready ? null : blockedLayer }, gates: { ssl: gate(true), - auth: gate(ready, "runtime_durable_adapter_auth_blocked"), - schema: gate(ready), - migration: gate(ready), - durability: gate(ready) + auth: gate(ready || !["auth"].includes(blockedLayer), blockedLayer === "auth" ? blocker : null), + schema: gate(ready || !["schema"].includes(blockedLayer), blockedLayer === "schema" ? blocker : null), + migration: gate(ready || !["migration"].includes(blockedLayer), blockedLayer === "migration" ? blocker : null), + durability: gate(ready, blocker) } }, readiness: { @@ -230,11 +303,14 @@ function apiPayload({ ready }) { status: ready ? "ready" : "blocked", durability: { ready, - blockedLayer: ready ? null : "auth" + status: ready ? "ready" : "blocked", + blocker: ready ? null : blocker, + blockedLayer: ready ? null : blockedLayer, + queryResult } }, blockerCodes: ready ? [] - : ["runtime_durable_adapter_auth_blocked"], + : [blocker], m3IoControl: { route: "/v1/m3/io", status: "available", diff --git a/web/hwlab-cloud-web/live-status.mjs b/web/hwlab-cloud-web/live-status.mjs index 95699fab..22158374 100644 --- a/web/hwlab-cloud-web/live-status.mjs +++ b/web/hwlab-cloud-web/live-status.mjs @@ -3,6 +3,26 @@ const ERROR_STATUSES = Object.freeze(["error", "failed", "failure"]); const RAW_DEGRADED_STATUSES = Object.freeze(["degraded"]); const BLOCKED_STATUSES = Object.freeze(["blocked", "unavailable", "rejected"]); const UNVERIFIED_STATUSES = Object.freeze(["", "unknown", "pending", "loading", "probing", "unverified", "not_observed"]); +const RUNTIME_DURABLE_READY_QUERY_RESULTS = Object.freeze([ + "durable_readiness_ready", + "runtime_durable_ready", + "query_ready", + "readiness_ready", + "ready", + "ok", + "pass", + "passed" +]); +const RUNTIME_DURABLE_BLOCKED_QUERY_RESULTS = Object.freeze({ + driver_missing: "runtime_durable_adapter_driver_missing", + ssl_negotiation_blocked: "runtime_durable_adapter_ssl_blocked", + auth_blocked: "runtime_durable_adapter_auth_blocked", + schema_blocked: "runtime_durable_adapter_schema_blocked", + migration_blocked: "runtime_durable_adapter_migration_blocked", + query_blocked: "runtime_durable_adapter_query_blocked", + not_ready: "runtime_durable_adapter_query_blocked" +}); +const RUNTIME_DURABLE_GATE_LAYERS = Object.freeze(["ssl", "auth", "schema", "migration", "durability"]); const READ_ONLY_REASON_CODES = Object.freeze([ "read_only", "read-only", @@ -480,70 +500,286 @@ function classifyM3StatusProbe(live = {}) { } function durableRuntimeReason(payload) { - const runtime = payload?.runtime; - const durability = payload?.readiness?.durability; - const runtimeDurable = runtime?.durable === true || durability?.durable === true; - const runtimeExplicitlyNonDurable = runtime?.durable === false || durability?.durable === false; - const runtimeReady = runtime?.ready === true || durability?.ready === true; - const durabilityReady = durability?.ready === true || normalizedStatus(durability?.status) === "ready"; - const liveRuntimeEvidence = runtime?.liveRuntimeEvidence === true || durability?.liveRuntimeEvidence === true; - if (runtimeDurable && runtimeReady && durabilityReady && !hasDurableRuntimeBlockedSignal(payload)) return null; - const reasonCode = firstNonEmpty( - runtime?.blocker, - durability?.blocker, - runtime?.durabilityContract?.blockedLayer, - durability?.blockedLayer, - blockedDurabilityQueryResult(runtime?.connection?.queryResult), - blockedDurabilityQueryResult(durability?.queryResult), - blockerCodeStartingWith(payload, "runtime_durable_adapter_") - ); - const blockedLayer = firstNonEmpty(runtime?.durabilityContract?.blockedLayer, durability?.blockedLayer); - if (!reasonCode && runtimeDurable && runtimeReady && liveRuntimeEvidence) return null; - if (runtimeExplicitlyNonDurable) { - return { - reasonCode: "runtime_durable_false", - reason: `runtime durable=false; adapter=${firstNonEmpty(runtime?.adapter, durability?.adapter, "unknown")}`, - layer: "runtime-durable" - }; - } + if (!hasRuntimeDurableSignal(payload)) return null; + const classification = classifyRuntimeDurableReadiness(payload); + if (classification.ready) return null; if ( - reasonCode && + classification.status === "blocked" && ( - isReadOnlyReason(reasonCode) || - String(reasonCode).endsWith("_blocked") || - normalizedStatus(blockedLayer).includes("durability") + isReadOnlyReason(classification.reasonCode) || + String(classification.reasonCode).endsWith("_blocked") || + normalizedStatus(classification.evidence?.blockedLayer).includes("durability") ) ) { return { - reasonCode, - reason: `runtime durable blocked at ${blockedLayer || "durability"}; queryResult=${firstNonEmpty(runtime?.connection?.queryResult, durability?.queryResult, "unknown")}`, - layer: blockedLayer || "runtime-durable" + reasonCode: classification.reasonCode, + reason: classification.reason, + layer: classification.evidence?.blockedLayer || "runtime-durable", + classification }; } return null; } -function hasDurableRuntimeBlockedSignal(payload) { +function hasRuntimeDurableSignal(payload = {}) { const runtime = payload?.runtime; const durability = payload?.readiness?.durability; - return [ - runtime?.blocker, - durability?.blocker, - runtime?.durabilityContract?.blockedLayer, - durability?.blockedLayer, - blockedDurabilityQueryResult(runtime?.connection?.queryResult), - blockedDurabilityQueryResult(durability?.queryResult), - blockerCodeStartingWith(payload, "runtime_durable_adapter_") - ].some(Boolean); + const dbRuntimeReadiness = payload?.db?.runtimeReadiness; + return Boolean( + blockerCodeStartingWith(payload, "runtime_durable_adapter_") || + hasOwnNonEmpty(runtime, ["adapter", "durable", "durableRequested", "ready", "blocker", "liveRuntimeEvidence", "connection", "durabilityContract", "gates"]) || + hasOwnNonEmpty(durability, ["adapter", "durable", "ready", "blocker", "blockedLayer", "queryResult", "liveRuntimeEvidence", "gates"]) || + hasOwnNonEmpty(dbRuntimeReadiness, ["adapter", "durable", "ready", "blocker", "blockedLayer", "queryResult", "liveRuntimeEvidence"]) + ); } -function blockedDurabilityQueryResult(value) { - const text = normalizedStatus(value); - if (!text || text === "durable_readiness_ready" || text === "ready") return ""; - if (text.endsWith("_blocked") || text.includes("blocked")) return String(value).trim(); +function hasOwnNonEmpty(object, fields) { + if (!object || typeof object !== "object") return false; + return fields.some((field) => Object.hasOwn(object, field) && object[field] !== undefined && object[field] !== null && object[field] !== ""); +} + +export function classifyRuntimeDurableReadiness(payload = {}, options = {}) { + const runtime = payload?.runtime ?? {}; + const durability = payload?.readiness?.durability ?? {}; + const dbRuntimeReadiness = payload?.db?.runtimeReadiness ?? {}; + const gates = runtime?.gates ?? durability?.gates ?? {}; + const gateBlocker = firstBlockedRuntimeGate(gates); + const queryResult = firstNonEmpty( + runtime?.connection?.queryResult, + durability?.queryResult, + dbRuntimeReadiness?.queryResult + ); + const directBlocker = firstNonEmpty( + runtime?.blocker, + durability?.blocker, + dbRuntimeReadiness?.blocker, + blockerCodeStartingWith(payload, "runtime_durable_adapter_") + ); + const queryBlocker = runtimeDurableBlockerFromQueryResult(queryResult); + const reasonCode = firstNonEmpty(directBlocker, gateBlocker?.blocker, queryBlocker); + const blockedLayer = firstNonEmpty( + runtime?.durabilityContract?.blockedLayer, + durability?.blockedLayer, + dbRuntimeReadiness?.blockedLayer, + gateBlocker?.layer, + runtimeDurableLayerFromBlocker(reasonCode), + runtimeDurableLayerFromQueryResult(queryResult) + ); + const adapter = firstNonEmpty(runtime?.adapter, durability?.adapter, dbRuntimeReadiness?.adapter, "unknown"); + const runtimeDurable = runtime?.durable === true || durability?.durable === true || dbRuntimeReadiness?.durable === true; + const runtimeExplicitlyNonDurable = runtime?.durable === false || durability?.durable === false || dbRuntimeReadiness?.durable === false; + const runtimeReady = runtime?.ready === true || durability?.runtimeReady === true || dbRuntimeReadiness?.ready === true; + const durabilityReady = durability?.ready === true || dbRuntimeReadiness?.ready === true || runtimeReady; + const liveRuntimeEvidence = + runtime?.liveRuntimeEvidence === true || + durability?.liveRuntimeEvidence === true || + dbRuntimeReadiness?.liveRuntimeEvidence === true; + const rawStatus = rawStatusFrom(payload); + const rawStatusBlocked = [...ERROR_STATUSES, ...BLOCKED_STATUSES, ...RAW_DEGRADED_STATUSES].includes(normalizedStatus(rawStatus)); + const readyQueryResult = isRuntimeDurableReadyQueryResult(queryResult); + const postgresRequired = options.requirePostgresAdapter === true; + const adapterReady = !postgresRequired || adapter === "postgres"; + const ready = adapterReady && + runtimeDurable && + runtimeReady && + durabilityReady && + liveRuntimeEvidence && + !reasonCode && + !rawStatusBlocked; + + if (ready) { + return { + status: "ready", + ready: true, + classification: "runtime_durable_ready", + reasonCode: "runtime_durable_ready", + reason: `runtime durable ready; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`, + impact: "M3 durable postflight may run because runtime persistence readiness is green.", + safeNextAction: "Continue with repo-owned DEV CD postflight; do not read or print Secret values.", + retryable: true, + readonly: false, + evidence: runtimeDurableEvidence({ + adapter, + runtimeDurable, + runtimeReady, + durabilityReady, + liveRuntimeEvidence, + blockedLayer: null, + queryResult, + readyQueryResult, + reasonCode: null, + gates, + rawStatus, + postgresRequired + }) + }; + } + + const nonReadyReasonCode = firstNonEmpty( + reasonCode, + postgresRequired && adapter !== "postgres" ? "runtime_durable_adapter_not_postgres" : "", + runtimeExplicitlyNonDurable ? "runtime_durable_false" : "", + runtimeDurable ? "" : "runtime_durable_adapter_missing", + runtimeReady ? "" : "runtime_durable_runtime_not_ready", + liveRuntimeEvidence ? "" : "runtime_durable_live_evidence_missing", + rawStatusBlocked ? rawStatus : "", + "runtime_durable_adapter_not_ready" + ); + const layer = firstNonEmpty( + blockedLayer, + runtimeDurableLayerFromBlocker(nonReadyReasonCode), + runtimeExplicitlyNonDurable ? "runtime-durable" : "", + "runtime-durable" + ); + return { + status: "blocked", + ready: false, + classification: nonReadyReasonCode, + reasonCode: nonReadyReasonCode, + reason: runtimeDurableBlockedReason({ reasonCode: nonReadyReasonCode, adapter, layer, queryResult }), + impact: "M3 true/false IO durable postflight is skipped or blocked so DEV CD cannot prove evidence persistence.", + safeNextAction: runtimeDurableSafeNextAction(nonReadyReasonCode, layer), + retryable: true, + readonly: true, + evidence: runtimeDurableEvidence({ + adapter, + runtimeDurable, + runtimeReady, + durabilityReady, + liveRuntimeEvidence, + blockedLayer: layer, + queryResult, + readyQueryResult, + reasonCode: nonReadyReasonCode, + gates, + rawStatus, + postgresRequired + }) + }; +} + +function firstBlockedRuntimeGate(gates = {}) { + for (const layer of RUNTIME_DURABLE_GATE_LAYERS) { + const gate = gates?.[layer]; + const status = normalizedStatus(gate?.status); + if (gate?.ready === false || status === "blocked" || status === "failed") { + return { + layer, + blocker: firstNonEmpty(gate?.blocker, runtimeDurableBlockerForLayer(layer)) + }; + } + } + return null; +} + +function runtimeDurableBlockerForLayer(layer) { + const text = normalizedStatus(layer); + if (text === "ssl") return "runtime_durable_adapter_ssl_blocked"; + if (text === "auth") return "runtime_durable_adapter_auth_blocked"; + if (text === "schema") return "runtime_durable_adapter_schema_blocked"; + if (text === "migration") return "runtime_durable_adapter_migration_blocked"; + if (text === "durability" || text === "durability_query") return "runtime_durable_adapter_query_blocked"; + return "runtime_durable_adapter_blocked"; +} + +function runtimeDurableLayerFromBlocker(blocker) { + const text = normalizedStatus(blocker); + if (!text) return ""; + if (text.includes("ssl")) return "ssl"; + if (text.includes("auth")) return "auth"; + if (text.includes("schema")) return "schema"; + if (text.includes("migration")) return "migration"; + if (text.includes("query")) return "durability_query"; + if (text.includes("missing") || text.includes("unconfigured") || text.includes("driver")) return "adapter"; return ""; } +function runtimeDurableLayerFromQueryResult(queryResult) { + const text = normalizedStatus(queryResult); + if (!text || isRuntimeDurableReadyQueryResult(text)) return ""; + if (text.includes("ssl")) return "ssl"; + if (text.includes("auth")) return "auth"; + if (text.includes("schema")) return "schema"; + if (text.includes("migration")) return "migration"; + if (text.includes("query")) return "durability_query"; + if (text.includes("driver")) return "adapter"; + return ""; +} + +function runtimeDurableBlockerFromQueryResult(queryResult) { + const text = normalizedStatus(queryResult); + if (!text || isRuntimeDurableReadyQueryResult(text)) return ""; + return RUNTIME_DURABLE_BLOCKED_QUERY_RESULTS[text] ?? ""; +} + +function isRuntimeDurableReadyQueryResult(queryResult) { + const text = normalizedStatus(queryResult); + return RUNTIME_DURABLE_READY_QUERY_RESULTS.includes(text); +} + +function runtimeDurableEvidence({ + adapter, + runtimeDurable, + runtimeReady, + durabilityReady, + liveRuntimeEvidence, + blockedLayer, + queryResult, + readyQueryResult, + reasonCode, + gates, + rawStatus, + postgresRequired +}) { + return { + adapter, + durable: runtimeDurable, + runtimeReady, + durabilityReady, + liveRuntimeEvidence, + blocker: reasonCode, + blockedLayer, + queryResult: queryResult || null, + readyQueryResult, + rawStatus: rawStatus || null, + postgresRequired, + gates: summarizeRuntimeGateStatuses(gates), + secretMaterialRead: false + }; +} + +function summarizeRuntimeGateStatuses(gates = {}) { + return Object.fromEntries(RUNTIME_DURABLE_GATE_LAYERS.map((layer) => [ + layer, + { + checked: gates?.[layer]?.checked === true, + ready: gates?.[layer]?.ready === true, + status: gates?.[layer]?.status ?? "unknown", + blocker: gates?.[layer]?.blocker ?? null + } + ])); +} + +function runtimeDurableBlockedReason({ reasonCode, adapter, layer, queryResult }) { + if (reasonCode === "runtime_durable_false") { + return `runtime durable=false; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`; + } + if (reasonCode === "runtime_durable_adapter_not_postgres") { + return `runtime durable adapter is not postgres; adapter=${adapter}; queryResult=${queryResult || "not_observed"}`; + } + return `runtime durable blocked at ${layer || "runtime-durable"}; blocker=${reasonCode}; queryResult=${queryResult || "unknown"}`; +} + +function runtimeDurableSafeNextAction(reasonCode, layer) { + const text = normalizedStatus(`${reasonCode} ${layer}`); + if (text.includes("auth")) return "Repair DEV DB auth/SecretRef metadata through repo-owned provisioning, then rerun postflight without printing Secret values."; + if (text.includes("schema") || text.includes("migration")) return "Run repo-owned DEV runtime provisioning and migration, then rerun postflight."; + if (text.includes("ssl")) return "Align DEV DB SSL mode with the repo-owned DEV contract, then rerun postflight."; + if (text.includes("query")) return "Inspect durable runtime query readiness, preserve evidence persistence, and rerun postflight."; + if (text.includes("postgres")) return "Run DEV with HWLAB_CLOUD_RUNTIME_ADAPTER=postgres and DB URLs injected only through SecretRefs."; + return "Restore durable runtime readiness through repo-owned DEV CD steps, then rerun postflight."; +} + function firstBlocker(payload) { const blockers = [ ...(Array.isArray(payload?.blockers) ? payload.blockers : []), @@ -626,13 +862,22 @@ function probeAttribution(probe) { return [ `${probe.serviceId} ${probe.apiPath}`, probe.httpStatus ? `HTTP ${probe.httpStatus}` : null, - probe.reasonCode, - probe.reason && probe.reason !== probe.reasonCode ? probe.reason : null, + displayReasonCode(probe.reasonCode), + probe.reason && probe.reason !== probe.reasonCode ? displayReasonText(probe.reason) : null, probe.traceId ? `trace=${probe.traceId}` : null, probe.evidenceSummary ].filter(Boolean).join(" / "); } +function displayReasonCode(reasonCode) { + const text = String(reasonCode ?? ""); + return text.startsWith("runtime_durable_adapter_") ? "runtime durable" : text; +} + +function displayReasonText(reason) { + return String(reason ?? "").replace(/\bruntime_durable_adapter_[a-z0-9_]+\b/giu, "runtime durable"); +} + function normalizedStatus(value) { return String(value ?? "").trim().toLowerCase(); } diff --git a/web/hwlab-cloud-web/scripts/live-status-contract.test.mjs b/web/hwlab-cloud-web/scripts/live-status-contract.test.mjs index 54c60f1e..4c9a3dfa 100644 --- a/web/hwlab-cloud-web/scripts/live-status-contract.test.mjs +++ b/web/hwlab-cloud-web/scripts/live-status-contract.test.mjs @@ -34,6 +34,25 @@ const okRest = Object.freeze({ serviceId: "hwlab-cloud-api", status: "ok", ready: true, + readiness: { + status: "ok", + ready: true, + durability: { + status: "ready", + ready: true, + queryResult: "durable_readiness_ready" + } + }, + runtime: { + adapter: "postgres", + status: "ready", + durable: true, + ready: true, + liveRuntimeEvidence: true, + connection: { + queryResult: "durable_readiness_ready" + } + }, codeAgent: { endpoint: "POST /v1/agent/chat", status: "available", @@ -52,6 +71,25 @@ const codexStdioRest = Object.freeze({ serviceId: "hwlab-cloud-api", status: "ok", ready: true, + readiness: { + status: "ok", + ready: true, + durability: { + status: "ready", + ready: true, + queryResult: "durable_readiness_ready" + } + }, + runtime: { + adapter: "postgres", + status: "ready", + durable: true, + ready: true, + liveRuntimeEvidence: true, + connection: { + queryResult: "durable_readiness_ready" + } + }, codeAgent: { endpoint: "POST /v1/agent/chat", status: "codex-stdio-feasible", @@ -78,6 +116,25 @@ const readonlyOnlyRest = Object.freeze({ serviceId: "hwlab-cloud-api", status: "degraded", ready: false, + readiness: { + status: "ok", + ready: true, + durability: { + status: "ready", + ready: true, + queryResult: "durable_readiness_ready" + } + }, + runtime: { + adapter: "postgres", + status: "ready", + durable: true, + ready: true, + liveRuntimeEvidence: true, + connection: { + queryResult: "durable_readiness_ready" + } + }, codeAgent: { endpoint: "POST /v1/agent/chat", status: "partial", @@ -232,6 +289,111 @@ test("classifies durable runtime degraded as explicit read-only mode", () => { assert.deepEqual(status.internalRawStatuses.filter((item) => item === "degraded"), ["degraded"]); }); +test("does not classify ready queryResult durable runtime as blocked", () => { + const status = classifyWorkbenchLiveStatus({ + healthLive: { + ok: true, + status: 200, + data: { + serviceId: "hwlab-cloud-api", + status: "ready", + ready: true, + runtime: { + adapter: "postgres", + durable: true, + ready: true, + liveRuntimeEvidence: true, + status: "ready", + blocker: null, + connection: { + queryResult: "durable_readiness_ready" + } + }, + readiness: { + status: "ready", + ready: true, + durability: { + status: "ready", + ready: true, + blocker: null, + blockedLayer: null, + queryResult: "durable_readiness_ready" + } + }, + blockerCodes: [] + } + }, + restIndex: okRest, + health: okApi, + adapter: okApi, + m3Control: okM3Control, + m3Status: okM3Status + }); + + assert.equal(status.kind, "pass"); + assert.equal(status.reasonCode, "ready"); + assert.equal(status.probes.find((probe) => probe.apiPath === "/health/live")?.reasonCode, "ready"); +}); + +test("classifies auth and migration durable blockers with shared runtime reason", () => { + for (const [reasonCode, queryResult, blockedLayer] of [ + ["runtime_durable_adapter_auth_blocked", "auth_blocked", "auth"], + ["runtime_durable_adapter_migration_blocked", "migration_blocked", "migration"] + ]) { + const status = classifyWorkbenchLiveStatus({ + healthLive: { + ok: true, + status: 200, + data: { + serviceId: "hwlab-cloud-api", + status: "degraded", + ready: false, + runtime: { + adapter: "postgres", + durable: false, + durableRequested: true, + ready: false, + liveRuntimeEvidence: false, + status: "blocked", + blocker: reasonCode, + connection: { queryResult }, + durabilityContract: { blockedLayer }, + gates: { + [blockedLayer]: { + checked: true, + ready: false, + status: "blocked", + blocker: reasonCode + } + } + }, + readiness: { + ready: false, + status: "blocked", + durability: { + ready: false, + status: "blocked", + blocker: reasonCode, + blockedLayer, + queryResult + } + }, + blockerCodes: [reasonCode] + } + }, + restIndex: okRest, + health: okApi, + adapter: okApi, + m3Control: okM3Control, + m3Status: okM3Status + }); + + assert.equal(status.kind, "readonly"); + assert.equal(status.reasonCode, reasonCode); + assert.match(status.detail, new RegExp(queryResult, "u")); + } +}); + test("classifies missing probes as unverified instead of degraded", () => { const status = classifyWorkbenchLiveStatus({});