Files
pikasTech-HWLAB/scripts/src/dev-runtime-postflight.mjs
T
Lyon 83e5f52c98 fix: harden dev cd runtime postflight
Summary:
- shared runtime durable readiness classifier for live-status and DEV postflight
- structured postflight blockers propagated through dev-cd apply
- contract coverage for ready queryResult, auth/schema/migration blockers, evidence persistence blockers, and bounded stdout

Verification:
- git diff --check
- node --check changed scripts/tests/live-status modules
- node --test scripts/src/dev-runtime-postflight.test.mjs scripts/src/dev-cd-apply.test.mjs web/hwlab-cloud-web/scripts/live-status-contract.test.mjs
- node scripts/dev-runtime-postflight.mjs --check
- node web/hwlab-cloud-web/scripts/check.mjs
- npm run check

No DEV apply/rollout/live health verification was run.
2026-05-23 22:15:04 +08:00

663 lines
23 KiB
JavaScript

import { createHash } from "node:crypto";
import { mkdir, writeFile } from "node:fs/promises";
import path from "node:path";
import { fileURLToPath } from "node:url";
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
import { classifyRuntimeDurableReadiness } from "../../web/hwlab-cloud-web/live-status.mjs";
import {
buildM3IoControlSourceReport,
runM3IoControlLiveReport
} from "./m3-io-control-e2e.mjs";
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const defaultReportPath = "reports/dev-gate/dev-runtime-postflight-report.json";
const issue = "pikasTech/HWLAB#311";
const defaultHealthUrl = `${DEV_ENDPOINT}/health/live`;
const defaultV1Url = `${DEV_ENDPOINT}/v1`;
const expectedM3Sequence = Object.freeze([
{ id: "write-do1-true", action: "do.write", resultValue: true },
{ id: "read-di1-true", action: "di.read", resultValue: true },
{ id: "write-do1-false", action: "do.write", resultValue: false },
{ id: "read-di1-false", action: "di.read", resultValue: false }
]);
export async function runDevRuntimePostflightCli(argv = process.argv.slice(2), options = {}) {
const args = parseArgs(argv);
if (args.help) {
process.stdout.write(`${usage()}\n`);
return 0;
}
const report = await buildDevRuntimePostflightReport(args, options);
if (args.writeReport) {
await writeReport(report, args.reportPath, options.repoRoot ?? repoRoot);
}
const printable = args.pretty || args.writeReport ? report : summarizeReport(report);
process.stdout.write(`${JSON.stringify(printable, null, 2)}\n`);
return report.conclusion.status === "pass" || !args.failOnBlocked ? 0 : 1;
}
export async function buildDevRuntimePostflightReport(args = {}, options = {}) {
const parsed = normalizeArgs(args);
const now = options.now ?? (() => new Date().toISOString());
const report = {
issue,
mode: parsed.live ? "dev-live" : "check",
generatedAt: now(),
target: {
environment: ENVIRONMENT_DEV,
namespace: "hwlab-dev",
prodAllowed: false,
apiHealthUrl: defaultHealthUrl,
apiV1Url: defaultV1Url,
m3Target: parsed.target
},
actions: {
healthLiveReadAttempted: false,
v1ReadAttempted: false,
m3MutationAttempted: false,
m3MutationAllowed: false
},
apiHealth: null,
apiV1: null,
m3: null,
blockers: [],
safety: buildSafety(parsed),
safetyRefusal: false
};
if (!parsed.live) {
const sourceReport = await (options.m3SourceBuilder ?? buildM3IoControlSourceReport)({
repoRoot: options.repoRoot ?? repoRoot
});
report.m3 = summarizeM3Report(sourceReport);
report.conclusion = {
status: sourceReport.summary?.status === "pass" ? "pass" : "blocked",
summary: "DEV runtime postflight source contract checked only; no live HTTP or M3 write was attempted.",
blockerCount: sourceReport.summary?.status === "pass" ? 0 : 1
};
if (sourceReport.summary?.status !== "pass") {
addBlocker(report, {
type: "contract_blocker",
scope: "m3-source-contract",
reason: "M3 postflight source contract is blocked.",
impact: "DEV CD cannot rely on the M3 durable postflight harness contract.",
safeNextAction: "Fix the source M3 IO control contract tests before running live DEV postflight.",
retryable: true,
evidence: {
classification: sourceReport.summary?.classification ?? "unknown"
}
});
}
return report;
}
if (!parsed.confirmDev || !parsed.confirmedNonProduction) {
addSafetyRefusal(report, "DEV runtime postflight live mode requires --live --confirm-dev --confirmed-non-production.");
return finalizeReport(report);
}
const httpGetJson = options.httpGetJson ?? defaultHttpGetJson;
report.actions.healthLiveReadAttempted = true;
report.actions.v1ReadAttempted = true;
const [health, v1] = await Promise.all([
httpGetJson(defaultHealthUrl, parsed.timeoutMs),
httpGetJson(defaultV1Url, parsed.timeoutMs)
]);
report.apiHealth = summarizeApiPayload("GET /health/live", health);
report.apiV1 = summarizeApiPayload("GET /v1", v1);
addApiReadinessBlockers(report);
if (report.blockers.length === 0) {
report.actions.m3MutationAllowed = true;
report.actions.m3MutationAttempted = true;
const m3Runner = options.m3Runner ?? defaultM3Runner;
report.m3 = summarizeM3Report(await m3Runner(parsed));
addM3Blockers(report);
} else {
report.m3 = {
status: "not_run",
trustedGreen: false,
classification: "runtime_durable_postflight_precondition_blocked",
operationCount: 0,
reason: "M3 true/false write/read postflight is skipped until /health/live and /v1 prove durable runtime readiness."
};
}
return finalizeReport(report);
}
export function parseArgs(argv = []) {
const args = {
live: false,
confirmDev: false,
confirmedNonProduction: false,
writeReport: false,
reportPath: defaultReportPath,
pretty: false,
failOnBlocked: false,
target: "api",
timeoutMs: 8000,
help: false
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--check") args.live = false;
else if (arg === "--live") args.live = true;
else if (arg === "--confirm-dev") args.confirmDev = true;
else if (arg === "--confirmed-non-production") args.confirmedNonProduction = true;
else if (arg === "--write-report") args.writeReport = true;
else if (arg === "--pretty") args.pretty = true;
else if (arg === "--fail-on-blocked") args.failOnBlocked = true;
else if (arg === "--target") {
index += 1;
args.target = requireArgValue(argv[index], "--target");
} else if (arg === "--timeout-ms") {
index += 1;
args.timeoutMs = parseTimeout(requireArgValue(argv[index], "--timeout-ms"));
} else if (arg === "--report") {
index += 1;
args.reportPath = requireArgValue(argv[index], "--report");
} else if (arg.startsWith("--report=")) {
args.reportPath = requireArgValue(arg.slice("--report=".length), "--report");
} else if (arg === "--help" || arg === "-h") {
args.help = true;
} else {
throw new Error(`unknown argument: ${arg}`);
}
}
if (!["api", "frontend"].includes(args.target)) {
throw new Error("--target must be api or frontend");
}
return args;
}
function normalizeArgs(args) {
return {
...parseArgs([]),
...args
};
}
async function defaultM3Runner(args) {
const sourceReport = await buildM3IoControlSourceReport({ repoRoot });
return runM3IoControlLiveReport({
flags: new Set(["--live", "--confirm-dev", "--confirmed-non-production"]),
live: true,
confirmDev: true,
confirmedNonProduction: true,
target: args.target,
frontendUrl: "http://74.48.78.17:16666/",
apiUrl: `${DEV_ENDPOINT}/v1/m3/io`,
timeoutMs: args.timeoutMs
}, sourceReport);
}
async function defaultHttpGetJson(url, timeoutMs) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), timeoutMs);
try {
const response = await fetch(url, {
method: "GET",
headers: {
accept: "application/json"
},
signal: controller.signal
});
const text = await response.text();
let json = null;
try {
json = text ? JSON.parse(text) : null;
} catch {
json = null;
}
return {
ok: response.ok,
status: response.status,
json,
error: response.ok ? null : json?.error?.message ?? response.statusText
};
} catch (error) {
return {
ok: false,
status: 0,
json: null,
error: error.name === "AbortError" ? `request timed out after ${timeoutMs}ms` : error.message
};
} finally {
clearTimeout(timer);
}
}
function summarizeApiPayload(check, response) {
const json = response.json ?? {};
const runtime = summarizeRuntime(json.runtime);
const runtimeDurableReadiness = classifyRuntimeDurableReadiness(json, { requirePostgresAdapter: true });
return {
check,
httpStatus: response.status,
reachable: response.ok === true && Boolean(response.json),
serviceId: json.serviceId ?? json.service?.id ?? null,
environment: json.environment ?? null,
status: json.status ?? null,
ready: json.ready === true,
runtime,
runtimeDurableReadiness,
db: summarizeDb(json.db),
readiness: {
ready: json.readiness?.ready === true,
status: json.readiness?.status ?? null,
durabilityReady: json.readiness?.durability?.ready === true,
durabilityBlockedLayer: json.readiness?.durability?.blockedLayer ?? json.runtime?.durabilityContract?.blockedLayer ?? null
},
blockerCodes: Array.isArray(json.blockerCodes) ? json.blockerCodes : [],
m3IoControl: json.m3IoControl
? {
route: json.m3IoControl.route ?? null,
status: json.m3IoControl.status ?? null,
enabled: json.m3IoControl.enabled === true || json.m3IoControl.status === "available",
contractVersion: json.m3IoControl.contractVersion ?? null,
directFrontendGatewayOrBoxAccess: json.m3IoControl.boundaries?.directFrontendGatewayOrBoxAccess === true,
genericHardwareRpcExposedToFrontend: json.m3IoControl.boundaries?.genericHardwareRpcExposedToFrontend === true
}
: null,
error: response.error ? redactFailureText(response.error) : null
};
}
function summarizeRuntime(runtime = {}) {
return {
adapter: runtime?.adapter ?? "unknown",
durable: runtime?.durable === true,
durableRequested: runtime?.durableRequested === true,
ready: runtime?.ready === true,
status: runtime?.status ?? null,
blocker: runtime?.blocker ?? null,
liveRuntimeEvidence: runtime?.liveRuntimeEvidence === true,
requiredEvidence: runtime?.durabilityContract?.requiredEvidence ?? null,
blockedLayer: runtime?.durabilityContract?.blockedLayer ?? null,
gates: summarizeRuntimeGates(runtime?.gates)
};
}
function summarizeRuntimeGates(gates = {}) {
return Object.fromEntries(["ssl", "auth", "schema", "migration", "durability"].map((name) => [
name,
{
checked: gates?.[name]?.checked === true,
ready: gates?.[name]?.ready === true,
status: gates?.[name]?.status ?? "unknown",
blocker: gates?.[name]?.blocker ?? null
}
]));
}
function summarizeDb(db = {}) {
return {
ready: db?.ready === true,
connected: db?.connected === true,
liveDbEvidence: db?.liveDbEvidence === true,
endpointSource: db?.endpointSource ?? null,
connectionResult: db?.connectionResult ?? db?.connection?.result ?? null,
endpointRedacted: true,
valueRedacted: true,
runtimeReadiness: {
ready: db?.runtimeReadiness?.ready === true,
status: db?.runtimeReadiness?.status ?? null,
blocker: db?.runtimeReadiness?.blocker ?? null,
queryResult: db?.runtimeReadiness?.queryResult ?? null,
requiredEvidence: db?.runtimeReadiness?.requiredEvidence ?? null
}
};
}
function summarizeM3Report(report = {}) {
const operations = Array.isArray(report.liveOperations) ? report.liveOperations : [];
const persistenceContract = summarizeM3PersistenceContract(operations);
return {
mode: report.mode ?? "unknown",
status: report.summary?.status ?? report.status ?? "unknown",
classification: report.summary?.classification ?? null,
trustedGreen: report.summary?.trustedGreen === true,
result: report.summary?.result ?? null,
operationCount: operations.length,
persisted: persistenceContract.ready,
persistenceContract,
operations: operations.map((operation) => ({
id: operation.id,
action: operation.action,
status: operation.status,
operationId: operation.operationId ?? null,
traceId: operation.traceId ?? null,
auditId: operation.auditId ?? null,
evidenceId: operation.evidenceId ?? null,
resultValue: operation.resultValue,
evidenceState: {
status: operation.evidenceState?.status ?? null,
durable: operation.evidenceState?.durable === true,
sourceKind: operation.evidenceState?.sourceKind ?? null,
writeStatus: operation.evidenceState?.writeStatus ?? null
},
blocker: sanitizeM3Blocker(operation.blocker)
}))
};
}
function summarizeM3PersistenceContract(operations = []) {
const observed = new Map(operations.map((operation) => [operation.id, operation]));
const sequence = expectedM3Sequence.map((expected) => {
const operation = observed.get(expected.id) ?? {};
const evidenceState = operation.evidenceState ?? {};
const idsPresent = Boolean(operation.operationId && operation.traceId && operation.auditId && operation.evidenceId);
const valueMatches = operation.resultValue === expected.resultValue;
const evidenceGreen = evidenceState.status === "green" &&
evidenceState.durable === true &&
evidenceState.sourceKind === "DEV-LIVE";
const persisted = evidenceState.writeStatus === "persisted";
return {
id: expected.id,
action: expected.action,
expectedValue: expected.resultValue,
observedValue: Object.hasOwn(operation, "resultValue") ? operation.resultValue : null,
status: operation.status ?? "missing",
idsPresent,
valueMatches,
evidenceGreen,
persisted
};
});
return {
contractVersion: "m3-do1-true-false-persisted-v1",
requiredRoute: "res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1",
requiredValues: [true, true, false, false],
ready: operations.length === expectedM3Sequence.length &&
sequence.every((item) => item.status === "succeeded" && item.idsPresent && item.valueMatches && item.evidenceGreen && item.persisted),
sequence
};
}
function addApiReadinessBlockers(report) {
for (const [scope, payload] of [["api-health-live", report.apiHealth], ["api-v1", report.apiV1]]) {
if (!payload.reachable) {
addBlocker(report, {
type: "runtime_blocker",
scope,
reason: `${payload.check} was not reachable.`,
impact: "DEV CD cannot prove cloud-api durable readiness, so M3 evidence persistence postflight cannot safely run.",
safeNextAction: "Restore DEV cloud-api reachability on the repo-owned public endpoint, then rerun postflight.",
retryable: true,
evidence: {
httpStatus: payload.httpStatus,
error: payload.error
}
});
continue;
}
if (payload.runtimeDurableReadiness?.ready !== true) {
addBlocker(report, {
type: "runtime_blocker",
scope,
reason: `${payload.check} did not prove durable runtime readiness: ${payload.runtimeDurableReadiness?.reason ?? "not ready"}.`,
impact: payload.runtimeDurableReadiness?.impact ?? "M3 durable postflight cannot prove evidence persistence.",
safeNextAction: payload.runtimeDurableReadiness?.safeNextAction ?? "Repair runtime durability through repo-owned DEV CD steps, then rerun postflight.",
retryable: payload.runtimeDurableReadiness?.retryable !== false,
evidence: {
runtime: payload.runtime,
runtimeDurableReadiness: payload.runtimeDurableReadiness,
blockerCodes: payload.blockerCodes
}
});
}
if (scope === "api-v1" && !apiV1M3ContractReady(payload)) {
addBlocker(report, {
type: "runtime_blocker",
scope,
reason: "GET /v1 did not expose the controlled M3 IO route required by durable postflight.",
impact: "DEV CD cannot run the M3 true/false IO persistence smoke through the cloud-api controlled boundary.",
safeNextAction: "Restore /v1 m3IoControl route metadata for /v1/m3/io without exposing direct frontend simulator access.",
retryable: true,
evidence: {
m3IoControl: payload.m3IoControl,
requiredRoute: "/v1/m3/io",
requiredContractVersion: "m3-io-control-v1"
}
});
}
}
}
function apiV1M3ContractReady(payload) {
const control = payload.m3IoControl;
return payload.check === "GET /v1" &&
control?.route === "/v1/m3/io" &&
control?.contractVersion === "m3-io-control-v1" &&
control?.enabled === true &&
control?.directFrontendGatewayOrBoxAccess !== true &&
control?.genericHardwareRpcExposedToFrontend !== true;
}
function addM3Blockers(report) {
if (report.m3?.persistenceContract?.ready !== true) {
addBlocker(report, {
type: "runtime_blocker",
scope: "m3-durable-postflight",
reason: "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.",
impact: "A later rollout could appear ready while losing operation, audit, or evidence persistence for M3 hardware IO.",
safeNextAction: "Inspect the failed M3 operation sequence, repair durable audit/evidence persistence, then rerun DEV runtime postflight.",
retryable: true,
evidence: {
persistenceContract: report.m3?.persistenceContract ?? null,
trustedGreen: report.m3?.trustedGreen === true,
operationCount: report.m3?.operationCount ?? 0
}
});
return;
}
if (report.m3?.trustedGreen !== true || report.m3?.status !== "pass") {
addBlocker(report, {
type: "runtime_blocker",
scope: "m3-durable-postflight",
reason: "M3 true/false durable postflight did not produce trusted green persisted evidence.",
impact: "M3 durable readiness cannot be accepted because trusted evidence is not green.",
safeNextAction: "Fix the M3 control path or trusted-record persistence and rerun DEV runtime postflight.",
retryable: true,
evidence: {
classification: report.m3?.classification ?? "unknown",
trustedGreen: report.m3?.trustedGreen === true,
operationCount: report.m3?.operationCount ?? 0
}
});
return;
}
for (const operation of report.m3.operations ?? []) {
if (!operation.operationId || !operation.auditId || !operation.evidenceId || operation.evidenceState?.status !== "green" || operation.evidenceState?.durable !== true) {
addBlocker(report, {
type: "runtime_blocker",
scope: "m3-durable-evidence",
reason: "M3 operation/audit/evidence durable fields were incomplete.",
impact: "The M3 postflight cannot prove every IO operation is bound to persisted audit and evidence records.",
safeNextAction: "Repair operation, audit, and evidence ID propagation before accepting DEV readiness.",
retryable: true,
evidence: {
operationIdPresent: Boolean(operation.operationId),
traceIdPresent: Boolean(operation.traceId),
auditIdPresent: Boolean(operation.auditId),
evidenceIdPresent: Boolean(operation.evidenceId),
evidenceState: operation.evidenceState
}
});
return;
}
}
}
function addSafetyRefusal(report, summary) {
report.safetyRefusal = true;
addBlocker(report, {
type: "safety_refusal",
scope: "runtime-postflight-live-boundary",
reason: summary,
impact: "Live DEV reads and M3 writes are not allowed without explicit non-production confirmation.",
safeNextAction: "Rerun with --live --confirm-dev --confirmed-non-production for DEV only, or use --check for source-only validation.",
retryable: true,
evidence: {
devOnly: true,
prodAllowed: false,
secretValuesPrinted: false
}
});
}
function addBlocker(report, {
type,
scope,
reason,
impact,
safeNextAction,
retryable,
evidence = {}
}) {
report.blockers.push({
type,
scope,
status: "open",
reason: oneLine(reason),
impact: oneLine(impact),
safeNextAction: oneLine(safeNextAction),
retryable: Boolean(retryable),
summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`),
sourceIssue: issue,
evidence
});
}
function finalizeReport(report) {
const openBlockers = report.blockers.filter((blocker) => blocker.status === "open");
report.conclusion = {
status: openBlockers.length === 0 ? "pass" : "blocked",
summary: openBlockers.length === 0
? "DEV runtime postflight passed: /health/live, /v1, and M3 true/false durable evidence are green."
: "DEV runtime postflight is blocked; M3 is skipped unless durable runtime readiness is proven first.",
blockerCount: openBlockers.length
};
return report;
}
function oneLine(value) {
return String(value ?? "").replace(/\s+/g, " ").trim();
}
function buildSafety(args) {
return {
devOnly: true,
environment: ENVIRONMENT_DEV,
prodAllowed: false,
liveHttpReads: args.live === true,
liveM3Writes: args.live === true,
liveM3WritesRequireDurableRuntimeReady: true,
readsKubernetesSecrets: false,
writesKubernetesSecrets: false,
secretValuesPrinted: false,
dbUrlValueRedacted: true,
endpointRedacted: true
};
}
function summarizeReport(report) {
return {
issue: report.issue,
mode: report.mode,
conclusion: report.conclusion,
actions: report.actions,
apiHealth: report.apiHealth,
apiV1: report.apiV1,
m3: report.m3,
blockers: report.blockers,
safety: report.safety
};
}
function sanitizeM3Blocker(blocker) {
if (!blocker || typeof blocker !== "object") {
return typeof blocker === "string" ? redactFailureText(blocker) : blocker ?? null;
}
return sanitizeRedactedValue(blocker);
}
function sanitizeRedactedValue(value) {
if (typeof value === "string") {
return redactFailureText(value);
}
if (Array.isArray(value)) {
return value.map(sanitizeRedactedValue);
}
if (value && typeof value === "object") {
return Object.fromEntries(Object.entries(value).map(([key, nested]) => [key, sanitizeRedactedValue(nested)]));
}
return value;
}
async function writeReport(report, reportPath, root) {
const absolute = path.resolve(root, reportPath);
await mkdir(path.dirname(absolute), { recursive: true });
await writeFile(absolute, `${JSON.stringify(report, null, 2)}\n`);
}
function parseTimeout(value) {
const parsed = Number.parseInt(value, 10);
if (!Number.isInteger(parsed) || parsed <= 0) {
throw new Error("--timeout-ms must be a positive integer");
}
return Math.min(parsed, 30000);
}
function requireArgValue(value, flag) {
if (typeof value !== "string" || value.trim() === "") {
throw new Error(`${flag} requires a value`);
}
return value;
}
function usage() {
return [
"Usage: node scripts/dev-runtime-postflight.mjs [--check|--live]",
"",
"Default --check validates source M3 postflight contracts only.",
"--live requires --confirm-dev --confirmed-non-production and reads /health/live plus /v1 before any M3 write.",
"M3 true/false write/read is skipped unless durable runtime readiness is already green.",
"Reports never print DB URLs, passwords, tokens, Secret values, or kubeconfig material."
].join("\n");
}
export function formatDevRuntimePostflightFailure(error) {
const message = redactFailureText(error instanceof Error ? error.message : String(error));
return {
issue,
conclusion: {
status: "blocked",
summary: "DEV runtime postflight command failed before producing a report."
},
error: message,
safety: {
devOnly: true,
prodAllowed: false,
secretValuesPrinted: false
},
trace: createHash("sha256").update(message).digest("hex").slice(0, 12)
};
}
function redactFailureText(value) {
return String(value ?? "")
.replace(/postgres(?:ql)?:\/\/[^\s"'<>]+/giu, "[redacted-postgres-url]")
.replace(/(password\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]")
.replace(/(token\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]")
.replace(/(kubeconfig\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]");
}