import { createHash } from "node:crypto"; import { execFileSync } from "node:child_process"; import { mkdir, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; import { buildM3IoControlSourceReport, runM3IoControlLiveReport } from "./m3-io-control-e2e.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const defaultReportPath = tempReportPath("dev-runtime-postflight-report.json"); const issue = "pikasTech/HWLAB#311"; const defaultHealthUrl = `${DEV_ENDPOINT}/health/live`; const defaultV1Url = `${DEV_ENDPOINT}/v1`; const expectedM3Sequence = Object.freeze([ { id: "write-do1-true", action: "do.write", resultValue: true }, { id: "read-di1-true", action: "di.read", resultValue: true }, { id: "write-do1-false", action: "do.write", resultValue: false }, { id: "read-di1-false", action: "di.read", resultValue: false } ]); function bunExecutable() { return process.env.HWLAB_BUN_BINARY || process.env.BUN_BINARY || "bun"; } function classifyRuntimeDurableReadiness(payload = {}, options = {}) { const script = [ "import { classifyRuntimeDurableReadiness } from './tools/src/runtime-durable-readiness.ts';", "const input = JSON.parse(await Bun.stdin.text());", "const output = classifyRuntimeDurableReadiness(input.payload, input.options);", "process.stdout.write(JSON.stringify(output));" ].join("\n"); const stdout = execFileSync(bunExecutable(), ["--eval", script], { cwd: repoRoot, input: JSON.stringify({ payload, options }), encoding: "utf8" }); return JSON.parse(stdout); } export async function runDevRuntimePostflightCli(argv = process.argv.slice(2), options = {}) { const args = parseArgs(argv); if (args.help) { process.stdout.write(`${usage()}\n`); return 0; } const report = await buildDevRuntimePostflightReport(args, options); if (args.writeReport) { await writeReport(report, args.reportPath, options.repoRoot ?? repoRoot); } const printable = args.pretty || args.writeReport ? report : summarizeReport(report); process.stdout.write(`${JSON.stringify(printable, null, 2)}\n`); return report.conclusion.status === "pass" || !args.failOnBlocked ? 0 : 1; } export async function buildDevRuntimePostflightReport(args = {}, options = {}) { const parsed = normalizeArgs(args); const now = options.now ?? (() => new Date().toISOString()); const report = { issue, mode: parsed.live ? "dev-live" : "check", generatedAt: now(), target: { environment: ENVIRONMENT_DEV, namespace: "hwlab-dev", prodAllowed: false, apiHealthUrl: defaultHealthUrl, apiV1Url: defaultV1Url, m3Target: parsed.target }, actions: { healthLiveReadAttempted: false, v1ReadAttempted: false, m3MutationAttempted: false, m3MutationAllowed: false }, apiHealth: null, apiV1: null, m3: null, blockers: [], safety: buildSafety(parsed), safetyRefusal: false }; if (!parsed.live) { const sourceReport = await (options.m3SourceBuilder ?? buildM3IoControlSourceReport)({ repoRoot: options.repoRoot ?? repoRoot }); report.m3 = summarizeM3Report(sourceReport); report.conclusion = { status: sourceReport.summary?.status === "pass" ? "pass" : "blocked", summary: "DEV runtime postflight source contract checked only; no live HTTP or M3 write was attempted.", blockerCount: sourceReport.summary?.status === "pass" ? 0 : 1 }; if (sourceReport.summary?.status !== "pass") { addBlocker(report, { type: "contract_blocker", scope: "m3-source-contract", reason: "M3 postflight source contract is blocked.", impact: "DEV CD cannot rely on the M3 durable postflight harness contract.", safeNextAction: "Fix the source M3 IO control contract tests before running live DEV postflight.", retryable: true, evidence: { classification: sourceReport.summary?.classification ?? "unknown" } }); } return report; } if (!parsed.confirmDev || !parsed.confirmedNonProduction) { addSafetyRefusal(report, "DEV runtime postflight live mode requires --live --confirm-dev --confirmed-non-production."); return finalizeReport(report); } const httpGetJson = options.httpGetJson ?? defaultHttpGetJson; report.actions.healthLiveReadAttempted = true; report.actions.v1ReadAttempted = true; const [health, v1] = await Promise.all([ httpGetJson(defaultHealthUrl, parsed.timeoutMs), httpGetJson(defaultV1Url, parsed.timeoutMs) ]); report.apiHealth = summarizeApiPayload("GET /health/live", health); report.apiV1 = summarizeApiPayload("GET /v1", v1); addApiReadinessBlockers(report); if (report.blockers.length === 0) { report.actions.m3MutationAllowed = true; report.actions.m3MutationAttempted = true; const m3Runner = options.m3Runner ?? defaultM3Runner; report.m3 = summarizeM3Report(await m3Runner(parsed)); addM3Blockers(report); } else { report.m3 = { status: "not_run", trustedGreen: false, classification: "runtime_durable_postflight_precondition_blocked", operationCount: 0, reason: "M3 true/false write/read postflight is skipped until /health/live and /v1 prove durable runtime readiness." }; } return finalizeReport(report); } export function parseArgs(argv = []) { const args = { live: false, confirmDev: false, confirmedNonProduction: false, writeReport: false, reportPath: defaultReportPath, pretty: false, failOnBlocked: false, target: "api", timeoutMs: 8000, help: false }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === "--check") args.live = false; else if (arg === "--live") args.live = true; else if (arg === "--confirm-dev") args.confirmDev = true; else if (arg === "--confirmed-non-production") args.confirmedNonProduction = true; else if (arg === "--pretty") args.pretty = true; else if (arg === "--fail-on-blocked") args.failOnBlocked = true; else if (arg === "--target") { index += 1; args.target = requireArgValue(argv[index], "--target"); } else if (arg === "--timeout-ms") { index += 1; args.timeoutMs = parseTimeout(requireArgValue(argv[index], "--timeout-ms")); } else if (arg === "--report") { index += 1; args.reportPath = requireArgValue(argv[index], "--report"); args.writeReport = true; } else if (arg.startsWith("--report=")) { args.reportPath = requireArgValue(arg.slice("--report=".length), "--report"); args.writeReport = true; } else if (arg === "--help" || arg === "-h") { args.help = true; } else { throw new Error(`unknown argument: ${arg}`); } } if (!["api", "frontend"].includes(args.target)) { throw new Error("--target must be api or frontend"); } return args; } function normalizeArgs(args) { return { ...parseArgs([]), ...args }; } async function defaultM3Runner(args) { const sourceReport = await buildM3IoControlSourceReport({ repoRoot }); return runM3IoControlLiveReport({ flags: new Set(["--live", "--confirm-dev", "--confirmed-non-production"]), live: true, confirmDev: true, confirmedNonProduction: true, target: args.target, frontendUrl: "http://74.48.78.17:16666/", apiUrl: `${DEV_ENDPOINT}/v1/m3/io`, timeoutMs: args.timeoutMs }, sourceReport); } async function defaultHttpGetJson(url, timeoutMs) { const controller = new AbortController(); const timer = setTimeout(() => controller.abort(), timeoutMs); try { const response = await fetch(url, { method: "GET", headers: { accept: "application/json" }, signal: controller.signal }); const text = await response.text(); let json = null; try { json = text ? JSON.parse(text) : null; } catch { json = null; } return { ok: response.ok, status: response.status, json, error: response.ok ? null : json?.error?.message ?? response.statusText }; } catch (error) { return { ok: false, status: 0, json: null, error: error.name === "AbortError" ? `request timed out after ${timeoutMs}ms` : error.message }; } finally { clearTimeout(timer); } } function summarizeApiPayload(check, response) { const json = response.json ?? {}; const runtime = summarizeRuntime(json.runtime); const runtimeDurableReadiness = classifyRuntimeDurableReadiness(json, { requirePostgresAdapter: true }); return { check, httpStatus: response.status, reachable: response.ok === true && Boolean(response.json), serviceId: json.serviceId ?? json.service?.id ?? null, environment: json.environment ?? null, status: json.status ?? null, ready: json.ready === true, runtime, runtimeDurableReadiness, db: summarizeDb(json.db), readiness: { ready: json.readiness?.ready === true, status: json.readiness?.status ?? null, durabilityReady: json.readiness?.durability?.ready === true, durabilityBlockedLayer: json.readiness?.durability?.blockedLayer ?? json.runtime?.durabilityContract?.blockedLayer ?? null }, blockerCodes: Array.isArray(json.blockerCodes) ? json.blockerCodes : [], m3IoControl: json.m3IoControl ? { route: json.m3IoControl.route ?? null, status: json.m3IoControl.status ?? null, enabled: json.m3IoControl.enabled === true || json.m3IoControl.status === "available", contractVersion: json.m3IoControl.contractVersion ?? null, directFrontendGatewayOrBoxAccess: json.m3IoControl.boundaries?.directFrontendGatewayOrBoxAccess === true, genericHardwareRpcExposedToFrontend: json.m3IoControl.boundaries?.genericHardwareRpcExposedToFrontend === true } : null, error: response.error ? redactFailureText(response.error) : null }; } function summarizeRuntime(runtime = {}) { return { adapter: runtime?.adapter ?? "unknown", durable: runtime?.durable === true, durableRequested: runtime?.durableRequested === true, ready: runtime?.ready === true, status: runtime?.status ?? null, blocker: runtime?.blocker ?? null, liveRuntimeEvidence: runtime?.liveRuntimeEvidence === true, requiredEvidence: runtime?.durabilityContract?.requiredEvidence ?? null, blockedLayer: runtime?.durabilityContract?.blockedLayer ?? null, gates: summarizeRuntimeGates(runtime?.gates) }; } function summarizeRuntimeGates(gates = {}) { return Object.fromEntries(["ssl", "auth", "schema", "migration", "durability"].map((name) => [ name, { checked: gates?.[name]?.checked === true, ready: gates?.[name]?.ready === true, status: gates?.[name]?.status ?? "unknown", blocker: gates?.[name]?.blocker ?? null } ])); } function summarizeDb(db = {}) { return { ready: db?.ready === true, connected: db?.connected === true, liveDbEvidence: db?.liveDbEvidence === true, endpointSource: db?.endpointSource ?? null, connectionResult: db?.connectionResult ?? db?.connection?.result ?? null, endpointRedacted: true, valueRedacted: true, runtimeReadiness: { ready: db?.runtimeReadiness?.ready === true, status: db?.runtimeReadiness?.status ?? null, blocker: db?.runtimeReadiness?.blocker ?? null, queryResult: db?.runtimeReadiness?.queryResult ?? null, requiredEvidence: db?.runtimeReadiness?.requiredEvidence ?? null } }; } function summarizeM3Report(report = {}) { const operations = Array.isArray(report.liveOperations) ? report.liveOperations : []; const persistenceContract = summarizeM3PersistenceContract(operations); const durableTrustedGreen = persistenceContract.ready === true; return { mode: report.mode ?? "unknown", status: durableTrustedGreen ? "pass" : report.summary?.status ?? report.status ?? "unknown", classification: durableTrustedGreen ? "trusted_green" : report.summary?.classification ?? null, reportedClassification: report.summary?.classification ?? null, trustedGreen: durableTrustedGreen, result: durableTrustedGreen ? "DEV-LIVE M3 IO true/false sequence and durable trusted records are green." : report.summary?.result ?? null, operationCount: operations.length, persisted: persistenceContract.ready, persistenceContract, operations: operations.map((operation) => ({ id: operation.id, action: operation.action, status: operation.status, operationId: operation.operationId ?? null, traceId: operation.traceId ?? null, auditId: operation.auditId ?? null, evidenceId: operation.evidenceId ?? null, resultValue: operation.resultValue, evidenceState: { status: operation.evidenceState?.status ?? null, durable: operation.evidenceState?.durable === true, sourceKind: operation.evidenceState?.sourceKind ?? null, writeStatus: operation.evidenceState?.writeStatus ?? null }, blocker: sanitizeM3Blocker(operation.blocker) })) }; } function summarizeM3PersistenceContract(operations = []) { const observed = new Map(operations.map((operation) => [operation.id, operation])); const sequence = expectedM3Sequence.map((expected) => { const operation = observed.get(expected.id) ?? {}; const evidenceState = operation.evidenceState ?? {}; const idsPresent = Boolean(operation.operationId && operation.traceId && operation.auditId && operation.evidenceId); const valueMatches = operation.resultValue === expected.resultValue; const evidenceGreen = evidenceState.status === "green" && evidenceState.durable === true && evidenceState.sourceKind === "DEV-LIVE"; const persisted = evidenceState.writeStatus === "persisted"; return { id: expected.id, action: expected.action, expectedValue: expected.resultValue, observedValue: Object.hasOwn(operation, "resultValue") ? operation.resultValue : null, status: operation.status ?? "missing", idsPresent, valueMatches, evidenceGreen, persisted }; }); return { contractVersion: "m3-do1-true-false-persisted-v1", requiredRoute: "res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1", requiredValues: [true, true, false, false], ready: operations.length === expectedM3Sequence.length && sequence.every((item) => item.status === "succeeded" && item.idsPresent && item.valueMatches && item.evidenceGreen && item.persisted), sequence }; } function addApiReadinessBlockers(report) { for (const [scope, payload] of [["api-health-live", report.apiHealth], ["api-v1", report.apiV1]]) { if (!payload.reachable) { addBlocker(report, { type: "runtime_blocker", scope, reason: `${payload.check} was not reachable.`, impact: "DEV CD cannot prove cloud-api durable readiness, so M3 evidence persistence postflight cannot safely run.", safeNextAction: "Restore DEV cloud-api reachability on the repo-owned public endpoint, then rerun postflight.", retryable: true, evidence: { httpStatus: payload.httpStatus, error: payload.error } }); continue; } if (payload.runtimeDurableReadiness?.ready !== true) { addBlocker(report, { type: "runtime_blocker", scope, reason: `${payload.check} did not prove durable runtime readiness: ${payload.runtimeDurableReadiness?.reason ?? "not ready"}.`, impact: payload.runtimeDurableReadiness?.impact ?? "M3 durable postflight cannot prove evidence persistence.", safeNextAction: payload.runtimeDurableReadiness?.safeNextAction ?? "Repair runtime durability through repo-owned DEV CD steps, then rerun postflight.", retryable: payload.runtimeDurableReadiness?.retryable !== false, evidence: { runtime: payload.runtime, runtimeDurableReadiness: payload.runtimeDurableReadiness, blockerCodes: payload.blockerCodes } }); } if (scope === "api-v1" && !apiV1M3ContractReady(payload)) { addBlocker(report, { type: "runtime_blocker", scope, reason: "GET /v1 did not expose the controlled M3 IO route required by durable postflight.", impact: "DEV CD cannot run the M3 true/false IO persistence smoke through the cloud-api controlled boundary.", safeNextAction: "Restore /v1 m3IoControl route metadata for /v1/m3/io without exposing direct frontend simulator access.", retryable: true, evidence: { m3IoControl: payload.m3IoControl, requiredRoute: "/v1/m3/io", requiredContractVersion: "m3-io-control-v1" } }); } } } function apiV1M3ContractReady(payload) { const control = payload.m3IoControl; return payload.check === "GET /v1" && control?.route === "/v1/m3/io" && control?.contractVersion === "m3-io-control-v1" && control?.enabled === true && control?.directFrontendGatewayOrBoxAccess !== true && control?.genericHardwareRpcExposedToFrontend !== true; } function addM3Blockers(report) { if (report.m3?.persistenceContract?.ready !== true) { addBlocker(report, { type: "runtime_blocker", scope: "m3-durable-postflight", reason: "M3 DO1 true/false durable postflight did not prove persisted operation/audit/evidence for every step.", impact: "A later rollout could appear ready while losing operation, audit, or evidence persistence for M3 hardware IO.", safeNextAction: "Inspect the failed M3 operation sequence, repair durable audit/evidence persistence, then rerun DEV runtime postflight.", retryable: true, evidence: { persistenceContract: report.m3?.persistenceContract ?? null, trustedGreen: report.m3?.trustedGreen === true, operationCount: report.m3?.operationCount ?? 0 } }); return; } if (report.m3?.trustedGreen !== true || report.m3?.status !== "pass") { addBlocker(report, { type: "runtime_blocker", scope: "m3-durable-postflight", reason: `M3 true/false durable postflight did not produce trusted green persisted evidence: ${m3DurableBlockerCode(report.m3)}.`, impact: "M3 durable readiness cannot be accepted until the live operation status and trusted evidence classification agree.", safeNextAction: "Fix the M3 control path or trusted-record persistence and rerun DEV runtime postflight.", retryable: true, evidence: { classification: report.m3?.classification ?? "unknown", trustedGreen: report.m3?.trustedGreen === true, operationCount: report.m3?.operationCount ?? 0 } }); return; } for (const operation of report.m3.operations ?? []) { if (!operation.operationId || !operation.auditId || !operation.evidenceId || operation.evidenceState?.status !== "green" || operation.evidenceState?.durable !== true) { addBlocker(report, { type: "runtime_blocker", scope: "m3-durable-evidence", reason: "M3 operation/audit/evidence durable fields were incomplete.", impact: "The M3 postflight cannot prove every IO operation is bound to persisted audit and evidence records.", safeNextAction: "Repair operation, audit, and evidence ID propagation before accepting DEV readiness.", retryable: true, evidence: { operationIdPresent: Boolean(operation.operationId), traceIdPresent: Boolean(operation.traceId), auditIdPresent: Boolean(operation.auditId), evidenceIdPresent: Boolean(operation.evidenceId), evidenceState: operation.evidenceState } }); return; } } } function m3DurableBlockerCode(m3 = {}) { if (m3.classification && m3.classification !== "source_contract_failed") { return m3.classification; } if (m3.operationCount === expectedM3Sequence.length && m3.persistenceContract?.ready === true) { return "trusted_green_summary_mismatch"; } if (m3.operationCount !== expectedM3Sequence.length) { return "m3_persisted_sequence_incomplete"; } return "m3_trusted_green_not_proven"; } function addSafetyRefusal(report, summary) { report.safetyRefusal = true; addBlocker(report, { type: "safety_refusal", scope: "runtime-postflight-live-boundary", reason: summary, impact: "Live DEV reads and M3 writes are not allowed without explicit non-production confirmation.", safeNextAction: "Rerun with --live --confirm-dev --confirmed-non-production for DEV only, or use --check for source-only validation.", retryable: true, evidence: { devOnly: true, prodAllowed: false, secretValuesPrinted: false } }); } function addBlocker(report, { type, scope, reason, impact, safeNextAction, retryable, evidence = {} }) { report.blockers.push({ type, scope, status: "open", reason: oneLine(reason), impact: oneLine(impact), safeNextAction: oneLine(safeNextAction), retryable: Boolean(retryable), summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`), sourceIssue: issue, evidence }); } function finalizeReport(report) { const openBlockers = report.blockers.filter((blocker) => blocker.status === "open"); report.conclusion = { status: openBlockers.length === 0 ? "pass" : "blocked", summary: openBlockers.length === 0 ? "DEV runtime postflight passed: /health/live, /v1, and M3 true/false durable evidence are green." : "DEV runtime postflight is blocked; M3 is skipped unless durable runtime readiness is proven first.", blockerCount: openBlockers.length }; return report; } function oneLine(value) { return String(value ?? "").replace(/\s+/g, " ").trim(); } function buildSafety(args) { return { devOnly: true, environment: ENVIRONMENT_DEV, prodAllowed: false, liveHttpReads: args.live === true, liveM3Writes: args.live === true, liveM3WritesRequireDurableRuntimeReady: true, readsKubernetesSecrets: false, writesKubernetesSecrets: false, secretValuesPrinted: false, dbUrlValueRedacted: true, endpointRedacted: true }; } function summarizeReport(report) { return { issue: report.issue, mode: report.mode, conclusion: report.conclusion, actions: report.actions, apiHealth: report.apiHealth, apiV1: report.apiV1, m3: report.m3, blockers: report.blockers, safety: report.safety }; } function sanitizeM3Blocker(blocker) { if (!blocker || typeof blocker !== "object") { return typeof blocker === "string" ? redactFailureText(blocker) : blocker ?? null; } return sanitizeRedactedValue(blocker); } function sanitizeRedactedValue(value) { if (typeof value === "string") { return redactFailureText(value); } if (Array.isArray(value)) { return value.map(sanitizeRedactedValue); } if (value && typeof value === "object") { return Object.fromEntries(Object.entries(value).map(([key, nested]) => [key, sanitizeRedactedValue(nested)])); } return value; } async function writeReport(report, reportPath, root) { const absolute = ensureNotRepoReportsPath(root, reportPath, "--report"); await mkdir(path.dirname(absolute), { recursive: true }); await writeFile(absolute, `${JSON.stringify(report, null, 2)}\n`); } function parseTimeout(value) { const parsed = Number.parseInt(value, 10); if (!Number.isInteger(parsed) || parsed <= 0) { throw new Error("--timeout-ms must be a positive integer"); } return Math.min(parsed, 30000); } function requireArgValue(value, flag) { if (typeof value !== "string" || value.trim() === "") { throw new Error(`${flag} requires a value`); } return value; } function usage() { return [ "Usage: node scripts/dev-runtime-postflight.mjs [--check|--live]", "", "Default --check validates source M3 postflight contracts only.", "--live requires --confirm-dev --confirmed-non-production and reads /health/live plus /v1 before any M3 write.", "M3 true/false write/read is skipped unless durable runtime readiness is already green.", "Reports never print DB URLs, passwords, tokens, Secret values, or kubeconfig material." ].join("\n"); } export function formatDevRuntimePostflightFailure(error) { const message = redactFailureText(error instanceof Error ? error.message : String(error)); return { issue, conclusion: { status: "blocked", summary: "DEV runtime postflight command failed before producing a report." }, error: message, safety: { devOnly: true, prodAllowed: false, secretValuesPrinted: false }, trace: createHash("sha256").update(message).digest("hex").slice(0, 12) }; } function redactFailureText(value) { return String(value ?? "") .replace(/postgres(?:ql)?:\/\/[^\s"'<>]+/giu, "[redacted-postgres-url]") .replace(/(password\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]") .replace(/(token\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]") .replace(/(kubeconfig\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]"); }