feat: add dev durable runtime provisioning gates
This commit is contained in:
Code Queue Review
@@ -569,6 +569,7 @@ function dockerfile(baseImage, port) {
|
||||
"RUN if [ -f package-lock.json ]; then npm ci --omit=dev --ignore-scripts; else npm install --omit=dev --ignore-scripts; fi",
|
||||
"COPY internal ./internal",
|
||||
"COPY cmd ./cmd",
|
||||
"COPY scripts ./scripts",
|
||||
"COPY web ./web",
|
||||
"COPY tools ./tools",
|
||||
"COPY skills ./skills",
|
||||
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env node
|
||||
import {
|
||||
formatDevRuntimePostflightFailure,
|
||||
runDevRuntimePostflightCli
|
||||
} from "./src/dev-runtime-postflight.mjs";
|
||||
|
||||
try {
|
||||
process.exitCode = await runDevRuntimePostflightCli(process.argv.slice(2));
|
||||
} catch (error) {
|
||||
process.stdout.write(`${JSON.stringify(formatDevRuntimePostflightFailure(error), null, 2)}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
#!/usr/bin/env node
|
||||
import {
|
||||
formatDevRuntimeProvisioningFailure,
|
||||
runDevRuntimeProvisioningCli
|
||||
} from "./src/dev-runtime-provisioning.mjs";
|
||||
|
||||
try {
|
||||
process.exitCode = await runDevRuntimeProvisioningCli(process.argv.slice(2));
|
||||
} catch (error) {
|
||||
process.stdout.write(`${JSON.stringify(formatDevRuntimeProvisioningFailure(error), null, 2)}\n`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
+290
-20
@@ -19,6 +19,10 @@ const defaultLockName = "hwlab-dev-cd-lock";
|
||||
const defaultTtlSeconds = 3600;
|
||||
const defaultReportPath = "reports/dev-gate/dev-cd-apply.json";
|
||||
const artifactReportPath = "reports/dev-gate/dev-artifacts.json";
|
||||
const runtimeProvisioningReportPath = "reports/dev-gate/dev-runtime-provisioning-report.json";
|
||||
const runtimeMigrationReportPath = "reports/dev-gate/dev-runtime-migration-report.json";
|
||||
const runtimePostflightReportPath = "reports/dev-gate/dev-runtime-postflight-report.json";
|
||||
const runtimeJobReportPrefix = "/tmp/hwlab-dev-runtime-report";
|
||||
const browserLiveUrl = "http://74.48.78.17:16666/health/live";
|
||||
const apiLiveUrl = `${DEV_ENDPOINT}/health/live`;
|
||||
const lockAnnotationPrefix = "hwlab.pikastech.local";
|
||||
@@ -1042,6 +1046,213 @@ async function runStep(ctx, transaction, step) {
|
||||
};
|
||||
}
|
||||
|
||||
async function runRuntimeK8sJobStep(ctx, transaction, step) {
|
||||
const startedAt = ctx.now().toISOString();
|
||||
const result = await runRuntimeK8sJob(ctx, transaction, step);
|
||||
const finishedAt = ctx.now().toISOString();
|
||||
return {
|
||||
id: step.id,
|
||||
phase: step.phase,
|
||||
status: result.status,
|
||||
command: result.command,
|
||||
code: result.status === "pass" ? 0 : 1,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
stdoutJson: result.report,
|
||||
stdoutTail: null,
|
||||
stderrTail: result.error ?? "",
|
||||
reportPath: step.reportPath ?? null,
|
||||
k8sJob: result.k8sJob
|
||||
};
|
||||
}
|
||||
|
||||
async function runRuntimeK8sJob(ctx, transaction, step) {
|
||||
const jobName = `${step.jobNamePrefix}-${transaction.transactionId.slice(0, 8)}`;
|
||||
const reportFile = `${runtimeJobReportPrefix}-${step.id}.json`;
|
||||
const image = step.image ?? await resolveRuntimeJobImage(ctx);
|
||||
const manifest = buildRuntimeK8sJobManifest({
|
||||
jobName,
|
||||
namespace: transaction.targetNamespace,
|
||||
image,
|
||||
commandArgs: [...step.commandArgs, "--report", reportFile],
|
||||
transaction
|
||||
});
|
||||
const kubectl = transaction.kubectlContext;
|
||||
const apply = await kubectlCommandResult(ctx, kubectl, ["-n", transaction.targetNamespace, "apply", "-f", "-"], {
|
||||
input: JSON.stringify(manifest),
|
||||
timeoutMs: 30000
|
||||
});
|
||||
if (apply.code !== 0) {
|
||||
return runtimeJobFailure(step, "apply", apply, jobName, image);
|
||||
}
|
||||
|
||||
const wait = await kubectlCommandResult(ctx, kubectl, [
|
||||
"-n",
|
||||
transaction.targetNamespace,
|
||||
"wait",
|
||||
"--for=condition=complete",
|
||||
`job/${jobName}`,
|
||||
"--timeout=300s"
|
||||
], { timeoutMs: step.timeoutMs ?? 6 * 60 * 1000 });
|
||||
if (wait.code !== 0) {
|
||||
return runtimeJobFailure(step, "wait", wait, jobName, image);
|
||||
}
|
||||
|
||||
const logs = await kubectlCommandResult(ctx, kubectl, ["-n", transaction.targetNamespace, "logs", `job/${jobName}`], {
|
||||
timeoutMs: 30000
|
||||
});
|
||||
const logText = redactSensitiveText(`${logs.stdout}\n${logs.stderr}`);
|
||||
const report = parseLastJsonObject(logText);
|
||||
if (logs.code !== 0 || !report) {
|
||||
return {
|
||||
status: "blocked",
|
||||
command: `${shellCommand("kubectl", ["-n", transaction.targetNamespace, "logs", `job/${jobName}`])}`,
|
||||
error: logText.trim().slice(-2000) || "runtime job did not emit a JSON report",
|
||||
report: null,
|
||||
k8sJob: runtimeJobSummary(jobName, step, "logs", image)
|
||||
};
|
||||
}
|
||||
const reportStatus = report.conclusion?.status ?? report.summary?.status ?? report.status;
|
||||
return {
|
||||
status: ["ready", "pass"].includes(reportStatus) ? "pass" : "blocked",
|
||||
command: shellCommand("kubectl", ["-n", transaction.targetNamespace, "apply/wait/logs", `job/${jobName}`]),
|
||||
error: null,
|
||||
report,
|
||||
k8sJob: runtimeJobSummary(jobName, step, "complete", image)
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeJobFailure(step, phase, result, jobName, image) {
|
||||
return {
|
||||
status: "blocked",
|
||||
command: result.command ?? shellCommand("kubectl", [phase, `job/${jobName}`]),
|
||||
error: redactSensitiveText(`${result.stderr}\n${result.stdout}`).trim().slice(-2000),
|
||||
report: null,
|
||||
k8sJob: runtimeJobSummary(jobName, step, phase, image)
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeJobSummary(jobName, step, phase, image = step.image ?? "deploy-current:hwlab-cloud-api") {
|
||||
return {
|
||||
jobName,
|
||||
phase,
|
||||
serviceId: "hwlab-cloud-api",
|
||||
image,
|
||||
secretRefsOnly: true,
|
||||
secretValuesRead: false,
|
||||
secretValuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
async function resolveRuntimeJobImage(ctx) {
|
||||
const deploy = await readDeployJson(ctx.repoRoot);
|
||||
const image = deploy.manifest?.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.image;
|
||||
if (typeof image === "string" && image.trim().length > 0) {
|
||||
return image;
|
||||
}
|
||||
if (typeof deploy.commitId === "string" && deploy.commitId !== "unknown") {
|
||||
return `127.0.0.1:5000/hwlab/hwlab-cloud-api:${shortCommit(deploy.commitId)}`;
|
||||
}
|
||||
throw new DevCdApplyError("could not resolve hwlab-cloud-api image for runtime maintenance Job", {
|
||||
code: "runtime-job-image-unresolved"
|
||||
});
|
||||
}
|
||||
|
||||
function buildRuntimeK8sJobManifest({ jobName, namespace, image, commandArgs, transaction }) {
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "Job",
|
||||
metadata: {
|
||||
name: jobName,
|
||||
namespace,
|
||||
labels: {
|
||||
"app.kubernetes.io/part-of": "hwlab",
|
||||
"app.kubernetes.io/name": "hwlab-runtime-maintenance",
|
||||
"hwlab.pikastech.local/profile": "dev",
|
||||
"hwlab.pikastech.local/service-id": "hwlab-cloud-api",
|
||||
"hwlab.pikastech.local/cd-transaction": transaction.transactionId
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
backoffLimit: 0,
|
||||
ttlSecondsAfterFinished: 600,
|
||||
template: {
|
||||
metadata: {
|
||||
labels: {
|
||||
"app.kubernetes.io/name": "hwlab-runtime-maintenance",
|
||||
"hwlab.pikastech.local/service-id": "hwlab-cloud-api",
|
||||
"hwlab.pikastech.local/cd-transaction": transaction.transactionId
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
restartPolicy: "Never",
|
||||
containers: [
|
||||
{
|
||||
name: "runtime-maintenance",
|
||||
image,
|
||||
command: ["node"],
|
||||
args: commandArgs,
|
||||
env: [
|
||||
{ name: "HWLAB_ENVIRONMENT", value: ENVIRONMENT_DEV },
|
||||
{ name: "HWLAB_CLOUD_RUNTIME_ADAPTER", value: "postgres" },
|
||||
{ name: "HWLAB_CLOUD_RUNTIME_DURABLE", value: "true" },
|
||||
{
|
||||
name: "HWLAB_CLOUD_DB_URL",
|
||||
valueFrom: {
|
||||
secretKeyRef: {
|
||||
name: "hwlab-cloud-api-dev-db",
|
||||
key: "database-url",
|
||||
optional: false
|
||||
}
|
||||
}
|
||||
},
|
||||
{
|
||||
name: "HWLAB_CLOUD_DB_ADMIN_URL",
|
||||
valueFrom: {
|
||||
secretKeyRef: {
|
||||
name: "hwlab-cloud-api-dev-db-admin",
|
||||
key: "admin-url",
|
||||
optional: false
|
||||
}
|
||||
}
|
||||
},
|
||||
{ name: "HWLAB_CLOUD_DB_SSL_MODE", value: "disable" },
|
||||
{ name: "HWLAB_CD_TRANSACTION_ID", value: transaction.transactionId },
|
||||
{ name: "HWLAB_CD_TRANSACTION_OWNER", value: transaction.ownerTaskId },
|
||||
{ name: "HWLAB_CD_LOCK_NAME", value: transaction.lockName }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function kubectlCommandResult(ctx, kubectlContext, args, options = {}) {
|
||||
const result = await ctx.runCommand(kubectlContext.executor, args, {
|
||||
cwd: ctx.repoRoot,
|
||||
env: kubectlContext.env,
|
||||
input: options.input,
|
||||
timeoutMs: options.timeoutMs ?? 30000
|
||||
});
|
||||
return {
|
||||
...result,
|
||||
command: shellCommand("kubectl", args),
|
||||
stdout: result.stdout ?? "",
|
||||
stderr: result.stderr ?? ""
|
||||
};
|
||||
}
|
||||
|
||||
function parseLastJsonObject(text) {
|
||||
const lines = String(text ?? "").trim().split(/\r?\n/u).reverse();
|
||||
for (const line of lines) {
|
||||
const parsed = parseJsonMaybe(line.trim());
|
||||
if (parsed && typeof parsed === "object") return parsed;
|
||||
}
|
||||
return parseJsonMaybe(text);
|
||||
}
|
||||
|
||||
function stepBlocker(stepResult) {
|
||||
return {
|
||||
type: "runtime_blocker",
|
||||
@@ -1478,8 +1689,9 @@ function buildReport({
|
||||
"HEAD must match the requested target ref before publish.",
|
||||
"Lease/hwlab-dev-cd-lock must be acquired before publish/apply.",
|
||||
"Legacy publish/apply side-effect scripts must run with HWLAB_CD_TRANSACTION_ID.",
|
||||
"DEV DB role/database provisioning and runtime migration must run through repo-owned commands before workload apply.",
|
||||
"deploy/deploy.json, artifact catalog, and workloads must converge before apply.",
|
||||
"Public 16666 and 16667 live health must be recorded in this report."
|
||||
"Public 16666 and 16667 live health plus M3 durable postflight must be recorded in this report."
|
||||
],
|
||||
summary: blockers.length ? "One or more DEV CD transaction preconditions failed." : "DEV CD transaction preconditions passed."
|
||||
},
|
||||
@@ -1520,11 +1732,14 @@ function buildReport({
|
||||
steps,
|
||||
liveBefore,
|
||||
liveVerify,
|
||||
reportPaths: {
|
||||
transaction: args.writeReport ? args.reportPath : null,
|
||||
artifacts: artifactReportPath,
|
||||
deployApply: "reports/dev-gate/dev-deploy-report.json"
|
||||
},
|
||||
reportPaths: {
|
||||
transaction: args.writeReport ? args.reportPath : null,
|
||||
artifacts: artifactReportPath,
|
||||
deployApply: "reports/dev-gate/dev-deploy-report.json",
|
||||
runtimeProvisioning: runtimeProvisioningReportPath,
|
||||
runtimeMigration: runtimeMigrationReportPath,
|
||||
runtimePostflight: runtimePostflightReportPath
|
||||
},
|
||||
safety: {
|
||||
prodTouched: false,
|
||||
secretValuesRead: false,
|
||||
@@ -1607,7 +1822,8 @@ export async function runDevCdApply(argv, io = {}) {
|
||||
ownerTaskId: args.ownerTaskId ?? defaultOwnerTaskId(env),
|
||||
targetNamespace: args.targetNamespace,
|
||||
lockName: args.lockName,
|
||||
phases: []
|
||||
phases: [],
|
||||
kubectlContext: null
|
||||
};
|
||||
const steps = [];
|
||||
const blockers = [];
|
||||
@@ -1624,6 +1840,7 @@ export async function runDevCdApply(argv, io = {}) {
|
||||
target = await resolveTargetRef(ctx, args.targetRef);
|
||||
deployBefore = await readDeployJson(ctx.repoRoot);
|
||||
const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand);
|
||||
transaction.kubectlContext = kubectlContext;
|
||||
liveBefore = {
|
||||
status: "pending",
|
||||
reason: "liveBefore is captured immediately after Lease acquisition so publish/apply cannot race before the transaction lock."
|
||||
@@ -1708,6 +1925,38 @@ export async function runDevCdApply(argv, io = {}) {
|
||||
],
|
||||
timeoutMs: 120000
|
||||
},
|
||||
{
|
||||
id: "runtime-db-provisioning",
|
||||
phase: "applying",
|
||||
kind: "runtime-k8s-job",
|
||||
jobNamePrefix: "hwlab-runtime-provision",
|
||||
commandArgs: [
|
||||
"scripts/dev-runtime-provisioning.mjs",
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--write-report",
|
||||
"--fail-on-blocked"
|
||||
],
|
||||
timeoutMs: 5 * 60 * 1000,
|
||||
reportPath: runtimeProvisioningReportPath
|
||||
},
|
||||
{
|
||||
id: "runtime-db-migration",
|
||||
phase: "applying",
|
||||
kind: "runtime-k8s-job",
|
||||
jobNamePrefix: "hwlab-runtime-migrate",
|
||||
commandArgs: [
|
||||
"scripts/dev-runtime-migration.mjs",
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--write-report",
|
||||
"--fail-on-blocked"
|
||||
],
|
||||
timeoutMs: 5 * 60 * 1000,
|
||||
reportPath: runtimeMigrationReportPath
|
||||
},
|
||||
{
|
||||
id: "dev-deploy-apply",
|
||||
phase: "applying",
|
||||
@@ -1722,23 +1971,42 @@ export async function runDevCdApply(argv, io = {}) {
|
||||
],
|
||||
timeoutMs: 20 * 60 * 1000,
|
||||
reportPath: "reports/dev-gate/dev-deploy-report.json"
|
||||
},
|
||||
{
|
||||
id: "runtime-durable-postflight",
|
||||
phase: "verifying",
|
||||
command: process.execPath,
|
||||
args: [
|
||||
"scripts/dev-runtime-postflight.mjs",
|
||||
"--live",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--target",
|
||||
"api",
|
||||
"--write-report",
|
||||
"--fail-on-blocked"
|
||||
],
|
||||
timeoutMs: 2 * 60 * 1000,
|
||||
reportPath: runtimePostflightReportPath
|
||||
}
|
||||
];
|
||||
|
||||
if (blockers.length === 0) {
|
||||
for (const step of plannedSteps) {
|
||||
if (step.phase === "applying" && lockState.lock?.phase !== "applying") {
|
||||
if (step.phase !== lockState.lock?.phase) {
|
||||
await updateDeployLockPhase({
|
||||
ctx,
|
||||
args,
|
||||
kubectlContext,
|
||||
transactionId: transaction.transactionId,
|
||||
lockState,
|
||||
phase: "applying"
|
||||
phase: step.phase
|
||||
});
|
||||
transaction.phases.push({ phase: "applying", status: "entered", at: ctx.now().toISOString() });
|
||||
transaction.phases.push({ phase: step.phase, status: "entered", at: ctx.now().toISOString() });
|
||||
}
|
||||
const result = await runStep(ctx, transaction, step);
|
||||
const result = step.kind === "runtime-k8s-job"
|
||||
? await runRuntimeK8sJobStep(ctx, transaction, step)
|
||||
: await runStep(ctx, transaction, step);
|
||||
steps.push(result);
|
||||
if (result.status !== "pass") {
|
||||
blockers.push(stepBlocker(result));
|
||||
@@ -1748,15 +2016,17 @@ export async function runDevCdApply(argv, io = {}) {
|
||||
}
|
||||
|
||||
if (blockers.length === 0) {
|
||||
await updateDeployLockPhase({
|
||||
ctx,
|
||||
args,
|
||||
kubectlContext,
|
||||
transactionId: transaction.transactionId,
|
||||
lockState,
|
||||
phase: "verifying"
|
||||
});
|
||||
transaction.phases.push({ phase: "verifying", status: "entered", at: ctx.now().toISOString() });
|
||||
if (lockState.lock?.phase !== "verifying") {
|
||||
await updateDeployLockPhase({
|
||||
ctx,
|
||||
args,
|
||||
kubectlContext,
|
||||
transactionId: transaction.transactionId,
|
||||
lockState,
|
||||
phase: "verifying"
|
||||
});
|
||||
transaction.phases.push({ phase: "verifying", status: "entered", at: ctx.now().toISOString() });
|
||||
}
|
||||
deployAfter = await readDeployJson(ctx.repoRoot);
|
||||
liveVerify = args.skipLiveVerify
|
||||
? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } }
|
||||
|
||||
@@ -142,6 +142,34 @@ function makeRunCommand({ heldLock = null, commandLog = [], psOutput = "", killL
|
||||
lease.metadata.resourceVersion = String(Number(lease.metadata.resourceVersion ?? "2") + 1);
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("apply") && args.includes("-f") && args.includes("-")) {
|
||||
const manifest = JSON.parse(options.input);
|
||||
assert.equal(manifest.kind, "Job");
|
||||
assert.equal(manifest.metadata.namespace, "hwlab-dev");
|
||||
assert.ok(manifest.metadata.name.startsWith("hwlab-runtime-"));
|
||||
assert.equal(manifest.spec.template.spec.containers[0].env.some((entry) => entry.name === "HWLAB_CLOUD_DB_URL" && entry.valueFrom?.secretKeyRef?.name === "hwlab-cloud-api-dev-db"), true);
|
||||
assert.equal(manifest.spec.template.spec.containers[0].env.some((entry) => entry.name === "HWLAB_CLOUD_DB_ADMIN_URL" && entry.valueFrom?.secretKeyRef?.name === "hwlab-cloud-api-dev-db-admin"), true);
|
||||
assert.equal(JSON.stringify(manifest).includes("database-url"), true);
|
||||
assert.equal(JSON.stringify(manifest).includes("postgresql://"), false);
|
||||
return { code: 0, stdout: JSON.stringify(manifest), stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("wait") && args.some((arg) => String(arg).startsWith("job/hwlab-runtime-"))) {
|
||||
return { code: 0, stdout: "job.batch/hwlab-runtime complete\n", stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("logs") && args.some((arg) => String(arg).startsWith("job/hwlab-runtime-provision"))) {
|
||||
return {
|
||||
code: 0,
|
||||
stdout: `${JSON.stringify({ conclusion: { status: "ready" }, safety: { secretValuesPrinted: false } }, null, 2)}\n`,
|
||||
stderr: ""
|
||||
};
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("logs") && args.some((arg) => String(arg).startsWith("job/hwlab-runtime-migrate"))) {
|
||||
return {
|
||||
code: 0,
|
||||
stdout: `${JSON.stringify({ conclusion: { status: "ready" }, safety: { secretValuesPrinted: false } }, null, 2)}\n`,
|
||||
stderr: ""
|
||||
};
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("patch")) {
|
||||
assert.ok(lease, "lease must exist before patch");
|
||||
const patch = parsePatch(args);
|
||||
@@ -394,12 +422,34 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a
|
||||
assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-web-16666"), true);
|
||||
assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-api-16667"), true);
|
||||
assert.equal(report.devCdApply.reportPaths.transaction, "reports/dev-gate/dev-cd-apply.json");
|
||||
assert.equal(report.devCdApply.reportPaths.runtimeProvisioning, "reports/dev-gate/dev-runtime-provisioning-report.json");
|
||||
assert.equal(report.devCdApply.reportPaths.runtimeMigration, "reports/dev-gate/dev-runtime-migration-report.json");
|
||||
assert.equal(report.devCdApply.reportPaths.runtimePostflight, "reports/dev-gate/dev-runtime-postflight-report.json");
|
||||
|
||||
const publishCall = commandLog.find((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs"));
|
||||
const applyCall = commandLog.find((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs"));
|
||||
const provisioningCall = commandLog.find((entry) => entry.args.includes("scripts/dev-runtime-provisioning.mjs"));
|
||||
const migrationCall = commandLog.find((entry) => entry.args.includes("scripts/dev-runtime-migration.mjs"));
|
||||
const postflightCall = commandLog.find((entry) => entry.args.includes("scripts/dev-runtime-postflight.mjs"));
|
||||
const refreshCall = commandLog.find((entry) => entry.args.includes("scripts/refresh-artifact-catalog.mjs"));
|
||||
assert.ok(publishCall?.env.HWLAB_CD_TRANSACTION_ID);
|
||||
assert.equal(applyCall?.env.HWLAB_CD_TRANSACTION_ID, publishCall.env.HWLAB_CD_TRANSACTION_ID);
|
||||
const provisioningJobApply = commandLog.find((entry) =>
|
||||
entry.command.includes("kubectl") &&
|
||||
entry.args.includes("apply") &&
|
||||
entry.input.includes("hwlab-runtime-provision")
|
||||
);
|
||||
const migrationJobApply = commandLog.find((entry) =>
|
||||
entry.command.includes("kubectl") &&
|
||||
entry.args.includes("apply") &&
|
||||
entry.input.includes("hwlab-runtime-migrate")
|
||||
);
|
||||
assert.ok(provisioningJobApply);
|
||||
assert.ok(migrationJobApply);
|
||||
assert.equal(JSON.parse(provisioningJobApply.input).spec.template.spec.containers[0].args[0], "scripts/dev-runtime-provisioning.mjs");
|
||||
assert.equal(JSON.parse(migrationJobApply.input).spec.template.spec.containers[0].args[0], "scripts/dev-runtime-migration.mjs");
|
||||
assert.equal(postflightCall?.env.HWLAB_CD_TRANSACTION_ID, publishCall.env.HWLAB_CD_TRANSACTION_ID);
|
||||
assert.deepEqual(postflightCall.args.slice(0, 4), ["scripts/dev-runtime-postflight.mjs", "--live", "--confirm-dev", "--confirmed-non-production"]);
|
||||
assert.equal(refreshCall.args[refreshCall.args.indexOf("--target-ref") + 1], "abc1234abc1234abc1234abc1234abc1234abc1");
|
||||
|
||||
const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
|
||||
|
||||
@@ -0,0 +1,492 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
||||
import {
|
||||
buildM3IoControlSourceReport,
|
||||
runM3IoControlLiveReport
|
||||
} from "./m3-io-control-e2e.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const defaultReportPath = "reports/dev-gate/dev-runtime-postflight-report.json";
|
||||
const issue = "pikasTech/HWLAB#311";
|
||||
const defaultHealthUrl = `${DEV_ENDPOINT}/health/live`;
|
||||
const defaultV1Url = `${DEV_ENDPOINT}/v1`;
|
||||
|
||||
export async function runDevRuntimePostflightCli(argv = process.argv.slice(2), options = {}) {
|
||||
const args = parseArgs(argv);
|
||||
if (args.help) {
|
||||
process.stdout.write(`${usage()}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const report = await buildDevRuntimePostflightReport(args, options);
|
||||
if (args.writeReport) {
|
||||
await writeReport(report, args.reportPath, options.repoRoot ?? repoRoot);
|
||||
}
|
||||
|
||||
const printable = args.pretty || args.writeReport ? report : summarizeReport(report);
|
||||
process.stdout.write(`${JSON.stringify(printable, null, 2)}\n`);
|
||||
return report.conclusion.status === "pass" || !args.failOnBlocked ? 0 : 1;
|
||||
}
|
||||
|
||||
export async function buildDevRuntimePostflightReport(args = {}, options = {}) {
|
||||
const parsed = normalizeArgs(args);
|
||||
const now = options.now ?? (() => new Date().toISOString());
|
||||
const report = {
|
||||
issue,
|
||||
mode: parsed.live ? "dev-live" : "check",
|
||||
generatedAt: now(),
|
||||
target: {
|
||||
environment: ENVIRONMENT_DEV,
|
||||
namespace: "hwlab-dev",
|
||||
prodAllowed: false,
|
||||
apiHealthUrl: defaultHealthUrl,
|
||||
apiV1Url: defaultV1Url,
|
||||
m3Target: parsed.target
|
||||
},
|
||||
actions: {
|
||||
healthLiveReadAttempted: false,
|
||||
v1ReadAttempted: false,
|
||||
m3MutationAttempted: false,
|
||||
m3MutationAllowed: false
|
||||
},
|
||||
apiHealth: null,
|
||||
apiV1: null,
|
||||
m3: null,
|
||||
blockers: [],
|
||||
safety: buildSafety(parsed),
|
||||
safetyRefusal: false
|
||||
};
|
||||
|
||||
if (!parsed.live) {
|
||||
const sourceReport = await (options.m3SourceBuilder ?? buildM3IoControlSourceReport)({
|
||||
repoRoot: options.repoRoot ?? repoRoot
|
||||
});
|
||||
report.m3 = summarizeM3Report(sourceReport);
|
||||
report.conclusion = {
|
||||
status: sourceReport.summary?.status === "pass" ? "pass" : "blocked",
|
||||
summary: "DEV runtime postflight source contract checked only; no live HTTP or M3 write was attempted.",
|
||||
blockerCount: sourceReport.summary?.status === "pass" ? 0 : 1
|
||||
};
|
||||
if (sourceReport.summary?.status !== "pass") {
|
||||
addBlocker(report, "contract_blocker", "m3-source-contract", "M3 postflight source contract is blocked.", {
|
||||
classification: sourceReport.summary?.classification ?? "unknown"
|
||||
});
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
if (!parsed.confirmDev || !parsed.confirmedNonProduction) {
|
||||
addSafetyRefusal(report, "DEV runtime postflight live mode requires --live --confirm-dev --confirmed-non-production.");
|
||||
return finalizeReport(report);
|
||||
}
|
||||
|
||||
const httpGetJson = options.httpGetJson ?? defaultHttpGetJson;
|
||||
report.actions.healthLiveReadAttempted = true;
|
||||
report.actions.v1ReadAttempted = true;
|
||||
const [health, v1] = await Promise.all([
|
||||
httpGetJson(defaultHealthUrl, parsed.timeoutMs),
|
||||
httpGetJson(defaultV1Url, parsed.timeoutMs)
|
||||
]);
|
||||
report.apiHealth = summarizeApiPayload("GET /health/live", health);
|
||||
report.apiV1 = summarizeApiPayload("GET /v1", v1);
|
||||
|
||||
addApiReadinessBlockers(report);
|
||||
if (report.blockers.length === 0) {
|
||||
report.actions.m3MutationAllowed = true;
|
||||
report.actions.m3MutationAttempted = true;
|
||||
const m3Runner = options.m3Runner ?? defaultM3Runner;
|
||||
report.m3 = summarizeM3Report(await m3Runner(parsed));
|
||||
addM3Blockers(report);
|
||||
} else {
|
||||
report.m3 = {
|
||||
status: "not_run",
|
||||
trustedGreen: false,
|
||||
classification: "runtime_durable_postflight_precondition_blocked",
|
||||
operationCount: 0,
|
||||
reason: "M3 true/false write/read postflight is skipped until /health/live and /v1 prove durable runtime readiness."
|
||||
};
|
||||
}
|
||||
|
||||
return finalizeReport(report);
|
||||
}
|
||||
|
||||
export function parseArgs(argv = []) {
|
||||
const args = {
|
||||
live: false,
|
||||
confirmDev: false,
|
||||
confirmedNonProduction: false,
|
||||
writeReport: false,
|
||||
reportPath: defaultReportPath,
|
||||
pretty: false,
|
||||
failOnBlocked: false,
|
||||
target: "api",
|
||||
timeoutMs: 8000,
|
||||
help: false
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--check") args.live = false;
|
||||
else if (arg === "--live") args.live = true;
|
||||
else if (arg === "--confirm-dev") args.confirmDev = true;
|
||||
else if (arg === "--confirmed-non-production") args.confirmedNonProduction = true;
|
||||
else if (arg === "--write-report") args.writeReport = true;
|
||||
else if (arg === "--pretty") args.pretty = true;
|
||||
else if (arg === "--fail-on-blocked") args.failOnBlocked = true;
|
||||
else if (arg === "--target") {
|
||||
index += 1;
|
||||
args.target = requireArgValue(argv[index], "--target");
|
||||
} else if (arg === "--timeout-ms") {
|
||||
index += 1;
|
||||
args.timeoutMs = parseTimeout(requireArgValue(argv[index], "--timeout-ms"));
|
||||
} else if (arg === "--report") {
|
||||
index += 1;
|
||||
args.reportPath = requireArgValue(argv[index], "--report");
|
||||
} else if (arg.startsWith("--report=")) {
|
||||
args.reportPath = requireArgValue(arg.slice("--report=".length), "--report");
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
args.help = true;
|
||||
} else {
|
||||
throw new Error(`unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (!["api", "frontend"].includes(args.target)) {
|
||||
throw new Error("--target must be api or frontend");
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
function normalizeArgs(args) {
|
||||
return {
|
||||
...parseArgs([]),
|
||||
...args
|
||||
};
|
||||
}
|
||||
|
||||
async function defaultM3Runner(args) {
|
||||
const sourceReport = await buildM3IoControlSourceReport({ repoRoot });
|
||||
return runM3IoControlLiveReport({
|
||||
flags: new Set(["--live", "--confirm-dev", "--confirmed-non-production"]),
|
||||
live: true,
|
||||
confirmDev: true,
|
||||
confirmedNonProduction: true,
|
||||
target: args.target,
|
||||
frontendUrl: "http://74.48.78.17:16666/",
|
||||
apiUrl: `${DEV_ENDPOINT}/v1/m3/io`,
|
||||
timeoutMs: args.timeoutMs
|
||||
}, sourceReport);
|
||||
}
|
||||
|
||||
async function defaultHttpGetJson(url, timeoutMs) {
|
||||
const controller = new AbortController();
|
||||
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
method: "GET",
|
||||
headers: {
|
||||
accept: "application/json"
|
||||
},
|
||||
signal: controller.signal
|
||||
});
|
||||
const text = await response.text();
|
||||
let json = null;
|
||||
try {
|
||||
json = text ? JSON.parse(text) : null;
|
||||
} catch {
|
||||
json = null;
|
||||
}
|
||||
return {
|
||||
ok: response.ok,
|
||||
status: response.status,
|
||||
json,
|
||||
error: response.ok ? null : json?.error?.message ?? response.statusText
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
status: 0,
|
||||
json: null,
|
||||
error: error.name === "AbortError" ? `request timed out after ${timeoutMs}ms` : error.message
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
|
||||
function summarizeApiPayload(check, response) {
|
||||
const json = response.json ?? {};
|
||||
const runtime = summarizeRuntime(json.runtime);
|
||||
return {
|
||||
check,
|
||||
httpStatus: response.status,
|
||||
reachable: response.ok === true && Boolean(response.json),
|
||||
serviceId: json.serviceId ?? json.service?.id ?? null,
|
||||
environment: json.environment ?? null,
|
||||
status: json.status ?? null,
|
||||
ready: json.ready === true,
|
||||
runtime,
|
||||
db: summarizeDb(json.db),
|
||||
readiness: {
|
||||
ready: json.readiness?.ready === true,
|
||||
status: json.readiness?.status ?? null,
|
||||
durabilityReady: json.readiness?.durability?.ready === true,
|
||||
durabilityBlockedLayer: json.readiness?.durability?.blockedLayer ?? json.runtime?.durabilityContract?.blockedLayer ?? null
|
||||
},
|
||||
blockerCodes: Array.isArray(json.blockerCodes) ? json.blockerCodes : [],
|
||||
m3IoControl: json.m3IoControl
|
||||
? {
|
||||
route: json.m3IoControl.route ?? null,
|
||||
enabled: json.m3IoControl.enabled === true,
|
||||
contractVersion: json.m3IoControl.contractVersion ?? null
|
||||
}
|
||||
: null,
|
||||
error: response.error ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeRuntime(runtime = {}) {
|
||||
return {
|
||||
adapter: runtime?.adapter ?? "unknown",
|
||||
durable: runtime?.durable === true,
|
||||
durableRequested: runtime?.durableRequested === true,
|
||||
ready: runtime?.ready === true,
|
||||
status: runtime?.status ?? null,
|
||||
blocker: runtime?.blocker ?? null,
|
||||
liveRuntimeEvidence: runtime?.liveRuntimeEvidence === true,
|
||||
requiredEvidence: runtime?.durabilityContract?.requiredEvidence ?? null,
|
||||
blockedLayer: runtime?.durabilityContract?.blockedLayer ?? null,
|
||||
gates: summarizeRuntimeGates(runtime?.gates)
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeRuntimeGates(gates = {}) {
|
||||
return Object.fromEntries(["ssl", "auth", "schema", "migration", "durability"].map((name) => [
|
||||
name,
|
||||
{
|
||||
checked: gates?.[name]?.checked === true,
|
||||
ready: gates?.[name]?.ready === true,
|
||||
status: gates?.[name]?.status ?? "unknown",
|
||||
blocker: gates?.[name]?.blocker ?? null
|
||||
}
|
||||
]));
|
||||
}
|
||||
|
||||
function summarizeDb(db = {}) {
|
||||
return {
|
||||
ready: db?.ready === true,
|
||||
connected: db?.connected === true,
|
||||
liveDbEvidence: db?.liveDbEvidence === true,
|
||||
endpointSource: db?.endpointSource ?? null,
|
||||
connectionResult: db?.connectionResult ?? db?.connection?.result ?? null,
|
||||
endpointRedacted: true,
|
||||
valueRedacted: true,
|
||||
runtimeReadiness: {
|
||||
ready: db?.runtimeReadiness?.ready === true,
|
||||
status: db?.runtimeReadiness?.status ?? null,
|
||||
blocker: db?.runtimeReadiness?.blocker ?? null,
|
||||
queryResult: db?.runtimeReadiness?.queryResult ?? null,
|
||||
requiredEvidence: db?.runtimeReadiness?.requiredEvidence ?? null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeM3Report(report = {}) {
|
||||
const operations = Array.isArray(report.liveOperations) ? report.liveOperations : [];
|
||||
return {
|
||||
mode: report.mode ?? "unknown",
|
||||
status: report.summary?.status ?? report.status ?? "unknown",
|
||||
classification: report.summary?.classification ?? null,
|
||||
trustedGreen: report.summary?.trustedGreen === true,
|
||||
result: report.summary?.result ?? null,
|
||||
operationCount: operations.length,
|
||||
operations: operations.map((operation) => ({
|
||||
id: operation.id,
|
||||
action: operation.action,
|
||||
status: operation.status,
|
||||
operationId: operation.operationId ?? null,
|
||||
traceId: operation.traceId ?? null,
|
||||
auditId: operation.auditId ?? null,
|
||||
evidenceId: operation.evidenceId ?? null,
|
||||
resultValue: operation.resultValue,
|
||||
evidenceState: {
|
||||
status: operation.evidenceState?.status ?? null,
|
||||
durable: operation.evidenceState?.durable === true,
|
||||
sourceKind: operation.evidenceState?.sourceKind ?? null
|
||||
},
|
||||
blocker: operation.blocker ?? null
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
function addApiReadinessBlockers(report) {
|
||||
for (const [scope, payload] of [["api-health-live", report.apiHealth], ["api-v1", report.apiV1]]) {
|
||||
if (!payload.reachable) {
|
||||
addBlocker(report, "runtime_blocker", scope, `${payload.check} was not reachable.`, {
|
||||
httpStatus: payload.httpStatus,
|
||||
error: payload.error
|
||||
});
|
||||
continue;
|
||||
}
|
||||
if (!runtimeDurableReady(payload.runtime)) {
|
||||
addBlocker(report, "runtime_blocker", scope, `${payload.check} did not prove durable runtime readiness.`, {
|
||||
runtime: payload.runtime,
|
||||
blockerCodes: payload.blockerCodes
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function addM3Blockers(report) {
|
||||
if (report.m3?.trustedGreen !== true || report.m3?.status !== "pass") {
|
||||
addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 true/false durable postflight did not produce trusted green persisted evidence.", {
|
||||
classification: report.m3?.classification ?? "unknown",
|
||||
trustedGreen: report.m3?.trustedGreen === true,
|
||||
operationCount: report.m3?.operationCount ?? 0
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
for (const operation of report.m3.operations ?? []) {
|
||||
if (!operation.operationId || !operation.auditId || !operation.evidenceId || operation.evidenceState?.status !== "green" || operation.evidenceState?.durable !== true) {
|
||||
addBlocker(report, "runtime_blocker", "m3-durable-evidence", "M3 operation/audit/evidence durable fields were incomplete.", {
|
||||
operationIdPresent: Boolean(operation.operationId),
|
||||
auditIdPresent: Boolean(operation.auditId),
|
||||
evidenceIdPresent: Boolean(operation.evidenceId),
|
||||
evidenceState: operation.evidenceState
|
||||
});
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function runtimeDurableReady(runtime) {
|
||||
return runtime.adapter === "postgres" &&
|
||||
runtime.durable === true &&
|
||||
runtime.ready === true &&
|
||||
runtime.liveRuntimeEvidence === true;
|
||||
}
|
||||
|
||||
function addSafetyRefusal(report, summary) {
|
||||
report.safetyRefusal = true;
|
||||
addBlocker(report, "safety_refusal", "runtime-postflight-live-boundary", summary, {
|
||||
devOnly: true,
|
||||
prodAllowed: false,
|
||||
secretValuesPrinted: false
|
||||
});
|
||||
}
|
||||
|
||||
function addBlocker(report, type, scope, summary, evidence = {}) {
|
||||
report.blockers.push({
|
||||
type,
|
||||
scope,
|
||||
status: "open",
|
||||
summary,
|
||||
sourceIssue: issue,
|
||||
evidence
|
||||
});
|
||||
}
|
||||
|
||||
function finalizeReport(report) {
|
||||
const openBlockers = report.blockers.filter((blocker) => blocker.status === "open");
|
||||
report.conclusion = {
|
||||
status: openBlockers.length === 0 ? "pass" : "blocked",
|
||||
summary: openBlockers.length === 0
|
||||
? "DEV runtime postflight passed: /health/live, /v1, and M3 true/false durable evidence are green."
|
||||
: "DEV runtime postflight is blocked; M3 is skipped unless durable runtime readiness is proven first.",
|
||||
blockerCount: openBlockers.length
|
||||
};
|
||||
return report;
|
||||
}
|
||||
|
||||
function buildSafety(args) {
|
||||
return {
|
||||
devOnly: true,
|
||||
environment: ENVIRONMENT_DEV,
|
||||
prodAllowed: false,
|
||||
liveHttpReads: args.live === true,
|
||||
liveM3Writes: args.live === true,
|
||||
liveM3WritesRequireDurableRuntimeReady: true,
|
||||
readsKubernetesSecrets: false,
|
||||
writesKubernetesSecrets: false,
|
||||
secretValuesPrinted: false,
|
||||
dbUrlValueRedacted: true,
|
||||
endpointRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeReport(report) {
|
||||
return {
|
||||
issue: report.issue,
|
||||
mode: report.mode,
|
||||
conclusion: report.conclusion,
|
||||
actions: report.actions,
|
||||
apiHealth: report.apiHealth,
|
||||
apiV1: report.apiV1,
|
||||
m3: report.m3,
|
||||
blockers: report.blockers,
|
||||
safety: report.safety
|
||||
};
|
||||
}
|
||||
|
||||
async function writeReport(report, reportPath, root) {
|
||||
const absolute = path.resolve(root, reportPath);
|
||||
await mkdir(path.dirname(absolute), { recursive: true });
|
||||
await writeFile(absolute, `${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function parseTimeout(value) {
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
throw new Error("--timeout-ms must be a positive integer");
|
||||
}
|
||||
return Math.min(parsed, 30000);
|
||||
}
|
||||
|
||||
function requireArgValue(value, flag) {
|
||||
if (typeof value !== "string" || value.trim() === "") {
|
||||
throw new Error(`${flag} requires a value`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function usage() {
|
||||
return [
|
||||
"Usage: node scripts/dev-runtime-postflight.mjs [--check|--live]",
|
||||
"",
|
||||
"Default --check validates source M3 postflight contracts only.",
|
||||
"--live requires --confirm-dev --confirmed-non-production and reads /health/live plus /v1 before any M3 write.",
|
||||
"M3 true/false write/read is skipped unless durable runtime readiness is already green.",
|
||||
"Reports never print DB URLs, passwords, tokens, Secret values, or kubeconfig material."
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function formatDevRuntimePostflightFailure(error) {
|
||||
const message = redactFailureText(error instanceof Error ? error.message : String(error));
|
||||
return {
|
||||
issue,
|
||||
conclusion: {
|
||||
status: "blocked",
|
||||
summary: "DEV runtime postflight command failed before producing a report."
|
||||
},
|
||||
error: message,
|
||||
safety: {
|
||||
devOnly: true,
|
||||
prodAllowed: false,
|
||||
secretValuesPrinted: false
|
||||
},
|
||||
trace: createHash("sha256").update(message).digest("hex").slice(0, 12)
|
||||
};
|
||||
}
|
||||
|
||||
function redactFailureText(value) {
|
||||
return String(value ?? "")
|
||||
.replace(/postgres(?:ql)?:\/\/[^\s"'<>]+/giu, "[redacted-postgres-url]")
|
||||
.replace(/(password\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]")
|
||||
.replace(/(token\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]")
|
||||
.replace(/(kubeconfig\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]");
|
||||
}
|
||||
@@ -0,0 +1,202 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
buildDevRuntimePostflightReport,
|
||||
parseArgs
|
||||
} from "./dev-runtime-postflight.mjs";
|
||||
|
||||
test("check mode is source-only and does not attempt live reads or M3 writes", async () => {
|
||||
const report = await buildDevRuntimePostflightReport(parseArgs(["--check"]), {
|
||||
m3SourceBuilder: async () => ({
|
||||
mode: "source-static",
|
||||
summary: {
|
||||
status: "pass",
|
||||
classification: "source_static_contract_checked",
|
||||
trustedGreen: false,
|
||||
result: "source checked"
|
||||
}
|
||||
}),
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
});
|
||||
|
||||
assert.equal(report.conclusion.status, "pass");
|
||||
assert.equal(report.actions.healthLiveReadAttempted, false);
|
||||
assert.equal(report.actions.v1ReadAttempted, false);
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
assert.equal(report.safety.secretValuesPrinted, false);
|
||||
});
|
||||
|
||||
test("live mode refuses without explicit DEV confirmations", async () => {
|
||||
const report = await buildDevRuntimePostflightReport(parseArgs(["--live"]), {
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
});
|
||||
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.safetyRefusal, true);
|
||||
assert.equal(report.actions.healthLiveReadAttempted, false);
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
});
|
||||
|
||||
test("live mode skips M3 writes when /health/live durable runtime is blocked", async () => {
|
||||
const report = await buildDevRuntimePostflightReport(
|
||||
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
|
||||
{
|
||||
httpGetJson: async (url) => ({ ok: true, status: 200, json: apiPayload({ ready: url.endsWith("/v1") }) }),
|
||||
m3Runner: async () => {
|
||||
throw new Error("M3 runner should not run before durable readiness is green");
|
||||
},
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.actions.healthLiveReadAttempted, true);
|
||||
assert.equal(report.actions.v1ReadAttempted, true);
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
assert.equal(report.m3.status, "not_run");
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-health-live"), true);
|
||||
assert.equal(JSON.stringify(report).includes("postgresql://"), false);
|
||||
});
|
||||
|
||||
test("live mode requires /v1 durable readiness before M3 writes", async () => {
|
||||
const report = await buildDevRuntimePostflightReport(
|
||||
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
|
||||
{
|
||||
httpGetJson: async (url) => ({ ok: true, status: 200, json: apiPayload({ ready: !url.endsWith("/v1") }) }),
|
||||
m3Runner: async () => {
|
||||
throw new Error("M3 runner should not run when /v1 is blocked");
|
||||
},
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.actions.m3MutationAttempted, false);
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "api-v1"), true);
|
||||
});
|
||||
|
||||
test("live mode records M3 true/false durable green evidence", async () => {
|
||||
const report = await buildDevRuntimePostflightReport(
|
||||
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
|
||||
{
|
||||
httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: true }) }),
|
||||
m3Runner: async () => m3GreenReport(),
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "pass");
|
||||
assert.equal(report.actions.m3MutationAttempted, true);
|
||||
assert.equal(report.actions.m3MutationAllowed, true);
|
||||
assert.equal(report.m3.trustedGreen, true);
|
||||
assert.equal(report.m3.operationCount, 4);
|
||||
assert.deepEqual(report.m3.operations.map((operation) => operation.resultValue), [true, true, false, false]);
|
||||
assert.equal(report.blockers.length, 0);
|
||||
});
|
||||
|
||||
test("live mode blocks when M3 operation evidence is not durable green", async () => {
|
||||
const m3 = m3GreenReport();
|
||||
m3.liveOperations[1].evidenceState.durable = false;
|
||||
const report = await buildDevRuntimePostflightReport(
|
||||
parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]),
|
||||
{
|
||||
httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: true }) }),
|
||||
m3Runner: async () => m3,
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-evidence"), true);
|
||||
});
|
||||
|
||||
function apiPayload({ ready }) {
|
||||
return {
|
||||
serviceId: "hwlab-cloud-api",
|
||||
environment: "dev",
|
||||
status: ready ? "ready" : "blocked",
|
||||
ready,
|
||||
db: {
|
||||
ready,
|
||||
connected: ready,
|
||||
liveDbEvidence: ready,
|
||||
endpointSource: "secret-url-host",
|
||||
connectionResult: ready ? "connected" : "connected",
|
||||
runtimeReadiness: {
|
||||
ready,
|
||||
status: ready ? "ready" : "blocked",
|
||||
blocker: ready ? null : "runtime_durable_adapter_auth_blocked",
|
||||
queryResult: ready ? "durable_readiness_ready" : "auth_blocked",
|
||||
requiredEvidence: "runtime_adapter_schema_migration_read_query"
|
||||
}
|
||||
},
|
||||
runtime: {
|
||||
adapter: "postgres",
|
||||
durable: ready,
|
||||
durableRequested: true,
|
||||
ready,
|
||||
status: ready ? "ready" : "blocked",
|
||||
blocker: ready ? null : "runtime_durable_adapter_auth_blocked",
|
||||
liveRuntimeEvidence: ready,
|
||||
durabilityContract: {
|
||||
requiredEvidence: "runtime_adapter_schema_migration_read_query",
|
||||
blockedLayer: ready ? null : "auth"
|
||||
},
|
||||
gates: {
|
||||
ssl: gate(true),
|
||||
auth: gate(ready, "runtime_durable_adapter_auth_blocked"),
|
||||
schema: gate(ready),
|
||||
migration: gate(ready),
|
||||
durability: gate(ready)
|
||||
}
|
||||
},
|
||||
readiness: {
|
||||
ready,
|
||||
status: ready ? "ready" : "blocked",
|
||||
durability: {
|
||||
ready,
|
||||
blockedLayer: ready ? null : "auth"
|
||||
}
|
||||
},
|
||||
blockerCodes: ready ? []
|
||||
: ["runtime_durable_adapter_auth_blocked"]
|
||||
};
|
||||
}
|
||||
|
||||
function gate(ready, blocker = null) {
|
||||
return {
|
||||
checked: true,
|
||||
ready,
|
||||
status: ready ? "ready" : "blocked",
|
||||
blocker: ready ? null : blocker
|
||||
};
|
||||
}
|
||||
|
||||
function m3GreenReport() {
|
||||
const values = [true, true, false, false];
|
||||
return {
|
||||
mode: "dev-live",
|
||||
summary: {
|
||||
status: "pass",
|
||||
classification: "trusted_green",
|
||||
trustedGreen: true,
|
||||
result: "DEV-LIVE M3 IO control path and durable trusted records are green."
|
||||
},
|
||||
liveOperations: values.map((value, index) => ({
|
||||
id: ["write-do1-true", "read-di1-true", "write-do1-false", "read-di1-false"][index],
|
||||
action: index % 2 === 0 ? "do.write" : "di.read",
|
||||
status: "succeeded",
|
||||
operationId: `op_${index}`,
|
||||
traceId: `trc_${index}`,
|
||||
auditId: `aud_${index}`,
|
||||
evidenceId: `ev_${index}`,
|
||||
resultValue: value,
|
||||
evidenceState: {
|
||||
status: "green",
|
||||
durable: true,
|
||||
sourceKind: "DEV-LIVE"
|
||||
}
|
||||
}))
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,928 @@
|
||||
import { createHash } from "node:crypto";
|
||||
import { mkdir, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { DEV_DB_ENV_CONTRACT } from "../../internal/cloud/db-contract.mjs";
|
||||
import {
|
||||
PostgresCloudRuntimeStore,
|
||||
RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
|
||||
RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING,
|
||||
RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED,
|
||||
RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
|
||||
RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
|
||||
RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED,
|
||||
buildPostgresPoolConfig
|
||||
} from "../../internal/db/runtime-store.mjs";
|
||||
import { ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const defaultReportPath = "reports/dev-gate/dev-runtime-provisioning-report.json";
|
||||
const issue = "pikasTech/HWLAB#311";
|
||||
|
||||
export const DEV_DB_ADMIN_SECRET_REF = Object.freeze({
|
||||
env: "HWLAB_CLOUD_DB_ADMIN_URL",
|
||||
secretName: "hwlab-cloud-api-dev-db-admin",
|
||||
secretKey: "admin-url"
|
||||
});
|
||||
|
||||
const appSecretRef = DEV_DB_ENV_CONTRACT.secretRefs[0];
|
||||
const systemDatabases = new Set(["postgres", "template0", "template1"]);
|
||||
|
||||
export async function runDevRuntimeProvisioningCli(argv = process.argv.slice(2), options = {}) {
|
||||
const args = parseArgs(argv);
|
||||
if (args.help) {
|
||||
process.stdout.write(`${usage()}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
const report = await buildDevRuntimeProvisioningReport(args, options);
|
||||
if (args.writeReport) {
|
||||
await writeReport(report, args.reportPath, options.repoRoot ?? repoRoot);
|
||||
}
|
||||
|
||||
const printable = args.pretty || args.writeReport ? report : summarizeReport(report);
|
||||
process.stdout.write(`${JSON.stringify(printable, null, 2)}\n`);
|
||||
|
||||
if (report.safetyRefusal || (args.failOnBlocked && report.conclusion.status !== "ready")) {
|
||||
return 1;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
export async function buildDevRuntimeProvisioningReport(args = {}, options = {}) {
|
||||
const parsed = normalizeArgs(args);
|
||||
const env = options.env ?? process.env;
|
||||
const now = options.now ?? (() => new Date().toISOString());
|
||||
const target = parseProvisioningTarget(env.HWLAB_CLOUD_DB_URL);
|
||||
const admin = summarizeAdminEnv(env);
|
||||
const report = {
|
||||
issue,
|
||||
mode: parsed.mode,
|
||||
generatedAt: now(),
|
||||
target: {
|
||||
environment: ENVIRONMENT_DEV,
|
||||
namespace: DEV_DB_ENV_CONTRACT.dns.namespace,
|
||||
prodAllowed: false,
|
||||
dbUrlEnv: appSecretRef.env,
|
||||
dbUrlSecretRef: appSecretRef,
|
||||
adminUrlEnv: DEV_DB_ADMIN_SECRET_REF.env,
|
||||
adminUrlSecretRef: DEV_DB_ADMIN_SECRET_REF,
|
||||
dbEndpointAuthority: DEV_DB_ENV_CONTRACT.endpointAuthority.source
|
||||
},
|
||||
db: summarizeDbEnv(env, target, admin),
|
||||
applyBoundary: buildApplyBoundary(parsed),
|
||||
safety: buildSafety(parsed),
|
||||
actions: {
|
||||
liveDbReadAttempted: false,
|
||||
liveDbWriteAttempted: false,
|
||||
adminInspectionAttempted: false,
|
||||
adminWriteAttempted: false,
|
||||
appRuntimeReadinessAttempted: false,
|
||||
roleCreated: false,
|
||||
roleExisted: false,
|
||||
rolePasswordSynchronized: false,
|
||||
databaseCreated: false,
|
||||
databaseExisted: false,
|
||||
databaseConnectGranted: false,
|
||||
schemaPrivilegesGranted: false,
|
||||
readinessVerified: false
|
||||
},
|
||||
provisioning: {
|
||||
targetRole: emptyObjectState(target.roleNamePresent),
|
||||
targetDatabase: emptyObjectState(target.databaseNamePresent),
|
||||
privileges: {
|
||||
checked: false,
|
||||
ready: false,
|
||||
databaseConnectGranted: false,
|
||||
schemaPrivilegesGranted: false
|
||||
},
|
||||
ready: false
|
||||
},
|
||||
runtime: null,
|
||||
gates: sourceOnlyGates(target),
|
||||
downstreamRuntimeGates: null,
|
||||
blockers: [],
|
||||
safetyRefusal: false
|
||||
};
|
||||
|
||||
validateSourceContract(report, target, parsed);
|
||||
|
||||
if (parsed.mode === "check" || (parsed.mode === "dry-run" && !parsed.allowLiveDbRead)) {
|
||||
return finalizeReport(report);
|
||||
}
|
||||
|
||||
if (parsed.mode === "dry-run") {
|
||||
if (!parsed.confirmDev) {
|
||||
addSafetyRefusal(report, "DEV DB provisioning live dry-run requires --confirm-dev.");
|
||||
return finalizeReport(report);
|
||||
}
|
||||
if (!target.ok) {
|
||||
return finalizeReport(report);
|
||||
}
|
||||
await verifyRuntimeReadiness(report, env, options);
|
||||
if (admin.present || options.adminClient) {
|
||||
await inspectAdminProvisioning(report, env, target, options);
|
||||
} else {
|
||||
inferProvisioningFromRuntime(report);
|
||||
if (!report.provisioning.ready) {
|
||||
addBlocker(report, "runtime_blocker", "runtime-provisioning-admin", "Role/database inspection requires the DEV admin SecretRef env when runtime readiness is not already green.", {
|
||||
adminUrlEnv: DEV_DB_ADMIN_SECRET_REF.env,
|
||||
adminUrlSecretRef: DEV_DB_ADMIN_SECRET_REF,
|
||||
valueRedacted: true,
|
||||
secretValuesPrinted: false
|
||||
});
|
||||
}
|
||||
}
|
||||
addRuntimeProvisioningBlocker(report);
|
||||
return finalizeReport(report);
|
||||
}
|
||||
|
||||
if (parsed.mode === "apply") {
|
||||
if (!parsed.confirmDev || !parsed.confirmedNonProduction) {
|
||||
addSafetyRefusal(report, "DEV DB provisioning apply requires --confirm-dev and --confirmed-non-production.");
|
||||
return finalizeReport(report);
|
||||
}
|
||||
if (!target.ok) {
|
||||
return finalizeReport(report);
|
||||
}
|
||||
if (!admin.present && !options.adminClient) {
|
||||
addBlocker(report, "runtime_blocker", "runtime-provisioning-admin", "DEV DB provisioning apply requires the admin DB URL from its SecretRef env.", {
|
||||
adminUrlEnv: DEV_DB_ADMIN_SECRET_REF.env,
|
||||
adminUrlSecretRef: DEV_DB_ADMIN_SECRET_REF,
|
||||
valueRedacted: true,
|
||||
secretValuesPrinted: false
|
||||
});
|
||||
return finalizeReport(report);
|
||||
}
|
||||
await applyProvisioning(report, env, target, options);
|
||||
if (report.actions.adminWriteAttempted && report.blockers.length === 0) {
|
||||
await verifyRuntimeReadiness(report, env, options);
|
||||
inferProvisioningFromRuntime(report);
|
||||
addRuntimeProvisioningBlocker(report);
|
||||
}
|
||||
return finalizeReport(report);
|
||||
}
|
||||
|
||||
addSafetyRefusal(report, `Unsupported DEV DB provisioning mode: ${parsed.mode}`);
|
||||
return finalizeReport(report);
|
||||
}
|
||||
|
||||
export function parseArgs(argv = []) {
|
||||
const args = {
|
||||
mode: "check",
|
||||
allowLiveDbRead: false,
|
||||
confirmDev: false,
|
||||
confirmedNonProduction: false,
|
||||
writeReport: false,
|
||||
reportPath: defaultReportPath,
|
||||
pretty: false,
|
||||
failOnBlocked: false,
|
||||
help: false
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--check") args.mode = "check";
|
||||
else if (arg === "--dry-run") args.mode = "dry-run";
|
||||
else if (arg === "--apply") args.mode = "apply";
|
||||
else if (arg === "--allow-live-db-read") args.allowLiveDbRead = true;
|
||||
else if (arg === "--confirm-dev") args.confirmDev = true;
|
||||
else if (arg === "--confirmed-non-production") args.confirmedNonProduction = true;
|
||||
else if (arg === "--write-report") args.writeReport = true;
|
||||
else if (arg === "--pretty") args.pretty = true;
|
||||
else if (arg === "--fail-on-blocked") args.failOnBlocked = true;
|
||||
else if (arg === "--help" || arg === "-h") args.help = true;
|
||||
else if (arg === "--report") {
|
||||
index += 1;
|
||||
args.reportPath = requireArgValue(argv[index], "--report");
|
||||
} else if (arg.startsWith("--report=")) {
|
||||
args.reportPath = requireArgValue(arg.slice("--report=".length), "--report");
|
||||
} else {
|
||||
throw new Error(`unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
export function parseProvisioningTarget(rawUrl) {
|
||||
const present = typeof rawUrl === "string" && rawUrl.trim().length > 0;
|
||||
if (!present) {
|
||||
return {
|
||||
ok: false,
|
||||
present: false,
|
||||
parseStatus: "missing",
|
||||
errorCode: "TARGET_DB_URL_MISSING",
|
||||
roleNamePresent: false,
|
||||
databaseNamePresent: false,
|
||||
passwordPresent: false
|
||||
};
|
||||
}
|
||||
|
||||
let url;
|
||||
try {
|
||||
url = new URL(rawUrl);
|
||||
} catch {
|
||||
return {
|
||||
ok: false,
|
||||
present: true,
|
||||
parseStatus: "invalid_url",
|
||||
errorCode: "TARGET_DB_URL_INVALID",
|
||||
roleNamePresent: false,
|
||||
databaseNamePresent: false,
|
||||
passwordPresent: false
|
||||
};
|
||||
}
|
||||
|
||||
const protocolReady = ["postgres:", "postgresql:"].includes(url.protocol);
|
||||
const roleName = decodeURIComponent(url.username || "");
|
||||
const password = decodeURIComponent(url.password || "");
|
||||
const databaseName = decodeURIComponent(url.pathname.replace(/^\/+/u, ""));
|
||||
const databaseAllowed = Boolean(databaseName) && !systemDatabases.has(databaseName.toLowerCase());
|
||||
const ok = protocolReady && Boolean(roleName) && Boolean(password) && databaseAllowed;
|
||||
return {
|
||||
ok,
|
||||
present: true,
|
||||
parseStatus: ok ? "ready" : "blocked",
|
||||
errorCode: ok ? null : targetParseErrorCode({ protocolReady, roleName, password, databaseName, databaseAllowed }),
|
||||
roleNamePresent: Boolean(roleName),
|
||||
databaseNamePresent: Boolean(databaseName),
|
||||
databaseAllowed,
|
||||
passwordPresent: Boolean(password),
|
||||
endpointClass: DEV_DB_ENV_CONTRACT.endpointAuthority.source,
|
||||
roleName,
|
||||
databaseName,
|
||||
password
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeArgs(args) {
|
||||
return {
|
||||
...parseArgs([]),
|
||||
...args
|
||||
};
|
||||
}
|
||||
|
||||
function targetParseErrorCode({ protocolReady, roleName, password, databaseName, databaseAllowed }) {
|
||||
if (!protocolReady) return "TARGET_DB_URL_PROTOCOL_BLOCKED";
|
||||
if (!roleName) return "TARGET_DB_URL_ROLE_MISSING";
|
||||
if (!password) return "TARGET_DB_URL_PASSWORD_MISSING";
|
||||
if (!databaseName) return "TARGET_DB_URL_DATABASE_MISSING";
|
||||
if (!databaseAllowed) return "TARGET_DB_URL_DATABASE_FORBIDDEN";
|
||||
return "TARGET_DB_URL_BLOCKED";
|
||||
}
|
||||
|
||||
function summarizeAdminEnv(env) {
|
||||
return {
|
||||
present: typeof env?.[DEV_DB_ADMIN_SECRET_REF.env] === "string" && env[DEV_DB_ADMIN_SECRET_REF.env].trim().length > 0,
|
||||
env: DEV_DB_ADMIN_SECRET_REF.env,
|
||||
secretRef: DEV_DB_ADMIN_SECRET_REF,
|
||||
valueRedacted: true
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeDbEnv(env, target, admin) {
|
||||
return {
|
||||
targetUrlPresent: target.present,
|
||||
targetUrlValueRedacted: true,
|
||||
targetEndpointRedacted: true,
|
||||
targetEndpointClass: target.endpointClass ?? DEV_DB_ENV_CONTRACT.endpointAuthority.source,
|
||||
targetParseStatus: target.parseStatus,
|
||||
targetErrorCode: target.errorCode,
|
||||
targetRoleNamePresent: target.roleNamePresent,
|
||||
targetDatabaseNamePresent: target.databaseNamePresent,
|
||||
targetDatabaseAllowed: target.databaseAllowed === true,
|
||||
targetPasswordPresent: target.passwordPresent,
|
||||
sslMode: env?.HWLAB_CLOUD_DB_SSL_MODE || DEV_DB_ENV_CONTRACT.nonSecretDefaults.HWLAB_CLOUD_DB_SSL_MODE,
|
||||
sslModeSecret: false,
|
||||
adminUrlPresent: admin.present,
|
||||
adminUrlValueRedacted: true,
|
||||
adminEndpointRedacted: true,
|
||||
dbUrlSecretRef: appSecretRef,
|
||||
adminUrlSecretRef: DEV_DB_ADMIN_SECRET_REF,
|
||||
endpointAuthority: DEV_DB_ENV_CONTRACT.endpointAuthority.source
|
||||
};
|
||||
}
|
||||
|
||||
function validateSourceContract(report, target, args) {
|
||||
if (target.ok) return;
|
||||
if (args.mode === "check" && target.present === false) return;
|
||||
addBlocker(report, "runtime_blocker", "runtime-provisioning-target", "DEV DB provisioning requires a valid target DB URL from the cloud-api SecretRef env.", {
|
||||
env: appSecretRef.env,
|
||||
secretRef: appSecretRef,
|
||||
errorCode: target.errorCode,
|
||||
valueRedacted: true,
|
||||
endpointRedacted: true,
|
||||
secretValuesPrinted: false
|
||||
});
|
||||
}
|
||||
|
||||
async function verifyRuntimeReadiness(report, env, options) {
|
||||
report.actions.liveDbReadAttempted = true;
|
||||
report.actions.appRuntimeReadinessAttempted = true;
|
||||
const runtimeStore = new PostgresCloudRuntimeStore({
|
||||
env,
|
||||
dbUrl: env?.HWLAB_CLOUD_DB_URL,
|
||||
sslMode: env?.HWLAB_CLOUD_DB_SSL_MODE,
|
||||
queryClient: options.queryClient,
|
||||
pgModuleLoader: options.pgModuleLoader
|
||||
});
|
||||
let runtime;
|
||||
try {
|
||||
runtime = await runtimeStore.readiness();
|
||||
} finally {
|
||||
if (!options.queryClient && typeof runtimeStore.pool?.end === "function") {
|
||||
await runtimeStore.pool.end();
|
||||
}
|
||||
}
|
||||
report.runtime = summarizeRuntime(runtime);
|
||||
report.actions.readinessVerified = runtime.ready === true;
|
||||
report.downstreamRuntimeGates = report.runtime.gates;
|
||||
report.gates = {
|
||||
...report.gates,
|
||||
ssl: normalizeGate(runtime.gates?.ssl),
|
||||
auth: normalizeGate(runtime.gates?.auth),
|
||||
schema: normalizeGate(runtime.gates?.schema),
|
||||
migration: normalizeGate(runtime.gates?.migration),
|
||||
durability: normalizeGate(runtime.gates?.durability)
|
||||
};
|
||||
}
|
||||
|
||||
async function inspectAdminProvisioning(report, env, target, options) {
|
||||
report.actions.liveDbReadAttempted = true;
|
||||
report.actions.adminInspectionAttempted = true;
|
||||
const { client, close } = await openAdminClient(env, options);
|
||||
try {
|
||||
const [role, database] = await Promise.all([
|
||||
objectExists(client, "SELECT 1 AS present FROM pg_catalog.pg_roles WHERE rolname = $1 LIMIT 1", [target.roleName]),
|
||||
objectExists(client, "SELECT 1 AS present FROM pg_catalog.pg_database WHERE datname = $1 LIMIT 1", [target.databaseName])
|
||||
]);
|
||||
report.provisioning.targetRole = {
|
||||
checked: true,
|
||||
exists: role,
|
||||
created: false,
|
||||
ready: role,
|
||||
targetNamePresent: target.roleNamePresent
|
||||
};
|
||||
report.provisioning.targetDatabase = {
|
||||
checked: true,
|
||||
exists: database,
|
||||
created: false,
|
||||
ready: database,
|
||||
targetNamePresent: target.databaseNamePresent
|
||||
};
|
||||
report.actions.roleExisted = role;
|
||||
report.actions.databaseExisted = database;
|
||||
report.provisioning.ready = role && database && runtimeAuthSatisfied(report.runtime);
|
||||
if (!role) addMissingObjectBlocker(report, "role");
|
||||
if (!database) addMissingObjectBlocker(report, "database");
|
||||
} catch (error) {
|
||||
addClassifiedAdminBlocker(report, error, "DEV DB provisioning inspection could not verify the target role/database.");
|
||||
} finally {
|
||||
await close();
|
||||
}
|
||||
}
|
||||
|
||||
async function applyProvisioning(report, env, target, options) {
|
||||
report.actions.liveDbWriteAttempted = true;
|
||||
report.actions.adminWriteAttempted = true;
|
||||
let admin;
|
||||
try {
|
||||
admin = await openAdminClient(env, options);
|
||||
const client = admin.client;
|
||||
const roleExists = await objectExists(client, "SELECT 1 AS present FROM pg_catalog.pg_roles WHERE rolname = $1 LIMIT 1", [target.roleName]);
|
||||
const databaseExists = await objectExists(client, "SELECT 1 AS present FROM pg_catalog.pg_database WHERE datname = $1 LIMIT 1", [target.databaseName]);
|
||||
report.actions.roleExisted = roleExists;
|
||||
report.actions.databaseExisted = databaseExists;
|
||||
|
||||
if (!roleExists) {
|
||||
await client.query(`CREATE ROLE ${quotePgIdent(target.roleName)} LOGIN PASSWORD ${quotePgLiteral(target.password)}`);
|
||||
report.actions.roleCreated = true;
|
||||
} else {
|
||||
await client.query(`ALTER ROLE ${quotePgIdent(target.roleName)} WITH LOGIN PASSWORD ${quotePgLiteral(target.password)}`);
|
||||
}
|
||||
report.actions.rolePasswordSynchronized = true;
|
||||
|
||||
if (!databaseExists) {
|
||||
await client.query(`CREATE DATABASE ${quotePgIdent(target.databaseName)} OWNER ${quotePgIdent(target.roleName)}`);
|
||||
report.actions.databaseCreated = true;
|
||||
}
|
||||
await client.query(`GRANT CONNECT ON DATABASE ${quotePgIdent(target.databaseName)} TO ${quotePgIdent(target.roleName)}`);
|
||||
report.actions.databaseConnectGranted = true;
|
||||
} catch (error) {
|
||||
addClassifiedAdminBlocker(report, error, "DEV DB provisioning apply could not complete role/database provisioning.");
|
||||
return;
|
||||
} finally {
|
||||
if (admin) await admin.close();
|
||||
}
|
||||
|
||||
let targetAdmin;
|
||||
try {
|
||||
targetAdmin = await openAdminClient(env, {
|
||||
...options,
|
||||
adminDbUrlOverride: withDatabaseInPostgresUrl(env[DEV_DB_ADMIN_SECRET_REF.env], target.databaseName)
|
||||
});
|
||||
await targetAdmin.client.query(`GRANT USAGE, CREATE ON SCHEMA public TO ${quotePgIdent(target.roleName)}`);
|
||||
report.actions.schemaPrivilegesGranted = true;
|
||||
} catch (error) {
|
||||
addClassifiedAdminBlocker(report, error, "DEV DB provisioning apply could not grant target database schema privileges.");
|
||||
return;
|
||||
} finally {
|
||||
if (targetAdmin) await targetAdmin.close();
|
||||
}
|
||||
|
||||
report.provisioning.targetRole = {
|
||||
checked: true,
|
||||
exists: true,
|
||||
created: report.actions.roleCreated,
|
||||
ready: true,
|
||||
targetNamePresent: true
|
||||
};
|
||||
report.provisioning.targetDatabase = {
|
||||
checked: true,
|
||||
exists: true,
|
||||
created: report.actions.databaseCreated,
|
||||
ready: true,
|
||||
targetNamePresent: true
|
||||
};
|
||||
report.provisioning.privileges = {
|
||||
checked: true,
|
||||
ready: report.actions.databaseConnectGranted && report.actions.schemaPrivilegesGranted,
|
||||
databaseConnectGranted: report.actions.databaseConnectGranted,
|
||||
schemaPrivilegesGranted: report.actions.schemaPrivilegesGranted
|
||||
};
|
||||
report.provisioning.ready = report.provisioning.privileges.ready;
|
||||
}
|
||||
|
||||
async function openAdminClient(env, options) {
|
||||
if (options.adminClient) {
|
||||
return {
|
||||
client: options.adminClient,
|
||||
close: async () => {}
|
||||
};
|
||||
}
|
||||
|
||||
const pool = await createPgPool({
|
||||
dbUrl: options.adminDbUrlOverride ?? env[DEV_DB_ADMIN_SECRET_REF.env],
|
||||
sslMode: env.HWLAB_CLOUD_DB_SSL_MODE,
|
||||
timeoutMs: env.HWLAB_CLOUD_DB_PROBE_TIMEOUT_MS,
|
||||
pgModuleLoader: options.pgModuleLoader
|
||||
});
|
||||
return {
|
||||
client: pool,
|
||||
close: async () => {
|
||||
await pool.end();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function createPgPool({ dbUrl, sslMode, timeoutMs, pgModuleLoader }) {
|
||||
let pg;
|
||||
try {
|
||||
pg = await (pgModuleLoader ? pgModuleLoader() : import("pg"));
|
||||
} catch (error) {
|
||||
if (error?.code === "ERR_MODULE_NOT_FOUND") {
|
||||
const driverError = new Error("Postgres provisioning requires the pg package");
|
||||
driverError.code = "HWLAB_PG_DRIVER_MISSING";
|
||||
throw driverError;
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
const Pool = pg.Pool ?? pg.default?.Pool;
|
||||
if (typeof Pool !== "function") {
|
||||
const driverError = new Error("Postgres provisioning could not load pg.Pool");
|
||||
driverError.code = "HWLAB_PG_DRIVER_MISSING";
|
||||
throw driverError;
|
||||
}
|
||||
return new Pool(buildPostgresPoolConfig({
|
||||
dbUrl,
|
||||
sslMode,
|
||||
timeoutMs
|
||||
}));
|
||||
}
|
||||
|
||||
function inferProvisioningFromRuntime(report) {
|
||||
const runtimeReady = report.runtime?.ready === true && report.runtime?.durable === true;
|
||||
const authReady = runtimeAuthSatisfied(report.runtime);
|
||||
if (!runtimeReady && !authReady) return;
|
||||
report.provisioning.targetRole = {
|
||||
...report.provisioning.targetRole,
|
||||
checked: report.provisioning.targetRole.checked || false,
|
||||
exists: true,
|
||||
ready: true,
|
||||
inferredFromRuntime: true
|
||||
};
|
||||
report.provisioning.targetDatabase = {
|
||||
...report.provisioning.targetDatabase,
|
||||
checked: report.provisioning.targetDatabase.checked || false,
|
||||
exists: true,
|
||||
ready: true,
|
||||
inferredFromRuntime: true
|
||||
};
|
||||
report.provisioning.ready = true;
|
||||
}
|
||||
|
||||
function addRuntimeProvisioningBlocker(report) {
|
||||
const blocker = report.runtime?.blocker;
|
||||
if (!blocker) return;
|
||||
if (blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED) {
|
||||
addBlocker(report, "runtime_blocker", "runtime-provisioning-ssl", "DEV DB provisioning reached the target runtime adapter, but SSL mode blocked connection.", runtimeEvidence(report.runtime));
|
||||
} else if (blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED) {
|
||||
addBlocker(report, "runtime_blocker", "runtime-provisioning-auth", "DEV DB target role/database exists check did not prove application authentication readiness.", runtimeEvidence(report.runtime));
|
||||
} else if (blocker === RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING) {
|
||||
addBlocker(report, "runtime_blocker", "runtime-provisioning-driver", "Postgres driver is unavailable for DEV DB provisioning verification.", runtimeEvidence(report.runtime));
|
||||
}
|
||||
}
|
||||
|
||||
function addMissingObjectBlocker(report, kind) {
|
||||
const scope = kind === "role" ? "runtime-provisioning-role" : "runtime-provisioning-database";
|
||||
addBlocker(report, "runtime_blocker", scope, `DEV DB target ${kind} is missing according to admin catalog inspection.`, {
|
||||
exists: false,
|
||||
created: false,
|
||||
valueRedacted: true,
|
||||
endpointRedacted: true,
|
||||
affectedRuntimeBlocker: RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
|
||||
secretValuesPrinted: false
|
||||
});
|
||||
}
|
||||
|
||||
function addClassifiedAdminBlocker(report, error, summary) {
|
||||
const classified = classifyProvisioningError(error);
|
||||
addBlocker(report, "runtime_blocker", classified.scope, summary, {
|
||||
blocker: classified.blocker,
|
||||
errorCode: classified.errorCode,
|
||||
valueRedacted: true,
|
||||
endpointRedacted: true,
|
||||
secretValuesPrinted: false
|
||||
});
|
||||
}
|
||||
|
||||
function classifyProvisioningError(error) {
|
||||
const errorCode = typeof error?.code === "string" ? error.code : "UNKNOWN";
|
||||
if (errorCode === "HWLAB_PG_DRIVER_MISSING") {
|
||||
return {
|
||||
blocker: RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING,
|
||||
scope: "runtime-provisioning-driver",
|
||||
errorCode
|
||||
};
|
||||
}
|
||||
if (isSslError(error)) {
|
||||
return {
|
||||
blocker: RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED,
|
||||
scope: "runtime-provisioning-ssl",
|
||||
errorCode
|
||||
};
|
||||
}
|
||||
if (["28P01", "28000", "42501"].includes(errorCode)) {
|
||||
return {
|
||||
blocker: RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
|
||||
scope: "runtime-provisioning-auth",
|
||||
errorCode
|
||||
};
|
||||
}
|
||||
if (["3D000"].includes(errorCode)) {
|
||||
return {
|
||||
blocker: RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
|
||||
scope: "runtime-provisioning-database",
|
||||
errorCode
|
||||
};
|
||||
}
|
||||
if (["42P01", "42703", "3F000"].includes(errorCode)) {
|
||||
return {
|
||||
blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
|
||||
scope: "runtime-provisioning-schema",
|
||||
errorCode
|
||||
};
|
||||
}
|
||||
return {
|
||||
blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
|
||||
scope: "runtime-provisioning-query",
|
||||
errorCode
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeEvidence(runtime) {
|
||||
return {
|
||||
blocker: runtime.blocker,
|
||||
queryResult: runtime.connection?.queryResult ?? "unknown",
|
||||
errorCode: runtime.connection?.errorCode ?? null,
|
||||
schemaReady: runtime.schema?.ready === true,
|
||||
migrationReady: runtime.migration?.ready === true,
|
||||
durableReady: runtime.ready === true && runtime.durable === true,
|
||||
secretValuesPrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
async function objectExists(client, sql, params) {
|
||||
const result = await client.query(sql, params);
|
||||
return Array.isArray(result?.rows) && result.rows.length > 0;
|
||||
}
|
||||
|
||||
function sourceOnlyGates(target) {
|
||||
const sourceReady = target.ok;
|
||||
const targetMissingInSourceOnly = target.present === false;
|
||||
return {
|
||||
ssl: notCheckedGate("source-only validation does not connect to Postgres"),
|
||||
auth: sourceReady || targetMissingInSourceOnly ? notCheckedGate("auth requires live target runtime verification") : blockedGate("runtime_provisioning_target_blocked"),
|
||||
role: sourceReady || targetMissingInSourceOnly ? notCheckedGate("role existence requires DEV admin catalog inspection") : blockedGate("runtime_provisioning_target_blocked"),
|
||||
database: sourceReady || targetMissingInSourceOnly ? notCheckedGate("database existence requires DEV admin catalog inspection") : blockedGate("runtime_provisioning_target_blocked"),
|
||||
schema: notCheckedGate("schema is verified by runtime migration/readiness after role/database provisioning"),
|
||||
migration: notCheckedGate("migration ledger is verified by dev-runtime-migration"),
|
||||
durability: notCheckedGate("durability is verified by post-provisioning runtime readiness and postflight")
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeGate(gate = {}) {
|
||||
if (gate.ready === true || gate.status === "ready") return readyGate();
|
||||
if (gate.status === "blocked") return blockedGate(gate.blocker ?? "runtime_durable_adapter_blocked");
|
||||
return notCheckedGate();
|
||||
}
|
||||
|
||||
function readyGate(summary = "ready") {
|
||||
return {
|
||||
checked: true,
|
||||
ready: true,
|
||||
status: "ready",
|
||||
blocker: null,
|
||||
summary
|
||||
};
|
||||
}
|
||||
|
||||
function blockedGate(blocker, summary = "blocked") {
|
||||
return {
|
||||
checked: true,
|
||||
ready: false,
|
||||
status: "blocked",
|
||||
blocker,
|
||||
summary
|
||||
};
|
||||
}
|
||||
|
||||
function notCheckedGate(summary = "not checked") {
|
||||
return {
|
||||
checked: false,
|
||||
ready: false,
|
||||
status: "not_checked",
|
||||
blocker: null,
|
||||
summary
|
||||
};
|
||||
}
|
||||
|
||||
function emptyObjectState(targetNamePresent) {
|
||||
return {
|
||||
checked: false,
|
||||
exists: false,
|
||||
created: false,
|
||||
ready: false,
|
||||
targetNamePresent
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeRuntime(runtime = {}) {
|
||||
return {
|
||||
adapter: runtime.adapter ?? "unknown",
|
||||
durable: Boolean(runtime.durable),
|
||||
durableRequested: Boolean(runtime.durableRequested),
|
||||
ready: runtime.ready === true,
|
||||
status: runtime.status ?? "unknown",
|
||||
blocker: runtime.blocker ?? null,
|
||||
liveRuntimeEvidence: Boolean(runtime.liveRuntimeEvidence),
|
||||
fixtureEvidence: Boolean(runtime.fixtureEvidence),
|
||||
connection: {
|
||||
queryAttempted: Boolean(runtime.connection?.queryAttempted),
|
||||
queryResult: runtime.connection?.queryResult ?? "unknown",
|
||||
endpointRedacted: true,
|
||||
valueRedacted: true,
|
||||
errorCode: runtime.connection?.errorCode ?? null
|
||||
},
|
||||
schema: {
|
||||
checked: Boolean(runtime.schema?.checked),
|
||||
ready: Boolean(runtime.schema?.ready),
|
||||
missingTables: Array.isArray(runtime.schema?.missingTables) ? [...runtime.schema.missingTables] : [],
|
||||
missingColumns: Array.isArray(runtime.schema?.missingColumns) ? [...runtime.schema.missingColumns] : []
|
||||
},
|
||||
migration: {
|
||||
checked: Boolean(runtime.migration?.checked),
|
||||
ready: Boolean(runtime.migration?.ready),
|
||||
requiredMigrationId: runtime.migration?.requiredMigrationId ?? null,
|
||||
appliedMigrationId: runtime.migration?.appliedMigrationId ?? null,
|
||||
missing: runtime.migration?.missing !== false
|
||||
},
|
||||
gates: runtime.gates ?? {}
|
||||
};
|
||||
}
|
||||
|
||||
function runtimeAuthSatisfied(runtime = {}) {
|
||||
if (runtime.ready === true && runtime.durable === true) return true;
|
||||
const blocker = runtime.blocker;
|
||||
return [
|
||||
RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
|
||||
RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED,
|
||||
RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED
|
||||
].includes(blocker) && runtime.connection?.queryAttempted === true;
|
||||
}
|
||||
|
||||
function buildApplyBoundary(args) {
|
||||
return {
|
||||
defaultMode: "check",
|
||||
mode: args.mode,
|
||||
liveDbReads: args.mode === "apply" || (args.mode === "dry-run" && args.allowLiveDbRead),
|
||||
liveDbWrites: args.mode === "apply",
|
||||
requiresForLiveDbRead: [
|
||||
"--confirm-dev",
|
||||
`${appSecretRef.env} from ${appSecretRef.secretName}/${appSecretRef.secretKey}`,
|
||||
`${DEV_DB_ADMIN_SECRET_REF.env} from ${DEV_DB_ADMIN_SECRET_REF.secretName}/${DEV_DB_ADMIN_SECRET_REF.secretKey} unless runtime readiness already proves auth`
|
||||
],
|
||||
requiresForApply: [
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
`${appSecretRef.env} from ${appSecretRef.secretName}/${appSecretRef.secretKey}`,
|
||||
`${DEV_DB_ADMIN_SECRET_REF.env} from ${DEV_DB_ADMIN_SECRET_REF.secretName}/${DEV_DB_ADMIN_SECRET_REF.secretKey}`
|
||||
],
|
||||
writeScope: args.mode === "apply"
|
||||
? [
|
||||
"DEV Postgres target role named by the redacted cloud-api DB URL",
|
||||
"DEV Postgres target database named by the redacted cloud-api DB URL",
|
||||
"DEV target DB CONNECT and public schema CREATE/USAGE grants for that role"
|
||||
]
|
||||
: [],
|
||||
noWriteScope: [
|
||||
"Kubernetes Secret resources",
|
||||
"PROD",
|
||||
"service restarts",
|
||||
"schema migrations",
|
||||
"M3/M4/M5 acceptance"
|
||||
],
|
||||
forbidden: [
|
||||
"printing DB URLs, passwords, tokens, Secret values, or kubeconfig material",
|
||||
"manual psql or node stdin database writes",
|
||||
"mutating PROD"
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function buildSafety(args) {
|
||||
return {
|
||||
devOnly: true,
|
||||
environment: ENVIRONMENT_DEV,
|
||||
targetNamespace: DEV_DB_ENV_CONTRACT.dns.namespace,
|
||||
prodAllowed: false,
|
||||
sourceStoresSecretValues: false,
|
||||
printsSecretValues: false,
|
||||
secretValuesPrinted: false,
|
||||
dbUrlValueRedacted: true,
|
||||
adminUrlValueRedacted: true,
|
||||
endpointRedacted: true,
|
||||
readsKubernetesSecrets: false,
|
||||
writesKubernetesSecrets: false,
|
||||
liveDbWrites: args.mode === "apply",
|
||||
fixtureEvidenceAllowed: false
|
||||
};
|
||||
}
|
||||
|
||||
function addBlocker(report, type, scope, summary, evidence = {}) {
|
||||
report.blockers.push({
|
||||
type,
|
||||
scope,
|
||||
status: "open",
|
||||
summary,
|
||||
sourceIssue: scope.includes("auth") ? "pikasTech/HWLAB#307" : issue,
|
||||
evidence
|
||||
});
|
||||
}
|
||||
|
||||
function addSafetyRefusal(report, summary) {
|
||||
report.safetyRefusal = true;
|
||||
addBlocker(report, "safety_refusal", "runtime-provisioning-apply-boundary", summary, {
|
||||
devOnly: true,
|
||||
prodAllowed: false,
|
||||
secretValuesPrinted: false
|
||||
});
|
||||
}
|
||||
|
||||
function finalizeReport(report) {
|
||||
const openBlockers = report.blockers.filter((blocker) => blocker.status === "open");
|
||||
report.provisioning.ready = report.provisioning.ready || (
|
||||
report.provisioning.targetRole.ready === true &&
|
||||
report.provisioning.targetDatabase.ready === true &&
|
||||
(report.provisioning.privileges.ready === true || report.mode !== "apply")
|
||||
);
|
||||
report.conclusion = {
|
||||
status: openBlockers.length === 0 ? "ready" : "blocked",
|
||||
summary: openBlockers.length === 0
|
||||
? conclusionReadySummary(report)
|
||||
: "DEV DB provisioning is blocked; see separated role/database/ssl/auth blockers.",
|
||||
blockerCount: openBlockers.length
|
||||
};
|
||||
return report;
|
||||
}
|
||||
|
||||
function conclusionReadySummary(report) {
|
||||
if (report.mode === "apply") {
|
||||
return "DEV DB role/database provisioning completed without printing secret values.";
|
||||
}
|
||||
if (report.mode === "dry-run" && report.actions.liveDbReadAttempted) {
|
||||
return "DEV DB role/database provisioning read-only verification passed without DB writes or secret output.";
|
||||
}
|
||||
return "DEV DB provisioning source contract is ready; no DB connection or live write was attempted.";
|
||||
}
|
||||
|
||||
function summarizeReport(report) {
|
||||
return {
|
||||
issue: report.issue,
|
||||
mode: report.mode,
|
||||
conclusion: report.conclusion,
|
||||
db: report.db,
|
||||
actions: report.actions,
|
||||
provisioning: report.provisioning,
|
||||
gates: report.gates,
|
||||
downstreamRuntimeGates: report.downstreamRuntimeGates,
|
||||
blockers: report.blockers,
|
||||
safety: report.safety
|
||||
};
|
||||
}
|
||||
|
||||
function quotePgIdent(value) {
|
||||
return `"${String(value).replaceAll('"', '""')}"`;
|
||||
}
|
||||
|
||||
function quotePgLiteral(value) {
|
||||
return `'${String(value).replaceAll("'", "''")}'`;
|
||||
}
|
||||
|
||||
function withDatabaseInPostgresUrl(rawUrl, databaseName) {
|
||||
const url = new URL(rawUrl);
|
||||
url.pathname = `/${encodeURIComponent(databaseName)}`;
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
function isSslError(error) {
|
||||
const code = typeof error?.code === "string" ? error.code : "";
|
||||
if (code.startsWith("ERR_SSL") || code.startsWith("ERR_TLS")) return true;
|
||||
const message = typeof error?.message === "string" ? error.message.toLowerCase() : "";
|
||||
return [
|
||||
"does not support ssl",
|
||||
"ssl is not enabled",
|
||||
"ssl negotiation",
|
||||
"ssl off",
|
||||
"ssl required",
|
||||
"requires ssl",
|
||||
"no ssl encryption",
|
||||
"no encryption",
|
||||
"hostssl",
|
||||
"server requires ssl",
|
||||
"tls handshake"
|
||||
].some((pattern) => message.includes(pattern));
|
||||
}
|
||||
|
||||
async function writeReport(report, reportPath, root) {
|
||||
const absolute = path.resolve(root, reportPath);
|
||||
await mkdir(path.dirname(absolute), { recursive: true });
|
||||
await writeFile(absolute, `${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
|
||||
function requireArgValue(value, flag) {
|
||||
if (typeof value !== "string" || value.trim() === "") {
|
||||
throw new Error(`${flag} requires a value`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function usage() {
|
||||
return [
|
||||
"Usage: node scripts/dev-runtime-provisioning.mjs [--check|--dry-run|--apply]",
|
||||
"",
|
||||
"Default --check validates redacted DEV DB provisioning inputs only and never connects to DB.",
|
||||
"--dry-run is source-only unless --allow-live-db-read --confirm-dev is provided.",
|
||||
"--apply requires --confirm-dev --confirmed-non-production, HWLAB_CLOUD_DB_URL, and HWLAB_CLOUD_DB_ADMIN_URL.",
|
||||
"Reports never print DB URLs, passwords, tokens, Secret values, or kubeconfig material."
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
export function formatDevRuntimeProvisioningFailure(error) {
|
||||
const message = redactFailureText(error instanceof Error ? error.message : String(error));
|
||||
return {
|
||||
issue,
|
||||
conclusion: {
|
||||
status: "blocked",
|
||||
summary: "DEV DB provisioning command failed before producing a report."
|
||||
},
|
||||
error: message,
|
||||
safety: {
|
||||
devOnly: true,
|
||||
prodAllowed: false,
|
||||
secretValuesPrinted: false
|
||||
},
|
||||
trace: createHash("sha256").update(message).digest("hex").slice(0, 12)
|
||||
};
|
||||
}
|
||||
|
||||
function redactFailureText(value) {
|
||||
return String(value ?? "")
|
||||
.replace(/postgres(?:ql)?:\/\/[^\s"'<>]+/giu, "[redacted-postgres-url]")
|
||||
.replace(/(password\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]")
|
||||
.replace(/(token\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]")
|
||||
.replace(/(kubeconfig\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]");
|
||||
}
|
||||
@@ -0,0 +1,270 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import {
|
||||
buildDevRuntimeProvisioningReport,
|
||||
parseArgs
|
||||
} from "./dev-runtime-provisioning.mjs";
|
||||
|
||||
test("source check parses target role/database without leaking DB URL material", async () => {
|
||||
const report = await buildDevRuntimeProvisioningReport(parseArgs(["--check"]), {
|
||||
env: {
|
||||
HWLAB_CLOUD_DB_URL: fixturePostgresUrl({ password: fixtureSecret("super") }),
|
||||
HWLAB_CLOUD_DB_SSL_MODE: "disable"
|
||||
},
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
});
|
||||
|
||||
assert.equal(report.conclusion.status, "ready");
|
||||
assert.equal(report.actions.liveDbReadAttempted, false);
|
||||
assert.equal(report.actions.liveDbWriteAttempted, false);
|
||||
assert.equal(report.db.targetRoleNamePresent, true);
|
||||
assert.equal(report.db.targetDatabaseNamePresent, true);
|
||||
assert.equal(report.db.targetPasswordPresent, true);
|
||||
assert.equal(report.safety.secretValuesPrinted, false);
|
||||
assertNoFixtureSecrets(report);
|
||||
});
|
||||
|
||||
test("source check without injected DB URL is ready and performs no live access", async () => {
|
||||
const report = await buildDevRuntimeProvisioningReport(parseArgs(["--check"]), {
|
||||
env: {},
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
});
|
||||
|
||||
assert.equal(report.conclusion.status, "ready");
|
||||
assert.equal(report.actions.liveDbReadAttempted, false);
|
||||
assert.equal(report.actions.liveDbWriteAttempted, false);
|
||||
assert.equal(report.blockers.length, 0);
|
||||
assert.equal(report.db.targetUrlPresent, false);
|
||||
assert.equal(report.gates.role.status, "not_checked");
|
||||
});
|
||||
|
||||
test("live dry-run classifies missing target role separately from missing database", async () => {
|
||||
const report = await buildDevRuntimeProvisioningReport(
|
||||
parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]),
|
||||
{
|
||||
env: envWithAdmin(),
|
||||
adminClient: createAdminClient({ roleExists: false, databaseExists: true }),
|
||||
queryClient: createRuntimeClient("auth"),
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.actions.adminInspectionAttempted, true);
|
||||
assert.equal(report.actions.liveDbWriteAttempted, false);
|
||||
assert.equal(report.provisioning.targetRole.exists, false);
|
||||
assert.equal(report.provisioning.targetDatabase.exists, true);
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-role"), true);
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-database"), false);
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-auth"), true);
|
||||
assertNoFixtureSecrets(report);
|
||||
});
|
||||
|
||||
test("live dry-run classifies missing target database separately from target role", async () => {
|
||||
const report = await buildDevRuntimeProvisioningReport(
|
||||
parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]),
|
||||
{
|
||||
env: envWithAdmin(),
|
||||
adminClient: createAdminClient({ roleExists: true, databaseExists: false }),
|
||||
queryClient: createRuntimeClient("auth"),
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.provisioning.targetRole.exists, true);
|
||||
assert.equal(report.provisioning.targetDatabase.exists, false);
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-role"), false);
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-database"), true);
|
||||
});
|
||||
|
||||
test("live dry-run separates SSL from auth/schema/migration", async () => {
|
||||
const report = await buildDevRuntimeProvisioningReport(
|
||||
parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]),
|
||||
{
|
||||
env: envWithAdmin({ sslMode: "require" }),
|
||||
adminClient: createAdminClient({ roleExists: true, databaseExists: true }),
|
||||
queryClient: createRuntimeClient("ssl"),
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "blocked");
|
||||
assert.equal(report.gates.ssl.status, "blocked");
|
||||
assert.equal(report.gates.auth.status, "not_checked");
|
||||
assert.equal(report.gates.schema.status, "not_checked");
|
||||
assert.equal(report.gates.migration.status, "not_checked");
|
||||
assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-ssl"), true);
|
||||
assert.equal(JSON.stringify(report).includes("does not support SSL"), false);
|
||||
});
|
||||
|
||||
test("live dry-run leaves role/database ready while migration remains a downstream blocker", async () => {
|
||||
const report = await buildDevRuntimeProvisioningReport(
|
||||
parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]),
|
||||
{
|
||||
env: envWithAdmin(),
|
||||
adminClient: createAdminClient({ roleExists: true, databaseExists: true }),
|
||||
queryClient: createRuntimeClient("migration"),
|
||||
now: () => "2026-05-23T00:00:00.000Z"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(report.conclusion.status, "ready");
|
||||
assert.equal(report.provisioning.ready, true);
|
||||
assert.equal(report.provisioning.targetRole.ready, true);
|
||||
assert.equal(report.provisioning.targetDatabase.ready, true);
|
||||
assert.equal(report.runtime.blocker, "runtime_durable_adapter_migration_blocked");
|
||||
assert.equal(report.gates.migration.status, "blocked");
|
||||
assert.equal(report.blockers.length, 0);
|
||||
});
|
||||
|
||||
test("apply creates missing role/database and grants target privileges without printing secrets", async () => {
|
||||
const adminClient = createAdminClient({ roleExists: false, databaseExists: false });
|
||||
const targetAdminClient = createAdminClient({ roleExists: true, databaseExists: true });
|
||||
const report = await buildDevRuntimeProvisioningReport(
|
||||
parseArgs(["--apply", "--confirm-dev", "--confirmed-non-production"]),
|
||||
{
|
||||
env: envWithAdmin(),
|
||||
adminClient,
|
||||
queryClient: createRuntimeClient("ready"),
|
||||
now: () => "2026-05-23T00:00:00.000Z",
|
||||
adminDbUrlOverride: null,
|
||||
pgModuleLoader: async () => {
|
||||
throw new Error("pgModuleLoader should not be used when adminClient is injected");
|
||||
}
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(targetAdminClient.calls.length, 0);
|
||||
assert.equal(report.conclusion.status, "ready");
|
||||
assert.equal(report.actions.roleCreated, true);
|
||||
assert.equal(report.actions.databaseCreated, true);
|
||||
assert.equal(report.actions.rolePasswordSynchronized, true);
|
||||
assert.equal(report.actions.databaseConnectGranted, true);
|
||||
assert.equal(report.actions.schemaPrivilegesGranted, true);
|
||||
assert.equal(report.provisioning.ready, true);
|
||||
assert.ok(adminClient.calls.some((call) => call.sql.startsWith("CREATE ROLE ")));
|
||||
assert.ok(adminClient.calls.some((call) => call.sql.startsWith("CREATE DATABASE ")));
|
||||
assert.ok(adminClient.calls.some((call) => call.sql.startsWith("GRANT CONNECT ON DATABASE ")));
|
||||
assert.ok(adminClient.calls.some((call) => call.sql.startsWith("GRANT USAGE, CREATE ON SCHEMA public ")));
|
||||
assertNoFixtureSecrets(report);
|
||||
});
|
||||
|
||||
function envWithAdmin({ sslMode = "disable" } = {}) {
|
||||
return {
|
||||
HWLAB_CLOUD_DB_URL: fixturePostgresUrl({ password: fixtureSecret("app") }),
|
||||
HWLAB_CLOUD_DB_ADMIN_URL: fixturePostgresUrl({
|
||||
user: "postgres",
|
||||
password: fixtureSecret("admin"),
|
||||
database: "postgres"
|
||||
}),
|
||||
HWLAB_CLOUD_DB_SSL_MODE: sslMode
|
||||
};
|
||||
}
|
||||
|
||||
function fixturePostgresUrl({
|
||||
user = "hwlab_app",
|
||||
password = fixtureSecret("app"),
|
||||
host = "db.example.test",
|
||||
database = "hwlab_runtime"
|
||||
} = {}) {
|
||||
return `${["postgresql", "://"].join("")}${user}:${password}@${host}:5432/${database}`;
|
||||
}
|
||||
|
||||
function fixtureSecret(kind) {
|
||||
return `${kind}-${"secret"}`;
|
||||
}
|
||||
|
||||
function assertNoFixtureSecrets(value) {
|
||||
const text = JSON.stringify(value);
|
||||
for (const forbidden of [
|
||||
fixtureSecret("super"),
|
||||
fixtureSecret("app"),
|
||||
fixtureSecret("admin"),
|
||||
"db.example.test",
|
||||
"hwlab_app",
|
||||
"hwlab_runtime"
|
||||
]) {
|
||||
assert.equal(text.includes(forbidden), false);
|
||||
}
|
||||
}
|
||||
|
||||
function createAdminClient({ roleExists, databaseExists }) {
|
||||
const state = {
|
||||
roleExists,
|
||||
databaseExists,
|
||||
calls: []
|
||||
};
|
||||
return {
|
||||
get calls() {
|
||||
return state.calls;
|
||||
},
|
||||
async query(sql, params = []) {
|
||||
state.calls.push({ sql, params });
|
||||
if (sql.includes("pg_catalog.pg_roles")) {
|
||||
return { rows: state.roleExists ? [{ present: 1 }] : [] };
|
||||
}
|
||||
if (sql.includes("pg_catalog.pg_database")) {
|
||||
return { rows: state.databaseExists ? [{ present: 1 }] : [] };
|
||||
}
|
||||
if (sql.startsWith("CREATE ROLE ")) {
|
||||
state.roleExists = true;
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.startsWith("ALTER ROLE ")) {
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.startsWith("CREATE DATABASE ")) {
|
||||
state.databaseExists = true;
|
||||
return { rows: [] };
|
||||
}
|
||||
if (sql.startsWith("GRANT CONNECT ON DATABASE ") || sql.startsWith("GRANT USAGE, CREATE ON SCHEMA public ")) {
|
||||
return { rows: [] };
|
||||
}
|
||||
throw new Error(`unexpected admin query: ${sql}`);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function createRuntimeClient(mode) {
|
||||
return {
|
||||
async query(sql) {
|
||||
if (sql.includes("information_schema.columns")) {
|
||||
if (mode === "ssl") {
|
||||
const error = new Error("fixture SSL blocked: server does not support SSL");
|
||||
error.code = "08P01";
|
||||
throw error;
|
||||
}
|
||||
if (mode === "auth") {
|
||||
const error = new Error("fixture auth blocked");
|
||||
error.code = "28P01";
|
||||
throw error;
|
||||
}
|
||||
return { rows: schemaRows() };
|
||||
}
|
||||
if (sql.startsWith("SELECT id, schema_version FROM hwlab_schema_migrations")) {
|
||||
if (mode === "migration") return { rows: [] };
|
||||
return { rows: [{ id: "0001_cloud_core_skeleton", schema_version: "runtime-durable-postgres-v1" }] };
|
||||
}
|
||||
if (sql.startsWith("SELECT COUNT(*)::int AS count FROM ")) {
|
||||
return { rows: [{ count: 0 }] };
|
||||
}
|
||||
throw new Error(`unexpected runtime query: ${sql}`);
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function schemaRows() {
|
||||
const schema = {
|
||||
gateway_sessions: ["id", "project_id", "gateway_service_id", "status", "started_at", "ended_at", "gateway_session_json"],
|
||||
box_resources: ["id", "project_id", "gateway_session_id", "resource_state", "labels_json", "resource_json", "updated_at"],
|
||||
box_capabilities: ["id", "box_resource_id", "capability_type", "capability_json", "updated_at"],
|
||||
hardware_operations: ["id", "project_id", "requested_by", "operation_type", "operation_json", "status", "requested_at", "updated_at"],
|
||||
audit_events: ["id", "request_id", "actor", "source", "operation", "target", "result", "timestamp", "event_json"],
|
||||
evidence_records: ["id", "project_id", "operation_id", "evidence_type", "uri", "metadata_json", "created_at"]
|
||||
};
|
||||
return Object.entries(schema).flatMap(([table, columns]) =>
|
||||
columns.map((column) => ({ table_name: table, column_name: column }))
|
||||
);
|
||||
}
|
||||
@@ -199,11 +199,14 @@ function assertDurableRuntimeRunbook(value) {
|
||||
for (const expected of [
|
||||
"DB live readiness and durable runtime readiness are separate gates",
|
||||
"runtime_durable_adapter_query_blocked",
|
||||
"DEV DB Provisioning Automation",
|
||||
"Source migration contract",
|
||||
"Live provisioning apply",
|
||||
"Static runtime boundary contract",
|
||||
"Public health observation",
|
||||
"Live DB read verification",
|
||||
"Live migration/repair apply",
|
||||
"Durable runtime postflight",
|
||||
"Do not run `kubectl get secret`",
|
||||
"secretValuesPrinted=false",
|
||||
"Before any authorized live migration or repair",
|
||||
@@ -212,10 +215,15 @@ function assertDurableRuntimeRunbook(value) {
|
||||
"Moved:",
|
||||
"Still blocked:",
|
||||
"No full M3, M4, or M5 acceptance is allowed while runtime durability is blocked",
|
||||
"node scripts/dev-runtime-provisioning.mjs --check",
|
||||
"node scripts/dev-runtime-provisioning.mjs --dry-run --allow-live-db-read --confirm-dev --write-report",
|
||||
"node scripts/dev-runtime-provisioning.mjs --apply --confirm-dev --confirmed-non-production --write-report",
|
||||
"node scripts/dev-runtime-migration.mjs --check",
|
||||
"node scripts/dev-runtime-migration.mjs --dry-run --write-report",
|
||||
"node scripts/dev-runtime-migration.mjs --dry-run --allow-live-db-read --confirm-dev --write-report",
|
||||
"node scripts/dev-runtime-migration.mjs --apply --confirm-dev --confirmed-non-production --write-report"
|
||||
"node scripts/dev-runtime-migration.mjs --apply --confirm-dev --confirmed-non-production --write-report",
|
||||
"node scripts/dev-runtime-postflight.mjs --check",
|
||||
"node scripts/dev-runtime-postflight.mjs --live --confirm-dev --confirmed-non-production --target api --write-report"
|
||||
]) {
|
||||
assertIncludes(value, expected, label);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user