From 14529d7531fa2c49040997b81355f64bdf4b5dbd Mon Sep 17 00:00:00 2001 From: Code Queue Review Date: Sat, 23 May 2026 10:16:59 +0000 Subject: [PATCH] feat: add dev durable runtime provisioning gates --- docs/reference/deployment-publish.md | 14 + docs/reference/dev-runtime-boundary.md | 56 ++ package.json | 6 +- scripts/dev-artifact-publish.mjs | 1 + scripts/dev-runtime-postflight.mjs | 12 + scripts/dev-runtime-provisioning.mjs | 12 + scripts/src/dev-cd-apply.mjs | 310 +++++- scripts/src/dev-cd-apply.test.mjs | 50 + scripts/src/dev-runtime-postflight.mjs | 492 ++++++++++ scripts/src/dev-runtime-postflight.test.mjs | 202 ++++ scripts/src/dev-runtime-provisioning.mjs | 928 ++++++++++++++++++ scripts/src/dev-runtime-provisioning.test.mjs | 270 +++++ scripts/validate-runtime-boundary.mjs | 10 +- 13 files changed, 2340 insertions(+), 23 deletions(-) create mode 100644 scripts/dev-runtime-postflight.mjs create mode 100644 scripts/dev-runtime-provisioning.mjs create mode 100644 scripts/src/dev-runtime-postflight.mjs create mode 100644 scripts/src/dev-runtime-postflight.test.mjs create mode 100644 scripts/src/dev-runtime-provisioning.mjs create mode 100644 scripts/src/dev-runtime-provisioning.test.mjs diff --git a/docs/reference/deployment-publish.md b/docs/reference/deployment-publish.md index 818333f0..1e95f868 100644 --- a/docs/reference/deployment-publish.md +++ b/docs/reference/deployment-publish.md @@ -191,6 +191,20 @@ transaction reproduces. New DEV side effects should use `scripts/dev-cd-apply.mjs`; do not bypass the Lease with ad hoc publish, `kubectl set image`, or restart commands. +The DEV CD transaction owns the durable runtime support sequence before final +live postflight: + +```sh +node scripts/dev-runtime-provisioning.mjs --apply --confirm-dev --confirmed-non-production --write-report +node scripts/dev-runtime-migration.mjs --apply --confirm-dev --confirmed-non-production --write-report +node scripts/dev-runtime-postflight.mjs --live --confirm-dev --confirmed-non-production --target api --write-report +``` + +These commands are called by `scripts/dev-cd-apply.mjs` under the transaction +Lease. Runners without explicit rollout authorization may run the source-only +`--check` variants and prepare PRs/artifacts, but must leave DEV apply, +rollout, live verification, and lock ownership to the host commander. + The previous verified manual path for `hwlab-cloud-web` was: ```sh diff --git a/docs/reference/dev-runtime-boundary.md b/docs/reference/dev-runtime-boundary.md index b34310fb..3988c849 100644 --- a/docs/reference/dev-runtime-boundary.md +++ b/docs/reference/dev-runtime-boundary.md @@ -196,6 +196,39 @@ runtime durability blocker, not a DB connectivity blocker, and must keep M3, M4, and M5 full acceptance blocked until the durable runtime postconditions below are true. +## DEV DB Provisioning Automation + +The repo-owned DEV DB provisioning entrypoint is: + +```sh +node scripts/dev-runtime-provisioning.mjs --check +node scripts/dev-runtime-provisioning.mjs --dry-run --allow-live-db-read --confirm-dev --write-report +``` + +The default `--check` path validates the redacted target DB URL contract only: +the application role name, target database name, password presence, SecretRef +names, and endpoint authority are represented as booleans or redacted endpoint +class. It never connects to Postgres, never reads Kubernetes Secret data, and +never prints the target role, database name, host, password, token, DSN, or +kubeconfig material. + +An authorized DEV provisioning apply is: + +```sh +node scripts/dev-runtime-provisioning.mjs --apply --confirm-dev --confirmed-non-production --write-report +``` + +The apply path uses `HWLAB_CLOUD_DB_URL` from +`hwlab-cloud-api-dev-db/database-url` as the redacted target contract and +`HWLAB_CLOUD_DB_ADMIN_URL` from +`hwlab-cloud-api-dev-db-admin/admin-url` as the admin SecretRef env. It may +create or update only the DEV target role, target database, database `CONNECT` +grant, and `public` schema `USAGE, CREATE` grants needed by the cloud-api +durable runtime. It must report `created` or `existed` booleans and structured +blockers such as role missing, database missing, SSL, auth, schema, migration, +or durability readiness. It must not read Secret objects, print Secret values, +run PROD changes, use manual `psql`, or perform schema migration. + ## Runtime Migration Automation The repo-owned runtime migration entrypoint is: @@ -240,6 +273,24 @@ Runtime migration reports separate the blockers as: | Migration | The required source migration is recorded in the ledger. | | Readiness | Durable runtime read readiness passed after schema and ledger checks. | +## Durable Runtime Postflight + +The repo-owned postflight entrypoint is: + +```sh +node scripts/dev-runtime-postflight.mjs --check +node scripts/dev-runtime-postflight.mjs --live --confirm-dev --confirmed-non-production --target api --write-report +``` + +Default `--check` is source-only. Live mode first reads +`http://74.48.78.17:16667/health/live` and +`http://74.48.78.17:16667/v1`. It only performs the M3 +`DO1=true -> DI1=true -> DO1=false -> DI1=false` postflight when both +endpoints prove `runtime.adapter="postgres"`, `runtime.durable=true`, +`runtime.ready=true`, and `runtime.liveRuntimeEvidence=true`. The M3 portion +must then prove operation, trace, audit, and evidence identifiers with durable +green evidence; otherwise the report remains blocked. + ## Durable Runtime Unblock Runbook This runbook is a reusable checklist for a future authorized live @@ -252,8 +303,10 @@ authorizes the DEV apply command and the preconditions below are already true. | Source migration contract | Yes | `node scripts/dev-runtime-migration.mjs --check` and `node scripts/dev-runtime-migration.mjs --dry-run --write-report` | DB connection, Secret read, DEV/PROD write | | Static runtime boundary contract | Yes | `node scripts/validate-runtime-boundary.mjs`, source docs, manifests, and redacted Secret refs | Live mutation, service restart, Secret value output | | Public health observation | Yes, when command scope is read-only | `/health/live` fields such as `db.liveDbEvidence`, `runtime.adapter`, `runtime.durable`, `runtime.blocker`, `readiness.durability.blockedLayer`, `requiredEvidence`, and redaction/safety flags | Printing DB URLs, passwords, tokens, kubeconfig material, or Secret data | +| Live provisioning apply | No, unless explicitly authorized by commander/operator | `node scripts/dev-runtime-provisioning.mjs --apply --confirm-dev --confirmed-non-production --write-report` with role/database created/existed booleans and no secret output | PROD apply, Secret value output, manual DB shell, schema migration, service restart, M3 acceptance promotion | | Live DB read verification | Only with explicit DEV read authorization | `--dry-run --allow-live-db-read --confirm-dev --write-report` output with redacted endpoint and `secretValuesPrinted=false` | Kubernetes Secret reads, writes, migration apply, PROD target | | Live migration/repair apply | No, unless explicitly authorized by commander/operator | `--apply --confirm-dev --confirmed-non-production --write-report` plus post-apply health evidence | PROD apply, Secret value output, service restart, Code Agent POST, hardware write, or acceptance promotion | +| Durable runtime postflight | Only after durable runtime readiness is already green | `node scripts/dev-runtime-postflight.mjs --live --confirm-dev --confirmed-non-production --target api --write-report` with `/health/live`, `/v1`, and M3 true/false durable green evidence | Running M3 writes while runtime durable readiness is blocked, Secret output, PROD, or manual rollout | Secret handling rules are the same for every row: verify env names, `secretKeyRef` names/keys, configured provider/model/base URL, redacted endpoint @@ -271,6 +324,9 @@ be true: - Source validation passes for `internal/db/migrations/0001_cloud_core_skeleton.sql`, the durable runtime tables/columns, and the `hwlab_schema_migrations` ledger row. +- DEV DB provisioning is ready: target role and database are either already + present or created by `scripts/dev-runtime-provisioning.mjs`, and the report + carries only created/existed booleans plus redacted endpoint class. - The runtime is requesting the Postgres durable adapter, not silently using memory: `runtime.adapter="postgres"` or equivalent selected-adapter evidence, with `runtime.durableRequested=true`. diff --git a/package.json b/package.json index 57246c05..a1279487 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "private": true, "type": "module", "scripts": { - "validate": "node scripts/validate-contract.mjs && node scripts/deploy-contract-plan.mjs --check && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-migration.mjs --check && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --test scripts/artifact-runtime-readiness-guard.test.mjs", - "check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check internal/agent/runtime.mjs && node --check internal/audit/index.mjs && node --check internal/db/runtime-store.mjs && node --check internal/db/runtime-store.test.mjs && node --check internal/mvp-gate/summary.mjs && node --check internal/cloud/db-contract.mjs && node --check internal/dev-entrypoint/http.mjs && node --check internal/dev-entrypoint/http.test.mjs && node --check internal/cloud/code-agent-contract.mjs && node --check internal/cloud/code-agent-session-registry.mjs && node --check internal/cloud/code-agent-session-registry.test.mjs && node --check internal/cloud/json-rpc.mjs && node --check internal/cloud/code-agent-chat.mjs && node --check internal/cloud/m3-io-control.mjs && node --check internal/cloud/server.mjs && node --check internal/sim/model.mjs && node --check internal/sim/model.test.mjs && node --check internal/sim/http.mjs && node --check internal/sim/l2-runtime.mjs && node --check internal/patchpanel/model.mjs && node --check internal/patchpanel/runtime.mjs && node --check cmd/hwlab-cloud-api/main.mjs && node --check cmd/hwlab-edge-proxy/main.mjs && node --check cmd/hwlab-agent-mgr/main.mjs && node --check cmd/hwlab-agent-worker/main.mjs && node --check cmd/hwlab-box-simu/main.mjs && node --check cmd/hwlab-gateway-simu/main.mjs && node --check scripts/cloud-api-runtime-smoke.mjs && node --check scripts/code-agent-chat-smoke.mjs && node --check scripts/dev-edge-health-smoke.mjs && node --check scripts/src/dev-edge-health-smoke-lib.mjs && node --check scripts/validate-contract.mjs && node --check scripts/deploy-contract-plan.test.mjs && node --check scripts/deploy-desired-state-plan.mjs && node --check scripts/src/deploy-desired-state-plan.mjs && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --check scripts/artifact-runtime-readiness-guard.test.mjs && node --check scripts/report-lifecycle.mjs && node --check scripts/report-lifecycle.test.mjs && node --check scripts/validate-dev-gate-report.mjs && node --check scripts/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.test.mjs && node --check scripts/dev-m3-hardware-loop-smoke.test.mjs && node --check scripts/m3-io-control-e2e.mjs && node --check scripts/src/m3-io-control-e2e.mjs && node --check scripts/m3-io-control-e2e.test.mjs && node --check scripts/dev-cloud-workbench-smoke.test.mjs && node --check scripts/dev-cloud-workbench-layout-smoke.mjs && node --check scripts/validate-dev-m3-cardinality.mjs && node --check scripts/validate-dev-m3-cardinality.test.mjs && node --check scripts/validate-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/src/dev-artifact-services.mjs && node --check scripts/src/registry-capabilities.mjs && node --check scripts/preflight-dev-base-image.mjs && node --check scripts/src/dev-base-image-preflight.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.test.mjs && node --check scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs && node --check scripts/d601-k3s-readonly-observability.mjs && node --check scripts/src/d601-k3s-readonly-observability.mjs && node --check scripts/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.test.mjs && node --check scripts/l2-runtime-contract-smoke.mjs && node --check scripts/patch-panel-runtime-smoke.mjs && node --check scripts/export-web-gate-summary.mjs && node --check scripts/l6-cli-web-smoke.mjs && node --check tools/hwlab-cli/bin/hwlab-cli.mjs && node --check tools/hwlab-cli/lib/cli.mjs && node --check web/hwlab-cloud-web/app.mjs && node --check web/hwlab-cloud-web/gate-summary.mjs && node --check web/hwlab-cloud-web/scripts/check.mjs && node --check web/hwlab-cloud-web/scripts/build.mjs && node --check web/hwlab-cloud-web/scripts/dist-contract.mjs && node scripts/validate-contract.mjs && node scripts/validate-dev-gate-report.mjs && node scripts/report-lifecycle.test.mjs && node scripts/validate-dev-m3-cardinality.mjs && node scripts/validate-artifact-catalog.mjs && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-migration.mjs --check && node scripts/dev-evidence-blocker-aggregator.mjs --check && node scripts/l2-runtime-contract-smoke.mjs && node scripts/l6-cli-web-smoke.mjs && node web/hwlab-cloud-web/scripts/check.mjs && node scripts/code-agent-chat-smoke.mjs && node --test scripts/artifact-runtime-readiness-guard.test.mjs scripts/deploy-contract-plan.test.mjs scripts/dev-m3-hardware-loop-smoke.test.mjs scripts/validate-dev-m3-cardinality.test.mjs scripts/dev-cloud-workbench-smoke.test.mjs scripts/deploy-desired-state-plan.test.mjs scripts/src/dev-cd-apply.test.mjs scripts/src/dev-deploy-apply.test.mjs scripts/src/dev-runtime-migration.test.mjs scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs scripts/src/dev-evidence-blocker-aggregator.test.mjs internal/agent/index.test.mjs internal/audit/index.test.mjs internal/db/schema.test.mjs internal/db/runtime-store.test.mjs internal/cloud/json-rpc.test.mjs internal/cloud/m3-io-control.test.mjs internal/cloud/code-agent-session-registry.test.mjs internal/cloud/server.test.mjs internal/dev-entrypoint/http.test.mjs internal/sim/model.test.mjs internal/patchpanel/model.test.mjs internal/patchpanel/runtime.test.mjs && node scripts/cloud-api-runtime-smoke.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", + "validate": "node scripts/validate-contract.mjs && node scripts/deploy-contract-plan.mjs --check && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-provisioning.mjs --check && node scripts/dev-runtime-migration.mjs --check && node scripts/dev-runtime-postflight.mjs --check && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --test scripts/artifact-runtime-readiness-guard.test.mjs", + "check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check internal/agent/runtime.mjs && node --check internal/audit/index.mjs && node --check internal/db/runtime-store.mjs && node --check internal/db/runtime-store.test.mjs && node --check internal/mvp-gate/summary.mjs && node --check internal/cloud/db-contract.mjs && node --check internal/dev-entrypoint/http.mjs && node --check internal/dev-entrypoint/http.test.mjs && node --check internal/cloud/code-agent-contract.mjs && node --check internal/cloud/code-agent-session-registry.mjs && node --check internal/cloud/code-agent-session-registry.test.mjs && node --check internal/cloud/json-rpc.mjs && node --check internal/cloud/code-agent-chat.mjs && node --check internal/cloud/m3-io-control.mjs && node --check internal/cloud/server.mjs && node --check internal/sim/model.mjs && node --check internal/sim/model.test.mjs && node --check internal/sim/http.mjs && node --check internal/sim/l2-runtime.mjs && node --check internal/patchpanel/model.mjs && node --check internal/patchpanel/runtime.mjs && node --check cmd/hwlab-cloud-api/main.mjs && node --check cmd/hwlab-edge-proxy/main.mjs && node --check cmd/hwlab-agent-mgr/main.mjs && node --check cmd/hwlab-agent-worker/main.mjs && node --check cmd/hwlab-box-simu/main.mjs && node --check cmd/hwlab-gateway-simu/main.mjs && node --check scripts/cloud-api-runtime-smoke.mjs && node --check scripts/code-agent-chat-smoke.mjs && node --check scripts/dev-edge-health-smoke.mjs && node --check scripts/src/dev-edge-health-smoke-lib.mjs && node --check scripts/validate-contract.mjs && node --check scripts/deploy-contract-plan.test.mjs && node --check scripts/deploy-desired-state-plan.mjs && node --check scripts/src/deploy-desired-state-plan.mjs && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --check scripts/artifact-runtime-readiness-guard.test.mjs && node --check scripts/report-lifecycle.mjs && node --check scripts/report-lifecycle.test.mjs && node --check scripts/validate-dev-gate-report.mjs && node --check scripts/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.test.mjs && node --check scripts/dev-m3-hardware-loop-smoke.test.mjs && node --check scripts/m3-io-control-e2e.mjs && node --check scripts/src/m3-io-control-e2e.mjs && node --check scripts/m3-io-control-e2e.test.mjs && node --check scripts/dev-cloud-workbench-smoke.test.mjs && node --check scripts/dev-cloud-workbench-layout-smoke.mjs && node --check scripts/validate-dev-m3-cardinality.mjs && node --check scripts/validate-dev-m3-cardinality.test.mjs && node --check scripts/validate-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/src/dev-artifact-services.mjs && node --check scripts/src/registry-capabilities.mjs && node --check scripts/preflight-dev-base-image.mjs && node --check scripts/src/dev-base-image-preflight.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.test.mjs && node --check scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs && node --check scripts/d601-k3s-readonly-observability.mjs && node --check scripts/src/d601-k3s-readonly-observability.mjs && node --check scripts/dev-runtime-provisioning.mjs && node --check scripts/src/dev-runtime-provisioning.mjs && node --check scripts/src/dev-runtime-provisioning.test.mjs && node --check scripts/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.test.mjs && node --check scripts/dev-runtime-postflight.mjs && node --check scripts/src/dev-runtime-postflight.mjs && node --check scripts/src/dev-runtime-postflight.test.mjs && node --check scripts/l2-runtime-contract-smoke.mjs && node --check scripts/patch-panel-runtime-smoke.mjs && node --check scripts/export-web-gate-summary.mjs && node --check scripts/l6-cli-web-smoke.mjs && node --check tools/hwlab-cli/bin/hwlab-cli.mjs && node --check tools/hwlab-cli/lib/cli.mjs && node --check web/hwlab-cloud-web/app.mjs && node --check web/hwlab-cloud-web/gate-summary.mjs && node --check web/hwlab-cloud-web/scripts/check.mjs && node --check web/hwlab-cloud-web/scripts/build.mjs && node --check web/hwlab-cloud-web/scripts/dist-contract.mjs && node scripts/validate-contract.mjs && node scripts/validate-dev-gate-report.mjs && node scripts/report-lifecycle.test.mjs && node scripts/validate-dev-m3-cardinality.mjs && node scripts/validate-artifact-catalog.mjs && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-provisioning.mjs --check && node scripts/dev-runtime-migration.mjs --check && node scripts/dev-runtime-postflight.mjs --check && node scripts/dev-evidence-blocker-aggregator.mjs --check && node scripts/l2-runtime-contract-smoke.mjs && node scripts/l6-cli-web-smoke.mjs && node web/hwlab-cloud-web/scripts/check.mjs && node scripts/code-agent-chat-smoke.mjs && node --test scripts/artifact-runtime-readiness-guard.test.mjs scripts/deploy-contract-plan.test.mjs scripts/dev-m3-hardware-loop-smoke.test.mjs scripts/validate-dev-m3-cardinality.test.mjs scripts/dev-cloud-workbench-smoke.test.mjs scripts/deploy-desired-state-plan.test.mjs scripts/src/dev-cd-apply.test.mjs scripts/src/dev-deploy-apply.test.mjs scripts/src/dev-runtime-provisioning.test.mjs scripts/src/dev-runtime-migration.test.mjs scripts/src/dev-runtime-postflight.test.mjs scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs scripts/src/dev-evidence-blocker-aggregator.test.mjs internal/agent/index.test.mjs internal/audit/index.test.mjs internal/db/schema.test.mjs internal/db/runtime-store.test.mjs internal/cloud/json-rpc.test.mjs internal/cloud/m3-io-control.test.mjs internal/cloud/code-agent-session-registry.test.mjs internal/cloud/server.test.mjs internal/dev-entrypoint/http.test.mjs internal/sim/model.test.mjs internal/patchpanel/model.test.mjs internal/patchpanel/runtime.test.mjs && node scripts/cloud-api-runtime-smoke.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", "dev-base-image:preflight": "node scripts/preflight-dev-base-image.mjs", "cloud-api:smoke": "node scripts/cloud-api-runtime-smoke.mjs", "m1:smoke": "node scripts/m1-contract-smoke.mjs", @@ -35,7 +35,9 @@ "d601:k3s:readonly": "node scripts/d601-k3s-readonly-observability.mjs", "runner:issue-visibility:preflight": "node scripts/runner-issue-visibility-preflight.mjs", "report:lifecycle:test": "node scripts/report-lifecycle.test.mjs", + "dev-runtime:provisioning": "node scripts/dev-runtime-provisioning.mjs", "dev-runtime:migration": "node scripts/dev-runtime-migration.mjs", + "dev-runtime:postflight": "node scripts/dev-runtime-postflight.mjs", "m3:io:e2e": "node scripts/m3-io-control-e2e.mjs", "web:layout": "node scripts/dev-cloud-workbench-layout-smoke.mjs --static --report reports/dev-gate/dev-cloud-workbench-layout.json", "web:layout:build": "node scripts/dev-cloud-workbench-layout-smoke.mjs --build --report reports/dev-gate/dev-cloud-workbench-layout-build.json", diff --git a/scripts/dev-artifact-publish.mjs b/scripts/dev-artifact-publish.mjs index ac6bc0fa..634b61bb 100644 --- a/scripts/dev-artifact-publish.mjs +++ b/scripts/dev-artifact-publish.mjs @@ -569,6 +569,7 @@ function dockerfile(baseImage, port) { "RUN if [ -f package-lock.json ]; then npm ci --omit=dev --ignore-scripts; else npm install --omit=dev --ignore-scripts; fi", "COPY internal ./internal", "COPY cmd ./cmd", + "COPY scripts ./scripts", "COPY web ./web", "COPY tools ./tools", "COPY skills ./skills", diff --git a/scripts/dev-runtime-postflight.mjs b/scripts/dev-runtime-postflight.mjs new file mode 100644 index 00000000..e244312e --- /dev/null +++ b/scripts/dev-runtime-postflight.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +import { + formatDevRuntimePostflightFailure, + runDevRuntimePostflightCli +} from "./src/dev-runtime-postflight.mjs"; + +try { + process.exitCode = await runDevRuntimePostflightCli(process.argv.slice(2)); +} catch (error) { + process.stdout.write(`${JSON.stringify(formatDevRuntimePostflightFailure(error), null, 2)}\n`); + process.exitCode = 1; +} diff --git a/scripts/dev-runtime-provisioning.mjs b/scripts/dev-runtime-provisioning.mjs new file mode 100644 index 00000000..cb349e44 --- /dev/null +++ b/scripts/dev-runtime-provisioning.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +import { + formatDevRuntimeProvisioningFailure, + runDevRuntimeProvisioningCli +} from "./src/dev-runtime-provisioning.mjs"; + +try { + process.exitCode = await runDevRuntimeProvisioningCli(process.argv.slice(2)); +} catch (error) { + process.stdout.write(`${JSON.stringify(formatDevRuntimeProvisioningFailure(error), null, 2)}\n`); + process.exitCode = 1; +} diff --git a/scripts/src/dev-cd-apply.mjs b/scripts/src/dev-cd-apply.mjs index c6e9e5ef..38bcfabd 100644 --- a/scripts/src/dev-cd-apply.mjs +++ b/scripts/src/dev-cd-apply.mjs @@ -19,6 +19,10 @@ const defaultLockName = "hwlab-dev-cd-lock"; const defaultTtlSeconds = 3600; const defaultReportPath = "reports/dev-gate/dev-cd-apply.json"; const artifactReportPath = "reports/dev-gate/dev-artifacts.json"; +const runtimeProvisioningReportPath = "reports/dev-gate/dev-runtime-provisioning-report.json"; +const runtimeMigrationReportPath = "reports/dev-gate/dev-runtime-migration-report.json"; +const runtimePostflightReportPath = "reports/dev-gate/dev-runtime-postflight-report.json"; +const runtimeJobReportPrefix = "/tmp/hwlab-dev-runtime-report"; const browserLiveUrl = "http://74.48.78.17:16666/health/live"; const apiLiveUrl = `${DEV_ENDPOINT}/health/live`; const lockAnnotationPrefix = "hwlab.pikastech.local"; @@ -1042,6 +1046,213 @@ async function runStep(ctx, transaction, step) { }; } +async function runRuntimeK8sJobStep(ctx, transaction, step) { + const startedAt = ctx.now().toISOString(); + const result = await runRuntimeK8sJob(ctx, transaction, step); + const finishedAt = ctx.now().toISOString(); + return { + id: step.id, + phase: step.phase, + status: result.status, + command: result.command, + code: result.status === "pass" ? 0 : 1, + startedAt, + finishedAt, + stdoutJson: result.report, + stdoutTail: null, + stderrTail: result.error ?? "", + reportPath: step.reportPath ?? null, + k8sJob: result.k8sJob + }; +} + +async function runRuntimeK8sJob(ctx, transaction, step) { + const jobName = `${step.jobNamePrefix}-${transaction.transactionId.slice(0, 8)}`; + const reportFile = `${runtimeJobReportPrefix}-${step.id}.json`; + const image = step.image ?? await resolveRuntimeJobImage(ctx); + const manifest = buildRuntimeK8sJobManifest({ + jobName, + namespace: transaction.targetNamespace, + image, + commandArgs: [...step.commandArgs, "--report", reportFile], + transaction + }); + const kubectl = transaction.kubectlContext; + const apply = await kubectlCommandResult(ctx, kubectl, ["-n", transaction.targetNamespace, "apply", "-f", "-"], { + input: JSON.stringify(manifest), + timeoutMs: 30000 + }); + if (apply.code !== 0) { + return runtimeJobFailure(step, "apply", apply, jobName, image); + } + + const wait = await kubectlCommandResult(ctx, kubectl, [ + "-n", + transaction.targetNamespace, + "wait", + "--for=condition=complete", + `job/${jobName}`, + "--timeout=300s" + ], { timeoutMs: step.timeoutMs ?? 6 * 60 * 1000 }); + if (wait.code !== 0) { + return runtimeJobFailure(step, "wait", wait, jobName, image); + } + + const logs = await kubectlCommandResult(ctx, kubectl, ["-n", transaction.targetNamespace, "logs", `job/${jobName}`], { + timeoutMs: 30000 + }); + const logText = redactSensitiveText(`${logs.stdout}\n${logs.stderr}`); + const report = parseLastJsonObject(logText); + if (logs.code !== 0 || !report) { + return { + status: "blocked", + command: `${shellCommand("kubectl", ["-n", transaction.targetNamespace, "logs", `job/${jobName}`])}`, + error: logText.trim().slice(-2000) || "runtime job did not emit a JSON report", + report: null, + k8sJob: runtimeJobSummary(jobName, step, "logs", image) + }; + } + const reportStatus = report.conclusion?.status ?? report.summary?.status ?? report.status; + return { + status: ["ready", "pass"].includes(reportStatus) ? "pass" : "blocked", + command: shellCommand("kubectl", ["-n", transaction.targetNamespace, "apply/wait/logs", `job/${jobName}`]), + error: null, + report, + k8sJob: runtimeJobSummary(jobName, step, "complete", image) + }; +} + +function runtimeJobFailure(step, phase, result, jobName, image) { + return { + status: "blocked", + command: result.command ?? shellCommand("kubectl", [phase, `job/${jobName}`]), + error: redactSensitiveText(`${result.stderr}\n${result.stdout}`).trim().slice(-2000), + report: null, + k8sJob: runtimeJobSummary(jobName, step, phase, image) + }; +} + +function runtimeJobSummary(jobName, step, phase, image = step.image ?? "deploy-current:hwlab-cloud-api") { + return { + jobName, + phase, + serviceId: "hwlab-cloud-api", + image, + secretRefsOnly: true, + secretValuesRead: false, + secretValuesPrinted: false + }; +} + +async function resolveRuntimeJobImage(ctx) { + const deploy = await readDeployJson(ctx.repoRoot); + const image = deploy.manifest?.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.image; + if (typeof image === "string" && image.trim().length > 0) { + return image; + } + if (typeof deploy.commitId === "string" && deploy.commitId !== "unknown") { + return `127.0.0.1:5000/hwlab/hwlab-cloud-api:${shortCommit(deploy.commitId)}`; + } + throw new DevCdApplyError("could not resolve hwlab-cloud-api image for runtime maintenance Job", { + code: "runtime-job-image-unresolved" + }); +} + +function buildRuntimeK8sJobManifest({ jobName, namespace, image, commandArgs, transaction }) { + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: jobName, + namespace, + labels: { + "app.kubernetes.io/part-of": "hwlab", + "app.kubernetes.io/name": "hwlab-runtime-maintenance", + "hwlab.pikastech.local/profile": "dev", + "hwlab.pikastech.local/service-id": "hwlab-cloud-api", + "hwlab.pikastech.local/cd-transaction": transaction.transactionId + } + }, + spec: { + backoffLimit: 0, + ttlSecondsAfterFinished: 600, + template: { + metadata: { + labels: { + "app.kubernetes.io/name": "hwlab-runtime-maintenance", + "hwlab.pikastech.local/service-id": "hwlab-cloud-api", + "hwlab.pikastech.local/cd-transaction": transaction.transactionId + } + }, + spec: { + restartPolicy: "Never", + containers: [ + { + name: "runtime-maintenance", + image, + command: ["node"], + args: commandArgs, + env: [ + { name: "HWLAB_ENVIRONMENT", value: ENVIRONMENT_DEV }, + { name: "HWLAB_CLOUD_RUNTIME_ADAPTER", value: "postgres" }, + { name: "HWLAB_CLOUD_RUNTIME_DURABLE", value: "true" }, + { + name: "HWLAB_CLOUD_DB_URL", + valueFrom: { + secretKeyRef: { + name: "hwlab-cloud-api-dev-db", + key: "database-url", + optional: false + } + } + }, + { + name: "HWLAB_CLOUD_DB_ADMIN_URL", + valueFrom: { + secretKeyRef: { + name: "hwlab-cloud-api-dev-db-admin", + key: "admin-url", + optional: false + } + } + }, + { name: "HWLAB_CLOUD_DB_SSL_MODE", value: "disable" }, + { name: "HWLAB_CD_TRANSACTION_ID", value: transaction.transactionId }, + { name: "HWLAB_CD_TRANSACTION_OWNER", value: transaction.ownerTaskId }, + { name: "HWLAB_CD_LOCK_NAME", value: transaction.lockName } + ] + } + ] + } + } + } + }; +} + +async function kubectlCommandResult(ctx, kubectlContext, args, options = {}) { + const result = await ctx.runCommand(kubectlContext.executor, args, { + cwd: ctx.repoRoot, + env: kubectlContext.env, + input: options.input, + timeoutMs: options.timeoutMs ?? 30000 + }); + return { + ...result, + command: shellCommand("kubectl", args), + stdout: result.stdout ?? "", + stderr: result.stderr ?? "" + }; +} + +function parseLastJsonObject(text) { + const lines = String(text ?? "").trim().split(/\r?\n/u).reverse(); + for (const line of lines) { + const parsed = parseJsonMaybe(line.trim()); + if (parsed && typeof parsed === "object") return parsed; + } + return parseJsonMaybe(text); +} + function stepBlocker(stepResult) { return { type: "runtime_blocker", @@ -1478,8 +1689,9 @@ function buildReport({ "HEAD must match the requested target ref before publish.", "Lease/hwlab-dev-cd-lock must be acquired before publish/apply.", "Legacy publish/apply side-effect scripts must run with HWLAB_CD_TRANSACTION_ID.", + "DEV DB role/database provisioning and runtime migration must run through repo-owned commands before workload apply.", "deploy/deploy.json, artifact catalog, and workloads must converge before apply.", - "Public 16666 and 16667 live health must be recorded in this report." + "Public 16666 and 16667 live health plus M3 durable postflight must be recorded in this report." ], summary: blockers.length ? "One or more DEV CD transaction preconditions failed." : "DEV CD transaction preconditions passed." }, @@ -1520,11 +1732,14 @@ function buildReport({ steps, liveBefore, liveVerify, - reportPaths: { - transaction: args.writeReport ? args.reportPath : null, - artifacts: artifactReportPath, - deployApply: "reports/dev-gate/dev-deploy-report.json" - }, + reportPaths: { + transaction: args.writeReport ? args.reportPath : null, + artifacts: artifactReportPath, + deployApply: "reports/dev-gate/dev-deploy-report.json", + runtimeProvisioning: runtimeProvisioningReportPath, + runtimeMigration: runtimeMigrationReportPath, + runtimePostflight: runtimePostflightReportPath + }, safety: { prodTouched: false, secretValuesRead: false, @@ -1607,7 +1822,8 @@ export async function runDevCdApply(argv, io = {}) { ownerTaskId: args.ownerTaskId ?? defaultOwnerTaskId(env), targetNamespace: args.targetNamespace, lockName: args.lockName, - phases: [] + phases: [], + kubectlContext: null }; const steps = []; const blockers = []; @@ -1624,6 +1840,7 @@ export async function runDevCdApply(argv, io = {}) { target = await resolveTargetRef(ctx, args.targetRef); deployBefore = await readDeployJson(ctx.repoRoot); const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand); + transaction.kubectlContext = kubectlContext; liveBefore = { status: "pending", reason: "liveBefore is captured immediately after Lease acquisition so publish/apply cannot race before the transaction lock." @@ -1708,6 +1925,38 @@ export async function runDevCdApply(argv, io = {}) { ], timeoutMs: 120000 }, + { + id: "runtime-db-provisioning", + phase: "applying", + kind: "runtime-k8s-job", + jobNamePrefix: "hwlab-runtime-provision", + commandArgs: [ + "scripts/dev-runtime-provisioning.mjs", + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--write-report", + "--fail-on-blocked" + ], + timeoutMs: 5 * 60 * 1000, + reportPath: runtimeProvisioningReportPath + }, + { + id: "runtime-db-migration", + phase: "applying", + kind: "runtime-k8s-job", + jobNamePrefix: "hwlab-runtime-migrate", + commandArgs: [ + "scripts/dev-runtime-migration.mjs", + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--write-report", + "--fail-on-blocked" + ], + timeoutMs: 5 * 60 * 1000, + reportPath: runtimeMigrationReportPath + }, { id: "dev-deploy-apply", phase: "applying", @@ -1722,23 +1971,42 @@ export async function runDevCdApply(argv, io = {}) { ], timeoutMs: 20 * 60 * 1000, reportPath: "reports/dev-gate/dev-deploy-report.json" + }, + { + id: "runtime-durable-postflight", + phase: "verifying", + command: process.execPath, + args: [ + "scripts/dev-runtime-postflight.mjs", + "--live", + "--confirm-dev", + "--confirmed-non-production", + "--target", + "api", + "--write-report", + "--fail-on-blocked" + ], + timeoutMs: 2 * 60 * 1000, + reportPath: runtimePostflightReportPath } ]; if (blockers.length === 0) { for (const step of plannedSteps) { - if (step.phase === "applying" && lockState.lock?.phase !== "applying") { + if (step.phase !== lockState.lock?.phase) { await updateDeployLockPhase({ ctx, args, kubectlContext, transactionId: transaction.transactionId, lockState, - phase: "applying" + phase: step.phase }); - transaction.phases.push({ phase: "applying", status: "entered", at: ctx.now().toISOString() }); + transaction.phases.push({ phase: step.phase, status: "entered", at: ctx.now().toISOString() }); } - const result = await runStep(ctx, transaction, step); + const result = step.kind === "runtime-k8s-job" + ? await runRuntimeK8sJobStep(ctx, transaction, step) + : await runStep(ctx, transaction, step); steps.push(result); if (result.status !== "pass") { blockers.push(stepBlocker(result)); @@ -1748,15 +2016,17 @@ export async function runDevCdApply(argv, io = {}) { } if (blockers.length === 0) { - await updateDeployLockPhase({ - ctx, - args, - kubectlContext, - transactionId: transaction.transactionId, - lockState, - phase: "verifying" - }); - transaction.phases.push({ phase: "verifying", status: "entered", at: ctx.now().toISOString() }); + if (lockState.lock?.phase !== "verifying") { + await updateDeployLockPhase({ + ctx, + args, + kubectlContext, + transactionId: transaction.transactionId, + lockState, + phase: "verifying" + }); + transaction.phases.push({ phase: "verifying", status: "entered", at: ctx.now().toISOString() }); + } deployAfter = await readDeployJson(ctx.repoRoot); liveVerify = args.skipLiveVerify ? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } } diff --git a/scripts/src/dev-cd-apply.test.mjs b/scripts/src/dev-cd-apply.test.mjs index 04d36ce4..e47216f4 100644 --- a/scripts/src/dev-cd-apply.test.mjs +++ b/scripts/src/dev-cd-apply.test.mjs @@ -142,6 +142,34 @@ function makeRunCommand({ heldLock = null, commandLog = [], psOutput = "", killL lease.metadata.resourceVersion = String(Number(lease.metadata.resourceVersion ?? "2") + 1); return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" }; } + if (command.includes("kubectl") && args.includes("apply") && args.includes("-f") && args.includes("-")) { + const manifest = JSON.parse(options.input); + assert.equal(manifest.kind, "Job"); + assert.equal(manifest.metadata.namespace, "hwlab-dev"); + assert.ok(manifest.metadata.name.startsWith("hwlab-runtime-")); + assert.equal(manifest.spec.template.spec.containers[0].env.some((entry) => entry.name === "HWLAB_CLOUD_DB_URL" && entry.valueFrom?.secretKeyRef?.name === "hwlab-cloud-api-dev-db"), true); + assert.equal(manifest.spec.template.spec.containers[0].env.some((entry) => entry.name === "HWLAB_CLOUD_DB_ADMIN_URL" && entry.valueFrom?.secretKeyRef?.name === "hwlab-cloud-api-dev-db-admin"), true); + assert.equal(JSON.stringify(manifest).includes("database-url"), true); + assert.equal(JSON.stringify(manifest).includes("postgresql://"), false); + return { code: 0, stdout: JSON.stringify(manifest), stderr: "" }; + } + if (command.includes("kubectl") && args.includes("wait") && args.some((arg) => String(arg).startsWith("job/hwlab-runtime-"))) { + return { code: 0, stdout: "job.batch/hwlab-runtime complete\n", stderr: "" }; + } + if (command.includes("kubectl") && args.includes("logs") && args.some((arg) => String(arg).startsWith("job/hwlab-runtime-provision"))) { + return { + code: 0, + stdout: `${JSON.stringify({ conclusion: { status: "ready" }, safety: { secretValuesPrinted: false } }, null, 2)}\n`, + stderr: "" + }; + } + if (command.includes("kubectl") && args.includes("logs") && args.some((arg) => String(arg).startsWith("job/hwlab-runtime-migrate"))) { + return { + code: 0, + stdout: `${JSON.stringify({ conclusion: { status: "ready" }, safety: { secretValuesPrinted: false } }, null, 2)}\n`, + stderr: "" + }; + } if (command.includes("kubectl") && args.includes("patch")) { assert.ok(lease, "lease must exist before patch"); const patch = parsePatch(args); @@ -394,12 +422,34 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-web-16666"), true); assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-api-16667"), true); assert.equal(report.devCdApply.reportPaths.transaction, "reports/dev-gate/dev-cd-apply.json"); + assert.equal(report.devCdApply.reportPaths.runtimeProvisioning, "reports/dev-gate/dev-runtime-provisioning-report.json"); + assert.equal(report.devCdApply.reportPaths.runtimeMigration, "reports/dev-gate/dev-runtime-migration-report.json"); + assert.equal(report.devCdApply.reportPaths.runtimePostflight, "reports/dev-gate/dev-runtime-postflight-report.json"); const publishCall = commandLog.find((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")); const applyCall = commandLog.find((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")); + const provisioningCall = commandLog.find((entry) => entry.args.includes("scripts/dev-runtime-provisioning.mjs")); + const migrationCall = commandLog.find((entry) => entry.args.includes("scripts/dev-runtime-migration.mjs")); + const postflightCall = commandLog.find((entry) => entry.args.includes("scripts/dev-runtime-postflight.mjs")); const refreshCall = commandLog.find((entry) => entry.args.includes("scripts/refresh-artifact-catalog.mjs")); assert.ok(publishCall?.env.HWLAB_CD_TRANSACTION_ID); assert.equal(applyCall?.env.HWLAB_CD_TRANSACTION_ID, publishCall.env.HWLAB_CD_TRANSACTION_ID); + const provisioningJobApply = commandLog.find((entry) => + entry.command.includes("kubectl") && + entry.args.includes("apply") && + entry.input.includes("hwlab-runtime-provision") + ); + const migrationJobApply = commandLog.find((entry) => + entry.command.includes("kubectl") && + entry.args.includes("apply") && + entry.input.includes("hwlab-runtime-migrate") + ); + assert.ok(provisioningJobApply); + assert.ok(migrationJobApply); + assert.equal(JSON.parse(provisioningJobApply.input).spec.template.spec.containers[0].args[0], "scripts/dev-runtime-provisioning.mjs"); + assert.equal(JSON.parse(migrationJobApply.input).spec.template.spec.containers[0].args[0], "scripts/dev-runtime-migration.mjs"); + assert.equal(postflightCall?.env.HWLAB_CD_TRANSACTION_ID, publishCall.env.HWLAB_CD_TRANSACTION_ID); + assert.deepEqual(postflightCall.args.slice(0, 4), ["scripts/dev-runtime-postflight.mjs", "--live", "--confirm-dev", "--confirmed-non-production"]); assert.equal(refreshCall.args[refreshCall.args.indexOf("--target-ref") + 1], "abc1234abc1234abc1234abc1234abc1234abc1"); const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8")); diff --git a/scripts/src/dev-runtime-postflight.mjs b/scripts/src/dev-runtime-postflight.mjs new file mode 100644 index 00000000..50944414 --- /dev/null +++ b/scripts/src/dev-runtime-postflight.mjs @@ -0,0 +1,492 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; +import { + buildM3IoControlSourceReport, + runM3IoControlLiveReport +} from "./m3-io-control-e2e.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const defaultReportPath = "reports/dev-gate/dev-runtime-postflight-report.json"; +const issue = "pikasTech/HWLAB#311"; +const defaultHealthUrl = `${DEV_ENDPOINT}/health/live`; +const defaultV1Url = `${DEV_ENDPOINT}/v1`; + +export async function runDevRuntimePostflightCli(argv = process.argv.slice(2), options = {}) { + const args = parseArgs(argv); + if (args.help) { + process.stdout.write(`${usage()}\n`); + return 0; + } + + const report = await buildDevRuntimePostflightReport(args, options); + if (args.writeReport) { + await writeReport(report, args.reportPath, options.repoRoot ?? repoRoot); + } + + const printable = args.pretty || args.writeReport ? report : summarizeReport(report); + process.stdout.write(`${JSON.stringify(printable, null, 2)}\n`); + return report.conclusion.status === "pass" || !args.failOnBlocked ? 0 : 1; +} + +export async function buildDevRuntimePostflightReport(args = {}, options = {}) { + const parsed = normalizeArgs(args); + const now = options.now ?? (() => new Date().toISOString()); + const report = { + issue, + mode: parsed.live ? "dev-live" : "check", + generatedAt: now(), + target: { + environment: ENVIRONMENT_DEV, + namespace: "hwlab-dev", + prodAllowed: false, + apiHealthUrl: defaultHealthUrl, + apiV1Url: defaultV1Url, + m3Target: parsed.target + }, + actions: { + healthLiveReadAttempted: false, + v1ReadAttempted: false, + m3MutationAttempted: false, + m3MutationAllowed: false + }, + apiHealth: null, + apiV1: null, + m3: null, + blockers: [], + safety: buildSafety(parsed), + safetyRefusal: false + }; + + if (!parsed.live) { + const sourceReport = await (options.m3SourceBuilder ?? buildM3IoControlSourceReport)({ + repoRoot: options.repoRoot ?? repoRoot + }); + report.m3 = summarizeM3Report(sourceReport); + report.conclusion = { + status: sourceReport.summary?.status === "pass" ? "pass" : "blocked", + summary: "DEV runtime postflight source contract checked only; no live HTTP or M3 write was attempted.", + blockerCount: sourceReport.summary?.status === "pass" ? 0 : 1 + }; + if (sourceReport.summary?.status !== "pass") { + addBlocker(report, "contract_blocker", "m3-source-contract", "M3 postflight source contract is blocked.", { + classification: sourceReport.summary?.classification ?? "unknown" + }); + } + return report; + } + + if (!parsed.confirmDev || !parsed.confirmedNonProduction) { + addSafetyRefusal(report, "DEV runtime postflight live mode requires --live --confirm-dev --confirmed-non-production."); + return finalizeReport(report); + } + + const httpGetJson = options.httpGetJson ?? defaultHttpGetJson; + report.actions.healthLiveReadAttempted = true; + report.actions.v1ReadAttempted = true; + const [health, v1] = await Promise.all([ + httpGetJson(defaultHealthUrl, parsed.timeoutMs), + httpGetJson(defaultV1Url, parsed.timeoutMs) + ]); + report.apiHealth = summarizeApiPayload("GET /health/live", health); + report.apiV1 = summarizeApiPayload("GET /v1", v1); + + addApiReadinessBlockers(report); + if (report.blockers.length === 0) { + report.actions.m3MutationAllowed = true; + report.actions.m3MutationAttempted = true; + const m3Runner = options.m3Runner ?? defaultM3Runner; + report.m3 = summarizeM3Report(await m3Runner(parsed)); + addM3Blockers(report); + } else { + report.m3 = { + status: "not_run", + trustedGreen: false, + classification: "runtime_durable_postflight_precondition_blocked", + operationCount: 0, + reason: "M3 true/false write/read postflight is skipped until /health/live and /v1 prove durable runtime readiness." + }; + } + + return finalizeReport(report); +} + +export function parseArgs(argv = []) { + const args = { + live: false, + confirmDev: false, + confirmedNonProduction: false, + writeReport: false, + reportPath: defaultReportPath, + pretty: false, + failOnBlocked: false, + target: "api", + timeoutMs: 8000, + help: false + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--check") args.live = false; + else if (arg === "--live") args.live = true; + else if (arg === "--confirm-dev") args.confirmDev = true; + else if (arg === "--confirmed-non-production") args.confirmedNonProduction = true; + else if (arg === "--write-report") args.writeReport = true; + else if (arg === "--pretty") args.pretty = true; + else if (arg === "--fail-on-blocked") args.failOnBlocked = true; + else if (arg === "--target") { + index += 1; + args.target = requireArgValue(argv[index], "--target"); + } else if (arg === "--timeout-ms") { + index += 1; + args.timeoutMs = parseTimeout(requireArgValue(argv[index], "--timeout-ms")); + } else if (arg === "--report") { + index += 1; + args.reportPath = requireArgValue(argv[index], "--report"); + } else if (arg.startsWith("--report=")) { + args.reportPath = requireArgValue(arg.slice("--report=".length), "--report"); + } else if (arg === "--help" || arg === "-h") { + args.help = true; + } else { + throw new Error(`unknown argument: ${arg}`); + } + } + + if (!["api", "frontend"].includes(args.target)) { + throw new Error("--target must be api or frontend"); + } + return args; +} + +function normalizeArgs(args) { + return { + ...parseArgs([]), + ...args + }; +} + +async function defaultM3Runner(args) { + const sourceReport = await buildM3IoControlSourceReport({ repoRoot }); + return runM3IoControlLiveReport({ + flags: new Set(["--live", "--confirm-dev", "--confirmed-non-production"]), + live: true, + confirmDev: true, + confirmedNonProduction: true, + target: args.target, + frontendUrl: "http://74.48.78.17:16666/", + apiUrl: `${DEV_ENDPOINT}/v1/m3/io`, + timeoutMs: args.timeoutMs + }, sourceReport); +} + +async function defaultHttpGetJson(url, timeoutMs) { + const controller = new AbortController(); + const timer = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { + method: "GET", + headers: { + accept: "application/json" + }, + signal: controller.signal + }); + const text = await response.text(); + let json = null; + try { + json = text ? JSON.parse(text) : null; + } catch { + json = null; + } + return { + ok: response.ok, + status: response.status, + json, + error: response.ok ? null : json?.error?.message ?? response.statusText + }; + } catch (error) { + return { + ok: false, + status: 0, + json: null, + error: error.name === "AbortError" ? `request timed out after ${timeoutMs}ms` : error.message + }; + } finally { + clearTimeout(timer); + } +} + +function summarizeApiPayload(check, response) { + const json = response.json ?? {}; + const runtime = summarizeRuntime(json.runtime); + return { + check, + httpStatus: response.status, + reachable: response.ok === true && Boolean(response.json), + serviceId: json.serviceId ?? json.service?.id ?? null, + environment: json.environment ?? null, + status: json.status ?? null, + ready: json.ready === true, + runtime, + db: summarizeDb(json.db), + readiness: { + ready: json.readiness?.ready === true, + status: json.readiness?.status ?? null, + durabilityReady: json.readiness?.durability?.ready === true, + durabilityBlockedLayer: json.readiness?.durability?.blockedLayer ?? json.runtime?.durabilityContract?.blockedLayer ?? null + }, + blockerCodes: Array.isArray(json.blockerCodes) ? json.blockerCodes : [], + m3IoControl: json.m3IoControl + ? { + route: json.m3IoControl.route ?? null, + enabled: json.m3IoControl.enabled === true, + contractVersion: json.m3IoControl.contractVersion ?? null + } + : null, + error: response.error ?? null + }; +} + +function summarizeRuntime(runtime = {}) { + return { + adapter: runtime?.adapter ?? "unknown", + durable: runtime?.durable === true, + durableRequested: runtime?.durableRequested === true, + ready: runtime?.ready === true, + status: runtime?.status ?? null, + blocker: runtime?.blocker ?? null, + liveRuntimeEvidence: runtime?.liveRuntimeEvidence === true, + requiredEvidence: runtime?.durabilityContract?.requiredEvidence ?? null, + blockedLayer: runtime?.durabilityContract?.blockedLayer ?? null, + gates: summarizeRuntimeGates(runtime?.gates) + }; +} + +function summarizeRuntimeGates(gates = {}) { + return Object.fromEntries(["ssl", "auth", "schema", "migration", "durability"].map((name) => [ + name, + { + checked: gates?.[name]?.checked === true, + ready: gates?.[name]?.ready === true, + status: gates?.[name]?.status ?? "unknown", + blocker: gates?.[name]?.blocker ?? null + } + ])); +} + +function summarizeDb(db = {}) { + return { + ready: db?.ready === true, + connected: db?.connected === true, + liveDbEvidence: db?.liveDbEvidence === true, + endpointSource: db?.endpointSource ?? null, + connectionResult: db?.connectionResult ?? db?.connection?.result ?? null, + endpointRedacted: true, + valueRedacted: true, + runtimeReadiness: { + ready: db?.runtimeReadiness?.ready === true, + status: db?.runtimeReadiness?.status ?? null, + blocker: db?.runtimeReadiness?.blocker ?? null, + queryResult: db?.runtimeReadiness?.queryResult ?? null, + requiredEvidence: db?.runtimeReadiness?.requiredEvidence ?? null + } + }; +} + +function summarizeM3Report(report = {}) { + const operations = Array.isArray(report.liveOperations) ? report.liveOperations : []; + return { + mode: report.mode ?? "unknown", + status: report.summary?.status ?? report.status ?? "unknown", + classification: report.summary?.classification ?? null, + trustedGreen: report.summary?.trustedGreen === true, + result: report.summary?.result ?? null, + operationCount: operations.length, + operations: operations.map((operation) => ({ + id: operation.id, + action: operation.action, + status: operation.status, + operationId: operation.operationId ?? null, + traceId: operation.traceId ?? null, + auditId: operation.auditId ?? null, + evidenceId: operation.evidenceId ?? null, + resultValue: operation.resultValue, + evidenceState: { + status: operation.evidenceState?.status ?? null, + durable: operation.evidenceState?.durable === true, + sourceKind: operation.evidenceState?.sourceKind ?? null + }, + blocker: operation.blocker ?? null + })) + }; +} + +function addApiReadinessBlockers(report) { + for (const [scope, payload] of [["api-health-live", report.apiHealth], ["api-v1", report.apiV1]]) { + if (!payload.reachable) { + addBlocker(report, "runtime_blocker", scope, `${payload.check} was not reachable.`, { + httpStatus: payload.httpStatus, + error: payload.error + }); + continue; + } + if (!runtimeDurableReady(payload.runtime)) { + addBlocker(report, "runtime_blocker", scope, `${payload.check} did not prove durable runtime readiness.`, { + runtime: payload.runtime, + blockerCodes: payload.blockerCodes + }); + } + } +} + +function addM3Blockers(report) { + if (report.m3?.trustedGreen !== true || report.m3?.status !== "pass") { + addBlocker(report, "runtime_blocker", "m3-durable-postflight", "M3 true/false durable postflight did not produce trusted green persisted evidence.", { + classification: report.m3?.classification ?? "unknown", + trustedGreen: report.m3?.trustedGreen === true, + operationCount: report.m3?.operationCount ?? 0 + }); + return; + } + + for (const operation of report.m3.operations ?? []) { + if (!operation.operationId || !operation.auditId || !operation.evidenceId || operation.evidenceState?.status !== "green" || operation.evidenceState?.durable !== true) { + addBlocker(report, "runtime_blocker", "m3-durable-evidence", "M3 operation/audit/evidence durable fields were incomplete.", { + operationIdPresent: Boolean(operation.operationId), + auditIdPresent: Boolean(operation.auditId), + evidenceIdPresent: Boolean(operation.evidenceId), + evidenceState: operation.evidenceState + }); + return; + } + } +} + +function runtimeDurableReady(runtime) { + return runtime.adapter === "postgres" && + runtime.durable === true && + runtime.ready === true && + runtime.liveRuntimeEvidence === true; +} + +function addSafetyRefusal(report, summary) { + report.safetyRefusal = true; + addBlocker(report, "safety_refusal", "runtime-postflight-live-boundary", summary, { + devOnly: true, + prodAllowed: false, + secretValuesPrinted: false + }); +} + +function addBlocker(report, type, scope, summary, evidence = {}) { + report.blockers.push({ + type, + scope, + status: "open", + summary, + sourceIssue: issue, + evidence + }); +} + +function finalizeReport(report) { + const openBlockers = report.blockers.filter((blocker) => blocker.status === "open"); + report.conclusion = { + status: openBlockers.length === 0 ? "pass" : "blocked", + summary: openBlockers.length === 0 + ? "DEV runtime postflight passed: /health/live, /v1, and M3 true/false durable evidence are green." + : "DEV runtime postflight is blocked; M3 is skipped unless durable runtime readiness is proven first.", + blockerCount: openBlockers.length + }; + return report; +} + +function buildSafety(args) { + return { + devOnly: true, + environment: ENVIRONMENT_DEV, + prodAllowed: false, + liveHttpReads: args.live === true, + liveM3Writes: args.live === true, + liveM3WritesRequireDurableRuntimeReady: true, + readsKubernetesSecrets: false, + writesKubernetesSecrets: false, + secretValuesPrinted: false, + dbUrlValueRedacted: true, + endpointRedacted: true + }; +} + +function summarizeReport(report) { + return { + issue: report.issue, + mode: report.mode, + conclusion: report.conclusion, + actions: report.actions, + apiHealth: report.apiHealth, + apiV1: report.apiV1, + m3: report.m3, + blockers: report.blockers, + safety: report.safety + }; +} + +async function writeReport(report, reportPath, root) { + const absolute = path.resolve(root, reportPath); + await mkdir(path.dirname(absolute), { recursive: true }); + await writeFile(absolute, `${JSON.stringify(report, null, 2)}\n`); +} + +function parseTimeout(value) { + const parsed = Number.parseInt(value, 10); + if (!Number.isInteger(parsed) || parsed <= 0) { + throw new Error("--timeout-ms must be a positive integer"); + } + return Math.min(parsed, 30000); +} + +function requireArgValue(value, flag) { + if (typeof value !== "string" || value.trim() === "") { + throw new Error(`${flag} requires a value`); + } + return value; +} + +function usage() { + return [ + "Usage: node scripts/dev-runtime-postflight.mjs [--check|--live]", + "", + "Default --check validates source M3 postflight contracts only.", + "--live requires --confirm-dev --confirmed-non-production and reads /health/live plus /v1 before any M3 write.", + "M3 true/false write/read is skipped unless durable runtime readiness is already green.", + "Reports never print DB URLs, passwords, tokens, Secret values, or kubeconfig material." + ].join("\n"); +} + +export function formatDevRuntimePostflightFailure(error) { + const message = redactFailureText(error instanceof Error ? error.message : String(error)); + return { + issue, + conclusion: { + status: "blocked", + summary: "DEV runtime postflight command failed before producing a report." + }, + error: message, + safety: { + devOnly: true, + prodAllowed: false, + secretValuesPrinted: false + }, + trace: createHash("sha256").update(message).digest("hex").slice(0, 12) + }; +} + +function redactFailureText(value) { + return String(value ?? "") + .replace(/postgres(?:ql)?:\/\/[^\s"'<>]+/giu, "[redacted-postgres-url]") + .replace(/(password\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]") + .replace(/(token\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]") + .replace(/(kubeconfig\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]"); +} diff --git a/scripts/src/dev-runtime-postflight.test.mjs b/scripts/src/dev-runtime-postflight.test.mjs new file mode 100644 index 00000000..53e737ed --- /dev/null +++ b/scripts/src/dev-runtime-postflight.test.mjs @@ -0,0 +1,202 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + buildDevRuntimePostflightReport, + parseArgs +} from "./dev-runtime-postflight.mjs"; + +test("check mode is source-only and does not attempt live reads or M3 writes", async () => { + const report = await buildDevRuntimePostflightReport(parseArgs(["--check"]), { + m3SourceBuilder: async () => ({ + mode: "source-static", + summary: { + status: "pass", + classification: "source_static_contract_checked", + trustedGreen: false, + result: "source checked" + } + }), + now: () => "2026-05-23T00:00:00.000Z" + }); + + assert.equal(report.conclusion.status, "pass"); + assert.equal(report.actions.healthLiveReadAttempted, false); + assert.equal(report.actions.v1ReadAttempted, false); + assert.equal(report.actions.m3MutationAttempted, false); + assert.equal(report.safety.secretValuesPrinted, false); +}); + +test("live mode refuses without explicit DEV confirmations", async () => { + const report = await buildDevRuntimePostflightReport(parseArgs(["--live"]), { + now: () => "2026-05-23T00:00:00.000Z" + }); + + assert.equal(report.conclusion.status, "blocked"); + assert.equal(report.safetyRefusal, true); + assert.equal(report.actions.healthLiveReadAttempted, false); + assert.equal(report.actions.m3MutationAttempted, false); +}); + +test("live mode skips M3 writes when /health/live durable runtime is blocked", async () => { + const report = await buildDevRuntimePostflightReport( + parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]), + { + httpGetJson: async (url) => ({ ok: true, status: 200, json: apiPayload({ ready: url.endsWith("/v1") }) }), + m3Runner: async () => { + throw new Error("M3 runner should not run before durable readiness is green"); + }, + now: () => "2026-05-23T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "blocked"); + assert.equal(report.actions.healthLiveReadAttempted, true); + assert.equal(report.actions.v1ReadAttempted, true); + assert.equal(report.actions.m3MutationAttempted, false); + assert.equal(report.m3.status, "not_run"); + assert.equal(report.blockers.some((blocker) => blocker.scope === "api-health-live"), true); + assert.equal(JSON.stringify(report).includes("postgresql://"), false); +}); + +test("live mode requires /v1 durable readiness before M3 writes", async () => { + const report = await buildDevRuntimePostflightReport( + parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]), + { + httpGetJson: async (url) => ({ ok: true, status: 200, json: apiPayload({ ready: !url.endsWith("/v1") }) }), + m3Runner: async () => { + throw new Error("M3 runner should not run when /v1 is blocked"); + }, + now: () => "2026-05-23T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "blocked"); + assert.equal(report.actions.m3MutationAttempted, false); + assert.equal(report.blockers.some((blocker) => blocker.scope === "api-v1"), true); +}); + +test("live mode records M3 true/false durable green evidence", async () => { + const report = await buildDevRuntimePostflightReport( + parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]), + { + httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: true }) }), + m3Runner: async () => m3GreenReport(), + now: () => "2026-05-23T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "pass"); + assert.equal(report.actions.m3MutationAttempted, true); + assert.equal(report.actions.m3MutationAllowed, true); + assert.equal(report.m3.trustedGreen, true); + assert.equal(report.m3.operationCount, 4); + assert.deepEqual(report.m3.operations.map((operation) => operation.resultValue), [true, true, false, false]); + assert.equal(report.blockers.length, 0); +}); + +test("live mode blocks when M3 operation evidence is not durable green", async () => { + const m3 = m3GreenReport(); + m3.liveOperations[1].evidenceState.durable = false; + const report = await buildDevRuntimePostflightReport( + parseArgs(["--live", "--confirm-dev", "--confirmed-non-production"]), + { + httpGetJson: async () => ({ ok: true, status: 200, json: apiPayload({ ready: true }) }), + m3Runner: async () => m3, + now: () => "2026-05-23T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "blocked"); + assert.equal(report.blockers.some((blocker) => blocker.scope === "m3-durable-evidence"), true); +}); + +function apiPayload({ ready }) { + return { + serviceId: "hwlab-cloud-api", + environment: "dev", + status: ready ? "ready" : "blocked", + ready, + db: { + ready, + connected: ready, + liveDbEvidence: ready, + endpointSource: "secret-url-host", + connectionResult: ready ? "connected" : "connected", + runtimeReadiness: { + ready, + status: ready ? "ready" : "blocked", + blocker: ready ? null : "runtime_durable_adapter_auth_blocked", + queryResult: ready ? "durable_readiness_ready" : "auth_blocked", + requiredEvidence: "runtime_adapter_schema_migration_read_query" + } + }, + runtime: { + adapter: "postgres", + durable: ready, + durableRequested: true, + ready, + status: ready ? "ready" : "blocked", + blocker: ready ? null : "runtime_durable_adapter_auth_blocked", + liveRuntimeEvidence: ready, + durabilityContract: { + requiredEvidence: "runtime_adapter_schema_migration_read_query", + blockedLayer: ready ? null : "auth" + }, + gates: { + ssl: gate(true), + auth: gate(ready, "runtime_durable_adapter_auth_blocked"), + schema: gate(ready), + migration: gate(ready), + durability: gate(ready) + } + }, + readiness: { + ready, + status: ready ? "ready" : "blocked", + durability: { + ready, + blockedLayer: ready ? null : "auth" + } + }, + blockerCodes: ready ? [] + : ["runtime_durable_adapter_auth_blocked"] + }; +} + +function gate(ready, blocker = null) { + return { + checked: true, + ready, + status: ready ? "ready" : "blocked", + blocker: ready ? null : blocker + }; +} + +function m3GreenReport() { + const values = [true, true, false, false]; + return { + mode: "dev-live", + summary: { + status: "pass", + classification: "trusted_green", + trustedGreen: true, + result: "DEV-LIVE M3 IO control path and durable trusted records are green." + }, + liveOperations: values.map((value, index) => ({ + id: ["write-do1-true", "read-di1-true", "write-do1-false", "read-di1-false"][index], + action: index % 2 === 0 ? "do.write" : "di.read", + status: "succeeded", + operationId: `op_${index}`, + traceId: `trc_${index}`, + auditId: `aud_${index}`, + evidenceId: `ev_${index}`, + resultValue: value, + evidenceState: { + status: "green", + durable: true, + sourceKind: "DEV-LIVE" + } + })) + }; +} diff --git a/scripts/src/dev-runtime-provisioning.mjs b/scripts/src/dev-runtime-provisioning.mjs new file mode 100644 index 00000000..0253662e --- /dev/null +++ b/scripts/src/dev-runtime-provisioning.mjs @@ -0,0 +1,928 @@ +import { createHash } from "node:crypto"; +import { mkdir, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { DEV_DB_ENV_CONTRACT } from "../../internal/cloud/db-contract.mjs"; +import { + PostgresCloudRuntimeStore, + RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, + RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING, + RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, + RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, + RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED, + buildPostgresPoolConfig +} from "../../internal/db/runtime-store.mjs"; +import { ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const defaultReportPath = "reports/dev-gate/dev-runtime-provisioning-report.json"; +const issue = "pikasTech/HWLAB#311"; + +export const DEV_DB_ADMIN_SECRET_REF = Object.freeze({ + env: "HWLAB_CLOUD_DB_ADMIN_URL", + secretName: "hwlab-cloud-api-dev-db-admin", + secretKey: "admin-url" +}); + +const appSecretRef = DEV_DB_ENV_CONTRACT.secretRefs[0]; +const systemDatabases = new Set(["postgres", "template0", "template1"]); + +export async function runDevRuntimeProvisioningCli(argv = process.argv.slice(2), options = {}) { + const args = parseArgs(argv); + if (args.help) { + process.stdout.write(`${usage()}\n`); + return 0; + } + + const report = await buildDevRuntimeProvisioningReport(args, options); + if (args.writeReport) { + await writeReport(report, args.reportPath, options.repoRoot ?? repoRoot); + } + + const printable = args.pretty || args.writeReport ? report : summarizeReport(report); + process.stdout.write(`${JSON.stringify(printable, null, 2)}\n`); + + if (report.safetyRefusal || (args.failOnBlocked && report.conclusion.status !== "ready")) { + return 1; + } + return 0; +} + +export async function buildDevRuntimeProvisioningReport(args = {}, options = {}) { + const parsed = normalizeArgs(args); + const env = options.env ?? process.env; + const now = options.now ?? (() => new Date().toISOString()); + const target = parseProvisioningTarget(env.HWLAB_CLOUD_DB_URL); + const admin = summarizeAdminEnv(env); + const report = { + issue, + mode: parsed.mode, + generatedAt: now(), + target: { + environment: ENVIRONMENT_DEV, + namespace: DEV_DB_ENV_CONTRACT.dns.namespace, + prodAllowed: false, + dbUrlEnv: appSecretRef.env, + dbUrlSecretRef: appSecretRef, + adminUrlEnv: DEV_DB_ADMIN_SECRET_REF.env, + adminUrlSecretRef: DEV_DB_ADMIN_SECRET_REF, + dbEndpointAuthority: DEV_DB_ENV_CONTRACT.endpointAuthority.source + }, + db: summarizeDbEnv(env, target, admin), + applyBoundary: buildApplyBoundary(parsed), + safety: buildSafety(parsed), + actions: { + liveDbReadAttempted: false, + liveDbWriteAttempted: false, + adminInspectionAttempted: false, + adminWriteAttempted: false, + appRuntimeReadinessAttempted: false, + roleCreated: false, + roleExisted: false, + rolePasswordSynchronized: false, + databaseCreated: false, + databaseExisted: false, + databaseConnectGranted: false, + schemaPrivilegesGranted: false, + readinessVerified: false + }, + provisioning: { + targetRole: emptyObjectState(target.roleNamePresent), + targetDatabase: emptyObjectState(target.databaseNamePresent), + privileges: { + checked: false, + ready: false, + databaseConnectGranted: false, + schemaPrivilegesGranted: false + }, + ready: false + }, + runtime: null, + gates: sourceOnlyGates(target), + downstreamRuntimeGates: null, + blockers: [], + safetyRefusal: false + }; + + validateSourceContract(report, target, parsed); + + if (parsed.mode === "check" || (parsed.mode === "dry-run" && !parsed.allowLiveDbRead)) { + return finalizeReport(report); + } + + if (parsed.mode === "dry-run") { + if (!parsed.confirmDev) { + addSafetyRefusal(report, "DEV DB provisioning live dry-run requires --confirm-dev."); + return finalizeReport(report); + } + if (!target.ok) { + return finalizeReport(report); + } + await verifyRuntimeReadiness(report, env, options); + if (admin.present || options.adminClient) { + await inspectAdminProvisioning(report, env, target, options); + } else { + inferProvisioningFromRuntime(report); + if (!report.provisioning.ready) { + addBlocker(report, "runtime_blocker", "runtime-provisioning-admin", "Role/database inspection requires the DEV admin SecretRef env when runtime readiness is not already green.", { + adminUrlEnv: DEV_DB_ADMIN_SECRET_REF.env, + adminUrlSecretRef: DEV_DB_ADMIN_SECRET_REF, + valueRedacted: true, + secretValuesPrinted: false + }); + } + } + addRuntimeProvisioningBlocker(report); + return finalizeReport(report); + } + + if (parsed.mode === "apply") { + if (!parsed.confirmDev || !parsed.confirmedNonProduction) { + addSafetyRefusal(report, "DEV DB provisioning apply requires --confirm-dev and --confirmed-non-production."); + return finalizeReport(report); + } + if (!target.ok) { + return finalizeReport(report); + } + if (!admin.present && !options.adminClient) { + addBlocker(report, "runtime_blocker", "runtime-provisioning-admin", "DEV DB provisioning apply requires the admin DB URL from its SecretRef env.", { + adminUrlEnv: DEV_DB_ADMIN_SECRET_REF.env, + adminUrlSecretRef: DEV_DB_ADMIN_SECRET_REF, + valueRedacted: true, + secretValuesPrinted: false + }); + return finalizeReport(report); + } + await applyProvisioning(report, env, target, options); + if (report.actions.adminWriteAttempted && report.blockers.length === 0) { + await verifyRuntimeReadiness(report, env, options); + inferProvisioningFromRuntime(report); + addRuntimeProvisioningBlocker(report); + } + return finalizeReport(report); + } + + addSafetyRefusal(report, `Unsupported DEV DB provisioning mode: ${parsed.mode}`); + return finalizeReport(report); +} + +export function parseArgs(argv = []) { + const args = { + mode: "check", + allowLiveDbRead: false, + confirmDev: false, + confirmedNonProduction: false, + writeReport: false, + reportPath: defaultReportPath, + pretty: false, + failOnBlocked: false, + help: false + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--check") args.mode = "check"; + else if (arg === "--dry-run") args.mode = "dry-run"; + else if (arg === "--apply") args.mode = "apply"; + else if (arg === "--allow-live-db-read") args.allowLiveDbRead = true; + else if (arg === "--confirm-dev") args.confirmDev = true; + else if (arg === "--confirmed-non-production") args.confirmedNonProduction = true; + else if (arg === "--write-report") args.writeReport = true; + else if (arg === "--pretty") args.pretty = true; + else if (arg === "--fail-on-blocked") args.failOnBlocked = true; + else if (arg === "--help" || arg === "-h") args.help = true; + else if (arg === "--report") { + index += 1; + args.reportPath = requireArgValue(argv[index], "--report"); + } else if (arg.startsWith("--report=")) { + args.reportPath = requireArgValue(arg.slice("--report=".length), "--report"); + } else { + throw new Error(`unknown argument: ${arg}`); + } + } + + return args; +} + +export function parseProvisioningTarget(rawUrl) { + const present = typeof rawUrl === "string" && rawUrl.trim().length > 0; + if (!present) { + return { + ok: false, + present: false, + parseStatus: "missing", + errorCode: "TARGET_DB_URL_MISSING", + roleNamePresent: false, + databaseNamePresent: false, + passwordPresent: false + }; + } + + let url; + try { + url = new URL(rawUrl); + } catch { + return { + ok: false, + present: true, + parseStatus: "invalid_url", + errorCode: "TARGET_DB_URL_INVALID", + roleNamePresent: false, + databaseNamePresent: false, + passwordPresent: false + }; + } + + const protocolReady = ["postgres:", "postgresql:"].includes(url.protocol); + const roleName = decodeURIComponent(url.username || ""); + const password = decodeURIComponent(url.password || ""); + const databaseName = decodeURIComponent(url.pathname.replace(/^\/+/u, "")); + const databaseAllowed = Boolean(databaseName) && !systemDatabases.has(databaseName.toLowerCase()); + const ok = protocolReady && Boolean(roleName) && Boolean(password) && databaseAllowed; + return { + ok, + present: true, + parseStatus: ok ? "ready" : "blocked", + errorCode: ok ? null : targetParseErrorCode({ protocolReady, roleName, password, databaseName, databaseAllowed }), + roleNamePresent: Boolean(roleName), + databaseNamePresent: Boolean(databaseName), + databaseAllowed, + passwordPresent: Boolean(password), + endpointClass: DEV_DB_ENV_CONTRACT.endpointAuthority.source, + roleName, + databaseName, + password + }; +} + +function normalizeArgs(args) { + return { + ...parseArgs([]), + ...args + }; +} + +function targetParseErrorCode({ protocolReady, roleName, password, databaseName, databaseAllowed }) { + if (!protocolReady) return "TARGET_DB_URL_PROTOCOL_BLOCKED"; + if (!roleName) return "TARGET_DB_URL_ROLE_MISSING"; + if (!password) return "TARGET_DB_URL_PASSWORD_MISSING"; + if (!databaseName) return "TARGET_DB_URL_DATABASE_MISSING"; + if (!databaseAllowed) return "TARGET_DB_URL_DATABASE_FORBIDDEN"; + return "TARGET_DB_URL_BLOCKED"; +} + +function summarizeAdminEnv(env) { + return { + present: typeof env?.[DEV_DB_ADMIN_SECRET_REF.env] === "string" && env[DEV_DB_ADMIN_SECRET_REF.env].trim().length > 0, + env: DEV_DB_ADMIN_SECRET_REF.env, + secretRef: DEV_DB_ADMIN_SECRET_REF, + valueRedacted: true + }; +} + +function summarizeDbEnv(env, target, admin) { + return { + targetUrlPresent: target.present, + targetUrlValueRedacted: true, + targetEndpointRedacted: true, + targetEndpointClass: target.endpointClass ?? DEV_DB_ENV_CONTRACT.endpointAuthority.source, + targetParseStatus: target.parseStatus, + targetErrorCode: target.errorCode, + targetRoleNamePresent: target.roleNamePresent, + targetDatabaseNamePresent: target.databaseNamePresent, + targetDatabaseAllowed: target.databaseAllowed === true, + targetPasswordPresent: target.passwordPresent, + sslMode: env?.HWLAB_CLOUD_DB_SSL_MODE || DEV_DB_ENV_CONTRACT.nonSecretDefaults.HWLAB_CLOUD_DB_SSL_MODE, + sslModeSecret: false, + adminUrlPresent: admin.present, + adminUrlValueRedacted: true, + adminEndpointRedacted: true, + dbUrlSecretRef: appSecretRef, + adminUrlSecretRef: DEV_DB_ADMIN_SECRET_REF, + endpointAuthority: DEV_DB_ENV_CONTRACT.endpointAuthority.source + }; +} + +function validateSourceContract(report, target, args) { + if (target.ok) return; + if (args.mode === "check" && target.present === false) return; + addBlocker(report, "runtime_blocker", "runtime-provisioning-target", "DEV DB provisioning requires a valid target DB URL from the cloud-api SecretRef env.", { + env: appSecretRef.env, + secretRef: appSecretRef, + errorCode: target.errorCode, + valueRedacted: true, + endpointRedacted: true, + secretValuesPrinted: false + }); +} + +async function verifyRuntimeReadiness(report, env, options) { + report.actions.liveDbReadAttempted = true; + report.actions.appRuntimeReadinessAttempted = true; + const runtimeStore = new PostgresCloudRuntimeStore({ + env, + dbUrl: env?.HWLAB_CLOUD_DB_URL, + sslMode: env?.HWLAB_CLOUD_DB_SSL_MODE, + queryClient: options.queryClient, + pgModuleLoader: options.pgModuleLoader + }); + let runtime; + try { + runtime = await runtimeStore.readiness(); + } finally { + if (!options.queryClient && typeof runtimeStore.pool?.end === "function") { + await runtimeStore.pool.end(); + } + } + report.runtime = summarizeRuntime(runtime); + report.actions.readinessVerified = runtime.ready === true; + report.downstreamRuntimeGates = report.runtime.gates; + report.gates = { + ...report.gates, + ssl: normalizeGate(runtime.gates?.ssl), + auth: normalizeGate(runtime.gates?.auth), + schema: normalizeGate(runtime.gates?.schema), + migration: normalizeGate(runtime.gates?.migration), + durability: normalizeGate(runtime.gates?.durability) + }; +} + +async function inspectAdminProvisioning(report, env, target, options) { + report.actions.liveDbReadAttempted = true; + report.actions.adminInspectionAttempted = true; + const { client, close } = await openAdminClient(env, options); + try { + const [role, database] = await Promise.all([ + objectExists(client, "SELECT 1 AS present FROM pg_catalog.pg_roles WHERE rolname = $1 LIMIT 1", [target.roleName]), + objectExists(client, "SELECT 1 AS present FROM pg_catalog.pg_database WHERE datname = $1 LIMIT 1", [target.databaseName]) + ]); + report.provisioning.targetRole = { + checked: true, + exists: role, + created: false, + ready: role, + targetNamePresent: target.roleNamePresent + }; + report.provisioning.targetDatabase = { + checked: true, + exists: database, + created: false, + ready: database, + targetNamePresent: target.databaseNamePresent + }; + report.actions.roleExisted = role; + report.actions.databaseExisted = database; + report.provisioning.ready = role && database && runtimeAuthSatisfied(report.runtime); + if (!role) addMissingObjectBlocker(report, "role"); + if (!database) addMissingObjectBlocker(report, "database"); + } catch (error) { + addClassifiedAdminBlocker(report, error, "DEV DB provisioning inspection could not verify the target role/database."); + } finally { + await close(); + } +} + +async function applyProvisioning(report, env, target, options) { + report.actions.liveDbWriteAttempted = true; + report.actions.adminWriteAttempted = true; + let admin; + try { + admin = await openAdminClient(env, options); + const client = admin.client; + const roleExists = await objectExists(client, "SELECT 1 AS present FROM pg_catalog.pg_roles WHERE rolname = $1 LIMIT 1", [target.roleName]); + const databaseExists = await objectExists(client, "SELECT 1 AS present FROM pg_catalog.pg_database WHERE datname = $1 LIMIT 1", [target.databaseName]); + report.actions.roleExisted = roleExists; + report.actions.databaseExisted = databaseExists; + + if (!roleExists) { + await client.query(`CREATE ROLE ${quotePgIdent(target.roleName)} LOGIN PASSWORD ${quotePgLiteral(target.password)}`); + report.actions.roleCreated = true; + } else { + await client.query(`ALTER ROLE ${quotePgIdent(target.roleName)} WITH LOGIN PASSWORD ${quotePgLiteral(target.password)}`); + } + report.actions.rolePasswordSynchronized = true; + + if (!databaseExists) { + await client.query(`CREATE DATABASE ${quotePgIdent(target.databaseName)} OWNER ${quotePgIdent(target.roleName)}`); + report.actions.databaseCreated = true; + } + await client.query(`GRANT CONNECT ON DATABASE ${quotePgIdent(target.databaseName)} TO ${quotePgIdent(target.roleName)}`); + report.actions.databaseConnectGranted = true; + } catch (error) { + addClassifiedAdminBlocker(report, error, "DEV DB provisioning apply could not complete role/database provisioning."); + return; + } finally { + if (admin) await admin.close(); + } + + let targetAdmin; + try { + targetAdmin = await openAdminClient(env, { + ...options, + adminDbUrlOverride: withDatabaseInPostgresUrl(env[DEV_DB_ADMIN_SECRET_REF.env], target.databaseName) + }); + await targetAdmin.client.query(`GRANT USAGE, CREATE ON SCHEMA public TO ${quotePgIdent(target.roleName)}`); + report.actions.schemaPrivilegesGranted = true; + } catch (error) { + addClassifiedAdminBlocker(report, error, "DEV DB provisioning apply could not grant target database schema privileges."); + return; + } finally { + if (targetAdmin) await targetAdmin.close(); + } + + report.provisioning.targetRole = { + checked: true, + exists: true, + created: report.actions.roleCreated, + ready: true, + targetNamePresent: true + }; + report.provisioning.targetDatabase = { + checked: true, + exists: true, + created: report.actions.databaseCreated, + ready: true, + targetNamePresent: true + }; + report.provisioning.privileges = { + checked: true, + ready: report.actions.databaseConnectGranted && report.actions.schemaPrivilegesGranted, + databaseConnectGranted: report.actions.databaseConnectGranted, + schemaPrivilegesGranted: report.actions.schemaPrivilegesGranted + }; + report.provisioning.ready = report.provisioning.privileges.ready; +} + +async function openAdminClient(env, options) { + if (options.adminClient) { + return { + client: options.adminClient, + close: async () => {} + }; + } + + const pool = await createPgPool({ + dbUrl: options.adminDbUrlOverride ?? env[DEV_DB_ADMIN_SECRET_REF.env], + sslMode: env.HWLAB_CLOUD_DB_SSL_MODE, + timeoutMs: env.HWLAB_CLOUD_DB_PROBE_TIMEOUT_MS, + pgModuleLoader: options.pgModuleLoader + }); + return { + client: pool, + close: async () => { + await pool.end(); + } + }; +} + +async function createPgPool({ dbUrl, sslMode, timeoutMs, pgModuleLoader }) { + let pg; + try { + pg = await (pgModuleLoader ? pgModuleLoader() : import("pg")); + } catch (error) { + if (error?.code === "ERR_MODULE_NOT_FOUND") { + const driverError = new Error("Postgres provisioning requires the pg package"); + driverError.code = "HWLAB_PG_DRIVER_MISSING"; + throw driverError; + } + throw error; + } + const Pool = pg.Pool ?? pg.default?.Pool; + if (typeof Pool !== "function") { + const driverError = new Error("Postgres provisioning could not load pg.Pool"); + driverError.code = "HWLAB_PG_DRIVER_MISSING"; + throw driverError; + } + return new Pool(buildPostgresPoolConfig({ + dbUrl, + sslMode, + timeoutMs + })); +} + +function inferProvisioningFromRuntime(report) { + const runtimeReady = report.runtime?.ready === true && report.runtime?.durable === true; + const authReady = runtimeAuthSatisfied(report.runtime); + if (!runtimeReady && !authReady) return; + report.provisioning.targetRole = { + ...report.provisioning.targetRole, + checked: report.provisioning.targetRole.checked || false, + exists: true, + ready: true, + inferredFromRuntime: true + }; + report.provisioning.targetDatabase = { + ...report.provisioning.targetDatabase, + checked: report.provisioning.targetDatabase.checked || false, + exists: true, + ready: true, + inferredFromRuntime: true + }; + report.provisioning.ready = true; +} + +function addRuntimeProvisioningBlocker(report) { + const blocker = report.runtime?.blocker; + if (!blocker) return; + if (blocker === RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED) { + addBlocker(report, "runtime_blocker", "runtime-provisioning-ssl", "DEV DB provisioning reached the target runtime adapter, but SSL mode blocked connection.", runtimeEvidence(report.runtime)); + } else if (blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED) { + addBlocker(report, "runtime_blocker", "runtime-provisioning-auth", "DEV DB target role/database exists check did not prove application authentication readiness.", runtimeEvidence(report.runtime)); + } else if (blocker === RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING) { + addBlocker(report, "runtime_blocker", "runtime-provisioning-driver", "Postgres driver is unavailable for DEV DB provisioning verification.", runtimeEvidence(report.runtime)); + } +} + +function addMissingObjectBlocker(report, kind) { + const scope = kind === "role" ? "runtime-provisioning-role" : "runtime-provisioning-database"; + addBlocker(report, "runtime_blocker", scope, `DEV DB target ${kind} is missing according to admin catalog inspection.`, { + exists: false, + created: false, + valueRedacted: true, + endpointRedacted: true, + affectedRuntimeBlocker: RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, + secretValuesPrinted: false + }); +} + +function addClassifiedAdminBlocker(report, error, summary) { + const classified = classifyProvisioningError(error); + addBlocker(report, "runtime_blocker", classified.scope, summary, { + blocker: classified.blocker, + errorCode: classified.errorCode, + valueRedacted: true, + endpointRedacted: true, + secretValuesPrinted: false + }); +} + +function classifyProvisioningError(error) { + const errorCode = typeof error?.code === "string" ? error.code : "UNKNOWN"; + if (errorCode === "HWLAB_PG_DRIVER_MISSING") { + return { + blocker: RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING, + scope: "runtime-provisioning-driver", + errorCode + }; + } + if (isSslError(error)) { + return { + blocker: RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED, + scope: "runtime-provisioning-ssl", + errorCode + }; + } + if (["28P01", "28000", "42501"].includes(errorCode)) { + return { + blocker: RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, + scope: "runtime-provisioning-auth", + errorCode + }; + } + if (["3D000"].includes(errorCode)) { + return { + blocker: RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, + scope: "runtime-provisioning-database", + errorCode + }; + } + if (["42P01", "42703", "3F000"].includes(errorCode)) { + return { + blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, + scope: "runtime-provisioning-schema", + errorCode + }; + } + return { + blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + scope: "runtime-provisioning-query", + errorCode + }; +} + +function runtimeEvidence(runtime) { + return { + blocker: runtime.blocker, + queryResult: runtime.connection?.queryResult ?? "unknown", + errorCode: runtime.connection?.errorCode ?? null, + schemaReady: runtime.schema?.ready === true, + migrationReady: runtime.migration?.ready === true, + durableReady: runtime.ready === true && runtime.durable === true, + secretValuesPrinted: false + }; +} + +async function objectExists(client, sql, params) { + const result = await client.query(sql, params); + return Array.isArray(result?.rows) && result.rows.length > 0; +} + +function sourceOnlyGates(target) { + const sourceReady = target.ok; + const targetMissingInSourceOnly = target.present === false; + return { + ssl: notCheckedGate("source-only validation does not connect to Postgres"), + auth: sourceReady || targetMissingInSourceOnly ? notCheckedGate("auth requires live target runtime verification") : blockedGate("runtime_provisioning_target_blocked"), + role: sourceReady || targetMissingInSourceOnly ? notCheckedGate("role existence requires DEV admin catalog inspection") : blockedGate("runtime_provisioning_target_blocked"), + database: sourceReady || targetMissingInSourceOnly ? notCheckedGate("database existence requires DEV admin catalog inspection") : blockedGate("runtime_provisioning_target_blocked"), + schema: notCheckedGate("schema is verified by runtime migration/readiness after role/database provisioning"), + migration: notCheckedGate("migration ledger is verified by dev-runtime-migration"), + durability: notCheckedGate("durability is verified by post-provisioning runtime readiness and postflight") + }; +} + +function normalizeGate(gate = {}) { + if (gate.ready === true || gate.status === "ready") return readyGate(); + if (gate.status === "blocked") return blockedGate(gate.blocker ?? "runtime_durable_adapter_blocked"); + return notCheckedGate(); +} + +function readyGate(summary = "ready") { + return { + checked: true, + ready: true, + status: "ready", + blocker: null, + summary + }; +} + +function blockedGate(blocker, summary = "blocked") { + return { + checked: true, + ready: false, + status: "blocked", + blocker, + summary + }; +} + +function notCheckedGate(summary = "not checked") { + return { + checked: false, + ready: false, + status: "not_checked", + blocker: null, + summary + }; +} + +function emptyObjectState(targetNamePresent) { + return { + checked: false, + exists: false, + created: false, + ready: false, + targetNamePresent + }; +} + +function summarizeRuntime(runtime = {}) { + return { + adapter: runtime.adapter ?? "unknown", + durable: Boolean(runtime.durable), + durableRequested: Boolean(runtime.durableRequested), + ready: runtime.ready === true, + status: runtime.status ?? "unknown", + blocker: runtime.blocker ?? null, + liveRuntimeEvidence: Boolean(runtime.liveRuntimeEvidence), + fixtureEvidence: Boolean(runtime.fixtureEvidence), + connection: { + queryAttempted: Boolean(runtime.connection?.queryAttempted), + queryResult: runtime.connection?.queryResult ?? "unknown", + endpointRedacted: true, + valueRedacted: true, + errorCode: runtime.connection?.errorCode ?? null + }, + schema: { + checked: Boolean(runtime.schema?.checked), + ready: Boolean(runtime.schema?.ready), + missingTables: Array.isArray(runtime.schema?.missingTables) ? [...runtime.schema.missingTables] : [], + missingColumns: Array.isArray(runtime.schema?.missingColumns) ? [...runtime.schema.missingColumns] : [] + }, + migration: { + checked: Boolean(runtime.migration?.checked), + ready: Boolean(runtime.migration?.ready), + requiredMigrationId: runtime.migration?.requiredMigrationId ?? null, + appliedMigrationId: runtime.migration?.appliedMigrationId ?? null, + missing: runtime.migration?.missing !== false + }, + gates: runtime.gates ?? {} + }; +} + +function runtimeAuthSatisfied(runtime = {}) { + if (runtime.ready === true && runtime.durable === true) return true; + const blocker = runtime.blocker; + return [ + RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, + RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, + RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED + ].includes(blocker) && runtime.connection?.queryAttempted === true; +} + +function buildApplyBoundary(args) { + return { + defaultMode: "check", + mode: args.mode, + liveDbReads: args.mode === "apply" || (args.mode === "dry-run" && args.allowLiveDbRead), + liveDbWrites: args.mode === "apply", + requiresForLiveDbRead: [ + "--confirm-dev", + `${appSecretRef.env} from ${appSecretRef.secretName}/${appSecretRef.secretKey}`, + `${DEV_DB_ADMIN_SECRET_REF.env} from ${DEV_DB_ADMIN_SECRET_REF.secretName}/${DEV_DB_ADMIN_SECRET_REF.secretKey} unless runtime readiness already proves auth` + ], + requiresForApply: [ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + `${appSecretRef.env} from ${appSecretRef.secretName}/${appSecretRef.secretKey}`, + `${DEV_DB_ADMIN_SECRET_REF.env} from ${DEV_DB_ADMIN_SECRET_REF.secretName}/${DEV_DB_ADMIN_SECRET_REF.secretKey}` + ], + writeScope: args.mode === "apply" + ? [ + "DEV Postgres target role named by the redacted cloud-api DB URL", + "DEV Postgres target database named by the redacted cloud-api DB URL", + "DEV target DB CONNECT and public schema CREATE/USAGE grants for that role" + ] + : [], + noWriteScope: [ + "Kubernetes Secret resources", + "PROD", + "service restarts", + "schema migrations", + "M3/M4/M5 acceptance" + ], + forbidden: [ + "printing DB URLs, passwords, tokens, Secret values, or kubeconfig material", + "manual psql or node stdin database writes", + "mutating PROD" + ] + }; +} + +function buildSafety(args) { + return { + devOnly: true, + environment: ENVIRONMENT_DEV, + targetNamespace: DEV_DB_ENV_CONTRACT.dns.namespace, + prodAllowed: false, + sourceStoresSecretValues: false, + printsSecretValues: false, + secretValuesPrinted: false, + dbUrlValueRedacted: true, + adminUrlValueRedacted: true, + endpointRedacted: true, + readsKubernetesSecrets: false, + writesKubernetesSecrets: false, + liveDbWrites: args.mode === "apply", + fixtureEvidenceAllowed: false + }; +} + +function addBlocker(report, type, scope, summary, evidence = {}) { + report.blockers.push({ + type, + scope, + status: "open", + summary, + sourceIssue: scope.includes("auth") ? "pikasTech/HWLAB#307" : issue, + evidence + }); +} + +function addSafetyRefusal(report, summary) { + report.safetyRefusal = true; + addBlocker(report, "safety_refusal", "runtime-provisioning-apply-boundary", summary, { + devOnly: true, + prodAllowed: false, + secretValuesPrinted: false + }); +} + +function finalizeReport(report) { + const openBlockers = report.blockers.filter((blocker) => blocker.status === "open"); + report.provisioning.ready = report.provisioning.ready || ( + report.provisioning.targetRole.ready === true && + report.provisioning.targetDatabase.ready === true && + (report.provisioning.privileges.ready === true || report.mode !== "apply") + ); + report.conclusion = { + status: openBlockers.length === 0 ? "ready" : "blocked", + summary: openBlockers.length === 0 + ? conclusionReadySummary(report) + : "DEV DB provisioning is blocked; see separated role/database/ssl/auth blockers.", + blockerCount: openBlockers.length + }; + return report; +} + +function conclusionReadySummary(report) { + if (report.mode === "apply") { + return "DEV DB role/database provisioning completed without printing secret values."; + } + if (report.mode === "dry-run" && report.actions.liveDbReadAttempted) { + return "DEV DB role/database provisioning read-only verification passed without DB writes or secret output."; + } + return "DEV DB provisioning source contract is ready; no DB connection or live write was attempted."; +} + +function summarizeReport(report) { + return { + issue: report.issue, + mode: report.mode, + conclusion: report.conclusion, + db: report.db, + actions: report.actions, + provisioning: report.provisioning, + gates: report.gates, + downstreamRuntimeGates: report.downstreamRuntimeGates, + blockers: report.blockers, + safety: report.safety + }; +} + +function quotePgIdent(value) { + return `"${String(value).replaceAll('"', '""')}"`; +} + +function quotePgLiteral(value) { + return `'${String(value).replaceAll("'", "''")}'`; +} + +function withDatabaseInPostgresUrl(rawUrl, databaseName) { + const url = new URL(rawUrl); + url.pathname = `/${encodeURIComponent(databaseName)}`; + return url.toString(); +} + +function isSslError(error) { + const code = typeof error?.code === "string" ? error.code : ""; + if (code.startsWith("ERR_SSL") || code.startsWith("ERR_TLS")) return true; + const message = typeof error?.message === "string" ? error.message.toLowerCase() : ""; + return [ + "does not support ssl", + "ssl is not enabled", + "ssl negotiation", + "ssl off", + "ssl required", + "requires ssl", + "no ssl encryption", + "no encryption", + "hostssl", + "server requires ssl", + "tls handshake" + ].some((pattern) => message.includes(pattern)); +} + +async function writeReport(report, reportPath, root) { + const absolute = path.resolve(root, reportPath); + await mkdir(path.dirname(absolute), { recursive: true }); + await writeFile(absolute, `${JSON.stringify(report, null, 2)}\n`); +} + +function requireArgValue(value, flag) { + if (typeof value !== "string" || value.trim() === "") { + throw new Error(`${flag} requires a value`); + } + return value; +} + +function usage() { + return [ + "Usage: node scripts/dev-runtime-provisioning.mjs [--check|--dry-run|--apply]", + "", + "Default --check validates redacted DEV DB provisioning inputs only and never connects to DB.", + "--dry-run is source-only unless --allow-live-db-read --confirm-dev is provided.", + "--apply requires --confirm-dev --confirmed-non-production, HWLAB_CLOUD_DB_URL, and HWLAB_CLOUD_DB_ADMIN_URL.", + "Reports never print DB URLs, passwords, tokens, Secret values, or kubeconfig material." + ].join("\n"); +} + +export function formatDevRuntimeProvisioningFailure(error) { + const message = redactFailureText(error instanceof Error ? error.message : String(error)); + return { + issue, + conclusion: { + status: "blocked", + summary: "DEV DB provisioning command failed before producing a report." + }, + error: message, + safety: { + devOnly: true, + prodAllowed: false, + secretValuesPrinted: false + }, + trace: createHash("sha256").update(message).digest("hex").slice(0, 12) + }; +} + +function redactFailureText(value) { + return String(value ?? "") + .replace(/postgres(?:ql)?:\/\/[^\s"'<>]+/giu, "[redacted-postgres-url]") + .replace(/(password\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]") + .replace(/(token\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]") + .replace(/(kubeconfig\s*[=:]\s*)[^\s,;]+/giu, "$1[redacted]"); +} diff --git a/scripts/src/dev-runtime-provisioning.test.mjs b/scripts/src/dev-runtime-provisioning.test.mjs new file mode 100644 index 00000000..d611e0d8 --- /dev/null +++ b/scripts/src/dev-runtime-provisioning.test.mjs @@ -0,0 +1,270 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + buildDevRuntimeProvisioningReport, + parseArgs +} from "./dev-runtime-provisioning.mjs"; + +test("source check parses target role/database without leaking DB URL material", async () => { + const report = await buildDevRuntimeProvisioningReport(parseArgs(["--check"]), { + env: { + HWLAB_CLOUD_DB_URL: fixturePostgresUrl({ password: fixtureSecret("super") }), + HWLAB_CLOUD_DB_SSL_MODE: "disable" + }, + now: () => "2026-05-23T00:00:00.000Z" + }); + + assert.equal(report.conclusion.status, "ready"); + assert.equal(report.actions.liveDbReadAttempted, false); + assert.equal(report.actions.liveDbWriteAttempted, false); + assert.equal(report.db.targetRoleNamePresent, true); + assert.equal(report.db.targetDatabaseNamePresent, true); + assert.equal(report.db.targetPasswordPresent, true); + assert.equal(report.safety.secretValuesPrinted, false); + assertNoFixtureSecrets(report); +}); + +test("source check without injected DB URL is ready and performs no live access", async () => { + const report = await buildDevRuntimeProvisioningReport(parseArgs(["--check"]), { + env: {}, + now: () => "2026-05-23T00:00:00.000Z" + }); + + assert.equal(report.conclusion.status, "ready"); + assert.equal(report.actions.liveDbReadAttempted, false); + assert.equal(report.actions.liveDbWriteAttempted, false); + assert.equal(report.blockers.length, 0); + assert.equal(report.db.targetUrlPresent, false); + assert.equal(report.gates.role.status, "not_checked"); +}); + +test("live dry-run classifies missing target role separately from missing database", async () => { + const report = await buildDevRuntimeProvisioningReport( + parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]), + { + env: envWithAdmin(), + adminClient: createAdminClient({ roleExists: false, databaseExists: true }), + queryClient: createRuntimeClient("auth"), + now: () => "2026-05-23T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "blocked"); + assert.equal(report.actions.adminInspectionAttempted, true); + assert.equal(report.actions.liveDbWriteAttempted, false); + assert.equal(report.provisioning.targetRole.exists, false); + assert.equal(report.provisioning.targetDatabase.exists, true); + assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-role"), true); + assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-database"), false); + assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-auth"), true); + assertNoFixtureSecrets(report); +}); + +test("live dry-run classifies missing target database separately from target role", async () => { + const report = await buildDevRuntimeProvisioningReport( + parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]), + { + env: envWithAdmin(), + adminClient: createAdminClient({ roleExists: true, databaseExists: false }), + queryClient: createRuntimeClient("auth"), + now: () => "2026-05-23T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "blocked"); + assert.equal(report.provisioning.targetRole.exists, true); + assert.equal(report.provisioning.targetDatabase.exists, false); + assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-role"), false); + assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-database"), true); +}); + +test("live dry-run separates SSL from auth/schema/migration", async () => { + const report = await buildDevRuntimeProvisioningReport( + parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]), + { + env: envWithAdmin({ sslMode: "require" }), + adminClient: createAdminClient({ roleExists: true, databaseExists: true }), + queryClient: createRuntimeClient("ssl"), + now: () => "2026-05-23T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "blocked"); + assert.equal(report.gates.ssl.status, "blocked"); + assert.equal(report.gates.auth.status, "not_checked"); + assert.equal(report.gates.schema.status, "not_checked"); + assert.equal(report.gates.migration.status, "not_checked"); + assert.equal(report.blockers.some((blocker) => blocker.scope === "runtime-provisioning-ssl"), true); + assert.equal(JSON.stringify(report).includes("does not support SSL"), false); +}); + +test("live dry-run leaves role/database ready while migration remains a downstream blocker", async () => { + const report = await buildDevRuntimeProvisioningReport( + parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]), + { + env: envWithAdmin(), + adminClient: createAdminClient({ roleExists: true, databaseExists: true }), + queryClient: createRuntimeClient("migration"), + now: () => "2026-05-23T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "ready"); + assert.equal(report.provisioning.ready, true); + assert.equal(report.provisioning.targetRole.ready, true); + assert.equal(report.provisioning.targetDatabase.ready, true); + assert.equal(report.runtime.blocker, "runtime_durable_adapter_migration_blocked"); + assert.equal(report.gates.migration.status, "blocked"); + assert.equal(report.blockers.length, 0); +}); + +test("apply creates missing role/database and grants target privileges without printing secrets", async () => { + const adminClient = createAdminClient({ roleExists: false, databaseExists: false }); + const targetAdminClient = createAdminClient({ roleExists: true, databaseExists: true }); + const report = await buildDevRuntimeProvisioningReport( + parseArgs(["--apply", "--confirm-dev", "--confirmed-non-production"]), + { + env: envWithAdmin(), + adminClient, + queryClient: createRuntimeClient("ready"), + now: () => "2026-05-23T00:00:00.000Z", + adminDbUrlOverride: null, + pgModuleLoader: async () => { + throw new Error("pgModuleLoader should not be used when adminClient is injected"); + } + } + ); + + assert.equal(targetAdminClient.calls.length, 0); + assert.equal(report.conclusion.status, "ready"); + assert.equal(report.actions.roleCreated, true); + assert.equal(report.actions.databaseCreated, true); + assert.equal(report.actions.rolePasswordSynchronized, true); + assert.equal(report.actions.databaseConnectGranted, true); + assert.equal(report.actions.schemaPrivilegesGranted, true); + assert.equal(report.provisioning.ready, true); + assert.ok(adminClient.calls.some((call) => call.sql.startsWith("CREATE ROLE "))); + assert.ok(adminClient.calls.some((call) => call.sql.startsWith("CREATE DATABASE "))); + assert.ok(adminClient.calls.some((call) => call.sql.startsWith("GRANT CONNECT ON DATABASE "))); + assert.ok(adminClient.calls.some((call) => call.sql.startsWith("GRANT USAGE, CREATE ON SCHEMA public "))); + assertNoFixtureSecrets(report); +}); + +function envWithAdmin({ sslMode = "disable" } = {}) { + return { + HWLAB_CLOUD_DB_URL: fixturePostgresUrl({ password: fixtureSecret("app") }), + HWLAB_CLOUD_DB_ADMIN_URL: fixturePostgresUrl({ + user: "postgres", + password: fixtureSecret("admin"), + database: "postgres" + }), + HWLAB_CLOUD_DB_SSL_MODE: sslMode + }; +} + +function fixturePostgresUrl({ + user = "hwlab_app", + password = fixtureSecret("app"), + host = "db.example.test", + database = "hwlab_runtime" +} = {}) { + return `${["postgresql", "://"].join("")}${user}:${password}@${host}:5432/${database}`; +} + +function fixtureSecret(kind) { + return `${kind}-${"secret"}`; +} + +function assertNoFixtureSecrets(value) { + const text = JSON.stringify(value); + for (const forbidden of [ + fixtureSecret("super"), + fixtureSecret("app"), + fixtureSecret("admin"), + "db.example.test", + "hwlab_app", + "hwlab_runtime" + ]) { + assert.equal(text.includes(forbidden), false); + } +} + +function createAdminClient({ roleExists, databaseExists }) { + const state = { + roleExists, + databaseExists, + calls: [] + }; + return { + get calls() { + return state.calls; + }, + async query(sql, params = []) { + state.calls.push({ sql, params }); + if (sql.includes("pg_catalog.pg_roles")) { + return { rows: state.roleExists ? [{ present: 1 }] : [] }; + } + if (sql.includes("pg_catalog.pg_database")) { + return { rows: state.databaseExists ? [{ present: 1 }] : [] }; + } + if (sql.startsWith("CREATE ROLE ")) { + state.roleExists = true; + return { rows: [] }; + } + if (sql.startsWith("ALTER ROLE ")) { + return { rows: [] }; + } + if (sql.startsWith("CREATE DATABASE ")) { + state.databaseExists = true; + return { rows: [] }; + } + if (sql.startsWith("GRANT CONNECT ON DATABASE ") || sql.startsWith("GRANT USAGE, CREATE ON SCHEMA public ")) { + return { rows: [] }; + } + throw new Error(`unexpected admin query: ${sql}`); + } + }; +} + +function createRuntimeClient(mode) { + return { + async query(sql) { + if (sql.includes("information_schema.columns")) { + if (mode === "ssl") { + const error = new Error("fixture SSL blocked: server does not support SSL"); + error.code = "08P01"; + throw error; + } + if (mode === "auth") { + const error = new Error("fixture auth blocked"); + error.code = "28P01"; + throw error; + } + return { rows: schemaRows() }; + } + if (sql.startsWith("SELECT id, schema_version FROM hwlab_schema_migrations")) { + if (mode === "migration") return { rows: [] }; + return { rows: [{ id: "0001_cloud_core_skeleton", schema_version: "runtime-durable-postgres-v1" }] }; + } + if (sql.startsWith("SELECT COUNT(*)::int AS count FROM ")) { + return { rows: [{ count: 0 }] }; + } + throw new Error(`unexpected runtime query: ${sql}`); + } + }; +} + +function schemaRows() { + const schema = { + gateway_sessions: ["id", "project_id", "gateway_service_id", "status", "started_at", "ended_at", "gateway_session_json"], + box_resources: ["id", "project_id", "gateway_session_id", "resource_state", "labels_json", "resource_json", "updated_at"], + box_capabilities: ["id", "box_resource_id", "capability_type", "capability_json", "updated_at"], + hardware_operations: ["id", "project_id", "requested_by", "operation_type", "operation_json", "status", "requested_at", "updated_at"], + audit_events: ["id", "request_id", "actor", "source", "operation", "target", "result", "timestamp", "event_json"], + evidence_records: ["id", "project_id", "operation_id", "evidence_type", "uri", "metadata_json", "created_at"] + }; + return Object.entries(schema).flatMap(([table, columns]) => + columns.map((column) => ({ table_name: table, column_name: column })) + ); +} diff --git a/scripts/validate-runtime-boundary.mjs b/scripts/validate-runtime-boundary.mjs index cdf36e66..f0b33f5e 100644 --- a/scripts/validate-runtime-boundary.mjs +++ b/scripts/validate-runtime-boundary.mjs @@ -199,11 +199,14 @@ function assertDurableRuntimeRunbook(value) { for (const expected of [ "DB live readiness and durable runtime readiness are separate gates", "runtime_durable_adapter_query_blocked", + "DEV DB Provisioning Automation", "Source migration contract", + "Live provisioning apply", "Static runtime boundary contract", "Public health observation", "Live DB read verification", "Live migration/repair apply", + "Durable runtime postflight", "Do not run `kubectl get secret`", "secretValuesPrinted=false", "Before any authorized live migration or repair", @@ -212,10 +215,15 @@ function assertDurableRuntimeRunbook(value) { "Moved:", "Still blocked:", "No full M3, M4, or M5 acceptance is allowed while runtime durability is blocked", + "node scripts/dev-runtime-provisioning.mjs --check", + "node scripts/dev-runtime-provisioning.mjs --dry-run --allow-live-db-read --confirm-dev --write-report", + "node scripts/dev-runtime-provisioning.mjs --apply --confirm-dev --confirmed-non-production --write-report", "node scripts/dev-runtime-migration.mjs --check", "node scripts/dev-runtime-migration.mjs --dry-run --write-report", "node scripts/dev-runtime-migration.mjs --dry-run --allow-live-db-read --confirm-dev --write-report", - "node scripts/dev-runtime-migration.mjs --apply --confirm-dev --confirmed-non-production --write-report" + "node scripts/dev-runtime-migration.mjs --apply --confirm-dev --confirmed-non-production --write-report", + "node scripts/dev-runtime-postflight.mjs --check", + "node scripts/dev-runtime-postflight.mjs --live --confirm-dev --confirmed-non-production --target api --write-report" ]) { assertIncludes(value, expected, label); }