diff --git a/deploy/deploy.json b/deploy/deploy.json index 016aebcb..d197870b 100644 --- a/deploy/deploy.json +++ b/deploy/deploy.json @@ -213,7 +213,10 @@ "replicas": 1, "env": { "HWLAB_ENVIRONMENT": "dev", - "HWLAB_API_BASE_URL": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667" + "HWLAB_API_BASE_URL": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667", + "HWLAB_COMMIT_ID": "7de6edd", + "HWLAB_IMAGE": "127.0.0.1:5000/hwlab/hwlab-cloud-web:7de6edd", + "HWLAB_IMAGE_TAG": "7de6edd" } }, { diff --git a/deploy/k8s/base/workloads.yaml b/deploy/k8s/base/workloads.yaml index 94aa6561..0ec16544 100644 --- a/deploy/k8s/base/workloads.yaml +++ b/deploy/k8s/base/workloads.yaml @@ -170,6 +170,18 @@ { "name": "HWLAB_API_BASE_URL", "value": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667" + }, + { + "name": "HWLAB_COMMIT_ID", + "value": "7de6edd" + }, + { + "name": "HWLAB_IMAGE", + "value": "127.0.0.1:5000/hwlab/hwlab-cloud-web:7de6edd" + }, + { + "name": "HWLAB_IMAGE_TAG", + "value": "7de6edd" } ], "readinessProbe": { diff --git a/docs/dev-deploy-apply.md b/docs/dev-deploy-apply.md index b6e2dd11..b9836c8b 100644 --- a/docs/dev-deploy-apply.md +++ b/docs/dev-deploy-apply.md @@ -31,7 +31,10 @@ Run a preflight that is expected to report blockers on the current skeleton: ```sh node --check scripts/dev-deploy-apply.mjs node --check scripts/src/dev-deploy-apply.mjs +node --check scripts/dev-runtime-migration.mjs +node --check scripts/src/dev-runtime-migration.mjs node scripts/deploy-desired-state-plan.mjs --check +node scripts/dev-runtime-migration.mjs --check node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked --write-report node scripts/validate-dev-gate-report.mjs ``` @@ -53,6 +56,35 @@ The script refuses PROD flags, secret-read flags, heavyweight e2e flags, and force-push flags. It also refuses to apply if the DEV artifact catalog still contains skeleton-only unpublished images. +Runtime DB migration is a separate DEV-only apply boundary. The source-only +check is safe for normal PR validation: + +```sh +node scripts/dev-runtime-migration.mjs --check +node scripts/dev-runtime-migration.mjs --dry-run --write-report +``` + +Read-only live DB verification requires explicit DEV confirmation and an +already injected `HWLAB_CLOUD_DB_URL`; the report redacts the value and keeps +auth, schema, migration ledger, and readiness blockers separate: + +```sh +node scripts/dev-runtime-migration.mjs --dry-run --allow-live-db-read --confirm-dev --write-report +``` + +A real migration write is not part of default deploy validation. It requires a +separately authorized DEV operation: + +```sh +node scripts/dev-runtime-migration.mjs --apply --confirm-dev --confirmed-non-production --write-report +``` + +That command may only apply +`internal/db/migrations/0001_cloud_core_skeleton.sql` to the DEV Postgres target +provided by `HWLAB_CLOUD_DB_URL`. It must not read Kubernetes Secret resources, +print the DB URL/password/token, mutate PROD, restart services, or claim +M3/M4/M5 acceptance. + `deploy-desired-state-plan.mjs` is not an apply preflight substitute. It is a read-only source planner and does not prove registry image existence, DEV cluster pullability, DEV apply, service restart, or M3 DEV-LIVE evidence. @@ -234,6 +266,13 @@ not a readiness gate or TCP probe target, until this repo owns its Service/Endpoint rollout contract. The local health gate reports DB runtime config without printing the connection string. +Durable runtime migration now has a repo-owned path: +`scripts/dev-runtime-migration.mjs --check` proves the source migration and +ledger contract without DB access; `--dry-run --allow-live-db-read +--confirm-dev` verifies live auth/schema/migration/readiness without writes; +`--apply --confirm-dev --confirmed-non-production` is the only DEV DB write +mode and remains outside default PR validation. + No PROD resource, secret value, UniDesk runtime substitute, browser e2e, heavyweight e2e, or force push is part of this flow. diff --git a/docs/dev-gate-preflight.md b/docs/dev-gate-preflight.md index 17b5301b..00b2b671 100644 --- a/docs/dev-gate-preflight.md +++ b/docs/dev-gate-preflight.md @@ -18,8 +18,11 @@ node --check scripts/src/dev-gate-preflight.mjs node --check scripts/src/registry-capabilities.mjs node --check scripts/deploy-desired-state-plan.mjs node --check scripts/src/deploy-desired-state-plan.mjs +node --check scripts/dev-runtime-migration.mjs +node --check scripts/src/dev-runtime-migration.mjs node --check scripts/refresh-artifact-catalog.mjs node scripts/deploy-desired-state-plan.mjs --check +node scripts/dev-runtime-migration.mjs --check node scripts/dev-gate-preflight.mjs ``` @@ -71,6 +74,10 @@ The preflight checks: parsed from that Secret URL. `cloud-api-db` is an optional desired alias only until this repo owns Service plus Endpoint or EndpointSlice manifests and rollout/apply contract. +- `scripts/dev-runtime-migration.mjs --check` validates the runtime migration + source and ledger contract without live DB access. Live read verification and + migration apply are separate DEV-only modes requiring explicit flags; default + preflight does not read DB secrets or write the DB. - The local cloud-api health payload reports DB env presence/missing status with redacted values only. This is a readiness gate, not live DB evidence. - `deploy/frp` and `deploy/master-edge` describe the D601-to-master DEV route diff --git a/docs/reference/dev-runtime-boundary.md b/docs/reference/dev-runtime-boundary.md index 6825bc22..c667444d 100644 --- a/docs/reference/dev-runtime-boundary.md +++ b/docs/reference/dev-runtime-boundary.md @@ -185,6 +185,50 @@ separate from runtime persistence. If runtime durability is blocked, layer, and `requiredEvidence` remains the durable adapter schema, migration, and read query contract. +## Runtime Migration Automation + +The repo-owned runtime migration entrypoint is: + +```sh +node scripts/dev-runtime-migration.mjs --check +node scripts/dev-runtime-migration.mjs --dry-run --write-report +``` + +Those default paths are source-only. They validate +`internal/db/migrations/0001_cloud_core_skeleton.sql`, the required +`hwlab_schema_migrations` ledger row, and the durable runtime table/column +contract without connecting to Postgres, reading Kubernetes Secrets, or writing +DEV/PROD state. + +An authorized operator may run a read-only live verification only after +injecting the DEV DB URL through the existing Secret/env path: + +```sh +node scripts/dev-runtime-migration.mjs --dry-run --allow-live-db-read --confirm-dev --write-report +``` + +A real DEV migration apply is intentionally separate and requires all explicit +write flags: + +```sh +node scripts/dev-runtime-migration.mjs --apply --confirm-dev --confirmed-non-production --write-report +``` + +The apply command is scoped to DEV Postgres schema objects declared by +`internal/db/migrations/0001_cloud_core_skeleton.sql` and the +`hwlab_schema_migrations` ledger row. It must not be run for PROD, must not +read Kubernetes Secret resources, must not print `HWLAB_CLOUD_DB_URL` or any +password/token value, and must not be described as M3/M4/M5 acceptance. + +Runtime migration reports separate the blockers as: + +| Gate | Meaning | +| --- | --- | +| Auth | DB driver/auth/authorization can run the readiness query. | +| Schema | Required durable runtime tables and columns are present. | +| Migration | The required source migration is recorded in the ledger. | +| Readiness | Durable runtime read readiness passed after schema and ledger checks. | + ## Runtime Substitution Ban UniDesk services, provider-gateway, backend-core, microservice proxies, local diff --git a/internal/cloud/code-agent-chat.mjs b/internal/cloud/code-agent-chat.mjs index 3fd75d2f..45e7ce9b 100644 --- a/internal/cloud/code-agent-chat.mjs +++ b/internal/cloud/code-agent-chat.mjs @@ -258,6 +258,7 @@ async function callOpenAiResponses({ providerPlan, message, conversationId, trac method: "POST", headers: { "content-type": "application/json", + accept: "text/event-stream", authorization: `Bearer ${env.OPENAI_API_KEY}` }, body: JSON.stringify({ @@ -274,11 +275,12 @@ async function callOpenAiResponses({ providerPlan, message, conversationId, trac ] } ], - store: false + store: false, + stream: true }), signal: controller.signal }); - payload = await response.json().catch(() => null); + payload = parseOpenAiResponseBody(await response.text()); } catch (error) { if (error.name === "AbortError") { throw providerUnavailable(`OpenAI Responses request timed out after ${effectiveTimeout(timeoutMs)}ms`, { @@ -296,8 +298,9 @@ async function callOpenAiResponses({ providerPlan, message, conversationId, trac clearTimeout(timer); } + const providerError = payload.error ?? payload.raw?.error ?? null; if (!response.ok) { - throw providerUnavailable(`OpenAI Responses returned HTTP ${response.status}: ${payload?.error?.message || "request rejected"}`, { + throw providerUnavailable(`OpenAI Responses returned HTTP ${response.status}: ${providerError?.message || "request rejected"}`, { provider: providerPlan.provider, model: providerPlan.model, backend: providerPlan.backend, @@ -305,7 +308,15 @@ async function callOpenAiResponses({ providerPlan, message, conversationId, trac }); } - const content = extractOpenAiOutputText(payload); + if (providerError) { + throw providerUnavailable(`OpenAI Responses stream error: ${providerError.message || "request rejected"}`, { + provider: providerPlan.provider, + model: providerPlan.model, + backend: providerPlan.backend + }); + } + + const content = payload.content || extractOpenAiOutputText(payload.raw); if (!content) { throw providerUnavailable("OpenAI Responses returned no assistant text", { provider: providerPlan.provider, @@ -321,7 +332,7 @@ async function callOpenAiResponses({ providerPlan, message, conversationId, trac content, usage: payload.usage ?? null, providerTrace: { - responseId: payload.id ?? null + responseId: payload.responseId ?? null } }; } @@ -477,6 +488,91 @@ function extractOpenAiOutputText(payload) { return chunks.join("\n").trim(); } +function parseOpenAiResponseBody(text) { + const raw = parseJsonOrNull(text); + if (raw) { + return { + raw, + content: extractOpenAiOutputText(raw), + responseId: raw.id ?? null, + model: raw.model ?? null, + usage: raw.usage ?? null, + error: raw.error ?? null + }; + } + + const stream = parseOpenAiResponsesSse(text); + return { + raw: stream.finalResponse, + content: stream.content, + responseId: stream.responseId, + model: stream.model, + usage: stream.usage, + error: stream.error + }; +} + +function parseOpenAiResponsesSse(text) { + const deltas = []; + let finalResponse = null; + let responseId = null; + let model = null; + let usage = null; + let error = null; + + for (const event of parseSseDataMessages(text)) { + const payload = parseJsonOrNull(event); + if (!payload) continue; + const response = payload.response && typeof payload.response === "object" ? payload.response : null; + if (response) { + finalResponse = response; + responseId = response.id ?? responseId; + model = response.model ?? model; + usage = response.usage ?? usage; + if (response.error) error = response.error; + } + if (payload.response_id) responseId = payload.response_id; + if (payload.item?.id) responseId = responseId ?? payload.item.id; + if (payload.error) error = payload.error; + if (payload.type === "error" && !payload.error) error = payload; + if (typeof payload.delta === "string") deltas.push(payload.delta); + else if (typeof payload.text === "string" && payload.type === "response.output_text.done" && deltas.length === 0) { + deltas.push(payload.text); + } + } + + const content = deltas.join("").trim() || extractOpenAiOutputText(finalResponse); + return { + finalResponse, + content, + responseId, + model, + usage, + error + }; +} + +function parseSseDataMessages(text) { + const messages = []; + for (const block of String(text ?? "").split(/\r?\n\r?\n/u)) { + const data = []; + for (const line of block.split(/\r?\n/u)) { + if (line.startsWith("data:")) data.push(line.slice(5).trimStart()); + } + const message = data.join("\n").trim(); + if (message && message !== "[DONE]") messages.push(message); + } + return messages; +} + +function parseJsonOrNull(value) { + try { + return JSON.parse(value); + } catch { + return null; + } +} + async function commandExists(command, env) { if (command.includes("/") || command.includes("\\")) { try { diff --git a/internal/cloud/server.test.mjs b/internal/cloud/server.test.mjs index bf6a5d77..6990737c 100644 --- a/internal/cloud/server.test.mjs +++ b/internal/cloud/server.test.mjs @@ -1,4 +1,5 @@ import assert from "node:assert/strict"; +import { createServer as createHttpServer } from "node:http"; import { createServer as createTcpServer } from "node:net"; import test from "node:test"; @@ -571,6 +572,113 @@ test("cloud api /v1/agent/chat returns structured completed Code Agent payload", } }); +test("cloud api /v1/agent/chat uses streaming OpenAI Responses and parses SSE text", async () => { + const providerRequests = []; + const providerServer = createHttpServer((request, response) => { + const chunks = []; + request.on("data", (chunk) => chunks.push(chunk)); + request.on("end", () => { + const bodyText = Buffer.concat(chunks).toString("utf8"); + const body = JSON.parse(bodyText); + providerRequests.push({ + method: request.method, + url: request.url, + authorizationPresent: Boolean(request.headers.authorization), + accept: request.headers.accept, + contentType: request.headers["content-type"], + body + }); + response.writeHead(200, { + "content-type": "text/event-stream" + }); + response.end([ + `data: ${JSON.stringify({ + type: "response.created", + response: { + id: "resp_server_test_stream", + model: body.model, + usage: null + } + })}`, + `data: ${JSON.stringify({ + type: "response.output_text.delta", + response_id: "resp_server_test_stream", + delta: "HWLAB Code Agent " + })}`, + `data: ${JSON.stringify({ + type: "response.output_text.delta", + response_id: "resp_server_test_stream", + delta: "streaming ready." + })}`, + `data: ${JSON.stringify({ + type: "response.completed", + response: { + id: "resp_server_test_stream", + model: body.model, + usage: null + } + })}`, + "data: [DONE]", + "" + ].join("\n\n")); + }); + }); + await new Promise((resolve) => providerServer.listen(0, "127.0.0.1", resolve)); + const providerPort = providerServer.address().port; + + const server = createCloudApiServer({ + env: { + OPENAI_API_KEY: "test-openai-key-material", + HWLAB_CODE_AGENT_PROVIDER: "openai", + HWLAB_CODE_AGENT_MODEL: "gpt-test", + HWLAB_CODE_AGENT_OPENAI_BASE_URL: `http://127.0.0.1:${providerPort}/v1/responses` + } + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + + try { + const { port } = server.address(); + const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, { + method: "POST", + headers: { + "content-type": "application/json", + "x-trace-id": "trc_server-test-agent-chat-stream" + }, + body: JSON.stringify({ + conversationId: "cnv_server-test-agent-chat-stream", + message: "请用一句话说明当前 HWLAB 工作台可以做什么。" + }) + }); + assert.equal(response.status, 200); + const payload = await response.json(); + assert.equal(payload.status, "completed"); + assert.equal(payload.conversationId, "cnv_server-test-agent-chat-stream"); + assert.equal(payload.traceId, "trc_server-test-agent-chat-stream"); + assert.equal(payload.provider, "openai-responses"); + assert.equal(payload.model, "gpt-test"); + assert.equal(payload.providerTrace.responseId, "resp_server_test_stream"); + assert.equal(payload.reply.content, "HWLAB Code Agent streaming ready."); + assert.equal(JSON.stringify(payload).includes("test-openai-key-material"), false); + + assert.equal(providerRequests.length, 1); + assert.equal(providerRequests[0].method, "POST"); + assert.equal(providerRequests[0].url, "/v1/responses"); + assert.equal(providerRequests[0].authorizationPresent, true); + assert.match(providerRequests[0].accept, /text\/event-stream/u); + assert.equal(providerRequests[0].contentType, "application/json"); + assert.equal(providerRequests[0].body.model, "gpt-test"); + assert.equal(providerRequests[0].body.store, false); + assert.equal(providerRequests[0].body.stream, true); + } finally { + await new Promise((resolve, reject) => { + server.close((error) => (error ? reject(error) : resolve())); + }); + await new Promise((resolve, reject) => { + providerServer.close((error) => (error ? reject(error) : resolve())); + }); + } +}); + test("cloud api /v1/agent/chat reports provider gaps without faking a reply", async () => { const server = createCloudApiServer({ env: { diff --git a/package.json b/package.json index 52e21b60..6842d168 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "private": true, "type": "module", "scripts": { - "validate": "node scripts/validate-contract.mjs && node scripts/deploy-contract-plan.mjs --check && node scripts/deploy-desired-state-plan.mjs --check", - "check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check internal/agent/runtime.mjs && node --check internal/audit/index.mjs && node --check internal/db/runtime-store.mjs && node --check internal/db/runtime-store.test.mjs && node --check internal/mvp-gate/summary.mjs && node --check internal/cloud/db-contract.mjs && node --check internal/cloud/code-agent-contract.mjs && node --check internal/cloud/json-rpc.mjs && node --check internal/cloud/code-agent-chat.mjs && node --check internal/cloud/server.mjs && node --check internal/sim/model.mjs && node --check internal/sim/http.mjs && node --check internal/sim/l2-runtime.mjs && node --check internal/patchpanel/model.mjs && node --check internal/patchpanel/runtime.mjs && node --check cmd/hwlab-cloud-api/main.mjs && node --check cmd/hwlab-agent-mgr/main.mjs && node --check cmd/hwlab-agent-worker/main.mjs && node --check cmd/hwlab-box-simu/main.mjs && node --check cmd/hwlab-gateway-simu/main.mjs && node --check scripts/cloud-api-runtime-smoke.mjs && node --check scripts/code-agent-chat-smoke.mjs && node --check scripts/dev-edge-health-smoke.mjs && node --check scripts/src/dev-edge-health-smoke-lib.mjs && node --check scripts/validate-contract.mjs && node --check scripts/deploy-desired-state-plan.mjs && node --check scripts/src/deploy-desired-state-plan.mjs && node --check scripts/report-lifecycle.mjs && node --check scripts/report-lifecycle.test.mjs && node --check scripts/validate-dev-gate-report.mjs && node --check scripts/dev-m3-hardware-loop-smoke.test.mjs && node --check scripts/dev-cloud-workbench-smoke.test.mjs && node --check scripts/validate-dev-m3-cardinality.mjs && node --check scripts/validate-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/src/dev-artifact-services.mjs && node --check scripts/src/registry-capabilities.mjs && node --check scripts/preflight-dev-base-image.mjs && node --check scripts/src/dev-base-image-preflight.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.mjs && node --check scripts/d601-k3s-readonly-observability.mjs && node --check scripts/src/d601-k3s-readonly-observability.mjs && node --check scripts/l2-runtime-contract-smoke.mjs && node --check scripts/patch-panel-runtime-smoke.mjs && node --check scripts/export-web-gate-summary.mjs && node --check scripts/l6-cli-web-smoke.mjs && node --check tools/hwlab-cli/bin/hwlab-cli.mjs && node --check tools/hwlab-cli/lib/cli.mjs && node --check web/hwlab-cloud-web/app.mjs && node --check web/hwlab-cloud-web/gate-summary.mjs && node --check web/hwlab-cloud-web/scripts/check.mjs && node --check web/hwlab-cloud-web/scripts/build.mjs && node --check web/hwlab-cloud-web/scripts/dist-contract.mjs && node scripts/validate-contract.mjs && node scripts/validate-dev-gate-report.mjs && node scripts/report-lifecycle.test.mjs && node scripts/validate-dev-m3-cardinality.mjs && node scripts/validate-artifact-catalog.mjs && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-evidence-blocker-aggregator.mjs --check && node scripts/l2-runtime-contract-smoke.mjs && node scripts/l6-cli-web-smoke.mjs && node web/hwlab-cloud-web/scripts/check.mjs && node scripts/code-agent-chat-smoke.mjs && node --test scripts/dev-m3-hardware-loop-smoke.test.mjs scripts/dev-cloud-workbench-smoke.test.mjs scripts/deploy-desired-state-plan.test.mjs scripts/src/dev-deploy-apply.test.mjs internal/agent/index.test.mjs internal/audit/index.test.mjs internal/db/schema.test.mjs internal/db/runtime-store.test.mjs internal/cloud/json-rpc.test.mjs internal/cloud/server.test.mjs internal/patchpanel/model.test.mjs internal/patchpanel/runtime.test.mjs && node scripts/cloud-api-runtime-smoke.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", + "validate": "node scripts/validate-contract.mjs && node scripts/deploy-contract-plan.mjs --check && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-migration.mjs --check", + "check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check internal/agent/runtime.mjs && node --check internal/audit/index.mjs && node --check internal/db/runtime-store.mjs && node --check internal/db/runtime-store.test.mjs && node --check internal/mvp-gate/summary.mjs && node --check internal/cloud/db-contract.mjs && node --check internal/cloud/code-agent-contract.mjs && node --check internal/cloud/json-rpc.mjs && node --check internal/cloud/code-agent-chat.mjs && node --check internal/cloud/server.mjs && node --check internal/sim/model.mjs && node --check internal/sim/http.mjs && node --check internal/sim/l2-runtime.mjs && node --check internal/patchpanel/model.mjs && node --check internal/patchpanel/runtime.mjs && node --check cmd/hwlab-cloud-api/main.mjs && node --check cmd/hwlab-agent-mgr/main.mjs && node --check cmd/hwlab-agent-worker/main.mjs && node --check cmd/hwlab-box-simu/main.mjs && node --check cmd/hwlab-gateway-simu/main.mjs && node --check scripts/cloud-api-runtime-smoke.mjs && node --check scripts/code-agent-chat-smoke.mjs && node --check scripts/dev-edge-health-smoke.mjs && node --check scripts/src/dev-edge-health-smoke-lib.mjs && node --check scripts/validate-contract.mjs && node --check scripts/deploy-desired-state-plan.mjs && node --check scripts/src/deploy-desired-state-plan.mjs && node --check scripts/report-lifecycle.mjs && node --check scripts/report-lifecycle.test.mjs && node --check scripts/validate-dev-gate-report.mjs && node --check scripts/dev-m3-hardware-loop-smoke.test.mjs && node --check scripts/dev-cloud-workbench-smoke.test.mjs && node --check scripts/validate-dev-m3-cardinality.mjs && node --check scripts/validate-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/src/dev-artifact-services.mjs && node --check scripts/src/registry-capabilities.mjs && node --check scripts/preflight-dev-base-image.mjs && node --check scripts/src/dev-base-image-preflight.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.mjs && node --check scripts/d601-k3s-readonly-observability.mjs && node --check scripts/src/d601-k3s-readonly-observability.mjs && node --check scripts/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.test.mjs && node --check scripts/l2-runtime-contract-smoke.mjs && node --check scripts/patch-panel-runtime-smoke.mjs && node --check scripts/export-web-gate-summary.mjs && node --check scripts/l6-cli-web-smoke.mjs && node --check tools/hwlab-cli/bin/hwlab-cli.mjs && node --check tools/hwlab-cli/lib/cli.mjs && node --check web/hwlab-cloud-web/app.mjs && node --check web/hwlab-cloud-web/gate-summary.mjs && node --check web/hwlab-cloud-web/scripts/check.mjs && node --check web/hwlab-cloud-web/scripts/build.mjs && node --check web/hwlab-cloud-web/scripts/dist-contract.mjs && node scripts/validate-contract.mjs && node scripts/validate-dev-gate-report.mjs && node scripts/report-lifecycle.test.mjs && node scripts/validate-dev-m3-cardinality.mjs && node scripts/validate-artifact-catalog.mjs && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-migration.mjs --check && node scripts/dev-evidence-blocker-aggregator.mjs --check && node scripts/l2-runtime-contract-smoke.mjs && node scripts/l6-cli-web-smoke.mjs && node web/hwlab-cloud-web/scripts/check.mjs && node scripts/code-agent-chat-smoke.mjs && node --test scripts/dev-m3-hardware-loop-smoke.test.mjs scripts/dev-cloud-workbench-smoke.test.mjs scripts/deploy-desired-state-plan.test.mjs scripts/src/dev-deploy-apply.test.mjs scripts/src/dev-runtime-migration.test.mjs internal/agent/index.test.mjs internal/audit/index.test.mjs internal/db/schema.test.mjs internal/db/runtime-store.test.mjs internal/cloud/json-rpc.test.mjs internal/cloud/server.test.mjs internal/patchpanel/model.test.mjs internal/patchpanel/runtime.test.mjs && node scripts/cloud-api-runtime-smoke.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", "dev-base-image:preflight": "node scripts/preflight-dev-base-image.mjs", "cloud-api:smoke": "node scripts/cloud-api-runtime-smoke.mjs", "m1:smoke": "node scripts/m1-contract-smoke.mjs", @@ -32,7 +32,8 @@ "docs:validate:m3-rollout": "node scripts/validate-m3-rollout-runbook.mjs", "d601:k3s:readonly": "node scripts/d601-k3s-readonly-observability.mjs", "runner:issue-visibility:preflight": "node scripts/runner-issue-visibility-preflight.mjs", - "report:lifecycle:test": "node scripts/report-lifecycle.test.mjs" + "report:lifecycle:test": "node scripts/report-lifecycle.test.mjs", + "dev-runtime:migration": "node scripts/dev-runtime-migration.mjs" }, "dependencies": { "pg": "^8.21.0" diff --git a/scripts/deploy-desired-state-plan.test.mjs b/scripts/deploy-desired-state-plan.test.mjs index 88bab249..92880d2a 100644 --- a/scripts/deploy-desired-state-plan.test.mjs +++ b/scripts/deploy-desired-state-plan.test.mjs @@ -7,6 +7,7 @@ import test from "node:test"; import { buildDesiredStatePlan } from "./src/deploy-desired-state-plan.mjs"; async function makeFixture({ + serviceId = "hwlab-cloud-api", commitId = "abc1234", catalogCommitId = commitId, catalogServiceCommitId = catalogCommitId, @@ -14,37 +15,41 @@ async function makeFixture({ deployEnvCommitId = commitId, deployEnvImageTag = commitId, deployEnvImage = null, + deployEnvMirrors = true, deploySkillsCommitId = null, workloadTag = commitId, workloadEnvCommitId = commitId, workloadEnvImageTag = commitId, workloadEnvImage = null, + workloadEnvMirrors = true, workloadSkillsCommitId = null } = {}) { const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-desired-state-")); await mkdir(path.join(root, "deploy/k8s/base"), { recursive: true }); await mkdir(path.join(root, "reports/dev-gate"), { recursive: true }); - const image = `127.0.0.1:5000/hwlab/hwlab-cloud-api:${commitId}`; - const catalogImage = `127.0.0.1:5000/hwlab/hwlab-cloud-api:${catalogImageTag}`; - const workloadImage = `127.0.0.1:5000/hwlab/hwlab-cloud-api:${workloadTag}`; + const image = `127.0.0.1:5000/hwlab/${serviceId}:${commitId}`; + const catalogImage = `127.0.0.1:5000/hwlab/${serviceId}:${catalogImageTag}`; + const workloadImage = `127.0.0.1:5000/hwlab/${serviceId}:${workloadTag}`; const deploy = { manifestVersion: "v1", environment: "dev", commitId, services: [ { - serviceId: "hwlab-cloud-api", + serviceId, image, namespace: "hwlab-dev", healthPath: "/health/live", profile: "dev", replicas: 1, - env: { - HWLAB_COMMIT_ID: deployEnvCommitId, - HWLAB_IMAGE: deployEnvImage ?? image, - HWLAB_IMAGE_TAG: deployEnvImageTag, - ...(deploySkillsCommitId ? { HWLAB_SKILLS_COMMIT_ID: deploySkillsCommitId } : {}) - } + env: deployEnvMirrors + ? { + HWLAB_COMMIT_ID: deployEnvCommitId, + HWLAB_IMAGE: deployEnvImage ?? image, + HWLAB_IMAGE_TAG: deployEnvImageTag, + ...(deploySkillsCommitId ? { HWLAB_SKILLS_COMMIT_ID: deploySkillsCommitId } : {}) + } + : {} } ] }; @@ -64,7 +69,7 @@ async function makeFixture({ }, services: [ { - serviceId: "hwlab-cloud-api", + serviceId, commitId: catalogServiceCommitId, image: catalogImage, imageTag: catalogImageTag, @@ -82,10 +87,10 @@ async function makeFixture({ apiVersion: "apps/v1", kind: "Deployment", metadata: { - name: "hwlab-cloud-api", + name: serviceId, namespace: "hwlab-dev", labels: { - "hwlab.pikastech.local/service-id": "hwlab-cloud-api" + "hwlab.pikastech.local/service-id": serviceId } }, spec: { @@ -93,14 +98,16 @@ async function makeFixture({ spec: { containers: [ { - name: "hwlab-cloud-api", + name: serviceId, image: workloadImage, - env: [ - { name: "HWLAB_COMMIT_ID", value: workloadEnvCommitId }, - { name: "HWLAB_IMAGE", value: workloadEnvImage ?? image }, - { name: "HWLAB_IMAGE_TAG", value: workloadEnvImageTag }, - ...(workloadSkillsCommitId ? [{ name: "HWLAB_SKILLS_COMMIT_ID", value: workloadSkillsCommitId }] : []) - ] + env: workloadEnvMirrors + ? [ + { name: "HWLAB_COMMIT_ID", value: workloadEnvCommitId }, + { name: "HWLAB_IMAGE", value: workloadEnvImage ?? image }, + { name: "HWLAB_IMAGE_TAG", value: workloadEnvImageTag }, + ...(workloadSkillsCommitId ? [{ name: "HWLAB_SKILLS_COMMIT_ID", value: workloadSkillsCommitId }] : []) + ] + : [] } ] } @@ -124,6 +131,33 @@ test("passes internally consistent desired-state", async () => { assert.deepEqual(plan.diagnostics, []); }); +test("passes when cloud-web owns runtime identity mirrors", async () => { + const repoRoot = await makeFixture({ serviceId: "hwlab-cloud-web" }); + const plan = await buildDesiredStatePlan({ repoRoot }); + assert.equal(plan.status, "pass"); + assert.equal(plan.summary.presentMirrorCount, 6); + assert.deepEqual(plan.diagnostics, []); +}); + +test("blocks when cloud-web runtime identity mirrors are missing", async () => { + const repoRoot = await makeFixture({ + serviceId: "hwlab-cloud-web", + deployEnvMirrors: false, + workloadEnvMirrors: false + }); + const plan = await buildDesiredStatePlan({ repoRoot }); + assert.equal(plan.status, "blocked"); + assert.equal(plan.summary.blockers, 6); + assert.ok(plan.diagnostics.some((diagnostic) => + diagnostic.code === "missing_mirror" && + diagnostic.path === "deploy/deploy.json.services.hwlab-cloud-web.env.HWLAB_COMMIT_ID" + )); + assert.ok(plan.diagnostics.some((diagnostic) => + diagnostic.code === "missing_mirror" && + diagnostic.path.endsWith(".env.HWLAB_IMAGE_TAG") + )); +}); + test("target tag review is read-only promotion_pending when current state is uniformly older", async () => { const repoRoot = await makeFixture(); const plan = await buildDesiredStatePlan({ repoRoot, targetTag: "def5678" }); diff --git a/scripts/dev-runtime-migration.mjs b/scripts/dev-runtime-migration.mjs new file mode 100644 index 00000000..aa48edb8 --- /dev/null +++ b/scripts/dev-runtime-migration.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +import { + formatDevRuntimeMigrationFailure, + runDevRuntimeMigrationCli +} from "./src/dev-runtime-migration.mjs"; + +try { + process.exitCode = await runDevRuntimeMigrationCli(process.argv.slice(2)); +} catch (error) { + process.stdout.write(`${JSON.stringify(formatDevRuntimeMigrationFailure(error), null, 2)}\n`); + process.exitCode = 1; +} diff --git a/scripts/src/deploy-desired-state-plan.mjs b/scripts/src/deploy-desired-state-plan.mjs index a7a04497..5ae3d64c 100644 --- a/scripts/src/deploy-desired-state-plan.mjs +++ b/scripts/src/deploy-desired-state-plan.mjs @@ -23,6 +23,9 @@ const commitMirrorEnvNames = new Set(["HWLAB_COMMIT_ID", "HWLAB_SKILLS_COMMIT_ID const imageMirrorEnvNames = new Set(["HWLAB_IMAGE"]); const tagMirrorEnvNames = new Set(["HWLAB_IMAGE_TAG"]); const digestMirrorEnvNames = new Set(["HWLAB_IMAGE_DIGEST"]); +const requiredMirrorEnvNamesByService = new Map([ + ["hwlab-cloud-web", ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"]] +]); const tagPattern = /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/; const commitPattern = /^[a-f0-9]{7,40}$/; @@ -212,14 +215,28 @@ function valueMatchesExpected(kind, actual, expected) { return Object.is(actual, expected); } -function inspectEnvMirrors(ctx, { source, basePath, serviceId, env, service, desiredCommitId }) { +function inspectEnvMirrors(ctx, { source, basePath, serviceId, env, service, desiredCommitId, requiredMirrorNames = [] }) { const present = {}; const missing = []; const diagnostics = []; + const required = new Set(requiredMirrorNames); for (const name of mirrorEnvNames) { if (!Object.hasOwn(env ?? {}, name)) { missing.push(name); + if (required.has(name)) { + const kind = observationKindForEnv(name); + const expected = expectedEnvValue(name, service, desiredCommitId); + const diagnostic = { + code: "missing_mirror", + path: `${basePath}.${name}`, + message: `${source} ${serviceId} must include ${name} so DEV applies own runtime ${kind} identity`, + expected, + actual: null + }; + diagnostics.push(diagnostic); + addDiagnostic(ctx, diagnostic); + } continue; } const actual = env[name]; @@ -600,7 +617,8 @@ export async function buildDesiredStatePlan(options = {}) { serviceId, env: deployService.env ?? {}, service: expectedService, - desiredCommitId + desiredCommitId, + requiredMirrorNames: requiredMirrorEnvNamesByService.get(serviceId) ?? [] }) }; } @@ -691,7 +709,8 @@ export async function buildDesiredStatePlan(options = {}) { serviceId, env: workload.env, service: expectedService, - desiredCommitId + desiredCommitId, + requiredMirrorNames: requiredMirrorEnvNamesByService.get(serviceId) ?? [] }) }); } diff --git a/scripts/src/dev-deploy-apply.mjs b/scripts/src/dev-deploy-apply.mjs index 4624243d..6ac3172a 100755 --- a/scripts/src/dev-deploy-apply.mjs +++ b/scripts/src/dev-deploy-apply.mjs @@ -56,6 +56,7 @@ const forbiddenActions = [ "heavy-master-e2e", "force-push" ]; +const runtimeIdentityEnvNames = ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"]; function parseArgs(argv) { const flags = new Set(argv.filter((arg) => arg.startsWith("--"))); @@ -407,6 +408,54 @@ function parseImageTag(image) { return image.slice(colonIndex + 1); } +function expectedRuntimeIdentityEnv(artifact) { + return { + HWLAB_COMMIT_ID: artifact.sourceCommitId, + HWLAB_IMAGE: artifact.image, + HWLAB_IMAGE_TAG: artifact.imageTag + }; +} + +function envObjectFromEnvList(envList) { + const env = {}; + if (!Array.isArray(envList)) return env; + for (const entry of envList) { + if (!entry?.name) continue; + env[entry.name] = Object.hasOwn(entry, "value") ? entry.value : null; + } + return env; +} + +function runtimeIdentityValueMatches(name, actual, expected) { + if (typeof actual !== "string" || typeof expected !== "string") return false; + if (name === "HWLAB_COMMIT_ID") return commitMatchesSource(actual, expected); + return actual === expected; +} + +export function compareRuntimeIdentityEnv(env, expected) { + const fields = runtimeIdentityEnvNames.map((name) => { + const actual = Object.hasOwn(env ?? {}, name) ? env[name] : null; + const expectedValue = expected?.[name] ?? null; + const status = runtimeIdentityValueMatches(name, actual, expectedValue) ? "match" : "drift"; + return { + name, + expected: expectedValue, + actual, + status + }; + }); + return { + status: fields.every((field) => field.status === "match") ? "match" : "drift", + matchesDesired: fields.every((field) => field.status === "match"), + fields, + drift: fields.filter((field) => field.status === "drift") + }; +} + +function formatRuntimeIdentityDrift(drift) { + return drift.map((field) => `${field.name} expected ${field.expected ?? "missing"} actual ${field.actual ?? "missing"}`).join("; "); +} + function deploymentRevision(deployment) { return deployment?.metadata?.annotations?.["deployment.kubernetes.io/revision"] ?? null; } @@ -420,13 +469,14 @@ function buildRolloutVerificationCommand(serviceId) { return `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout status deployment/${serviceId} --timeout=180s`; } -async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blockers) { +function buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase) { const artifact = artifactIdentityForService(deploy, catalog, serviceId); const commandArgs = ["-n", namespace, "get", "deployment", serviceId, "-o", "json"]; - const base = { + return { serviceId, namespace, status: "not_evaluated", + observationPhase, sourceCommitId: artifact.sourceCommitId, image: artifact.image ?? "unknown", imageTag: artifact.imageTag ?? "unknown", @@ -436,10 +486,29 @@ async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blo rolloutRevision: null, liveImage: null, imageMatchesDesired: false, + expectedRuntimeEnv: expectedRuntimeIdentityEnv(artifact), + liveRuntimeEnv: null, + runtimeEnvMatchesDesired: false, + runtimeEnvDrift: [], verificationCommand: buildRolloutVerificationCommand(serviceId), readCommand: kubectlCommand(kubectl, commandArgs), kubeconfig: kubectl.kubeconfig }; +} + +function skippedDeploymentRolloutObservation(kubectl, deploy, catalog, serviceId, observationPhase, reason) { + return { + ...buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase), + status: "not_evaluated", + blocker: reason + }; +} + +async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blockers, options = {}) { + const observationPhase = options.observationPhase ?? "read"; + const blockRuntimeEnvDrift = options.blockRuntimeEnvDrift === true; + const commandArgs = ["-n", namespace, "get", "deployment", serviceId, "-o", "json"]; + const base = buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase); if (kubectl.status !== "ready") { addBlocker(blockers, "environment_blocker", `rollout-read-${serviceId}`, kubectl.reason); @@ -464,12 +533,27 @@ async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blo try { const deployment = JSON.parse(result.stdout); const liveImage = containerImageForDeployment(deployment, serviceId); + const liveRuntimeEnv = compareRuntimeIdentityEnv( + envObjectFromEnvList(deployment?.spec?.template?.spec?.containers?.find((container) => container.name === serviceId)?.env), + base.expectedRuntimeEnv + ); + if (blockRuntimeEnvDrift && !liveRuntimeEnv.matchesDesired) { + addBlocker( + blockers, + "runtime_blocker", + `rollout-runtime-env-${observationPhase}-${serviceId}`, + `DEV Deployment ${serviceId} runtime identity env drift after apply: ${formatRuntimeIdentityDrift(liveRuntimeEnv.drift)}` + ); + } return { ...base, status: "observed", rolloutRevision: deploymentRevision(deployment), liveImage, - imageMatchesDesired: Boolean(artifact.image && liveImage === artifact.image) + imageMatchesDesired: Boolean(base.image && liveImage === base.image), + liveRuntimeEnv, + runtimeEnvMatchesDesired: liveRuntimeEnv.matchesDesired, + runtimeEnvDrift: liveRuntimeEnv.drift }; } catch (error) { const summary = oneLine(error.message); @@ -498,6 +582,12 @@ function blockerHint(blocker) { if (blocker.scope.startsWith("rollout-read-")) { return "Restore read-only Deployment visibility in hwlab-dev so rollout revision, image tag, digest, and source commit can be reported after apply."; } + if (blocker.scope.startsWith("deploy-runtime-env-") || blocker.scope.startsWith("k8s-runtime-env-")) { + return "Update deploy/deploy.json and deploy/k8s/base/workloads.yaml so the service owns HWLAB_COMMIT_ID, HWLAB_IMAGE, and HWLAB_IMAGE_TAG."; + } + if (blocker.scope.startsWith("rollout-runtime-env-")) { + return "Reconcile the DEV Deployment runtime identity env to the desired source image and rerun post-apply read-only verification."; + } if (blocker.scope === "dev-health" || blocker.scope === "dev-health-identity") { return "Restore the public DEV health path and confirm it identifies HWLAB dev, not a substitute runtime."; } @@ -603,6 +693,28 @@ function commitMatchesSource(commitId, sourceCommitId) { return sourceCommitId.startsWith(commitId) || commitId.startsWith(sourceCommitId); } +function validateDeployRuntimeIdentityEnv(deploy, catalog, serviceId, blockers) { + const deployService = deploy?.services?.find((service) => service.serviceId === serviceId) ?? null; + if (!deployService) return null; + const artifact = artifactIdentityForService(deploy, catalog, serviceId); + const expected = expectedRuntimeIdentityEnv(artifact); + const comparison = compareRuntimeIdentityEnv(deployService.env ?? {}, expected); + if (!comparison.matchesDesired) { + addBlocker( + blockers, + "contract_blocker", + `deploy-runtime-env-${serviceId}`, + `${serviceId} deploy env does not own desired runtime identity: ${formatRuntimeIdentityDrift(comparison.drift)}` + ); + } + return { + serviceId, + source: "deploy", + expected, + ...comparison + }; +} + function validateDeployAndCatalog(deploy, catalog, sourceCommitId, blockers) { const artifactEvidence = []; if (!deploy || !catalog) return artifactEvidence; @@ -649,6 +761,7 @@ function validateDeployAndCatalog(deploy, catalog, sourceCommitId, blockers) { publishState: catalogService.publishState }); } + validateDeployRuntimeIdentityEnv(deploy, catalog, "hwlab-cloud-web", blockers); const requiredServices = (catalog.services ?? []).filter((service) => service.artifactRequired !== false); const disabledServices = (catalog.services ?? []).filter((service) => service.artifactRequired === false); @@ -668,7 +781,7 @@ function validateDeployAndCatalog(deploy, catalog, sourceCommitId, blockers) { return artifactEvidence; } -function validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, blockers) { +function validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers) { const items = [...listItems(namespaceDoc), ...listItems(workloads), ...listItems(services), ...listItems(healthContract)]; if (devKustomization?.namespace !== namespace) { addBlocker(blockers, "safety_blocker", "dev-kustomization", "DEV kustomization must pin namespace hwlab-dev"); @@ -684,8 +797,10 @@ function validateK8s(devKustomization, namespaceDoc, workloads, services, health } const expectedImages = new Map((deploy?.services ?? []).map((service) => [service.serviceId, service.image])); + const cloudWebExpectedRuntimeEnv = expectedRuntimeIdentityEnv(artifactIdentityForService(deploy, catalog, "hwlab-cloud-web")); const workloadItems = listItems(workloads); const images = []; + const runtimeIdentityEnv = []; for (const item of workloadItems) { for (const container of containersFor(item)) { const serviceId = serviceIdFor(item, container); @@ -694,6 +809,25 @@ function validateK8s(devKustomization, namespaceDoc, workloads, services, health if (expectedImage && container.image !== expectedImage) { addBlocker(blockers, "contract_blocker", `k8s-image-${serviceId}`, `${serviceId} image differs in k8s workload`); } + if (serviceId === "hwlab-cloud-web") { + const comparison = compareRuntimeIdentityEnv(envObjectFromEnvList(container.env), cloudWebExpectedRuntimeEnv); + runtimeIdentityEnv.push({ + serviceId, + kind: item.kind, + name: item.metadata?.name, + container: container.name, + expected: cloudWebExpectedRuntimeEnv, + ...comparison + }); + if (!comparison.matchesDesired) { + addBlocker( + blockers, + "contract_blocker", + `k8s-runtime-env-${serviceId}`, + `${serviceId} workload env does not own desired runtime identity: ${formatRuntimeIdentityDrift(comparison.drift)}` + ); + } + } } } return { @@ -702,7 +836,8 @@ function validateK8s(devKustomization, namespaceDoc, workloads, services, health services: listItems(services).length, configMaps: listItems(healthContract).filter((item) => item.kind === "ConfigMap").length }, - images + images, + runtimeIdentityEnv }; } @@ -1160,7 +1295,7 @@ export async function runDevDeployApply(argv, io = {}) { const commitId = resolveApplySourceCommit(deploy, catalog, gitHeadCommitId); const artifactEvidence = validateDeployAndCatalog(deploy, catalog, commitId, blockers); - const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, blockers); + const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers); const cloudApiDb = await checkCloudApiDb(deploy, workloads, services, blockers); if (!(await readText("internal/cloud/server.mjs")).includes('url.pathname === "/health/live"')) { addBlocker(blockers, "contract_blocker", "cloud-api-health-path", "cloud-api does not implement the k8s /health/live path"); @@ -1168,7 +1303,10 @@ export async function runDevDeployApply(argv, io = {}) { const kubectl = await resolveD601Kubectl(); const clusterObservation = await observeCluster(kubectl, blockers); - const cloudWebRollout = await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers); + const cloudWebRolloutBeforeApply = await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, { + observationPhase: "before_apply", + blockRuntimeEnvDrift: false + }); const liveProbe = args.skipLiveProbe ? { status: "not_run", reason: "--skip-live-probe" } : await probeDevEndpoint(blockers); const templateJobReplacements = await buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers); const applyStep = await runApplyStep( @@ -1177,6 +1315,20 @@ export async function runDevDeployApply(argv, io = {}) { blockers, templateJobReplacements ); + const cloudWebRolloutAfterApply = args.apply && applyStep.status === "pass" + ? await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, { + observationPhase: "after_apply", + blockRuntimeEnvDrift: true + }) + : skippedDeploymentRolloutObservation( + kubectl, + deploy, + catalog, + "hwlab-cloud-web", + "after_apply", + args.apply ? "apply did not complete successfully; post-apply rollout env verification was not run" : "dry-run mode; no post-apply rollout env verification" + ); + const cloudWebRollout = args.apply ? cloudWebRolloutAfterApply : cloudWebRolloutBeforeApply; const status = blockers.length > 0 ? "blocked" : "pass"; const artifactPlan = buildArtifactPlan(deploy, catalog, commitId, artifactEvidence); const workloadPlan = buildWorkloadPlan(workloads); @@ -1264,6 +1416,8 @@ export async function runDevDeployApply(argv, io = {}) { templateJobReplacementPolicy: devTemplateJobReplacementPolicy, templateJobReplacements, cloudWebRollout, + cloudWebRolloutBeforeApply, + cloudWebRolloutAfterApply, applyStep, manualCommands, rollbackHint: buildRollbackHint(workloads), diff --git a/scripts/src/dev-deploy-apply.test.mjs b/scripts/src/dev-deploy-apply.test.mjs index 1f00eba7..016f208b 100644 --- a/scripts/src/dev-deploy-apply.test.mjs +++ b/scripts/src/dev-deploy-apply.test.mjs @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import test from "node:test"; -import { decideDevTemplateJobReplacement, resolveApplySourceCommit } from "./dev-deploy-apply.mjs"; +import { compareRuntimeIdentityEnv, decideDevTemplateJobReplacement, resolveApplySourceCommit } from "./dev-deploy-apply.mjs"; test("allowlisted suspended template Job image change plans replacement", () => { const decision = decideDevTemplateJobReplacement({ @@ -86,3 +86,44 @@ test("apply source commit follows deploy/catalog artifact identity", () => { "cb35ada68606" ); }); + +test("runtime identity env comparator accepts matching commit image and tag", () => { + const comparison = compareRuntimeIdentityEnv( + { + HWLAB_COMMIT_ID: "7de6edd", + HWLAB_IMAGE: "127.0.0.1:5000/hwlab/hwlab-cloud-web:7de6edd", + HWLAB_IMAGE_TAG: "7de6edd" + }, + { + HWLAB_COMMIT_ID: "7de6edd2c41fb50dcb007353780338be8832b27e", + HWLAB_IMAGE: "127.0.0.1:5000/hwlab/hwlab-cloud-web:7de6edd", + HWLAB_IMAGE_TAG: "7de6edd" + } + ); + + assert.equal(comparison.status, "match"); + assert.equal(comparison.matchesDesired, true); + assert.deepEqual(comparison.drift, []); +}); + +test("runtime identity env comparator detects stale cloud-web env drift", () => { + const comparison = compareRuntimeIdentityEnv( + { + HWLAB_COMMIT_ID: "c7de474", + HWLAB_IMAGE: "127.0.0.1:5000/hwlab/hwlab-cloud-web:c7de474", + HWLAB_IMAGE_TAG: "c7de474" + }, + { + HWLAB_COMMIT_ID: "7de6edd", + HWLAB_IMAGE: "127.0.0.1:5000/hwlab/hwlab-cloud-web:7de6edd", + HWLAB_IMAGE_TAG: "7de6edd" + } + ); + + assert.equal(comparison.status, "drift"); + assert.equal(comparison.matchesDesired, false); + assert.deepEqual( + comparison.drift.map((field) => field.name), + ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"] + ); +}); diff --git a/scripts/src/dev-runtime-migration.mjs b/scripts/src/dev-runtime-migration.mjs new file mode 100644 index 00000000..7696da44 --- /dev/null +++ b/scripts/src/dev-runtime-migration.mjs @@ -0,0 +1,672 @@ +import { createHash } from "node:crypto"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS, + CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS +} from "../../internal/db/schema.mjs"; +import { + PostgresCloudRuntimeStore, + RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, + RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING, + RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, + RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED +} from "../../internal/db/runtime-store.mjs"; +import { DEV_DB_ENV_CONTRACT } from "../../internal/cloud/db-contract.mjs"; +import { ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const migrationPath = "internal/db/migrations/0001_cloud_core_skeleton.sql"; +const defaultReportPath = "reports/dev-gate/dev-runtime-migration-report.json"; +const issue = "pikasTech/HWLAB#164"; + +export async function runDevRuntimeMigrationCli(argv = process.argv.slice(2), options = {}) { + const args = parseArgs(argv); + if (args.help) { + process.stdout.write(`${usage()}\n`); + return 0; + } + + const report = await buildDevRuntimeMigrationReport(args, options); + if (args.writeReport) { + await writeReport(report, args.reportPath, options.repoRoot ?? repoRoot); + } + + const printable = args.pretty || args.writeReport ? report : summarizeReport(report); + process.stdout.write(`${JSON.stringify(printable, null, 2)}\n`); + + if (report.safetyRefusal || (args.failOnBlocked && report.conclusion.status !== "ready")) { + return 1; + } + return 0; +} + +export async function buildDevRuntimeMigrationReport(args = {}, options = {}) { + const parsed = normalizeArgs(args); + const env = options.env ?? process.env; + const now = options.now ?? (() => new Date().toISOString()); + const root = options.repoRoot ?? repoRoot; + const sql = await readFile(path.join(root, migrationPath), "utf8"); + const sourceCheck = validateMigrationSource(sql); + const db = summarizeDbEnv(env); + const report = { + issue, + mode: parsed.mode, + generatedAt: now(), + target: { + environment: ENVIRONMENT_DEV, + namespace: DEV_DB_ENV_CONTRACT.dns.namespace, + prodAllowed: false, + dbUrlEnv: "HWLAB_CLOUD_DB_URL", + dbUrlSecretRef: DEV_DB_ENV_CONTRACT.secretRefs[0], + dbEndpointAuthority: DEV_DB_ENV_CONTRACT.endpointAuthority.source + }, + migration: { + id: CLOUD_CORE_MIGRATION_ID, + path: migrationPath, + sha256: sha256(sql), + schemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + ledgerTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + runtimeAdapter: "postgres" + }, + db, + sourceCheck, + applyBoundary: buildApplyBoundary(parsed), + safety: buildSafety(parsed), + actions: { + sourceValidated: sourceCheck.ready, + liveDbReadAttempted: false, + liveDbWriteAttempted: false, + migrationApplied: false, + readinessVerified: false + }, + gates: sourceOnlyGates(sourceCheck), + blockers: [], + safetyRefusal: false + }; + + if (!sourceCheck.ready) { + addBlocker(report, "contract_blocker", "runtime-migration-source", "Runtime migration source does not match the durable runtime schema contract.", { + missing: sourceCheck.missing + }); + } + + if (parsed.mode === "check" || (parsed.mode === "dry-run" && !parsed.allowLiveDbRead)) { + return finalizeReport(report); + } + + if (parsed.mode === "dry-run") { + if (!parsed.confirmDev) { + addSafetyRefusal(report, "dry-run live DB verification requires --confirm-dev."); + return finalizeReport(report); + } + if (!db.urlPresent && !options.queryClient) { + addBlocker(report, "runtime_blocker", "runtime-migration-auth", "Live DB read verification requires HWLAB_CLOUD_DB_URL from the DEV Secret reference.", { + env: "HWLAB_CLOUD_DB_URL", + valueRedacted: true + }); + return finalizeReport(report); + } + await verifyRuntimeReadiness(report, env, options); + return finalizeReport(report); + } + + if (parsed.mode === "apply") { + if (!parsed.confirmDev || !parsed.confirmedNonProduction) { + addSafetyRefusal(report, "DEV runtime migration apply requires --confirm-dev and --confirmed-non-production."); + return finalizeReport(report); + } + if (!db.urlPresent && !options.queryClient) { + addBlocker(report, "runtime_blocker", "runtime-migration-auth", "Runtime migration apply requires HWLAB_CLOUD_DB_URL from the DEV Secret reference.", { + env: "HWLAB_CLOUD_DB_URL", + valueRedacted: true + }); + return finalizeReport(report); + } + if (!sourceCheck.ready) { + addSafetyRefusal(report, "Refusing to apply because source migration validation is blocked."); + return finalizeReport(report); + } + await applyMigration(report, sql, env, options); + if (!report.safetyRefusal && report.actions.migrationApplied) { + await verifyRuntimeReadiness(report, env, options); + } + return finalizeReport(report); + } + + addSafetyRefusal(report, `Unsupported runtime migration mode: ${parsed.mode}`); + return finalizeReport(report); +} + +export function parseArgs(argv = []) { + const args = { + mode: "check", + allowLiveDbRead: false, + confirmDev: false, + confirmedNonProduction: false, + writeReport: false, + reportPath: defaultReportPath, + pretty: false, + failOnBlocked: false, + help: false + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--check") args.mode = "check"; + else if (arg === "--dry-run") args.mode = "dry-run"; + else if (arg === "--apply") args.mode = "apply"; + else if (arg === "--allow-live-db-read") args.allowLiveDbRead = true; + else if (arg === "--confirm-dev") args.confirmDev = true; + else if (arg === "--confirmed-non-production") args.confirmedNonProduction = true; + else if (arg === "--write-report") args.writeReport = true; + else if (arg === "--pretty") args.pretty = true; + else if (arg === "--fail-on-blocked") args.failOnBlocked = true; + else if (arg === "--help" || arg === "-h") args.help = true; + else if (arg === "--report") { + index += 1; + args.reportPath = requireArgValue(argv[index], "--report"); + } else if (arg.startsWith("--report=")) { + args.reportPath = requireArgValue(arg.slice("--report=".length), "--report"); + } else { + throw new Error(`unknown argument: ${arg}`); + } + } + + return args; +} + +export function validateMigrationSource(sql) { + const missing = []; + for (const [table, columns] of Object.entries(CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS)) { + const tableMatch = sql.match(new RegExp(`CREATE TABLE IF NOT EXISTS ${table}\\s*\\(([\\s\\S]*?)\\);`, "u")); + if (!tableMatch) { + missing.push(`table:${table}`); + continue; + } + for (const column of columns) { + if (!new RegExp(`\\b${column}\\b`, "u").test(tableMatch[1])) { + missing.push(`column:${table}.${column}`); + } + } + } + + const ledgerMatch = sql.match( + new RegExp(`CREATE TABLE IF NOT EXISTS ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE}\\s*\\(([\\s\\S]*?)\\);`, "u") + ); + if (!ledgerMatch) { + missing.push(`table:${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE}`); + } else { + for (const column of CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS) { + if (!new RegExp(`\\b${column}\\b`, "u").test(ledgerMatch[1])) { + missing.push(`column:${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE}.${column}`); + } + } + } + + if (!new RegExp(`'${CLOUD_CORE_MIGRATION_ID}'`, "u").test(sql)) { + missing.push(`ledger-id:${CLOUD_CORE_MIGRATION_ID}`); + } + if (!new RegExp(`'${CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION}'`, "u").test(sql)) { + missing.push(`schema-version:${CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION}`); + } + if (!/ON CONFLICT\s*\(id\)\s*DO UPDATE/iu.test(sql)) { + missing.push("ledger-upsert:on-conflict-id"); + } + + return { + ready: missing.length === 0, + checked: true, + migrationPath, + requiredMigrationId: CLOUD_CORE_MIGRATION_ID, + requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + requiredTables: Object.keys(CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS), + requiredLedgerTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + missing, + connectsToDatabase: false, + fixtureEvidence: false + }; +} + +function normalizeArgs(args) { + return { + ...parseArgs([]), + ...args + }; +} + +async function verifyRuntimeReadiness(report, env, options) { + report.actions.liveDbReadAttempted = true; + const runtimeStore = new PostgresCloudRuntimeStore({ + env, + dbUrl: env?.HWLAB_CLOUD_DB_URL, + sslMode: env?.HWLAB_CLOUD_DB_SSL_MODE, + queryClient: options.queryClient, + pgModuleLoader: options.pgModuleLoader + }); + let runtime; + try { + runtime = await runtimeStore.readiness(); + } finally { + if (!options.queryClient && typeof runtimeStore.pool?.end === "function") { + await runtimeStore.pool.end(); + } + } + report.runtime = summarizeRuntime(runtime); + report.actions.readinessVerified = runtime.ready === true; + report.gates = gatesFromRuntime(runtime); + + if (runtime.ready !== true) { + addBlocker(report, "runtime_blocker", blockerScope(runtime.blocker), runtime.reason ?? "Runtime durable readiness is blocked.", { + blocker: runtime.blocker ?? "runtime_durable_adapter_blocked", + adapter: runtime.adapter, + durable: Boolean(runtime.durable), + schemaReady: Boolean(runtime.schema?.ready), + migrationReady: Boolean(runtime.migration?.ready), + readinessReady: runtime.ready === true, + queryResult: runtime.connection?.queryResult ?? "unknown", + secretValuesPrinted: false + }); + } +} + +async function applyMigration(report, sql, env, options) { + report.actions.liveDbWriteAttempted = true; + let client; + let closeClient = async () => {}; + try { + if (options.queryClient) { + client = options.queryClient; + } else { + const migrationClient = await createMigrationClient(env, options); + client = migrationClient.client; + closeClient = migrationClient.close; + } + await client.query("BEGIN"); + await client.query(sql); + await client.query("COMMIT"); + report.actions.migrationApplied = true; + } catch (error) { + await rollbackQuietly(client); + const classified = classifyMigrationError(error); + addBlocker(report, "runtime_blocker", blockerScope(classified.blocker), classified.summary, { + blocker: classified.blocker, + errorCode: classified.errorCode, + secretValuesPrinted: false + }); + } finally { + await closeClient(); + } +} + +async function createMigrationClient(env, options) { + const pool = await createPgPool(env, options); + let client; + try { + client = await pool.connect(); + } catch (error) { + await pool.end(); + throw error; + } + return { + client, + close: async () => { + try { + client.release(); + } finally { + await pool.end(); + } + } + }; +} + +async function createPgPool(env, options) { + let pg; + try { + pg = await (options.pgModuleLoader ? options.pgModuleLoader() : import("pg")); + } catch (error) { + if (error?.code === "ERR_MODULE_NOT_FOUND") { + const driverError = new Error("Postgres migration apply requires the pg package"); + driverError.code = "HWLAB_PG_DRIVER_MISSING"; + throw driverError; + } + throw error; + } + const Pool = pg.Pool ?? pg.default?.Pool; + if (typeof Pool !== "function") { + const driverError = new Error("Postgres migration apply could not load pg.Pool"); + driverError.code = "HWLAB_PG_DRIVER_MISSING"; + throw driverError; + } + return new Pool({ + connectionString: env.HWLAB_CLOUD_DB_URL, + ssl: env.HWLAB_CLOUD_DB_SSL_MODE === "disable" ? false : { rejectUnauthorized: false }, + connectionTimeoutMillis: normalizeTimeoutMs(env.HWLAB_CLOUD_DB_PROBE_TIMEOUT_MS) + }); +} + +async function rollbackQuietly(client) { + if (!client) return; + try { + await client.query("ROLLBACK"); + } catch { + // The original migration failure is reported without leaking connection details. + } +} + +function gatesFromRuntime(runtime = {}) { + const gates = runtime.gates ?? {}; + return { + auth: normalizeGate(gates.auth), + schema: normalizeGate(gates.schema), + migration: normalizeGate(gates.migration), + readiness: normalizeGate(gates.durability) + }; +} + +function sourceOnlyGates(sourceCheck) { + return { + auth: notCheckedGate("source-only validation does not read DB credentials or connect to Postgres"), + schema: sourceCheck.ready + ? readyGate("source migration declares required durable runtime tables and columns") + : blockedGate("runtime_durable_schema_source_blocked"), + migration: sourceCheck.ready + ? readyGate("source migration declares the required ledger upsert") + : blockedGate("runtime_durable_migration_source_blocked"), + readiness: notCheckedGate("runtime readiness requires a separate authorized DEV DB read or apply") + }; +} + +function normalizeGate(gate = {}) { + if (gate.ready === true) return readyGate(); + if (gate.status === "blocked") return blockedGate(gate.blocker ?? "runtime_durable_adapter_blocked"); + return notCheckedGate(); +} + +function readyGate(summary = "ready") { + return { + checked: true, + ready: true, + status: "ready", + blocker: null, + summary + }; +} + +function blockedGate(blocker, summary = "blocked") { + return { + checked: true, + ready: false, + status: "blocked", + blocker, + summary + }; +} + +function notCheckedGate(summary = "not checked") { + return { + checked: false, + ready: false, + status: "not_checked", + blocker: null, + summary + }; +} + +function summarizeRuntime(runtime = {}) { + return { + adapter: runtime.adapter ?? "unknown", + durable: Boolean(runtime.durable), + durableRequested: Boolean(runtime.durableRequested), + ready: runtime.ready === true, + status: runtime.status ?? "unknown", + blocker: runtime.blocker ?? null, + liveRuntimeEvidence: Boolean(runtime.liveRuntimeEvidence), + fixtureEvidence: Boolean(runtime.fixtureEvidence), + connection: { + queryAttempted: Boolean(runtime.connection?.queryAttempted), + queryResult: runtime.connection?.queryResult ?? "unknown", + endpointRedacted: true, + valueRedacted: true, + errorCode: runtime.connection?.errorCode ?? null + }, + schema: summarizeRuntimeSchema(runtime.schema), + migration: summarizeRuntimeMigration(runtime.migration) + }; +} + +function summarizeRuntimeSchema(schema = {}) { + return { + checked: Boolean(schema.checked), + ready: Boolean(schema.ready), + requiredTables: Array.isArray(schema.requiredTables) ? [...schema.requiredTables] : Object.keys(CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS), + missingTables: Array.isArray(schema.missingTables) ? [...schema.missingTables] : [], + missingColumns: Array.isArray(schema.missingColumns) ? [...schema.missingColumns] : [] + }; +} + +function summarizeRuntimeMigration(migration = {}) { + return { + checked: Boolean(migration.checked), + ready: Boolean(migration.ready), + table: migration.table ?? CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + requiredMigrationId: migration.requiredMigrationId ?? CLOUD_CORE_MIGRATION_ID, + requiredSchemaVersion: migration.requiredSchemaVersion ?? CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + appliedMigrationId: migration.appliedMigrationId ?? null, + appliedSchemaVersion: migration.appliedSchemaVersion ?? null, + missing: migration.missing !== false + }; +} + +function summarizeDbEnv(env = {}) { + const urlPresent = typeof env.HWLAB_CLOUD_DB_URL === "string" && env.HWLAB_CLOUD_DB_URL.trim().length > 0; + const sslMode = env.HWLAB_CLOUD_DB_SSL_MODE || DEV_DB_ENV_CONTRACT.nonSecretDefaults.HWLAB_CLOUD_DB_SSL_MODE; + return { + requiredEnv: DEV_DB_ENV_CONTRACT.requiredEnv.map((name) => ({ + name, + present: name === "HWLAB_CLOUD_DB_URL" ? urlPresent : typeof env[name] === "string" && env[name].trim().length > 0, + redacted: name === "HWLAB_CLOUD_DB_URL", + source: name === "HWLAB_CLOUD_DB_URL" ? "k8s-secret-ref" : "runtime-env" + })), + urlPresent, + urlValueRedacted: true, + sslMode, + sslModeSecret: false, + secretRef: DEV_DB_ENV_CONTRACT.secretRefs[0], + endpointAuthority: DEV_DB_ENV_CONTRACT.endpointAuthority.source, + optionalPublicDnsAliasRequired: DEV_DB_ENV_CONTRACT.dns.requiredForReadiness + }; +} + +function buildApplyBoundary(args) { + return { + defaultMode: "check", + mode: args.mode, + liveDbReads: args.mode === "apply" || (args.mode === "dry-run" && args.allowLiveDbRead), + liveDbWrites: args.mode === "apply", + requiresForLiveDbRead: ["--confirm-dev", "HWLAB_CLOUD_DB_URL from hwlab-cloud-api-dev-db/database-url"], + requiresForApply: ["--apply", "--confirm-dev", "--confirmed-non-production", "HWLAB_CLOUD_DB_URL from hwlab-cloud-api-dev-db/database-url"], + writeScope: args.mode === "apply" + ? [ + "DEV Postgres schema objects declared by internal/db/migrations/0001_cloud_core_skeleton.sql", + "DEV hwlab_schema_migrations ledger row 0001_cloud_core_skeleton" + ] + : [], + noWriteScope: [ + "Kubernetes Secret resources", + "PROD", + "service restarts", + "M3/M4/M5 acceptance" + ], + forbidden: [ + "printing DB URLs, passwords, tokens, Secret values, or kubeconfig material", + "using fixture output as DEV-LIVE runtime evidence", + "mutating PROD" + ] + }; +} + +function buildSafety(args) { + return { + devOnly: true, + environment: ENVIRONMENT_DEV, + targetNamespace: DEV_DB_ENV_CONTRACT.dns.namespace, + prodAllowed: false, + sourceStoresSecretValues: false, + printsSecretValues: false, + dbUrlValueRedacted: true, + endpointRedacted: true, + readsKubernetesSecrets: false, + writesKubernetesSecrets: false, + liveDbWrites: args.mode === "apply", + fixtureEvidenceAllowed: false + }; +} + +function addBlocker(report, type, scope, summary, evidence = {}) { + report.blockers.push({ + type, + scope, + status: "open", + summary, + sourceIssue: scope.includes("auth") ? "pikasTech/HWLAB#49" : issue, + evidence + }); +} + +function addSafetyRefusal(report, summary) { + report.safetyRefusal = true; + addBlocker(report, "safety_refusal", "runtime-migration-apply-boundary", summary, { + devOnly: true, + prodAllowed: false, + secretValuesPrinted: false + }); +} + +function finalizeReport(report) { + const openBlockers = report.blockers.filter((blocker) => blocker.status === "open"); + report.conclusion = { + status: openBlockers.length === 0 ? "ready" : "blocked", + summary: openBlockers.length === 0 + ? conclusionReadySummary(report) + : "DEV runtime migration automation is blocked; see separated auth/schema/migration/readiness blockers.", + blockerCount: openBlockers.length + }; + return report; +} + +function conclusionReadySummary(report) { + if (report.mode === "apply") { + return "DEV runtime migration applied and durable runtime readiness was verified without printing secret values."; + } + if (report.mode === "dry-run" && report.actions.liveDbReadAttempted) { + return "DEV runtime migration read-only verification passed without DB writes or secret output."; + } + return "Runtime migration source contract is ready; no DB connection or live write was attempted."; +} + +function blockerScope(blocker) { + if (blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED) return "runtime-migration-auth"; + if (blocker === RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED) return "runtime-migration-schema"; + if (blocker === RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED) return "runtime-migration-ledger"; + if (blocker === RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING) return "runtime-migration-driver"; + return "runtime-migration-readiness"; +} + +function classifyMigrationError(error) { + const errorCode = typeof error?.code === "string" ? error.code : "UNKNOWN"; + if (errorCode === "HWLAB_PG_DRIVER_MISSING") { + return { + blocker: RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING, + errorCode, + summary: "Postgres driver is unavailable for DEV runtime migration apply." + }; + } + if (["28P01", "28000", "42501"].includes(errorCode)) { + return { + blocker: RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, + errorCode, + summary: "DEV runtime migration reached Postgres but auth or authorization blocked apply." + }; + } + if (["42P01", "42703", "3F000"].includes(errorCode)) { + return { + blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, + errorCode, + summary: "DEV runtime migration apply was blocked by schema/search_path state." + }; + } + return { + blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + errorCode, + summary: "DEV runtime migration apply could not complete." + }; +} + +async function writeReport(report, reportPath, root) { + const absolute = path.resolve(root, reportPath); + await mkdir(path.dirname(absolute), { recursive: true }); + await writeFile(absolute, `${JSON.stringify(report, null, 2)}\n`); +} + +function summarizeReport(report) { + return { + issue: report.issue, + mode: report.mode, + conclusion: report.conclusion, + migration: report.migration, + actions: report.actions, + gates: report.gates, + blockers: report.blockers, + safety: report.safety + }; +} + +function normalizeTimeoutMs(value) { + const parsed = Number.parseInt(value ?? "", 10); + return Number.isInteger(parsed) && parsed > 0 ? parsed : 1200; +} + +function requireArgValue(value, flag) { + if (typeof value !== "string" || value.trim() === "") { + throw new Error(`${flag} requires a value`); + } + return value; +} + +function sha256(value) { + return createHash("sha256").update(value).digest("hex"); +} + +function usage() { + return [ + "Usage: node scripts/dev-runtime-migration.mjs [--check|--dry-run|--apply]", + "", + "Default --check validates the source migration contract only and never connects to DB.", + "--dry-run is source-only unless --allow-live-db-read --confirm-dev is provided.", + "--apply requires --confirm-dev --confirmed-non-production and HWLAB_CLOUD_DB_URL.", + "Reports never print DB URLs, passwords, tokens, Secret values, or kubeconfig material." + ].join("\n"); +} + +export function formatDevRuntimeMigrationFailure(error) { + return { + issue, + conclusion: { + status: "blocked", + summary: "DEV runtime migration command failed before producing a report." + }, + error: error instanceof Error ? error.message : String(error), + safety: { + devOnly: true, + prodAllowed: false, + secretValuesPrinted: false + } + }; +} diff --git a/scripts/src/dev-runtime-migration.test.mjs b/scripts/src/dev-runtime-migration.test.mjs new file mode 100644 index 00000000..f4a006da --- /dev/null +++ b/scripts/src/dev-runtime-migration.test.mjs @@ -0,0 +1,238 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + buildDevRuntimeMigrationReport, + parseArgs +} from "./dev-runtime-migration.mjs"; +import { + CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS +} from "../../internal/db/schema.mjs"; + +test("source check is non-secret and does not attempt live DB access", async () => { + const report = await buildDevRuntimeMigrationReport(parseArgs(["--check"]), { + env: { + HWLAB_CLOUD_DB_URL: "postgresql://hwlab:secret-password@db.example.test:5432/hwlab", + HWLAB_CLOUD_DB_SSL_MODE: "require" + }, + now: () => "2026-05-22T00:00:00.000Z" + }); + + assert.equal(report.conclusion.status, "ready"); + assert.equal(report.actions.liveDbReadAttempted, false); + assert.equal(report.actions.liveDbWriteAttempted, false); + assert.equal(report.gates.auth.status, "not_checked"); + assert.equal(report.gates.schema.status, "ready"); + assert.equal(report.gates.migration.status, "ready"); + assert.equal(report.gates.readiness.status, "not_checked"); + assert.equal(report.safety.sourceStoresSecretValues, false); + assert.equal(report.safety.printsSecretValues, false); + assert.equal(JSON.stringify(report).includes("secret-password"), false); + assert.equal(JSON.stringify(report).includes("db.example.test"), false); +}); + +test("apply mode refuses missing DEV confirmation before touching query client", async () => { + const queryClient = createFakeQueryClient({ migrationReady: false }); + const report = await buildDevRuntimeMigrationReport(parseArgs(["--apply"]), { + env: { + HWLAB_CLOUD_DB_URL: "postgresql://hwlab:redacted@example.invalid:5432/hwlab", + HWLAB_CLOUD_DB_SSL_MODE: "require" + }, + queryClient, + now: () => "2026-05-22T00:00:00.000Z" + }); + + assert.equal(report.conclusion.status, "blocked"); + assert.equal(report.safetyRefusal, true); + assert.equal(report.actions.liveDbWriteAttempted, false); + assert.equal(queryClient.calls.length, 0); +}); + +test("live dry-run separates migration ledger blocker from auth and schema", async () => { + const report = await buildDevRuntimeMigrationReport( + parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]), + { + env: { + HWLAB_CLOUD_DB_URL: "postgresql://hwlab:redacted@example.invalid:5432/hwlab", + HWLAB_CLOUD_DB_SSL_MODE: "require" + }, + queryClient: createFakeQueryClient({ migrationReady: false }), + now: () => "2026-05-22T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "blocked"); + assert.equal(report.actions.liveDbReadAttempted, true); + assert.equal(report.actions.liveDbWriteAttempted, false); + assert.equal(report.gates.auth.status, "ready"); + assert.equal(report.gates.schema.status, "ready"); + assert.equal(report.gates.migration.status, "blocked"); + assert.equal(report.gates.readiness.status, "not_checked"); + assert.equal(report.blockers[0].scope, "runtime-migration-ledger"); + assert.equal(report.runtime.migration.missing, true); +}); + +test("apply mode records ledger and verifies durable runtime readiness", async () => { + const queryClient = createFakeQueryClient({ migrationReady: false }); + const report = await buildDevRuntimeMigrationReport( + parseArgs(["--apply", "--confirm-dev", "--confirmed-non-production"]), + { + env: { + HWLAB_CLOUD_DB_URL: "postgresql://hwlab:redacted@example.invalid:5432/hwlab", + HWLAB_CLOUD_DB_SSL_MODE: "require" + }, + queryClient, + now: () => "2026-05-22T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "ready"); + assert.equal(report.actions.liveDbWriteAttempted, true); + assert.equal(report.actions.migrationApplied, true); + assert.equal(report.actions.readinessVerified, true); + assert.equal(report.gates.auth.status, "ready"); + assert.equal(report.gates.schema.status, "ready"); + assert.equal(report.gates.migration.status, "ready"); + assert.equal(report.gates.readiness.status, "ready"); + assert.equal(report.runtime.migration.appliedMigrationId, CLOUD_CORE_MIGRATION_ID); + assert.equal(report.runtime.migration.appliedSchemaVersion, CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION); + assert.ok(queryClient.calls.some((call) => call.sql === "BEGIN")); + assert.ok(queryClient.calls.some((call) => call.sql.includes("INSERT INTO hwlab_schema_migrations"))); + assert.ok(queryClient.calls.some((call) => call.sql === "COMMIT")); + assert.equal(JSON.stringify(report).includes("redacted@example.invalid"), false); +}); + +test("live dry-run closes pg pool after readiness verification", async () => { + const backingClient = createFakeQueryClient({ migrationReady: true }); + const pools = []; + const report = await buildDevRuntimeMigrationReport( + parseArgs(["--dry-run", "--allow-live-db-read", "--confirm-dev"]), + { + env: { + HWLAB_CLOUD_DB_URL: "postgresql://hwlab:redacted@example.invalid:5432/hwlab", + HWLAB_CLOUD_DB_SSL_MODE: "require" + }, + pgModuleLoader: async () => createFakePgModule({ backingClient, pools }), + now: () => "2026-05-22T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "ready"); + assert.equal(pools.length, 1); + assert.ok(pools[0].calls.some((call) => call.target === "pool" && call.sql.includes("information_schema.columns"))); + assert.ok(pools[0].calls.some((call) => call.type === "end")); +}); + +test("apply mode uses a dedicated pg client transaction", async () => { + const backingClient = createFakeQueryClient({ migrationReady: false }); + const pools = []; + const report = await buildDevRuntimeMigrationReport( + parseArgs(["--apply", "--confirm-dev", "--confirmed-non-production"]), + { + env: { + HWLAB_CLOUD_DB_URL: "postgresql://hwlab:redacted@example.invalid:5432/hwlab", + HWLAB_CLOUD_DB_SSL_MODE: "require" + }, + pgModuleLoader: async () => createFakePgModule({ backingClient, pools }), + now: () => "2026-05-22T00:00:00.000Z" + } + ); + + assert.equal(report.conclusion.status, "ready"); + assert.equal(pools.length, 2); + assert.ok(pools[0].calls.some((call) => call.type === "connect")); + assert.ok(pools[0].calls.some((call) => call.target === "client" && call.sql === "BEGIN")); + assert.ok(pools[0].calls.some((call) => call.target === "client" && call.sql.includes("INSERT INTO hwlab_schema_migrations"))); + assert.ok(pools[0].calls.some((call) => call.target === "client" && call.sql === "COMMIT")); + assert.equal(pools[0].calls.some((call) => call.target === "pool"), false); + assert.ok(pools[0].calls.some((call) => call.type === "release")); + assert.ok(pools[0].calls.some((call) => call.type === "end")); + assert.ok(pools[1].calls.some((call) => call.target === "pool" && call.sql.includes("information_schema.columns"))); + assert.ok(pools[1].calls.some((call) => call.type === "end")); +}); + +function createFakeQueryClient({ migrationReady }) { + const state = { + migrationReady, + calls: [] + }; + return { + get calls() { + return state.calls; + }, + async query(sql, params = []) { + state.calls.push({ sql, params }); + if (sql === "BEGIN" || sql === "COMMIT" || sql === "ROLLBACK") { + return { rows: [] }; + } + if (sql.includes("INSERT INTO hwlab_schema_migrations")) { + state.migrationReady = true; + return { rows: [] }; + } + if (sql.includes("information_schema.columns")) { + return { rows: schemaRows() }; + } + if (sql.startsWith("SELECT id, schema_version FROM hwlab_schema_migrations")) { + return { + rows: state.migrationReady + ? [ + { + id: CLOUD_CORE_MIGRATION_ID, + schema_version: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION + } + ] + : [] + }; + } + if (sql.startsWith("SELECT COUNT(*)::int AS count FROM ")) { + return { rows: [{ count: 0 }] }; + } + throw new Error(`unexpected query: ${sql}`); + } + }; +} + +function createFakePgModule({ backingClient, pools }) { + return { + Pool: class FakePool { + constructor(config) { + this.config = config; + this.calls = []; + pools.push(this); + } + + async connect() { + this.calls.push({ type: "connect" }); + return { + query: async (sql, params = []) => { + this.calls.push({ target: "client", sql, params }); + return backingClient.query(sql, params); + }, + release: () => { + this.calls.push({ type: "release" }); + } + }; + } + + async query(sql, params = []) { + this.calls.push({ target: "pool", sql, params }); + return backingClient.query(sql, params); + } + + async end() { + this.calls.push({ type: "end" }); + } + } + }; +} + +function schemaRows() { + return Object.entries(CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS).flatMap(([table, columns]) => + columns.map((column) => ({ + table_name: table, + column_name: column + })) + ); +} diff --git a/web/hwlab-cloud-web/index.html b/web/hwlab-cloud-web/index.html index ee98288c..b1746487 100644 --- a/web/hwlab-cloud-web/index.html +++ b/web/hwlab-cloud-web/index.html @@ -213,7 +213,37 @@ SOURCE -
+
+
+ SOURCE 兜底状态 +
+
+
+ Gateway/Box SOURCE fixture + SOURCE +
+ 离线查看 + 等待 app.mjs 或同源只读 counts 覆盖；当前仅展示 SOURCE 拓扑摘要。 +
+
+
+ hwlab-patch-panel + SOURCE +
+ SOURCE route + res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 仅为源码兜底显示。 +
+
+
+ DO1 -> patch-panel -> DI1 + BLOCKED +
+ waiting-for-dev-live + 缺少 patch-panel live report 与 operation/trace/audit/evidence IDs，不能标为 DEV-LIVE。 +
+
+
+
@@ -250,7 +280,18 @@ trace/evidence - + + + res_boxsimu_1 + DO1 + hwlab-patch-panel + res_boxsimu_2 + DI1 + 阻塞 BLOCKED + SOURCE / required M3 route + m3-route-required / waiting-for-dev-live + +