diff --git a/AGENTS.md b/AGENTS.md index 941632e4..b0448b9f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -58,6 +58,8 @@ HWLAB 是硬件实验室运行面和控制面项目。本文是 agent、指挥 - Cloud Web M3 只读护栏：`npm run web:m3-readonly` - Cloud Workbench 布局/遮挡 smoke：`npm run web:layout`；local-build 用 `npm run web:layout:build`；DEV deploy 后用 `npm run web:layout:live`。 - DEV artifact 发布预检：`npm run dev-artifact:preflight` +- DEV artifact CI 发布：`HWLAB_CI_ARTIFACT_RUN_ID= npm run dev-artifact:publish` +- DEV 依赖 runtime base 构建：`npm run dev-runtime-base:build` - DEV CD 单事务发布/应用/验证：`npm run dev-cd:apply` - runner GitHub 可见性预检：`npm run runner:issue-visibility:preflight` - D601 k3s 只读观测：`npm run d601:k3s:readonly` diff --git a/docs/dev-artifact-publish.md b/docs/dev-artifact-publish.md index aff1911c..68227e68 100644 --- a/docs/dev-artifact-publish.md +++ b/docs/dev-artifact-publish.md @@ -2,10 +2,16 @@ `scripts/dev-artifact-publish.mjs` is the DEV-only artifact preflight/build backend for D601 local/internal registry artifacts. Publish side effects are -normally entered through `scripts/dev-cd-apply.mjs`, which sets the transaction -environment and serializes publish/apply/verify with the DEV CD Lease lock. -The artifact script does not deploy workloads, read secrets, enable PROD, or -push to GHCR/Docker Hub/other third-party registries. +normally entered by a CI artifact job identity, or by the legacy +`scripts/dev-cd-apply.mjs` transaction path during transition. The artifact +script does not deploy workloads, read secrets, enable PROD, or push to +GHCR/Docker Hub/other third-party registries. + +长期口径：CI 产物报告不是人工维护的镜像库状态，也不是 CD 放行的本地状态源。 +它是 CI 自动生成的一次性审计记录。CD 必须从 `deploy/deploy.json` 读取发布控制 +意图，从 `deploy/artifact-catalog.dev.json` 读取 digest 期望，并直接向 registry +manifest 验证目标 tag/digest 是否存在。`/tmp/hwlab-dev-gate/dev-artifacts.json` +只能用于排障、刷新 catalog 和审计追溯。 ## Scope @@ -20,9 +26,10 @@ push to GHCR/Docker Hub/other third-party registries. `--pull=false`. - Node runtime dependencies are installed from the source `package.json` and `package-lock.json` inside the DEV image build with `npm ci --omit=dev - --ignore-scripts` when the lockfile is present. This includes the Postgres - `pg` driver required by the - durable cloud-api runtime adapter. + --ignore-scripts` when the lockfile is present, unless the selected base + image already provides `/opt/hwlab-node-runtime-base/node_modules`. This + includes the Postgres `pg` driver required by the durable cloud-api runtime + adapter. - Image tag: first seven characters of the Git commit used as build source. - Required image labels: - `hwlab.pikastech.local/repo` @@ -40,7 +47,9 @@ node --check scripts/src/dev-base-image-preflight.mjs node --check scripts/src/dev-artifact-services.mjs node --check scripts/src/registry-capabilities.mjs node --check scripts/dev-artifact-publish.mjs +node --check scripts/dev-runtime-base-image.mjs node --check scripts/refresh-artifact-catalog.mjs +node --test scripts/refresh-artifact-catalog.test.mjs ``` Base image preflight: @@ -61,22 +70,31 @@ Build only: node scripts/dev-artifact-publish.mjs --build ``` -Build and publish to the D601 local/internal registry inside the DEV CD -transaction: +Build only one or more selected services for a CI artifact task: + +```sh +node scripts/dev-artifact-publish.mjs --build --services hwlab-cloud-web +``` + +`--services` narrows the build/publish report to selected service IDs. It is a +CI/artifact acceleration surface; the normal DEV rollout still consumes +repo-owned desired state through `scripts/dev-cd-apply.mjs`. + +Build and publish to the D601 local/internal registry inside a CI artifact job: + +```sh +HWLAB_CI_ARTIFACT_RUN_ID= node scripts/dev-artifact-publish.mjs --publish +``` + +The legacy transaction entry remains available during migration: ```sh node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json ``` -The underlying single-step command is kept only for transaction internals and -specialized tests that explicitly inject the transaction environment: - -```sh -node scripts/dev-artifact-publish.mjs --publish -``` - -Direct `--publish` without `HWLAB_CD_TRANSACTION_ID` is rejected with -`cd-transaction-required`; this is a bypass guard, not a second lock. +Direct `--publish` without `HWLAB_CI_ARTIFACT_RUN_ID` or +`HWLAB_CD_TRANSACTION_ID` is rejected with `cd-transaction-required`; this is a +bypass guard, not a second lock. Daily Cloud Web source validation does not require ignored `dist/` to exist: @@ -107,12 +125,32 @@ registry prefix. The script rejects third-party registry hosts and any prefix that names PROD. Use `--base-image` only to override `HWLAB_DEV_BASE_IMAGE` with an approved local image for that run. +Docker build output is visible by default so a slow layer is observable from the +runner log. Use `--quiet-build` only when a caller already captures the full +Docker log elsewhere. The artifact report records per-service +`dockerBuildDurationMs`, `cloudWebBuildDurationMs`, and `publishDurationMs`. + +An optional dependency base image can be prepared with: + +```sh +node scripts/dev-runtime-base-image.mjs --dry-run +node scripts/dev-runtime-base-image.mjs +``` + +The default tag is +`127.0.0.1:5000/hwlab/hwlab-node-runtime-base:deps-`. +Selecting that image through `HWLAB_DEV_BASE_IMAGE` lets service builds reuse a +preinstalled `node_modules` layer instead of repeating `npm ci` for every +service. + ## Report -The script writes `/tmp/hwlab-dev-gate/dev-artifacts.json`. The file is kept in the -existing DEV gate report envelope for compatibility with +The script writes `/tmp/hwlab-dev-gate/dev-artifacts.json`. The file is kept in +the existing DEV gate report envelope for compatibility with `scripts/validate-dev-gate-report.mjs`, while the `artifactPublish` object is -the HWLAB#35-specific publish record. +the HWLAB#35-specific publish record. This report is evidence, not desired +state. A later CD rollout must verify catalog images directly against the +registry instead of trusting this file. Each service record contains: diff --git a/docs/dev-base-image-preflight.md b/docs/dev-base-image-preflight.md index 71fe0cb9..e471ba27 100644 --- a/docs/dev-base-image-preflight.md +++ b/docs/dev-base-image-preflight.md @@ -57,10 +57,10 @@ Selection order is intentionally narrow: tag such as `127.0.0.1:5000/hwlab/hwlab-dev-base:node20-bookworm-slim`. 4. If neither source is available, the result is a blocker. -Explicit base image values must match local `node:20-*`, `hwlab-dev-base:*`, or -`hwlab-node20-base:*` references. They must not reference UniDesk, Code Queue, -`backend-core`, `provider-gateway`, `microservice-proxy`, or an HWLAB runtime -service image. A service image such as +Explicit base image values must match local `node:20-*`, `hwlab-dev-base:*`, +`hwlab-node20-base:*`, or `hwlab-node-runtime-base:*` references. They must not +reference UniDesk, Code Queue, `backend-core`, `provider-gateway`, +`microservice-proxy`, or an HWLAB runtime service image. A service image such as `127.0.0.1:5000/hwlab/hwlab-cloud-api:` is an artifact output, not a builder base image. @@ -113,6 +113,19 @@ docker tag node:20-bookworm-slim 127.0.0.1:5000/hwlab/hwlab-dev-base:node20-book HWLAB_DEV_BASE_IMAGE=127.0.0.1:5000/hwlab/hwlab-dev-base:node20-bookworm-slim node scripts/preflight-dev-base-image.mjs ``` +Optional dependency runtime base provision: + +```sh +node scripts/dev-runtime-base-image.mjs --dry-run +node scripts/dev-runtime-base-image.mjs +HWLAB_DEV_BASE_IMAGE=127.0.0.1:5000/hwlab/hwlab-node-runtime-base:deps- node scripts/preflight-dev-base-image.mjs +``` + +This runtime base caches `package.json` / `package-lock.json` dependencies in +`/opt/hwlab-node-runtime-base/node_modules`. Service artifact builds copy that +directory when present, which avoids repeating the same `npm ci` layer for each +service. + Offline preload path when D601 cannot pull directly: ```sh diff --git a/docs/reference/deployment-publish.md b/docs/reference/deployment-publish.md index 9d9b613e..413dcb41 100644 --- a/docs/reference/deployment-publish.md +++ b/docs/reference/deployment-publish.md @@ -43,7 +43,7 @@ truth. | Stage | Current HWLAB state | Required next behavior | | --- | --- | --- | | 1. Long-term reference | This file records DEV deployment, Cloud Web publish, API/edge/health, frp, D601 registry, k3s rollout, rollback, and verification rules. `AGENTS.md` indexes it. | Keep process issues as sources only; update this reference when a deployment rule changes. | -| 2. CLI or controlled scripts | `scripts/dev-cd-apply.mjs` is the only formal DEV CD side-effect entrypoint for publish, desired-state refresh, apply, and live verify. Other controlled scripts still cover contract planning, dry-run planning, preflight, single-step internals, k3s read-only visibility, and Cloud Web checks. Rollback is documented as commands/report hints but is not yet one unified CLI command. | Converge check, plan, rollback, status, and verify into HWLAB CLI while preserving the single transaction lock for side effects backed by `deploy/deploy.json`. | +| 2. CLI or controlled scripts | `scripts/dev-artifact-publish.mjs` is the formal CI artifact publish entrypoint under CI identity; `scripts/dev-cd-apply.mjs` is the formal DEV CD side-effect entrypoint for desired-state apply and live verify. Other controlled scripts still cover contract planning, dry-run planning, preflight, single-step internals, k3s read-only visibility, and Cloud Web checks. Rollback is documented as commands/report hints but is not yet one unified CLI command. | Converge check, plan, rollback, status, and verify into HWLAB CLI while preserving CI/CD separation and the single CD transaction lock backed by `deploy/deploy.json`. | | 3. UniDesk CI/CD plus images | DEV services are imageized and published to the D601 registry. D601 rollout still includes manual bridge steps for Cloud Web and frp/k3s execution. | Move build, publish, desired-state refresh, rollout, post-deploy validation, and report upload into standard UniDesk CI/CD jobs without replacing HWLAB runtime. | ## DEV CD Transaction @@ -54,8 +54,10 @@ truth. node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json ``` -该命令是唯一允许一次性发布 DEV artifact、从发布报告刷新 desired -state、apply 到 `hwlab-dev`、并复验公开 `16666/16667` 的正式写路径。 +该命令是唯一允许按 `deploy.json` 对 `hwlab-dev` 执行 desired-state apply、并复验 +公开 `16666/16667` 的正式 CD 写路径。镜像发布属于 CI artifact 职责；过渡期 +latest-main 兼容路径仍可由该命令调用 artifact publish，但正式 rollout 默认应消费 +已经发布并经过 registry manifest 验证的 artifact。 事务开始时读取 `deploy/deploy.json`，记录 manifest hash 和 target commit。 在获取 `Lease/hwlab-dev/hwlab-dev-cd-lock` 或运行任何 publish、runtime Job、 apply、rollout 类副作用前，脚本必须先完成 apply-only preflight：强制 @@ -90,14 +92,17 @@ checkout promotion parent 或手工拼装发布环境。 `origin/main` 控制入口，`target.promotionCommit` 是从 `deploy/deploy.json` 解析出的真实 promotion commit，`target.artifactBoundary` 记录最近一次触碰 `deploy/deploy.json`、`deploy/artifact-catalog.dev.json`、 -`deploy/k8s/base/workloads.yaml`、`/tmp/hwlab-dev-gate/dev-artifacts.json` 的 -first-parent artifact merge commit，以及该 merge commit 的 promotion parent -和 artifact/report parent。`artifactCatalog`、`artifactReport`、`deployJson` -分别记录 hash、commit、digest 计数和 publish source。若 deploy/catalog/report -或 artifact merge parent 不能同时指向同一个 promotion commit，状态降级为 -`degraded`，真实 `--apply` 在获取 Lease 前以 artifact-boundary blocker 停止。 -这保证 `deploy/deploy.json` 仍是唯一发布控制源；Lease 只串行化事务，不保存第 -二套 desired-state。 +`deploy/k8s/base/workloads.yaml` 的 first-parent desired-state commit，以及该 +commit 的 promotion parent。`artifactCatalog`、`artifactReport`、`deployJson` +分别记录 hash、commit、digest 计数和 publish source，但只有 +`deploy/deploy.json`、artifact catalog、workload manifest 和 registry manifest +共同决定 CD 是否可执行；`/tmp/hwlab-dev-gate/dev-artifacts.json` 只是 CI 审计快照。 +若 deploy/catalog/workloads 或 desired-state parent 不能同时指向同一个 +promotion commit，状态降级为 `degraded`，真实 `--apply` 在获取 Lease 前以 +artifact-boundary blocker 停止。缺失或过期的 CI 产物报告只能产生诊断提示，不得 +阻塞已经由 desired-state 和 registry 证明一致的 CD rollout。这保证 +`deploy/deploy.json` 仍是唯一发布控制源；Lease 只串行化事务，不保存第二套 +desired-state。 Lease 只是发布平面的互斥锁，不是 desired-state 来源。Lease annotations 至少记录 `promotionCommit`、`deployJsonHash`、`ownerTaskId`、 @@ -145,7 +150,7 @@ repo-owned 入口，不能绕过它们直接拼接 publish、apply、rollout 或 | 读取 DEV CD 状态 | `node scripts/dev-cd-apply.mjs --status` | 不写入、不获取 Lease、不读取 Secret value。 | | 事务 dry-run preflight | `node scripts/dev-cd-apply.mjs --dry-run` | 不 publish、不 apply、不 rollout。 | | 检查 desired-state 收敛 | `node scripts/deploy-desired-state-plan.mjs --promotion-commit --check` | 只读 source/report。 | -| 执行 DEV CD 事务 | `node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json` | 仅 DEV，在 `Lease/hwlab-dev/hwlab-dev-cd-lock` 下 publish/apply/verify。 | +| 执行 DEV CD 事务 | `node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json` | 仅 DEV，在 `Lease/hwlab-dev/hwlab-dev-cd-lock` 下 apply/verify；legacy latest-main publish 仅作过渡兼容。 | | 读取 D601 k3s 证据 | `node scripts/d601-k3s-readonly-observability.mjs` | 只读可见性报告；不 rollout、不输出 Secret value。 | 使用 master-side wrapper 时，稳定形态是： @@ -179,9 +184,10 @@ desired-state 来源关系固定如下： | 来源 | 权威范围 | | --- | --- | | `deploy/deploy.json` | DEV environment、namespace、endpoint、service set、promotion commit 和事务消费的 image tag。 | -| `deploy/artifact-catalog.dev.json` | 同一 promotion commit 的不可变 artifact digest/tag 证据。 | +| `deploy/artifact-catalog.dev.json` | 同一 promotion commit 的不可变 artifact digest/tag 期望；CD 必须直接向 registry 校验 tag/digest 是否真实存在。 | | `deploy/k8s/base/workloads.yaml` | Kubernetes workload desired state，必须镜像 `deploy.json` 和 artifact catalog 后才能 apply。 | -| `/tmp/hwlab-dev-gate/*.json` | 只作为证据快照；不能覆盖 repo desired state。 | +| Registry manifest | 镜像事实源；回答目标 tag/digest 是否存在，不能决定哪一组服务应该上线。 | +| `/tmp/hwlab-dev-gate/*.json` | 只作为 CI/CD 审计、排障和耗时统计快照；不能覆盖 repo desired state，也不能成为 CD 放行硬依赖。 | | `Lease/hwlab-dev/hwlab-dev-cd-lock` | 只作为事务互斥锁和审计状态；永远不是 desired-state 来源。 | 只改文档的 commit 如果没有修改 `deploy/deploy.json`、 @@ -204,6 +210,75 @@ live health verification 或看板/status 收口权限。Runner 发布 artifact 输出必须包含 tag、digest 和 report 证据，并把 desired-state convergence 与 live rollout evidence 分开。 +## CI/CD 职责拆分 + +长期稳定规则是 CI 负责构建和发布镜像，CD 只消费已经构建好的镜像和 +repo-owned desired state。CD 入口不得临场重新解释 source、手工拼装 build +env、重新构建镜像或把构建失败伪装成 rollout blocker。 + +CI/CD 三源分工固定如下： + +| 来源 | 职责 | 不能承担 | +| --- | --- | --- | +| `deploy/deploy.json` | 唯一发布控制源，声明 promotion commit、service image tag 和 DEV 目标。 | 证明镜像真实存在。 | +| Registry manifest | 镜像事实源，证明目标 tag/digest 可被 registry 读取，并与 catalog 记录一致。 | 决定哪一批服务应该作为一个发布集合上线。 | +| CI 产物报告 | 自动生成的一次性审计记录，包含构建、推送、digest、耗时和日志摘要。 | 作为手工维护状态表，或成为 CD rollout 的必要本地文件。 | + +`deploy/artifact-catalog.dev.json` 是 `deploy.json` 与 registry 之间的 repo-owned +digest 期望表；`deploy/k8s/base/workloads.yaml` 是最终 Kubernetes desired state。 +CD 在 apply 前必须根据 catalog 的 image tag/digest 直接读取 registry manifest， +确认镜像事实存在且 digest 匹配。过期、缺失或路径漂移的 +`/tmp/hwlab-dev-gate/dev-artifacts.json` 只能帮助定位 CI 问题，不得推翻 +`deploy.json + catalog + registry` 已经证明一致的发布结论。 + +过渡期如果 `scripts/dev-cd-apply.mjs` 发现 `deploy/deploy.json` 已经指向一个 +通过 desired-state check 的 promotion commit，应走 `promotionSource=deploy-json` +路径，只执行 desired-state 校验、runtime 前置、apply、postflight 和 live verify， +不得重新 publish 当前 HEAD。未来 CI 常态化后，latest-main target-ref publish +路径应降级为受限兼容路径，正式 DEV rollout 默认走已发布 artifact 的 +deploy-json promotion。 + +CI/artifact 层可以使用以下加速面： + +- `scripts/dev-artifact-publish.mjs --build --services `：只构建受影响服务，避免 web-only 变更重建所有服务； +- `scripts/dev-artifact-publish.mjs --publish --services `：只在 artifact 任务边界内发布选定服务并记录 digest； +- `scripts/dev-artifact-publish.mjs --publish --concurrency `：并行构建/推送服务镜像，默认 4，最高 8； +- `scripts/dev-runtime-base-image.mjs`：按 `package.json` / `package-lock.json` + hash 构建 `hwlab-node-runtime-base:deps-`，让后续服务镜像复用依赖层。 + +CD 层不得把上述 CI/artifact 加速面当作 rollout 授权。只有当 desired-state +文件已经收敛、registry manifest 直接验证通过，CD 才能拿 Lease 执行 DEV apply。 +CD apply 内部可以用 `--rollout-concurrency ` 或 `HWLAB_DEV_CD_CONCURRENCY` +并行执行 registry manifest 验证、DEV allowlist 清理和 post-apply rollout status； +并发只加速同一事务内的独立检查，不创建第二个 CD 控制源。 + +## Git Repo Owner 权限治理 + +GitHub repo owner/admin 权限是治理权限，不是日常构建、合并、发布和清理权限。 +长期目标是把 owner 从常规交付链路中解耦，避免 owner 账号或 owner-owned +工作区成为单点阻塞，也避免 root/owner 文件污染让 runner、CI 或 CD 无法继续。 + +稳定分工如下： + +| 权限域 | 可以做什么 | 不应承担什么 | +| --- | --- | --- | +| Repo owner/admin | 维护 branch protection、required checks、runner/App/token 授权、紧急权限恢复和审计规则。 | 日常手工 build、长期持有脏 worktree、替 runner 修权限污染、绕过 PR 合并主线。 | +| CI build identity | 从受保护主线或 PR commit 构建镜像，写 CI artifact、digest report 和受控 PR/commit。 | DEV apply、live rollout、看板收口、读取 Secret value。 | +| CD deploy identity | 读取已合并 desired-state 和 registry artifact，获取 DEV Lease，执行 apply/verify。 | 构建镜像、修改 source、改写 `deploy.json` promotion commit。 | +| Code Queue runner | 在任务分支完成代码/文档/desired-state PR，运行只读 status/preflight 和授权的 artifact 任务。 | 占用 owner 权限、写 main、污染共享发布 worktree、承担长期 CD lock。 | + +所有自动化应使用可审计、可轮换的 GitHub App 或 fine-grained token；token 只授予 +所需 repo 和最小 scope。日常主线合并通过 branch protection 和 required CI +结果完成，owner/admin 只处理策略变更、紧急解锁和权限恢复。至少保留两个维护者 +具备 owner/admin 恢复能力，避免单人离线造成 repo 或部署链路停摆。 + +共享发布目录只允许作为缓存或 clean mirror，不允许成为 source of truth。任何 +CI/CD 写操作都应在 ephemeral worktree 或明确 owner 的 clean worktree 中执行； +进入写路径前必须检查 `git status --short`、`.git` 可读写权限、工作区 owner/group +和 expected remote commit。发现 root-owned 文件、不可快进、未知本地修改或权限 +不一致时，应结构化报 `environment_blocker`，由权限治理流程修复；不得直接在污染 +工作区继续 build、apply 或 push。 + ## SecretRef Preflight apply 前还必须把必要 SecretRef 作为 preflight 条件处理，而不是等待 @@ -269,11 +344,11 @@ value。 完整 JSON。DEV apply、rollout 和 live verification 仍由 host commander 统一执行；runner 默认只使用 status/dry-run 观测面，不竞争 DEV CD lock。 -旧的 side-effect 命令已经降为事务内部步骤。直接手动执行 -`scripts/dev-artifact-publish.mjs --publish` 或 -`scripts/dev-deploy-apply.mjs --apply` 且没有 `HWLAB_CD_TRANSACTION_ID` 时， -会以 `cd-transaction-required` 拒绝；preflight、build、dry-run、check 和 -read-only 模式仍可用。 +旧的 deploy side-effect 命令已经降为事务内部步骤。直接手动执行 +`scripts/dev-deploy-apply.mjs --apply` 且没有 `HWLAB_CD_TRANSACTION_ID` 时，会以 +`cd-transaction-required` 拒绝。artifact publish 是 CI 职责，正式入口需要 +`HWLAB_CI_ARTIFACT_RUN_ID`；过渡期也接受 `HWLAB_CD_TRANSACTION_ID`。preflight、 +build、dry-run、check 和 read-only 模式仍可用。 ## Artifact Publish @@ -283,15 +358,16 @@ DEV artifacts publish to the D601 local/internal registry: 127.0.0.1:5000/hwlab/* ``` -The single-step publish command is a transaction-internal side-effect step: +The single-step publish command is a CI artifact side-effect step: ```sh -node scripts/dev-artifact-publish.mjs --publish +HWLAB_CI_ARTIFACT_RUN_ID= node scripts/dev-artifact-publish.mjs --publish ``` -It only runs in side-effect mode when `HWLAB_CD_TRANSACTION_ID` is set by -`scripts/dev-cd-apply.mjs`. Use `node scripts/dev-cd-apply.mjs --apply ---confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json` for normal DEV CD. +It only runs in side-effect mode when `HWLAB_CI_ARTIFACT_RUN_ID` is set by the +CI artifact job, or when the legacy `HWLAB_CD_TRANSACTION_ID` is set by +`scripts/dev-cd-apply.mjs` during transition. Normal DEV CD must consume +`deploy.json` and registry-verified catalog state; it should not rebuild images. For Cloud Web source freshness before the transaction: @@ -496,9 +572,10 @@ use the repo-owned scripts below instead of ad hoc shell fragments. | Plan desired image/workload state | `node scripts/deploy-desired-state-plan.mjs --plan --pretty` and `node scripts/deploy-desired-state-plan.mjs --promotion-commit --check` | Fold into `hwlab deploy plan --env dev --file deploy/deploy.json` with a promotion commit gate. | | Observe DEV CD status | `node scripts/dev-cd-apply.mjs`, `node scripts/dev-cd-apply.mjs --status`, or `node scripts/dev-cd-apply.mjs --dry-run` | Keep read-only for host and runners; no Lease mutation, artifact publish, apply, rollout, report write, or Secret read. | | Preflight artifact publish | `node scripts/dev-artifact-publish.mjs --preflight` | Keep as backend for `hwlab artifact publish --preflight`. | +| Prepare dependency runtime base | `node scripts/dev-runtime-base-image.mjs --dry-run` and `node scripts/dev-runtime-base-image.mjs` | Move into a CI base-image job keyed by dependency hash; CD only consumes the resulting image tag. | | Publish/apply/verify DEV | `node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json` | Keep as the single side-effect transaction and run from repo-owned DEV CI/CD. | -| Publish DEV images | `node scripts/dev-artifact-publish.mjs --publish` inside `HWLAB_CD_TRANSACTION_ID` | Internal step only; direct side-effect calls are rejected. | -| Refresh catalog/desired state | `node scripts/refresh-artifact-catalog.mjs --target-ref origin/main --publish-report /tmp/hwlab-dev-gate/dev-artifacts.json` | Internal transaction step after publish; read/write desired-state only from repo files. | +| Publish DEV images | `HWLAB_CI_ARTIFACT_RUN_ID= node scripts/dev-artifact-publish.mjs --publish` | CI artifact step only; legacy CD transaction env remains transitional. | +| Refresh catalog/desired state | `node scripts/refresh-artifact-catalog.mjs --target-ref origin/main --publish-report /tmp/hwlab-dev-gate/dev-artifacts.json` | CI promotion step after publish; report is input evidence for catalog refresh, not CD rollout state. | | Apply DEV rollout | `node scripts/dev-deploy-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json` inside `HWLAB_CD_TRANSACTION_ID` | Internal transaction step; direct side-effect calls are rejected. | | Roll back DEV Deployment | `KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev rollout undo deployment/` | Add `hwlab deploy rollback --env dev --service --to-revision `. | | Verify status | `curl` checks plus `node scripts/d601-k3s-readonly-observability.mjs` | Add `hwlab deploy verify --env dev --report /tmp/hwlab-dev-gate/report.json`. | @@ -512,9 +589,10 @@ converge manual operations into CLI plus `deploy/deploy.json`, then run that surface from UniDesk CI/CD: - render and verify frps/frpc from the deploy manifest; -- publish images and write digest reports; -- refresh catalog/workload desired state from the publish report; +- publish images and write digest reports as CI audit evidence; +- refresh catalog/workload desired state from a verified CI publish report; - apply DEV with immutable Job replacement policy; +- verify catalog tag/digest directly against registry manifests before apply; - verify `16666` Cloud Web and `16667/health/live` API; - invalidate or regenerate active reports after endpoint or artifact changes. diff --git a/package.json b/package.json index e8dcc51a..c6574b01 100644 --- a/package.json +++ b/package.json @@ -5,8 +5,9 @@ "type": "module", "scripts": { "validate": "node scripts/repo-reports-guard.mjs && node scripts/validate-contract.mjs && node scripts/deploy-contract-plan.mjs --check && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-provisioning.mjs --check && node scripts/dev-runtime-migration.mjs --check && node scripts/dev-runtime-postflight.mjs --check && node scripts/rpt004-mvp-e2e-harness.mjs --check --no-write && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --test scripts/artifact-runtime-readiness-guard.test.mjs", - "check": "node scripts/repo-reports-guard.mjs && node --check scripts/repo-reports-guard.mjs && node --check scripts/src/report-paths.mjs && node --check internal/protocol/index.mjs && node --check internal/build-metadata.mjs && node --check internal/agent/index.mjs && node --check internal/agent/runtime.mjs && node --check internal/audit/index.mjs && node --check internal/db/runtime-store.mjs && node --check internal/db/runtime-store.test.mjs && node --check internal/mvp-gate/summary.mjs && node --check internal/cloud/db-contract.mjs && node --check internal/dev-entrypoint/http.mjs && node --check internal/dev-entrypoint/http.test.mjs && node --check internal/cloud/code-agent-contract.mjs && node --check internal/cloud/code-agent-session-registry.mjs && node --check internal/cloud/code-agent-session-registry.test.mjs && node --check internal/cloud/code-agent-trace-store.mjs && node --check internal/cloud/codex-stdio-session.mjs && node --check internal/cloud/json-rpc.mjs && node --check internal/cloud/code-agent-chat.mjs && node --check internal/cloud/m3-io-control.mjs && node --check internal/cloud/server.mjs && node --check internal/sim/model.mjs && node --check internal/sim/model.test.mjs && node --check internal/sim/http.mjs && node --check internal/sim/l2-runtime.mjs && node --check internal/patchpanel/model.mjs && node --check internal/patchpanel/runtime.mjs && node --check cmd/hwlab-cloud-api/main.mjs && node --check cmd/hwlab-edge-proxy/main.mjs && node --check cmd/hwlab-agent-mgr/main.mjs && node --check cmd/hwlab-agent-worker/main.mjs && node --check cmd/hwlab-box-simu/main.mjs && node --check cmd/hwlab-gateway/main.mjs && node --check cmd/hwlab-gateway-simu/main.mjs && node --check scripts/cloud-api-runtime-smoke.mjs && node --check scripts/code-agent-chat-smoke.mjs && node --check scripts/dev-edge-health-smoke.mjs && node --check scripts/src/dev-edge-health-smoke-lib.mjs && node --check scripts/validate-contract.mjs && node --check scripts/deploy-contract-plan.test.mjs && node --check scripts/deploy-desired-state-plan.mjs && node --check scripts/src/deploy-desired-state-plan.mjs && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --check scripts/artifact-runtime-readiness-guard.test.mjs && node --check scripts/report-lifecycle.mjs && node --check scripts/report-lifecycle.test.mjs && node --check scripts/validate-dev-gate-report.mjs && node --check scripts/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.test.mjs && node --check scripts/dev-m3-hardware-loop-smoke.test.mjs && node --check scripts/m3-io-control-e2e.mjs && node --check scripts/src/m3-io-control-e2e.mjs && node --check scripts/m3-io-control-e2e.test.mjs && node --check scripts/dev-cloud-workbench-smoke.test.mjs && node --check scripts/dev-cloud-workbench-layout-smoke.mjs && node --check scripts/rpt004-mvp-e2e-harness.mjs && node --check scripts/src/rpt004-mvp-e2e-harness.mjs && node --check scripts/src/rpt004-mvp-e2e-harness.test.mjs && node --check scripts/validate-dev-m3-cardinality.mjs && node --check scripts/validate-dev-m3-cardinality.test.mjs && node --check scripts/validate-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/src/dev-artifact-services.mjs && node --check scripts/src/registry-capabilities.mjs && node --check scripts/preflight-dev-base-image.mjs && node --check scripts/src/dev-base-image-preflight.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.test.mjs && node --check scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs && node --check scripts/d601-k3s-readonly-observability.mjs && node --check scripts/src/d601-k3s-readonly-observability.mjs && node --check scripts/dev-runtime-provisioning.mjs && node --check scripts/src/dev-runtime-provisioning.mjs && node --check scripts/src/dev-runtime-provisioning.test.mjs && node --check scripts/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.test.mjs && node --check scripts/dev-runtime-postflight.mjs && node --check scripts/src/dev-runtime-postflight.mjs && node --check scripts/src/dev-runtime-postflight.test.mjs && node --check scripts/l2-runtime-contract-smoke.mjs && node --check scripts/patch-panel-runtime-smoke.mjs && node --check scripts/export-web-gate-summary.mjs && node --check scripts/l6-cli-web-smoke.mjs && node --check skills/hwlab-agent-runtime/scripts/hwlab-agent-runtime-cli.mjs && node --check skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs && node --check skills/hwlab-agent-runtime/scripts/m3-io-skill-client.test.mjs && node --check tools/hwlab-cli/bin/hwlab-cli.mjs && node --check tools/hwlab-cli/lib/cli.mjs && node --check tools/hwlab-cli/lib/cli.test.mjs && node --check web/hwlab-cloud-web/app.mjs && node --check web/hwlab-cloud-web/code-agent-facts.mjs && node --check web/hwlab-cloud-web/code-agent-facts.test.mjs && node --check web/hwlab-cloud-web/code-agent-status.mjs && node --check web/hwlab-cloud-web/code-agent-status.test.mjs && node --check web/hwlab-cloud-web/code-agent-m3-evidence.mjs && node --check web/hwlab-cloud-web/code-agent-m3-evidence.test.mjs && node --check web/hwlab-cloud-web/live-status.mjs && node --check web/hwlab-cloud-web/wiring-status.mjs && node --check web/hwlab-cloud-web/gate-summary.mjs && node --check web/hwlab-cloud-web/scripts/check.mjs && node --check web/hwlab-cloud-web/scripts/live-status-contract.test.mjs && node --check web/hwlab-cloud-web/scripts/build.mjs && node --check web/hwlab-cloud-web/scripts/dist-contract.mjs && node scripts/validate-contract.mjs && node scripts/validate-dev-gate-report.mjs && node scripts/report-lifecycle.test.mjs && node scripts/validate-dev-m3-cardinality.mjs && node scripts/validate-artifact-catalog.mjs && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-provisioning.mjs --check && node scripts/dev-runtime-migration.mjs --check && node scripts/dev-runtime-postflight.mjs --check && node scripts/rpt004-mvp-e2e-harness.mjs --check --no-write && node scripts/dev-evidence-blocker-aggregator.mjs --check && node scripts/l2-runtime-contract-smoke.mjs && node scripts/l6-cli-web-smoke.mjs && node web/hwlab-cloud-web/scripts/check.mjs && node scripts/code-agent-chat-smoke.mjs && node --test web/hwlab-cloud-web/code-agent-facts.test.mjs web/hwlab-cloud-web/code-agent-status.test.mjs web/hwlab-cloud-web/wiring-status.test.mjs web/hwlab-cloud-web/code-agent-m3-evidence.test.mjs scripts/artifact-runtime-readiness-guard.test.mjs scripts/deploy-contract-plan.test.mjs scripts/dev-m3-hardware-loop-smoke.test.mjs scripts/validate-dev-m3-cardinality.test.mjs scripts/dev-cloud-workbench-smoke.test.mjs web/hwlab-cloud-web/scripts/live-status-contract.test.mjs scripts/deploy-desired-state-plan.test.mjs scripts/src/dev-cd-apply.test.mjs scripts/src/dev-deploy-apply.test.mjs scripts/src/dev-runtime-provisioning.test.mjs scripts/src/dev-runtime-migration.test.mjs scripts/src/dev-runtime-postflight.test.mjs scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs scripts/src/dev-evidence-blocker-aggregator.test.mjs skills/hwlab-agent-runtime/scripts/m3-io-skill-client.test.mjs tools/hwlab-cli/lib/cli.test.mjs internal/agent/index.test.mjs internal/audit/index.test.mjs internal/db/schema.test.mjs internal/db/runtime-store.test.mjs internal/cloud/json-rpc.test.mjs internal/cloud/m3-io-control.test.mjs internal/cloud/code-agent-session-registry.test.mjs internal/cloud/server.test.mjs internal/dev-entrypoint/http.test.mjs internal/sim/model.test.mjs internal/patchpanel/model.test.mjs internal/patchpanel/runtime.test.mjs && node scripts/cloud-api-runtime-smoke.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", + "check": "node scripts/repo-reports-guard.mjs && node --check scripts/repo-reports-guard.mjs && node --check scripts/src/report-paths.mjs && node --check internal/protocol/index.mjs && node --check internal/build-metadata.mjs && node --check internal/agent/index.mjs && node --check internal/agent/runtime.mjs && node --check internal/audit/index.mjs && node --check internal/db/runtime-store.mjs && node --check internal/db/runtime-store.test.mjs && node --check internal/mvp-gate/summary.mjs && node --check internal/cloud/db-contract.mjs && node --check internal/dev-entrypoint/http.mjs && node --check internal/dev-entrypoint/http.test.mjs && node --check internal/cloud/code-agent-contract.mjs && node --check internal/cloud/code-agent-session-registry.mjs && node --check internal/cloud/code-agent-session-registry.test.mjs && node --check internal/cloud/code-agent-trace-store.mjs && node --check internal/cloud/codex-stdio-session.mjs && node --check internal/cloud/json-rpc.mjs && node --check internal/cloud/code-agent-chat.mjs && node --check internal/cloud/m3-io-control.mjs && node --check internal/cloud/server.mjs && node --check internal/sim/model.mjs && node --check internal/sim/model.test.mjs && node --check internal/sim/http.mjs && node --check internal/sim/l2-runtime.mjs && node --check internal/patchpanel/model.mjs && node --check internal/patchpanel/runtime.mjs && node --check cmd/hwlab-cloud-api/main.mjs && node --check cmd/hwlab-edge-proxy/main.mjs && node --check cmd/hwlab-agent-mgr/main.mjs && node --check cmd/hwlab-agent-worker/main.mjs && node --check cmd/hwlab-box-simu/main.mjs && node --check cmd/hwlab-gateway/main.mjs && node --check cmd/hwlab-gateway-simu/main.mjs && node --check scripts/cloud-api-runtime-smoke.mjs && node --check scripts/code-agent-chat-smoke.mjs && node --check scripts/dev-edge-health-smoke.mjs && node --check scripts/src/dev-edge-health-smoke-lib.mjs && node --check scripts/validate-contract.mjs && node --check scripts/deploy-contract-plan.test.mjs && node --check scripts/deploy-desired-state-plan.mjs && node --check scripts/src/deploy-desired-state-plan.mjs && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --check scripts/artifact-runtime-readiness-guard.test.mjs && node --check scripts/report-lifecycle.mjs && node --check scripts/report-lifecycle.test.mjs && node --check scripts/validate-dev-gate-report.mjs && node --check scripts/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.test.mjs && node --check scripts/dev-m3-hardware-loop-smoke.test.mjs && node --check scripts/m3-io-control-e2e.mjs && node --check scripts/src/m3-io-control-e2e.mjs && node --check scripts/m3-io-control-e2e.test.mjs && node --check scripts/dev-cloud-workbench-smoke.test.mjs && node --check scripts/dev-cloud-workbench-layout-smoke.mjs && node --check scripts/rpt004-mvp-e2e-harness.mjs && node --check scripts/src/rpt004-mvp-e2e-harness.mjs && node --check scripts/src/rpt004-mvp-e2e-harness.test.mjs && node --check scripts/validate-dev-m3-cardinality.mjs && node --check scripts/validate-dev-m3-cardinality.test.mjs && node --check scripts/validate-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.test.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/dev-runtime-base-image.mjs && node --check scripts/src/dev-artifact-services.mjs && node --check scripts/src/registry-capabilities.mjs && node --check scripts/preflight-dev-base-image.mjs && node --check scripts/src/dev-base-image-preflight.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.test.mjs && node --check scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs && node --check scripts/d601-k3s-readonly-observability.mjs && node --check scripts/src/d601-k3s-readonly-observability.mjs && node --check scripts/dev-runtime-provisioning.mjs && node --check scripts/src/dev-runtime-provisioning.mjs && node --check scripts/src/dev-runtime-provisioning.test.mjs && node --check scripts/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.test.mjs && node --check scripts/dev-runtime-postflight.mjs && node --check scripts/src/dev-runtime-postflight.mjs && node --check scripts/src/dev-runtime-postflight.test.mjs && node --check scripts/l2-runtime-contract-smoke.mjs && node --check scripts/patch-panel-runtime-smoke.mjs && node --check scripts/export-web-gate-summary.mjs && node --check scripts/l6-cli-web-smoke.mjs && node --check skills/hwlab-agent-runtime/scripts/hwlab-agent-runtime-cli.mjs && node --check skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs && node --check skills/hwlab-agent-runtime/scripts/m3-io-skill-client.test.mjs && node --check tools/hwlab-cli/bin/hwlab-cli.mjs && node --check tools/hwlab-cli/lib/cli.mjs && node --check tools/hwlab-cli/lib/cli.test.mjs && node --check web/hwlab-cloud-web/app.mjs && node --check web/hwlab-cloud-web/code-agent-facts.mjs && node --check web/hwlab-cloud-web/code-agent-facts.test.mjs && node --check web/hwlab-cloud-web/code-agent-status.mjs && node --check web/hwlab-cloud-web/code-agent-status.test.mjs && node --check web/hwlab-cloud-web/code-agent-m3-evidence.mjs && node --check web/hwlab-cloud-web/code-agent-m3-evidence.test.mjs && node --check web/hwlab-cloud-web/live-status.mjs && node --check web/hwlab-cloud-web/wiring-status.mjs && node --check web/hwlab-cloud-web/gate-summary.mjs && node --check web/hwlab-cloud-web/scripts/check.mjs && node --check web/hwlab-cloud-web/scripts/live-status-contract.test.mjs && node --check web/hwlab-cloud-web/scripts/build.mjs && node --check web/hwlab-cloud-web/scripts/dist-contract.mjs && node scripts/validate-contract.mjs && node scripts/validate-dev-gate-report.mjs && node scripts/report-lifecycle.test.mjs && node scripts/validate-dev-m3-cardinality.mjs && node scripts/validate-artifact-catalog.mjs && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-provisioning.mjs --check && node scripts/dev-runtime-migration.mjs --check && node scripts/dev-runtime-postflight.mjs --check && node scripts/rpt004-mvp-e2e-harness.mjs --check --no-write && node scripts/dev-evidence-blocker-aggregator.mjs --check && node scripts/l2-runtime-contract-smoke.mjs && node scripts/l6-cli-web-smoke.mjs && node web/hwlab-cloud-web/scripts/check.mjs && node scripts/code-agent-chat-smoke.mjs && node --test web/hwlab-cloud-web/code-agent-facts.test.mjs web/hwlab-cloud-web/code-agent-status.test.mjs web/hwlab-cloud-web/wiring-status.test.mjs web/hwlab-cloud-web/code-agent-m3-evidence.test.mjs scripts/artifact-runtime-readiness-guard.test.mjs scripts/deploy-contract-plan.test.mjs scripts/dev-m3-hardware-loop-smoke.test.mjs scripts/validate-dev-m3-cardinality.test.mjs scripts/dev-cloud-workbench-smoke.test.mjs web/hwlab-cloud-web/scripts/live-status-contract.test.mjs scripts/deploy-desired-state-plan.test.mjs scripts/refresh-artifact-catalog.test.mjs scripts/src/dev-cd-apply.test.mjs scripts/src/dev-deploy-apply.test.mjs scripts/src/dev-runtime-provisioning.test.mjs scripts/src/dev-runtime-migration.test.mjs scripts/src/dev-runtime-postflight.test.mjs scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs scripts/src/dev-evidence-blocker-aggregator.test.mjs skills/hwlab-agent-runtime/scripts/m3-io-skill-client.test.mjs tools/hwlab-cli/lib/cli.test.mjs internal/agent/index.test.mjs internal/audit/index.test.mjs internal/db/schema.test.mjs internal/db/runtime-store.test.mjs internal/cloud/json-rpc.test.mjs internal/cloud/m3-io-control.test.mjs internal/cloud/code-agent-session-registry.test.mjs internal/cloud/server.test.mjs internal/dev-entrypoint/http.test.mjs internal/sim/model.test.mjs internal/patchpanel/model.test.mjs internal/patchpanel/runtime.test.mjs && node scripts/cloud-api-runtime-smoke.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", "dev-base-image:preflight": "node scripts/preflight-dev-base-image.mjs", + "dev-runtime-base:build": "node scripts/dev-runtime-base-image.mjs", "cloud-api:smoke": "node scripts/cloud-api-runtime-smoke.mjs", "m1:smoke": "node scripts/m1-contract-smoke.mjs", "l2:smoke": "node scripts/l2-runtime-contract-smoke.mjs", @@ -30,7 +31,7 @@ "artifact-catalog:refresh-blocked": "node scripts/refresh-artifact-catalog.mjs --target-ref origin/main --blocked", "dev-artifact:preflight": "node scripts/dev-artifact-publish.mjs --preflight", "dev-cd:apply": "node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/dev-cd-apply.json", - "dev-artifact:publish": "node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/dev-cd-apply.json", + "dev-artifact:publish": "node -e \"if (!process.env.HWLAB_CI_ARTIFACT_RUN_ID) { console.error('Set HWLAB_CI_ARTIFACT_RUN_ID before DEV artifact publish.'); process.exit(2); }\" && node scripts/dev-artifact-publish.mjs --publish", "docs:validate:m3-rollout": "node scripts/validate-m3-rollout-runbook.mjs", "d601:k3s:readonly": "node scripts/d601-k3s-readonly-observability.mjs", "runner:issue-visibility:preflight": "node scripts/runner-issue-visibility-preflight.mjs", diff --git a/scripts/dev-artifact-publish.mjs b/scripts/dev-artifact-publish.mjs index 102385cd..b1dac952 100644 --- a/scripts/dev-artifact-publish.mjs +++ b/scripts/dev-artifact-publish.mjs @@ -16,7 +16,10 @@ import { resolveDevArtifactServices, serviceInventoryFromServices } from "./src/dev-artifact-services.mjs"; -import { requireDevCdTransactionForSideEffect } from "./src/dev-cd-transaction-guard.mjs"; +import { + devArtifactProducer, + requireDevArtifactSideEffectEnv +} from "./src/dev-cd-transaction-guard.mjs"; import { runDevBaseImagePreflight } from "./src/dev-base-image-preflight.mjs"; import { probeRegistryCapabilities } from "./src/registry-capabilities.mjs"; import { inspectCloudWebDistFreshness } from "../web/hwlab-cloud-web/scripts/dist-contract.mjs"; @@ -66,7 +69,9 @@ function parseArgs(argv) { baseImage: defaultBaseImage, reportPath: defaultReportPath, services: [...SERVICE_IDS], - emitReport: true + emitReport: true, + quietBuild: false, + concurrency: parseConcurrency(process.env.HWLAB_DEV_ARTIFACT_CONCURRENCY, 4) }; for (let index = 0; index < argv.length; index += 1) { @@ -79,6 +84,8 @@ function parseArgs(argv) { else if (arg === "--report") args.reportPath = readOption(argv, ++index, arg); else if (arg === "--services") args.services = readOption(argv, ++index, arg).split(",").map((item) => item.trim()).filter(Boolean); else if (arg === "--no-report") args.emitReport = false; + else if (arg === "--quiet-build") args.quietBuild = true; + else if (arg === "--concurrency") args.concurrency = parseConcurrency(readOption(argv, ++index, arg), 4); else if (arg === "--help" || arg === "-h") args.help = true; else throw new Error(`unknown argument ${arg}`); } @@ -94,6 +101,12 @@ function readOption(argv, index, name) { return value; } +function parseConcurrency(value, fallback) { + const parsed = Number.parseInt(String(value ?? ""), 10); + if (!Number.isInteger(parsed)) return fallback; + return Math.min(Math.max(parsed, 1), 8); +} + function printHelp() { console.log([ "usage: node scripts/dev-artifact-publish.mjs [--preflight|--build|--publish]", @@ -105,7 +118,9 @@ function printHelp() { " --base-image IMAGE override HWLAB_DEV_BASE_IMAGE for this run; must already be local", " --services LIST comma-separated service IDs; default: all frozen DEV services", ` --report PATH default: ${defaultReportPath}`, - " --no-report print JSON without updating the temporary artifact JSON" + " --no-report print JSON without updating the temporary artifact JSON", + " --quiet-build keep docker build quiet; default keeps build output visible", + " --concurrency N parallel service build/push workers; default: 4, max: 8" ].join("\n")); } @@ -114,7 +129,7 @@ async function readJson(relativePath) { } function emit(event, payload = {}) { - process.stderr.write(`${JSON.stringify({ event, ...payload })}\n`); + process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...payload })}\n`); } function blocker({ type, scope, summary, next }) { @@ -186,6 +201,7 @@ function commandLine(command, args) { } async function run(command, args, options = {}) { + const startedAt = Date.now(); const child = spawn(command, args, { cwd: options.cwd ?? repoRoot, env: options.env ?? process.env, @@ -210,7 +226,8 @@ async function run(command, args, options = {}) { command: commandLine(command, args), code, stdout, - stderr + stderr, + durationMs: Date.now() - startedAt }; } @@ -802,6 +819,28 @@ function dockerfile(baseImage, port) { return [ `FROM ${baseImage}`, "WORKDIR /app", + "ENV CODEX_HOME=/codex-home", + "ENV HWLAB_CODE_AGENT_CODEX_HOME=/codex-home", + "ENV HWLAB_CODE_AGENT_WORKSPACE=/workspace/hwlab", + "ENV HWLAB_CODE_AGENT_CODEX_WORKSPACE=/workspace/hwlab", + "ENV HWLAB_CODE_AGENT_CODEX_SANDBOX=workspace-write", + "ENV HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED=1", + "ENV HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR=repo-owned", + "ENV HWLAB_CODE_AGENT_CODEX_COMMAND=/app/node_modules/.bin/codex", + "ENV HWLAB_CODE_AGENT_PROVIDER=codex-stdio", + "ENV HWLAB_CODE_AGENT_SKILLS_DIRS=/app/skills:/root/.agents/skills:/home/ubuntu/.agents/skills", + "COPY package.json ./package.json", + "COPY package-lock.json* ./", + "RUN if [ -d /opt/hwlab-node-runtime-base/node_modules ]; then cp -a /opt/hwlab-node-runtime-base/node_modules ./node_modules; elif [ -f package-lock.json ]; then npm ci --omit=dev --ignore-scripts; else npm install --omit=dev --ignore-scripts; fi", + "COPY internal ./internal", + "COPY cmd ./cmd", + "COPY scripts ./scripts", + "COPY web ./web", + "COPY tools ./tools", + "COPY skills ./skills", + "COPY deploy ./deploy", + "RUN mkdir -p /workspace /codex-home && rm -rf /workspace/hwlab && ln -s /app /workspace/hwlab && chmod -R a+rwX /app /workspace /codex-home && test -x /app/node_modules/.bin/codex && /app/node_modules/.bin/codex --version >/tmp/hwlab-codex-version.txt && (test -x /app/node_modules/@openai/codex-linux-x64/vendor/x86_64-unknown-linux-musl/codex/codex || test -x /app/node_modules/@openai/codex-linux-arm64/vendor/aarch64-unknown-linux-musl/codex/codex)", + `RUN node -e "const fs=require('node:fs'); fs.writeFileSync('/usr/local/bin/hwlab-dev-artifact-runtime.mjs', Buffer.from('${runtimeScriptBase64()}', 'base64')); fs.chmodSync('/usr/local/bin/hwlab-dev-artifact-runtime.mjs', 0o755);"`, "ARG HWLAB_ENVIRONMENT", "ARG HWLAB_SERVICE_ID", "ARG HWLAB_ARTIFACT_KIND", @@ -816,16 +855,6 @@ function dockerfile(baseImage, port) { "ARG HWLAB_BUILD_PROVENANCE", "ARG PORT", "ARG HWLAB_PORT", - "ENV CODEX_HOME=/codex-home", - "ENV HWLAB_CODE_AGENT_CODEX_HOME=/codex-home", - "ENV HWLAB_CODE_AGENT_WORKSPACE=/workspace/hwlab", - "ENV HWLAB_CODE_AGENT_CODEX_WORKSPACE=/workspace/hwlab", - "ENV HWLAB_CODE_AGENT_CODEX_SANDBOX=workspace-write", - "ENV HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED=1", - "ENV HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR=repo-owned", - "ENV HWLAB_CODE_AGENT_CODEX_COMMAND=/app/node_modules/.bin/codex", - "ENV HWLAB_CODE_AGENT_PROVIDER=codex-stdio", - "ENV HWLAB_CODE_AGENT_SKILLS_DIRS=/app/skills:/root/.agents/skills:/home/ubuntu/.agents/skills", "ENV HWLAB_ENVIRONMENT=$HWLAB_ENVIRONMENT", "ENV HWLAB_SERVICE_ID=$HWLAB_SERVICE_ID", "ENV HWLAB_ARTIFACT_KIND=$HWLAB_ARTIFACT_KIND", @@ -840,18 +869,6 @@ function dockerfile(baseImage, port) { "ENV HWLAB_BUILD_PROVENANCE=$HWLAB_BUILD_PROVENANCE", "ENV PORT=$PORT", "ENV HWLAB_PORT=$HWLAB_PORT", - "COPY package.json ./package.json", - "COPY package-lock.json* ./", - "RUN if [ -f package-lock.json ]; then npm ci --omit=dev --ignore-scripts; else npm install --omit=dev --ignore-scripts; fi", - "COPY internal ./internal", - "COPY cmd ./cmd", - "COPY scripts ./scripts", - "COPY web ./web", - "COPY tools ./tools", - "COPY skills ./skills", - "COPY deploy ./deploy", - "RUN mkdir -p /workspace /codex-home && rm -rf /workspace/hwlab && ln -s /app /workspace/hwlab && chmod -R a+rwX /app /workspace /codex-home && test -x /app/node_modules/.bin/codex && /app/node_modules/.bin/codex --version >/tmp/hwlab-codex-version.txt && (test -x /app/node_modules/@openai/codex-linux-x64/vendor/x86_64-unknown-linux-musl/codex/codex || test -x /app/node_modules/@openai/codex-linux-arm64/vendor/aarch64-unknown-linux-musl/codex/codex)", - `RUN node -e "const fs=require('node:fs'); fs.writeFileSync('/usr/local/bin/hwlab-dev-artifact-runtime.mjs', Buffer.from('${runtimeScriptBase64()}', 'base64')); fs.chmodSync('/usr/local/bin/hwlab-dev-artifact-runtime.mjs', 0o755);"`, `EXPOSE ${port}`, "CMD [\"node\", \"/usr/local/bin/hwlab-dev-artifact-runtime.mjs\"]", "" @@ -1022,7 +1039,8 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC summary: "cloud web dist build failed before Docker build", next: "Run node web/hwlab-cloud-web/scripts/build.mjs, fix the static asset build, then rerun --publish." }), - logTail: tailText(`${build.stdout}\n${build.stderr}`) + logTail: tailText(`${build.stdout}\n${build.stderr}`), + cloudWebBuildDurationMs: build.durationMs }; } @@ -1048,7 +1066,8 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC service = { ...service, - distFreshness + distFreshness, + cloudWebBuildDurationMs: build.durationMs }; } @@ -1083,11 +1102,9 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC ]; const argsList = [ "build", - "--pull=false", - "--quiet", - "--build-arg", - `BASE_IMAGE=${args.baseImage}` + "--pull=false" ]; + if (args.quietBuild) argsList.push("--quiet"); for (const [name, value] of labels) { argsList.push("--label", `${name}=${value}`); @@ -1113,9 +1130,11 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC summary: `docker build failed for ${service.serviceId}`, next: `Inspect the Docker build output for ${service.serviceId}, then fix its copied source or base image.` }), - logTail: tailText(`${result.stdout}\n${result.stderr}`) + logTail: tailText(`${result.stdout}\n${result.stderr}`), + dockerBuildDurationMs: result.durationMs }; } + const imageInspect = await run("docker", ["image", "inspect", "--format", "{{.Id}}", ref]); return { ...service, @@ -1124,12 +1143,17 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC buildCreatedAt, buildSource, status: "built", - localImageId: result.stdout.trim(), + localImageId: imageInspect.code === 0 ? imageInspect.stdout.trim() : null, + dockerBuildDurationMs: result.durationMs, + dockerImageInspectDurationMs: imageInspect.durationMs, + dockerImageInspectStatus: imageInspect.code === 0 ? "pass" : "failed", + dockerBuildLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200), digest: "not_published" }; } async function runWithInput(command, args, input) { + const startedAt = Date.now(); const child = spawn(command, args, { cwd: repoRoot, env: process.env, @@ -1155,7 +1179,8 @@ async function runWithInput(command, args, input) { command: commandLine(command, args), code, stdout, - stderr + stderr, + durationMs: Date.now() - startedAt }; } @@ -1175,7 +1200,8 @@ async function publishService(artifact) { summary: `docker push failed for ${artifact.image}`, next: "Verify the D601 local/internal registry is reachable from the Docker daemon and rerun --publish." }), - logTail: tailText(`${result.stdout}\n${result.stderr}`) + logTail: tailText(`${result.stdout}\n${result.stderr}`), + publishDurationMs: result.durationMs }; } @@ -1191,7 +1217,8 @@ async function publishService(artifact) { summary: `docker push completed for ${artifact.image} but did not return an immutable digest`, next: "Verify registry digest reporting, then rerun --publish; do not update the catalog with a digest until a sha256 registry digest is observed." }), - pushLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200) + pushLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200), + publishDurationMs: result.durationMs }; } @@ -1200,7 +1227,8 @@ async function publishService(artifact) { status: "published", digest, repositoryDigest: `${repositoryFromImageRef(artifact.image)}@${digest}`, - pushLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200) + pushLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200), + publishDurationMs: result.durationMs }; } @@ -1288,13 +1316,34 @@ function publishPlanFor({ args, commitId, shortCommit, mode, artifacts, fatalBlo repositoryDigest: artifact.repositoryDigest ?? null, digestStatus: /^sha256:[a-f0-9]{64}$/u.test(digest) ? "real" : "placeholder", status: artifact.status, + dockerBuildDurationMs: artifact.dockerBuildDurationMs ?? null, + cloudWebBuildDurationMs: artifact.cloudWebBuildDurationMs ?? null, + publishDurationMs: artifact.publishDurationMs ?? null, notPublishedReason: artifactNotPublishedReason(artifact, mode, fatalBlocked) }; }) }; } -function createReport({ args, repo, commitId, shortCommit, mode, services, artifacts, blockers, baseImagePreflight, registryCapabilities }) { +function sumDurations(artifacts, field) { + return artifacts.reduce((sum, artifact) => sum + (Number.isFinite(artifact[field]) ? artifact[field] : 0), 0); +} + +async function mapWithConcurrency(items, concurrency, worker) { + const results = new Array(items.length); + let nextIndex = 0; + const workerCount = Math.min(Math.max(concurrency, 1), items.length); + await Promise.all(Array.from({ length: workerCount }, async () => { + while (nextIndex < items.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await worker(items[index], index); + } + })); + return results; +} + +function createReport({ args, repo, commitId, shortCommit, mode, services, artifacts, blockers, baseImagePreflight, registryCapabilities, producer }) { const status = reportStatus(mode, artifacts, blockers); const fatalBlocked = hasFatalBlocker(blockers); const requiredArtifacts = artifacts.filter((artifact) => artifact.artifactRequired); @@ -1331,6 +1380,8 @@ function createReport({ args, repo, commitId, shortCommit, mode, services, artif "node --check scripts/src/registry-capabilities.mjs", "node --check scripts/preflight-dev-base-image.mjs", "node --check scripts/src/dev-base-image-preflight.mjs", + "node --check scripts/dev-runtime-base-image.mjs", + "node --test scripts/refresh-artifact-catalog.test.mjs", "node scripts/preflight-dev-base-image.mjs", "node scripts/dev-artifact-publish.mjs --preflight --no-report", "node --check scripts/validate-dev-gate-report.mjs", @@ -1377,8 +1428,12 @@ function createReport({ args, repo, commitId, shortCommit, mode, services, artif repo, sourceCommitId: commitId, registryPrefix: args.registryPrefix, + producer, registryCapabilities: summarizeRegistryCapabilities(registryCapabilities), baseImage: args.baseImage ?? null, + selectedServiceIds: args.services, + quietBuild: args.quietBuild, + concurrency: args.concurrency, baseImagePreflight: summarizeBaseImagePreflight(baseImagePreflight), environment: ENVIRONMENT_DEV, buildCreatedAt: artifacts.find((artifact) => artifact.buildCreatedAt)?.buildCreatedAt ?? null, @@ -1389,6 +1444,11 @@ function createReport({ args, repo, commitId, shortCommit, mode, services, artif disabledServiceCount: serviceInventory.disabledServiceCount, builtCount, publishedCount, + timings: { + dockerBuildDurationMs: sumDurations(artifacts, "dockerBuildDurationMs"), + cloudWebBuildDurationMs: sumDurations(artifacts, "cloudWebBuildDurationMs"), + publishDurationMs: sumDurations(artifacts, "publishDurationMs") + }, generatedAt: new Date().toISOString(), services: artifacts.map((artifact) => ({ serviceId: artifact.serviceId, @@ -1408,7 +1468,13 @@ function createReport({ args, repo, commitId, shortCommit, mode, services, artif artifactRequired: artifact.artifactRequired, artifactScope: artifact.artifactScope, notPublishedReason: artifactNotPublishedReason(artifact, mode, fatalBlocked), - localImageId: artifact.localImageId + localImageId: artifact.localImageId, + dockerBuildDurationMs: artifact.dockerBuildDurationMs ?? null, + cloudWebBuildDurationMs: artifact.cloudWebBuildDurationMs ?? null, + publishDurationMs: artifact.publishDurationMs ?? null, + dockerBuildLogTail: artifact.dockerBuildLogTail ?? null, + logTail: artifact.logTail ?? null, + pushLogTail: artifact.pushLogTail ?? null })), blockers }, @@ -1429,7 +1495,7 @@ async function main() { return; } if (args.mode === "publish") { - const transactionGuard = requireDevCdTransactionForSideEffect({ + const transactionGuard = requireDevArtifactSideEffectEnv({ script: "scripts/dev-artifact-publish.mjs", mode: "--publish" }); @@ -1461,6 +1527,7 @@ async function main() { const shortCommit = commitId.slice(0, 7); const buildCreatedAt = new Date().toISOString(); const repo = repoLabelFromRemote(remoteUrl); + const producer = devArtifactProducer(process.env); const services = await resolveServices(args.services, catalog); let blockers = await preflight({ args, services, catalog, deployManifest, baseImagePreflight, registryCapabilities }); @@ -1478,12 +1545,18 @@ async function main() { }); } else if (args.mode === "build" || args.mode === "publish") { const artifactByServiceId = new Map(artifacts.map((artifact) => [artifact.serviceId, artifact])); - for (const service of services.filter((item) => item.artifactRequired)) { + await mapWithConcurrency(services.filter((item) => item.artifactRequired), args.concurrency, async (service) => { emit("build_start", { serviceId: service.serviceId, image: imageRef(args.registryPrefix, service.serviceId, shortCommit) }); const artifact = await buildService({ args, repo, commitId, shortCommit, service, buildCreatedAt }); artifactByServiceId.set(service.serviceId, artifact); if (artifact.blocker) blockers.push(artifact.blocker); - emit("build_done", { serviceId: service.serviceId, status: artifact.status, image: artifact.image }); + emit("build_done", { + serviceId: service.serviceId, + status: artifact.status, + image: artifact.image, + dockerBuildDurationMs: artifact.dockerBuildDurationMs ?? null, + cloudWebBuildDurationMs: artifact.cloudWebBuildDurationMs ?? null + }); if (args.mode === "publish" && artifact.status === "built") { emit("publish_start", { serviceId: service.serviceId, image: artifact.image }); @@ -1494,14 +1567,15 @@ async function main() { serviceId: service.serviceId, status: published.status, image: published.image, - digest: published.digest + digest: published.digest, + publishDurationMs: published.publishDurationMs ?? null }); } - } + }); artifacts = services.map((service) => artifactByServiceId.get(service.serviceId)); } - const report = createReport({ args, repo, commitId, shortCommit, mode: args.mode, services, artifacts, blockers, baseImagePreflight, registryCapabilities }); + const report = createReport({ args, repo, commitId, shortCommit, mode: args.mode, services, artifacts, blockers, baseImagePreflight, registryCapabilities, producer }); if (args.emitReport) { await writeReport(args.reportPath, report); } @@ -1514,6 +1588,9 @@ async function main() { registryPrefix: args.registryPrefix, registryCapabilities: summarizeRegistryCapabilities(registryCapabilities), baseImagePreflight: summarizeBaseImagePreflight(baseImagePreflight), + selectedServiceIds: report.artifactPublish.selectedServiceIds, + concurrency: report.artifactPublish.concurrency, + timings: report.artifactPublish.timings, publishPlan: report.artifactPublish.publishPlan, services: report.artifactPublish.services.map((service) => ({ serviceId: service.serviceId, @@ -1523,6 +1600,11 @@ async function main() { buildCreatedAt: service.buildCreatedAt, digest: service.digest, distFreshness: service.distFreshness, + dockerBuildDurationMs: service.dockerBuildDurationMs, + cloudWebBuildDurationMs: service.cloudWebBuildDurationMs, + publishDurationMs: service.publishDurationMs, + logTail: service.logTail, + pushLogTail: service.pushLogTail, notPublishedReason: service.notPublishedReason })), blockers diff --git a/scripts/dev-runtime-base-image.mjs b/scripts/dev-runtime-base-image.mjs new file mode 100644 index 00000000..b9204995 --- /dev/null +++ b/scripts/dev-runtime-base-image.mjs @@ -0,0 +1,301 @@ +#!/usr/bin/env node +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import { createHash } from "node:crypto"; +import { readFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + runDevBaseImagePreflight, + summarizeForRuntimeBase +} from "./src/dev-base-image-preflight.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const defaultRegistryPrefix = + process.env.HWLAB_DEV_REGISTRY_PREFIX || process.env.HWLAB_DEV_REGISTRY || "127.0.0.1:5000/hwlab"; +const defaultBaseImage = process.env.HWLAB_DEV_RUNTIME_BASE_PARENT || process.env.HWLAB_DEV_BASE_IMAGE || null; + +function parseArgs(argv) { + const args = { + registryPrefix: defaultRegistryPrefix, + baseImage: defaultBaseImage, + tag: null, + build: true, + push: false, + help: false + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--registry-prefix") args.registryPrefix = readOption(argv, ++index, arg); + else if (arg === "--base-image") args.baseImage = readOption(argv, ++index, arg); + else if (arg === "--tag") args.tag = readOption(argv, ++index, arg); + else if (arg === "--dry-run" || arg === "--plan") args.build = false; + else if (arg === "--push") args.push = true; + else if (arg === "--help" || arg === "-h") args.help = true; + else throw new Error(`unknown argument ${arg}`); + } + + return args; +} + +function readOption(argv, index, name) { + const value = argv[index]; + if (!value || value.startsWith("--")) { + throw new Error(`${name} requires a value`); + } + return value; +} + +function usage() { + return [ + "usage: node scripts/dev-runtime-base-image.mjs [--dry-run] [--push] [--base-image IMAGE] [--tag IMAGE]", + "", + "Build an optional D601-local HWLAB Node runtime base image with package dependencies preinstalled.", + "", + "options:", + " --registry-prefix PREFIX default: 127.0.0.1:5000/hwlab", + " --base-image IMAGE approved local parent image; default resolved by DEV base preflight", + " --tag IMAGE output image tag; default: /hwlab-node-runtime-base:deps-", + " --dry-run print the build plan without building", + " --push push the resulting base image to the local/internal registry" + ].join("\n"); +} + +function normalizeRegistryPrefix(prefix) { + const normalized = prefix.replace(/\/+$/u, ""); + const [hostPort, ...pathParts] = normalized.split("/"); + const host = hostPort.split(":")[0].toLowerCase(); + assert.ok(hostPort && pathParts.length > 0, "registry prefix must include host and namespace"); + assert.ok(!/prod|production/iu.test(normalized), "registry prefix must not include prod"); + assert.ok( + host === "localhost" || + host === "127.0.0.1" || + /^10\./u.test(host) || + /^192\.168\./u.test(host) || + /^172\.(1[6-9]|2\d|3[0-1])\./u.test(host) || + host.endsWith(".local") || + host.endsWith(".svc") || + host.endsWith(".cluster.local"), + "registry prefix must target localhost or a private/internal host" + ); + return normalized; +} + +async function fileOrEmpty(relativePath) { + try { + return await readFile(path.join(repoRoot, relativePath), "utf8"); + } catch (error) { + if (error?.code === "ENOENT") return ""; + throw error; + } +} + +async function dependencyHash() { + const packageJson = await fileOrEmpty("package.json"); + const packageLockJson = await fileOrEmpty("package-lock.json"); + assert.ok(packageJson.trim(), "package.json is required"); + const hash = createHash("sha256"); + hash.update("package.json\0"); + hash.update(packageJson); + hash.update("\0package-lock.json\0"); + hash.update(packageLockJson); + return hash.digest("hex"); +} + +function dockerfile(parentImage, depsHash) { + return [ + `FROM ${parentImage}`, + "WORKDIR /opt/hwlab-node-runtime-base", + `LABEL hwlab.pikastech.local/runtime-base-deps-hash=${depsHash}`, + "COPY package.json ./package.json", + "COPY package-lock.json* ./", + "RUN if [ -f package-lock.json ]; then npm ci --omit=dev --ignore-scripts; else npm install --omit=dev --ignore-scripts; fi", + "" + ].join("\n"); +} + +function commandLine(command, args) { + return [command, ...args].map((part) => (/\s/u.test(part) ? JSON.stringify(part) : part)).join(" "); +} + +function runWithInput(command, args, input) { + const startedAt = Date.now(); + return new Promise((resolve) => { + const child = spawn(command, args, { + cwd: repoRoot, + env: process.env, + stdio: ["pipe", "pipe", "pipe"] + }); + let stdout = ""; + let stderr = ""; + child.stdin.end(input); + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", (error) => { + resolve({ + command: commandLine(command, args), + code: 127, + stdout, + stderr: error.message, + durationMs: Date.now() - startedAt + }); + }); + child.on("close", (code) => { + resolve({ + command: commandLine(command, args), + code: code ?? 1, + stdout, + stderr, + durationMs: Date.now() - startedAt + }); + }); + }); +} + +function run(command, args) { + const startedAt = Date.now(); + return new Promise((resolve) => { + const child = spawn(command, args, { + cwd: repoRoot, + env: process.env, + stdio: ["ignore", "pipe", "pipe"] + }); + let stdout = ""; + let stderr = ""; + child.stdout.on("data", (chunk) => { + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { + stderr += chunk; + }); + child.on("error", (error) => { + resolve({ + command: commandLine(command, args), + code: 127, + stdout, + stderr: error.message, + durationMs: Date.now() - startedAt + }); + }); + child.on("close", (code) => { + resolve({ + command: commandLine(command, args), + code: code ?? 1, + stdout, + stderr, + durationMs: Date.now() - startedAt + }); + }); + }); +} + +function tailText(value, maxLength = 2500) { + const text = value.trim(); + if (text.length <= maxLength) return text; + return text.slice(text.length - maxLength); +} + +function inspectDigest(pushOutput) { + const matches = [...pushOutput.matchAll(/digest:\s+(sha256:[a-f0-9]{64})/giu)]; + return matches.length ? matches[matches.length - 1][1] : null; +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (args.help) { + console.log(usage()); + return; + } + + args.registryPrefix = normalizeRegistryPrefix(args.registryPrefix); + const depsHash = await dependencyHash(); + const image = args.tag ?? `${args.registryPrefix}/hwlab-node-runtime-base:deps-${depsHash.slice(0, 12)}`; + const baseImagePreflight = await runDevBaseImagePreflight({ + env: args.baseImage ? { ...process.env, HWLAB_DEV_BASE_IMAGE: args.baseImage } : process.env + }); + const parentImage = baseImagePreflight.localTag; + const plan = { + status: args.build ? "planned" : "dry-run", + image, + parentImage, + depsHash, + push: args.push, + baseImagePreflight: summarizeForRuntimeBase(baseImagePreflight) + }; + + if (!baseImagePreflight.publishUsable) { + console.log(JSON.stringify({ + ...plan, + status: "blocked", + blockers: baseImagePreflight.blockers, + nextSteps: baseImagePreflight.nextSteps + }, null, 2)); + process.exitCode = 2; + return; + } + + if (!args.build) { + console.log(JSON.stringify(plan, null, 2)); + return; + } + + const build = await runWithInput( + "docker", + ["build", "--pull=false", "-t", image, "-f", "-", "."], + dockerfile(parentImage, depsHash) + ); + if (build.code !== 0) { + console.log(JSON.stringify({ + ...plan, + status: "build_failed", + command: build.command, + durationMs: build.durationMs, + logTail: tailText(`${build.stdout}\n${build.stderr}`) + }, null, 2)); + process.exitCode = 1; + return; + } + + const inspect = await run("docker", ["image", "inspect", "--format", "{{.Id}}", image]); + let push = null; + if (args.push) { + const pushResult = await run("docker", ["push", image]); + const digest = pushResult.code === 0 ? inspectDigest(`${pushResult.stdout}\n${pushResult.stderr}`) : null; + push = { + status: pushResult.code === 0 && digest ? "published" : "blocked", + command: pushResult.command, + code: pushResult.code, + durationMs: pushResult.durationMs, + digest, + logTail: tailText(`${pushResult.stdout}\n${pushResult.stderr}`, 1200) + }; + } + + const status = push ? push.status : "built"; + console.log(JSON.stringify({ + ...plan, + status, + imageId: inspect.code === 0 ? inspect.stdout.trim() : null, + build: { + command: build.command, + durationMs: build.durationMs, + logTail: tailText(`${build.stdout}\n${build.stderr}`, 1200) + }, + push + }, null, 2)); + process.exitCode = status === "blocked" ? 2 : 0; +} + +main().catch((error) => { + console.error(JSON.stringify({ + status: "failed", + error: error instanceof Error ? error.message : String(error) + }, null, 2)); + process.exitCode = 1; +}); diff --git a/scripts/refresh-artifact-catalog.mjs b/scripts/refresh-artifact-catalog.mjs index cc39bebd..38981579 100644 --- a/scripts/refresh-artifact-catalog.mjs +++ b/scripts/refresh-artifact-catalog.mjs @@ -77,7 +77,8 @@ function usage() { } async function readJson(relativePath) { - return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8")); + const filePath = path.isAbsolute(relativePath) ? relativePath : path.join(repoRoot, relativePath); + return JSON.parse(await readFile(filePath, "utf8")); } async function writeJson(relativePath, value) { diff --git a/scripts/refresh-artifact-catalog.test.mjs b/scripts/refresh-artifact-catalog.test.mjs new file mode 100644 index 00000000..0d45bbc8 --- /dev/null +++ b/scripts/refresh-artifact-catalog.test.mjs @@ -0,0 +1,85 @@ +import assert from "node:assert/strict"; +import { execFile } from "node:child_process"; +import { mkdtemp, writeFile } from "node:fs/promises"; +import os from "node:os"; +import path from "node:path"; +import test from "node:test"; +import { promisify } from "node:util"; + +import { SERVICE_IDS } from "../internal/protocol/index.mjs"; + +const execFileAsync = promisify(execFile); +const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), ".."); + +async function git(args) { + const result = await execFileAsync("git", args, { + cwd: repoRoot, + timeout: 5000, + maxBuffer: 1024 * 1024 + }); + return result.stdout.trim(); +} + +function digestFor(index) { + return `sha256:${String(index + 1).padStart(64, "0")}`; +} + +test("refresh accepts an absolute publish report path", async () => { + const commitId = await git(["rev-parse", "HEAD"]); + const shortCommitId = await git(["rev-parse", "--short=7", "HEAD"]); + const tempDir = await mkdtemp(path.join(os.tmpdir(), "hwlab-refresh-")); + const reportPath = path.join(tempDir, "dev-artifacts.json"); + await writeFile(reportPath, `${JSON.stringify({ + reportVersion: "v1", + taskId: "dev-artifact-publish", + commitId: shortCommitId, + artifactPublish: { + status: "published", + mode: "publish", + sourceCommitId: commitId, + registryPrefix: "127.0.0.1:5000/hwlab", + serviceCount: SERVICE_IDS.length, + requiredServiceCount: SERVICE_IDS.length, + disabledServiceCount: 0, + publishedCount: SERVICE_IDS.length, + publishPlan: { + version: "v2", + services: SERVICE_IDS.map((serviceId, index) => ({ + serviceId, + required: true, + digest: digestFor(index), + image: `127.0.0.1:5000/hwlab/${serviceId}:${shortCommitId}`, + imageTag: shortCommitId + })) + }, + services: SERVICE_IDS.map((serviceId, index) => ({ + serviceId, + status: "published", + artifactRequired: true, + image: `127.0.0.1:5000/hwlab/${serviceId}:${shortCommitId}`, + imageTag: shortCommitId, + digest: digestFor(index), + repositoryDigest: `127.0.0.1:5000/hwlab/${serviceId}@${digestFor(index)}`, + buildCreatedAt: "2026-05-24T00:00:00.000Z", + buildSource: `pikasTech/HWLAB@${commitId}` + })) + } + }, null, 2)}\n`); + + const result = await execFileAsync(process.execPath, [ + "scripts/refresh-artifact-catalog.mjs", + "--target-ref", + "HEAD", + "--publish-report", + reportPath, + "--no-write" + ], { + cwd: repoRoot, + timeout: 15000, + maxBuffer: 10 * 1024 * 1024 + }); + const payload = JSON.parse(result.stdout); + assert.equal(payload.status, "published"); + assert.equal(payload.publishedCount, SERVICE_IDS.length); + assert.deepEqual(payload.wrote, []); +}); diff --git a/scripts/src/dev-base-image-preflight.mjs b/scripts/src/dev-base-image-preflight.mjs index 57e9b6db..0d340ef0 100644 --- a/scripts/src/dev-base-image-preflight.mjs +++ b/scripts/src/dev-base-image-preflight.mjs @@ -21,7 +21,9 @@ const allowedExplicitBaseImages = Object.freeze([ "hwlab-dev-base:*", "*/hwlab-dev-base:*", "hwlab-node20-base:*", - "*/hwlab-node20-base:*" + "*/hwlab-node20-base:*", + "hwlab-node-runtime-base:*", + "*/hwlab-node-runtime-base:*" ]); function baseImageRecommendation(envValue = recommendedNode20Image) { @@ -131,7 +133,7 @@ function isNode20BaseRef(imageRef) { function isHwlabDevBaseRef(imageRef) { const lowerRef = imageRef.toLowerCase(); - return /(^|\/)(hwlab-dev-base|hwlab-node20-base)(?=[:@]|$)/u.test(lowerRef); + return /(^|\/)(hwlab-dev-base|hwlab-node20-base|hwlab-node-runtime-base)(?=[:@]|$)/u.test(lowerRef); } function isTaggedImage(image) { @@ -612,3 +614,17 @@ export function formatPreflightResult(result) { export function exitCodeForPreflight(result) { return result.publishUsable ? 0 : 2; } + +export function summarizeForRuntimeBase(result) { + return { + status: result.status, + publishUsable: result.publishUsable, + imageSource: result.imageSource, + requestedImage: result.requestedImage, + localTag: result.localTag, + imageId: result.imageId, + blockers: result.blockers, + nextSteps: result.nextSteps, + containerEngine: result.containerEngine + }; +} diff --git a/scripts/src/dev-cd-apply.mjs b/scripts/src/dev-cd-apply.mjs index 8f217c08..d8b0ff7d 100644 --- a/scripts/src/dev-cd-apply.mjs +++ b/scripts/src/dev-cd-apply.mjs @@ -513,19 +513,24 @@ async function resolveArtifactBoundary(ctx, control, promotion, deployBefore, ar const reportCommitMatches = commitMatches(reportSource, promotion.commitId); const deployCommitMatches = commitMatches(deployBefore.commitId, promotion.commitId); const isMergeCommit = parents.length >= 2; - const status = ( + const desiredStateStatus = ( deployCommitMatches && catalogCommitMatches && - reportCommitMatches && ( promotionParentMatches || artifactParentMatches || commitMatches(mergeCommitId, promotion.commitId) ) ) ? "pass" : "degraded"; + const reportStatus = artifactEvidence.report.exists + ? (reportCommitMatches ? "matches_promotion" : "stale_or_unrelated") + : "missing"; return { - status, + status: desiredStateStatus, + reportStatus, + reportIsReleaseGate: false, + reportPolicy: "CI artifact reports are audit snapshots; deploy.json, artifact catalog, workloads, and registry manifests are the CD release gate.", controlCommit: { commitId: control.commitId, shortCommitId: control.shortCommitId @@ -557,6 +562,7 @@ async function resolveArtifactBoundary(ctx, control, promotion, deployBefore, ar deployCommitMatches, catalogCommitMatches, reportCommitMatches, + reportExists: artifactEvidence.report.exists, desiredStatePaths: artifactDesiredStatePaths } }; @@ -2335,7 +2341,7 @@ function buildReport({ "docs/dev-deploy-apply.md", "docs/artifact-catalog.md" ], - summary: "DEV CD publish/apply/verify side effects are serialized by one Kubernetes Lease and use deploy/deploy.json as the apply desired-state source." + summary: "DEV CD apply/verify side effects are serialized by one Kubernetes Lease and use deploy/deploy.json as the publish-control desired-state source; CI artifact reports are audit snapshots." }, validationCommands: [ "node --check scripts/dev-cd-apply.mjs", @@ -2367,8 +2373,8 @@ function buildReport({ "DEV CD apply must reject docker-desktop, desktop-control-plane, 127.0.0.1:11700, bare kubectl ambiguity, and second hwlab-dev control-plane signals before mutation.", "DEV CD apply must verify required SecretRef existence and key names without reading or printing Secret values.", "HEAD must match the requested target ref before publish.", - "Lease/hwlab-dev-cd-lock must be acquired before publish/apply.", - "Legacy publish/apply side-effect scripts must run with HWLAB_CD_TRANSACTION_ID.", + "Lease/hwlab-dev-cd-lock must be acquired before DEV apply/verify side effects.", + "Artifact publish must run under CI artifact identity, with HWLAB_CD_TRANSACTION_ID accepted only for the legacy transition path.", "DEV DB role/database provisioning and runtime migration must run through repo-owned commands before workload apply.", "deploy/deploy.json, artifact catalog, and workloads must converge before apply.", "When deploy/deploy.json already points to a published promotion commit, HEAD may be a later control commit and the transaction must consume that promotion without republishing HEAD.", diff --git a/scripts/src/dev-cd-apply.test.mjs b/scripts/src/dev-cd-apply.test.mjs index 5e3f3e6d..5b891b41 100644 --- a/scripts/src/dev-cd-apply.test.mjs +++ b/scripts/src/dev-cd-apply.test.mjs @@ -587,6 +587,56 @@ test("dev-cd status accepts a published deploy-json promotion under a later cont assertNoReadOnlySideEffects(commandLog); }); +test("dev-cd status treats stale CI artifact report as audit-only when desired state matches", async () => { + const repoRoot = await makeRepo({ commitId: "abc1234" }); + await writeFile( + artifactReportFixturePath, + `${JSON.stringify({ + status: "published", + commitId: "9999999", + artifactPublish: { + status: "published", + sourceCommitId: "9999999", + serviceCount: 0, + requiredServiceCount: 0, + publishedCount: 0, + services: [] + } + }, null, 2)}\n` + ); + const commandLog = []; + let output = ""; + const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ + commandLog, + targetCommitId: laterControlCommit, + headCommitId: laterControlCommit, + extraGitRefs: { + abc1234: publishedCommit + }, + latestArtifactMergeCommit: artifactMergeCommit, + commitParents: { + [artifactMergeCommit]: [publishedCommit, artifactReportCommit], + [artifactReportCommit]: [publishedCommit] + } + }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const status = JSON.parse(output); + assert.equal(code, 0); + assert.equal(status.status, "pass"); + assert.equal(status.target.artifactBoundary.status, "pass"); + assert.equal(status.target.artifactBoundary.reportStatus, "stale_or_unrelated"); + assert.equal(status.target.artifactBoundary.reportIsReleaseGate, false); + assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false); + assertNoReadOnlySideEffects(commandLog); +}); + test("dev-cd status exposes moving origin/main when checkout is behind a deploy-json promotion control ref", async () => { const repoRoot = await makeRepo({ commitId: "abc1234" }); const commandLog = []; @@ -603,8 +653,8 @@ test("dev-cd status exposes moving origin/main when checkout is behind a deploy- }, latestArtifactMergeCommit: artifactMergeCommit, commitParents: { - [artifactMergeCommit]: [publishedCommit, artifactReportCommit], - [artifactReportCommit]: [publishedCommit] + [artifactMergeCommit]: [laterControlCommit, artifactReportCommit], + [artifactReportCommit]: [laterControlCommit] } }), httpGetJson: makeHttpGetJson(), @@ -666,7 +716,7 @@ test("apply stops before lock acquisition when checkout is behind deploy-json pr assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false); }); -test("dev-cd status degrades artifact merge provenance when catalog/report do not match deploy-json promotion", async () => { +test("dev-cd status degrades artifact merge provenance when catalog does not match deploy-json promotion", async () => { const repoRoot = await makeRepo({ commitId: "abc1234" }); await writeFile( path.join(repoRoot, "deploy/artifact-catalog.dev.json"), @@ -751,8 +801,8 @@ test("apply stops before lock acquisition when artifact merge provenance mismatc }, latestArtifactMergeCommit: artifactMergeCommit, commitParents: { - [artifactMergeCommit]: [publishedCommit, artifactReportCommit], - [artifactReportCommit]: [publishedCommit] + [artifactMergeCommit]: [laterControlCommit, artifactReportCommit], + [artifactReportCommit]: [laterControlCommit] } }), httpGetJson: makeHttpGetJson(), diff --git a/scripts/src/dev-cd-transaction-guard.mjs b/scripts/src/dev-cd-transaction-guard.mjs index adb74715..35693ee7 100644 --- a/scripts/src/dev-cd-transaction-guard.mjs +++ b/scripts/src/dev-cd-transaction-guard.mjs @@ -5,11 +5,45 @@ const transactionEnvNames = [ "HWLAB_CD_TRANSACTION_OWNER", "HWLAB_CD_LOCK_NAME" ]; +const ciArtifactEnvNames = [ + "HWLAB_CI_ARTIFACT_RUN_ID", + "HWLAB_CI_ARTIFACT_RUN_OWNER" +]; export function hasDevCdTransactionEnv(env = process.env) { return typeof env.HWLAB_CD_TRANSACTION_ID === "string" && env.HWLAB_CD_TRANSACTION_ID.trim().length > 0; } +export function hasDevCiArtifactEnv(env = process.env) { + return typeof env.HWLAB_CI_ARTIFACT_RUN_ID === "string" && env.HWLAB_CI_ARTIFACT_RUN_ID.trim().length > 0; +} + +export function hasDevArtifactSideEffectEnv(env = process.env) { + return hasDevCdTransactionEnv(env) || hasDevCiArtifactEnv(env); +} + +export function devArtifactProducer(env = process.env) { + if (hasDevCiArtifactEnv(env)) { + return { + kind: "ci-artifact", + runId: env.HWLAB_CI_ARTIFACT_RUN_ID, + owner: env.HWLAB_CI_ARTIFACT_RUN_OWNER ?? null + }; + } + if (hasDevCdTransactionEnv(env)) { + return { + kind: "cd-transaction-legacy", + runId: env.HWLAB_CD_TRANSACTION_ID, + owner: env.HWLAB_CD_TRANSACTION_OWNER ?? null + }; + } + return { + kind: "missing", + runId: null, + owner: null + }; +} + export function devCdTransactionGuardFailure({ script, mode, @@ -32,7 +66,25 @@ export function devCdTransactionGuardFailure({ }; } +export function devArtifactSideEffectGuardFailure({ + script, + mode, + requiredEnv = "HWLAB_CI_ARTIFACT_RUN_ID or HWLAB_CD_TRANSACTION_ID" +}) { + return { + ...devCdTransactionGuardFailure({ script, mode, requiredEnv }), + acceptedEnv: [...ciArtifactEnvNames, ...transactionEnvNames], + entrypoint: `HWLAB_CI_ARTIFACT_RUN_ID= node scripts/dev-artifact-publish.mjs --publish --report ${tempReportPath("dev-artifacts.json")}`, + summary: `${script} ${mode} is a DEV artifact side-effect step and must run inside a CI artifact run or the legacy DEV CD transaction.` + }; +} + export function requireDevCdTransactionForSideEffect({ env = process.env, script, mode }) { if (hasDevCdTransactionEnv(env)) return null; return devCdTransactionGuardFailure({ script, mode }); } + +export function requireDevArtifactSideEffectEnv({ env = process.env, script, mode }) { + if (hasDevArtifactSideEffectEnv(env)) return null; + return devArtifactSideEffectGuardFailure({ script, mode }); +} diff --git a/scripts/src/dev-deploy-apply.mjs b/scripts/src/dev-deploy-apply.mjs index 4c99a743..db1bc086 100755 --- a/scripts/src/dev-deploy-apply.mjs +++ b/scripts/src/dev-deploy-apply.mjs @@ -1,6 +1,6 @@ #!/usr/bin/env node import { constants as fsConstants } from "node:fs"; -import { access, readFile, writeFile } from "node:fs/promises"; +import { access, mkdir, readFile, writeFile } from "node:fs/promises"; import { request as httpRequest } from "node:http"; import path from "node:path"; import { fileURLToPath } from "node:url"; @@ -88,6 +88,8 @@ const forbiddenActions = [ "force-push" ]; const runtimeIdentityEnvNames = ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"]; +const digestPattern = /^sha256:[a-f0-9]{64}$/u; +const defaultCdConcurrency = parseBoundedInteger(process.env.HWLAB_DEV_CD_CONCURRENCY, 4, 1, 8); export function parseArgs(argv) { const flags = new Set(); @@ -95,6 +97,7 @@ export function parseArgs(argv) { let kubeconfig = null; let kubeconfigSpecified = false; let reportPath = defaultReportPath; + let rolloutConcurrency = defaultCdConcurrency; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; @@ -141,6 +144,22 @@ export function parseArgs(argv) { } continue; } + if (arg === "--rollout-concurrency") { + flags.add("--rollout-concurrency"); + const next = argv[index + 1]; + if (typeof next !== "string" || next.startsWith("--")) { + errors.push("--rollout-concurrency requires a numeric value"); + } else { + rolloutConcurrency = parseBoundedInteger(next, defaultCdConcurrency, 1, 8); + index += 1; + } + continue; + } + if (arg.startsWith("--rollout-concurrency=")) { + flags.add("--rollout-concurrency"); + rolloutConcurrency = parseBoundedInteger(arg.slice("--rollout-concurrency=".length), defaultCdConcurrency, 1, 8); + continue; + } flags.add(arg); } @@ -151,6 +170,7 @@ export function parseArgs(argv) { skipLiveProbe: flags.has("--skip-live-probe"), writeReport: flags.has("--report-output"), reportPath, + rolloutConcurrency, kubeconfig, kubeconfigSpecified, errors, @@ -158,6 +178,12 @@ export function parseArgs(argv) { }; } +function parseBoundedInteger(value, fallback, min, max) { + const parsed = Number.parseInt(String(value ?? ""), 10); + if (!Number.isInteger(parsed)) return fallback; + return Math.min(Math.max(parsed, min), max); +} + function addBlocker(blockers, type, scope, summary) { const key = `${type}::${scope}`; if (!blockers.some((blocker) => `${blocker.type}::${blocker.scope}` === key)) { @@ -169,6 +195,21 @@ function oneLine(value) { return String(value).replace(/\s+/g, " ").trim(); } +async function mapWithConcurrency(items, concurrency, worker) { + if (items.length === 0) return []; + const results = new Array(items.length); + let nextIndex = 0; + const workerCount = Math.min(Math.max(concurrency, 1), items.length); + await Promise.all(Array.from({ length: workerCount }, async () => { + while (nextIndex < items.length) { + const index = nextIndex; + nextIndex += 1; + results[index] = await worker(items[index], index); + } + })); + return results; +} + async function readJson(relativePath, blockers) { try { return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8")); @@ -675,6 +716,146 @@ function parseImageTag(image) { return image.slice(colonIndex + 1); } +function parseTaggedImageReference(image) { + if (typeof image !== "string" || image.includes("@")) return null; + const firstSlash = image.indexOf("/"); + const lastSlash = image.lastIndexOf("/"); + const colonIndex = image.lastIndexOf(":"); + if (firstSlash <= 0 || colonIndex <= lastSlash) return null; + return { + image, + registry: image.slice(0, firstSlash), + repository: image.slice(firstSlash + 1, colonIndex), + tag: image.slice(colonIndex + 1) + }; +} + +function registryManifestUrlForImage(image) { + const parsed = parseTaggedImageReference(image); + if (!parsed) return null; + const repositoryPath = parsed.repository.split("/").map((part) => encodeURIComponent(part)).join("/"); + return { + ...parsed, + url: `http://${parsed.registry}/v2/${repositoryPath}/manifests/${encodeURIComponent(parsed.tag)}` + }; +} + +function registryAcceptHeader() { + return [ + "application/vnd.docker.distribution.manifest.v2+json", + "application/vnd.oci.image.manifest.v1+json", + "application/vnd.docker.distribution.manifest.list.v2+json", + "application/vnd.oci.image.index.v1+json", + "application/vnd.docker.distribution.manifest.v1+json" + ].join(", "); +} + +function httpRequestText(url, { method = "GET", headers = {}, timeoutMs = 15000 } = {}) { + return new Promise((resolve, reject) => { + const request = httpRequest(url, { method, headers, timeout: timeoutMs }, (response) => { + response.setEncoding("utf8"); + let body = ""; + response.on("data", (chunk) => { + body += chunk; + if (body.length > 4096) { + request.destroy(new Error("registry manifest response exceeded 4096 bytes during CD verification")); + } + }); + response.on("end", () => resolve({ + statusCode: response.statusCode ?? 0, + headers: response.headers, + body + })); + }); + request.on("timeout", () => { + request.destroy(new Error(`timeout after ${timeoutMs}ms`)); + }); + request.on("error", reject); + request.end(); + }); +} + +async function verifyCatalogRegistryManifest(service, blockers) { + const image = service.image; + const expectedDigest = service.digest; + const parsed = registryManifestUrlForImage(image); + const base = { + serviceId: service.serviceId, + image, + imageTag: service.imageTag ?? parseImageTag(image), + expectedDigest, + url: parsed?.url ?? null + }; + + if (!parsed) { + const reason = `${service.serviceId} image is not a tagged registry image`; + addBlocker(blockers, "contract_blocker", `registry-manifest-${service.serviceId}`, reason); + return { ...base, status: "blocked", reason }; + } + if (!digestPattern.test(expectedDigest ?? "")) { + const reason = `${service.serviceId} catalog digest is not an immutable sha256 digest`; + addBlocker(blockers, "contract_blocker", `registry-manifest-${service.serviceId}`, reason); + return { ...base, status: "blocked", reason }; + } + + try { + const response = await httpRequestText(parsed.url, { + method: "HEAD", + headers: { + Accept: registryAcceptHeader() + }, + timeoutMs: 15000 + }); + const observedDigest = String(response.headers["docker-content-digest"] ?? "").trim(); + const okStatus = response.statusCode >= 200 && response.statusCode < 300; + const digestMatches = observedDigest === expectedDigest; + if (!okStatus || !digestMatches) { + const reason = !okStatus + ? `${service.serviceId} registry manifest returned HTTP ${response.statusCode}` + : `${service.serviceId} registry digest mismatch: expected ${expectedDigest}, observed ${observedDigest || "missing"}`; + addBlocker(blockers, "environment_blocker", `registry-manifest-${service.serviceId}`, reason); + return { + ...base, + status: "blocked", + reason, + httpStatus: response.statusCode, + observedDigest, + digestMatches + }; + } + return { + ...base, + status: "pass", + httpStatus: response.statusCode, + observedDigest, + digestMatches: true + }; + } catch (error) { + const reason = `${service.serviceId} registry manifest read failed: ${oneLine(error.message)}`; + addBlocker(blockers, "environment_blocker", `registry-manifest-${service.serviceId}`, reason); + return { + ...base, + status: "blocked", + reason + }; + } +} + +async function verifyCatalogRegistryManifests(catalog, blockers, concurrency = defaultCdConcurrency) { + const services = (catalog?.services ?? []).filter((service) => service.artifactRequired !== false); + const results = await mapWithConcurrency(services, concurrency, async (service) => + verifyCatalogRegistryManifest(service, blockers) + ); + return { + status: results.every((result) => result.status === "pass") ? "pass" : "blocked", + source: "registry-manifest", + releaseGate: true, + serviceCount: results.length, + concurrency, + results + }; +} + function expectedRuntimeIdentityEnv(artifact) { return { HWLAB_COMMIT_ID: artifact.sourceCommitId, @@ -1675,9 +1856,9 @@ async function buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads, return cleanups; } -async function executeDevTemplateJobReplacements(kubectl, replacements, blockers) { +async function executeDevTemplateJobReplacements(kubectl, replacements, blockers, concurrency = defaultCdConcurrency) { const planned = replacements.filter((replacement) => replacement.replace && replacement.result === "planned"); - for (const replacement of planned) { + await mapWithConcurrency(planned, concurrency, async (replacement) => { const commandArgs = [ "-n", replacement.namespace, @@ -1694,17 +1875,17 @@ async function executeDevTemplateJobReplacements(kubectl, replacements, blockers replacement.result = "delete_failed"; replacement.reason = `Failed to delete live suspended template Job before apply: ${oneLine(commandOutput(result))}`; addBlocker(blockers, "environment_blocker", `template-job-replace-${replacement.jobName}`, replacement.reason); - continue; + return; } replacement.result = "deleted_pending_recreate"; replacement.reason = "Live suspended template Job was deleted; kubectl apply must recreate it from the desired manifest."; - } + }); return planned.length; } -async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, blockers) { +async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, blockers, concurrency = defaultCdConcurrency) { const planned = cleanups.filter((cleanup) => cleanup.cleanup && cleanup.result === "planned"); - for (const cleanup of planned) { + await mapWithConcurrency(planned, concurrency, async (cleanup) => { const commandArgs = [ "-n", cleanup.namespace, @@ -1721,14 +1902,107 @@ async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, bl cleanup.result = "delete_failed"; cleanup.reason = `Failed to delete stale legacy simulator Deployment before apply: ${oneLine(commandOutput(result))}`; addBlocker(blockers, "environment_blocker", `legacy-simulator-deployment-cleanup-${cleanup.deploymentName}`, cleanup.reason); - continue; + return; } cleanup.result = "deleted_pending_apply"; cleanup.reason = "Live stale legacy simulator Deployment was deleted; kubectl apply must leave the desired indexed StatefulSet in place."; - } + }); return planned.length; } +function rolloutApiName(kind) { + if (kind === "Deployment") return "deployment"; + if (kind === "StatefulSet") return "statefulset"; + return null; +} + +function rolloutTargetsFromWorkloads(workloads) { + return listItems(workloads) + .map((item) => { + const apiName = rolloutApiName(item?.kind); + if (!apiName) return null; + return { + kind: item.kind, + apiName, + name: item?.metadata?.name ?? "unknown", + namespace: item?.metadata?.namespace ?? namespace, + replicas: replicaPlan(item), + serviceIds: uniqueStrings(containersFor(item).map((container) => serviceIdFor(item, container))) + }; + }) + .filter(Boolean); +} + +async function verifyRolloutTarget(kubectl, target, blockers) { + const commandArgs = [ + "-n", + target.namespace, + "rollout", + "status", + `${target.apiName}/${target.name}`, + "--timeout=180s" + ]; + const base = { + ...target, + command: kubectlCommand(kubectl, commandArgs) + }; + if (kubectl.status !== "ready") { + addBlocker(blockers, "environment_blocker", `rollout-status-${target.name}`, kubectl.reason); + return { + ...base, + status: "blocked", + reason: kubectl.reason + }; + } + const result = await kubectlResult(kubectl, commandArgs, 190000); + if (!result.ok) { + const reason = oneLine(commandOutput(result)); + addBlocker(blockers, "runtime_blocker", `rollout-status-${target.name}`, reason); + return { + ...base, + status: "blocked", + reason, + stdout: result.redactedStdout ?? result.stdout, + stderr: result.redactedStderr ?? result.stderr + }; + } + return { + ...base, + status: "pass", + stdout: result.redactedStdout ?? result.stdout, + stderr: result.redactedStderr ?? result.stderr + }; +} + +async function verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep) { + const targets = rolloutTargetsFromWorkloads(workloads); + if (!args.apply) { + return { + status: "not_run", + reason: "dry-run mode", + concurrency: args.rolloutConcurrency, + targets + }; + } + if (applyStep.status !== "pass") { + return { + status: "not_run", + reason: "apply did not complete", + concurrency: args.rolloutConcurrency, + targets + }; + } + const results = await mapWithConcurrency(targets, args.rolloutConcurrency, async (target) => + verifyRolloutTarget(kubectl, target, blockers) + ); + return { + status: results.every((result) => result.status === "pass") ? "pass" : "blocked", + concurrency: args.rolloutConcurrency, + targetCount: targets.length, + results + }; +} + function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) { const plannedNames = replacementsNeeded.map((replacement) => replacement.jobName); if (result.ok || plannedNames.length === 0) return false; @@ -1758,8 +2032,8 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements, le const replacementsNeeded = templateJobReplacements.filter((replacement) => replacement.replace); if (args.apply) { - const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers); - const cleanupCount = await executeDevLegacySimulatorDeploymentCleanups(kubectl, legacySimulatorDeploymentCleanups, blockers); + const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers, args.rolloutConcurrency); + const cleanupCount = await executeDevLegacySimulatorDeploymentCleanups(kubectl, legacySimulatorDeploymentCleanups, blockers, args.rolloutConcurrency); mutationAttempted = replaceCount > 0 || cleanupCount > 0; if (blockers.length > 0) { return { @@ -1858,6 +2132,7 @@ export async function runDevDeployApply(argv, io = {}) { const commitId = resolveApplySourceCommit(deploy, catalog, gitHeadCommitId); const artifactEvidence = validateDeployAndCatalog(deploy, catalog, commitId, blockers); + const registryManifests = await verifyCatalogRegistryManifests(catalog, blockers, args.rolloutConcurrency); const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers); const cloudApiDb = await checkCloudApiDb(deploy, workloads, services, blockers); const codeAgentProviderDesiredState = inspectCodeAgentProviderDesiredState(deploy, workloads); @@ -1886,33 +2161,41 @@ export async function runDevDeployApply(argv, io = {}) { templateJobReplacements, legacySimulatorDeploymentCleanups ); - const cloudWebRolloutAfterApply = args.apply && applyStep.status === "pass" - ? await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, { + let rolloutStatusAfterApply; + let cloudWebRolloutAfterApply; + let codeAgentProviderLiveAfterApply; + if (args.apply && applyStep.status === "pass") { + [rolloutStatusAfterApply, cloudWebRolloutAfterApply, codeAgentProviderLiveAfterApply] = await Promise.all([ + verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep), + observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, { observationPhase: "after_apply", blockRuntimeEnvDrift: true - }) - : skippedDeploymentRolloutObservation( - kubectl, - deploy, - catalog, - "hwlab-cloud-web", - "after_apply", - args.apply ? "apply did not complete successfully; post-apply rollout env verification was not run" : "dry-run mode; no post-apply rollout env verification" - ); - const cloudWebRollout = args.apply ? cloudWebRolloutAfterApply : cloudWebRolloutBeforeApply; - const codeAgentProviderLiveAfterApply = args.apply && applyStep.status === "pass" - ? await observeCodeAgentProviderLiveDeployment(kubectl, blockers, { + }), + observeCodeAgentProviderLiveDeployment(kubectl, blockers, { phase: "after_apply", blockOnMismatch: true }) - : { - phase: "after_apply", - status: "not_run", - reason: args.apply ? "apply did not complete" : "dry-run mode", - secretValuesRead: false, - kubernetesSecretDataRead: false, - valuesRedacted: true - }; + ]); + } else { + rolloutStatusAfterApply = await verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep); + cloudWebRolloutAfterApply = skippedDeploymentRolloutObservation( + kubectl, + deploy, + catalog, + "hwlab-cloud-web", + "after_apply", + args.apply ? "apply did not complete successfully; post-apply rollout env verification was not run" : "dry-run mode; no post-apply rollout env verification" + ); + codeAgentProviderLiveAfterApply = { + phase: "after_apply", + status: "not_run", + reason: args.apply ? "apply did not complete" : "dry-run mode", + secretValuesRead: false, + kubernetesSecretDataRead: false, + valuesRedacted: true + }; + } + const cloudWebRollout = args.apply ? cloudWebRolloutAfterApply : cloudWebRolloutBeforeApply; const status = blockers.length > 0 ? "blocked" : "pass"; const artifactPlan = buildArtifactPlan(deploy, catalog, commitId, artifactEvidence); const workloadPlan = buildWorkloadPlan(workloads); @@ -1964,6 +2247,7 @@ export async function runDevDeployApply(argv, io = {}) { evidence: [ `artifact services checked: ${artifactEvidence.length}`, `expected artifact commit: ${artifactPlan.expectedArtifactCommit}`, + `registry manifests: ${registryManifests.status} (${registryManifests.serviceCount} services)`, `namespace: ${namespace}`, `workloads planned: ${workloadPlan.length}`, `kubectl executor: ${kubectl.executor ?? "missing"}`, @@ -1972,6 +2256,7 @@ export async function runDevDeployApply(argv, io = {}) { `live health: ${liveProbe.status}`, `code agent provider desired-state: ${codeAgentProviderDesiredState.status}`, `code agent provider live env before apply: ${codeAgentProviderLiveBeforeApply.status}`, + `rollout status concurrency: ${args.rolloutConcurrency}`, `template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}`, `legacy simulator deployment cleanups: ${legacySimulatorDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}` ], @@ -1980,7 +2265,7 @@ export async function runDevDeployApply(argv, io = {}) { devPreconditions: { status, requirements: [ - "DEV artifact catalog must contain CI publish evidence and registry digests", + "DEV artifact catalog must contain CI publish expectations and registry digests; CD must verify those tag/digest pairs directly against registry manifests", "kubectl must be available for D601 hwlab-dev and must not target PROD", "hwlab-cloud-api /health/live must report serviceId, dev environment, and DB env readiness without exposing secret values", "hwlab-cloud-api must declare and preserve OPENAI_API_KEY from hwlab-code-agent-provider/openai-api-key plus the DEV Code Agent egress proxy env without reading Secret values", @@ -2009,6 +2294,7 @@ export async function runDevDeployApply(argv, io = {}) { cloudWebRollout, cloudWebRolloutBeforeApply, cloudWebRolloutAfterApply, + rolloutStatusAfterApply, codeAgentProvider: { desiredState: codeAgentProviderDesiredState, liveBeforeApply: codeAgentProviderLiveBeforeApply, @@ -2024,6 +2310,7 @@ export async function runDevDeployApply(argv, io = {}) { rollbackHint: buildRollbackHint(kubectl, workloads), remainingBlockers, artifactEvidence, + registryManifests, cloudApiDb, k8sManifest, clusterObservation, diff --git a/scripts/src/dev-deploy-apply.test.mjs b/scripts/src/dev-deploy-apply.test.mjs index 706479f9..f939305b 100644 --- a/scripts/src/dev-deploy-apply.test.mjs +++ b/scripts/src/dev-deploy-apply.test.mjs @@ -15,8 +15,10 @@ import { resolveDevKubeconfigSelection } from "./dev-deploy-apply.mjs"; import { + devArtifactProducer, devCdTransactionGuardFailure, hasDevCdTransactionEnv, + requireDevArtifactSideEffectEnv, requireDevCdTransactionForSideEffect } from "./dev-cd-transaction-guard.mjs"; @@ -141,6 +143,32 @@ test("DEV CD transaction guard rejects direct legacy side-effect calls", () => { ); }); +test("DEV artifact publish guard accepts CI artifact identity without CD lock", () => { + const ciEnv = { + HWLAB_CI_ARTIFACT_RUN_ID: "ci-artifact-123", + HWLAB_CI_ARTIFACT_RUN_OWNER: "tekton" + }; + assert.equal(requireDevArtifactSideEffectEnv({ + env: ciEnv, + script: "scripts/dev-artifact-publish.mjs", + mode: "--publish" + }), null); + assert.deepEqual(devArtifactProducer(ciEnv), { + kind: "ci-artifact", + runId: "ci-artifact-123", + owner: "tekton" + }); + + const failure = requireDevArtifactSideEffectEnv({ + env: {}, + script: "scripts/dev-artifact-publish.mjs", + mode: "--publish" + }); + assert.equal(failure.error, "cd-transaction-required"); + assert.match(failure.requiredEnv, /HWLAB_CI_ARTIFACT_RUN_ID/u); + assert.match(failure.entrypoint, /HWLAB_CI_ARTIFACT_RUN_ID/u); +}); + test("matching allowlisted suspended template Job image does not replace", () => { const image = "127.0.0.1:5000/hwlab/hwlab-cli:73b379f"; const decision = decideDevTemplateJobReplacement({