feat: parallelize dev ci cd artifact flow
This commit is contained in:
Codex
@@ -58,6 +58,8 @@ HWLAB 是硬件实验室运行面和控制面项目。本文是 agent、指挥
|
||||
- Cloud Web M3 只读护栏：`npm run web:m3-readonly`
|
||||
- Cloud Workbench 布局/遮挡 smoke：`npm run web:layout`；local-build 用 `npm run web:layout:build`；DEV deploy 后用 `npm run web:layout:live`。
|
||||
- DEV artifact 发布预检：`npm run dev-artifact:preflight`
|
||||
- DEV artifact CI 发布：`HWLAB_CI_ARTIFACT_RUN_ID=<run-id> npm run dev-artifact:publish`
|
||||
- DEV 依赖 runtime base 构建：`npm run dev-runtime-base:build`
|
||||
- DEV CD 单事务发布/应用/验证：`npm run dev-cd:apply`
|
||||
- runner GitHub 可见性预检：`npm run runner:issue-visibility:preflight`
|
||||
- D601 k3s 只读观测：`npm run d601:k3s:readonly`
|
||||
|
||||
@@ -2,10 +2,16 @@
|
||||
|
||||
`scripts/dev-artifact-publish.mjs` is the DEV-only artifact preflight/build
|
||||
backend for D601 local/internal registry artifacts. Publish side effects are
|
||||
normally entered through `scripts/dev-cd-apply.mjs`, which sets the transaction
|
||||
environment and serializes publish/apply/verify with the DEV CD Lease lock.
|
||||
The artifact script does not deploy workloads, read secrets, enable PROD, or
|
||||
push to GHCR/Docker Hub/other third-party registries.
|
||||
normally entered by a CI artifact job identity, or by the legacy
|
||||
`scripts/dev-cd-apply.mjs` transaction path during transition. The artifact
|
||||
script does not deploy workloads, read secrets, enable PROD, or push to
|
||||
GHCR/Docker Hub/other third-party registries.
|
||||
|
||||
长期口径：CI 产物报告不是人工维护的镜像库状态，也不是 CD 放行的本地状态源。
|
||||
它是 CI 自动生成的一次性审计记录。CD 必须从 `deploy/deploy.json` 读取发布控制
|
||||
意图，从 `deploy/artifact-catalog.dev.json` 读取 digest 期望，并直接向 registry
|
||||
manifest 验证目标 tag/digest 是否存在。`/tmp/hwlab-dev-gate/dev-artifacts.json`
|
||||
只能用于排障、刷新 catalog 和审计追溯。
|
||||
|
||||
## Scope
|
||||
|
||||
@@ -20,9 +26,10 @@ push to GHCR/Docker Hub/other third-party registries.
|
||||
`--pull=false`.
|
||||
- Node runtime dependencies are installed from the source `package.json` and
|
||||
`package-lock.json` inside the DEV image build with `npm ci --omit=dev
|
||||
--ignore-scripts` when the lockfile is present. This includes the Postgres
|
||||
`pg` driver required by the
|
||||
durable cloud-api runtime adapter.
|
||||
--ignore-scripts` when the lockfile is present, unless the selected base
|
||||
image already provides `/opt/hwlab-node-runtime-base/node_modules`. This
|
||||
includes the Postgres `pg` driver required by the durable cloud-api runtime
|
||||
adapter.
|
||||
- Image tag: first seven characters of the Git commit used as build source.
|
||||
- Required image labels:
|
||||
- `hwlab.pikastech.local/repo`
|
||||
@@ -40,7 +47,9 @@ node --check scripts/src/dev-base-image-preflight.mjs
|
||||
node --check scripts/src/dev-artifact-services.mjs
|
||||
node --check scripts/src/registry-capabilities.mjs
|
||||
node --check scripts/dev-artifact-publish.mjs
|
||||
node --check scripts/dev-runtime-base-image.mjs
|
||||
node --check scripts/refresh-artifact-catalog.mjs
|
||||
node --test scripts/refresh-artifact-catalog.test.mjs
|
||||
```
|
||||
|
||||
Base image preflight:
|
||||
@@ -61,22 +70,31 @@ Build only:
|
||||
node scripts/dev-artifact-publish.mjs --build
|
||||
```
|
||||
|
||||
Build and publish to the D601 local/internal registry inside the DEV CD
|
||||
transaction:
|
||||
Build only one or more selected services for a CI artifact task:
|
||||
|
||||
```sh
|
||||
node scripts/dev-artifact-publish.mjs --build --services hwlab-cloud-web
|
||||
```
|
||||
|
||||
`--services` narrows the build/publish report to selected service IDs. It is a
|
||||
CI/artifact acceleration surface; the normal DEV rollout still consumes
|
||||
repo-owned desired state through `scripts/dev-cd-apply.mjs`.
|
||||
|
||||
Build and publish to the D601 local/internal registry inside a CI artifact job:
|
||||
|
||||
```sh
|
||||
HWLAB_CI_ARTIFACT_RUN_ID=<run-id> node scripts/dev-artifact-publish.mjs --publish
|
||||
```
|
||||
|
||||
The legacy transaction entry remains available during migration:
|
||||
|
||||
```sh
|
||||
node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json
|
||||
```
|
||||
|
||||
The underlying single-step command is kept only for transaction internals and
|
||||
specialized tests that explicitly inject the transaction environment:
|
||||
|
||||
```sh
|
||||
node scripts/dev-artifact-publish.mjs --publish
|
||||
```
|
||||
|
||||
Direct `--publish` without `HWLAB_CD_TRANSACTION_ID` is rejected with
|
||||
`cd-transaction-required`; this is a bypass guard, not a second lock.
|
||||
Direct `--publish` without `HWLAB_CI_ARTIFACT_RUN_ID` or
|
||||
`HWLAB_CD_TRANSACTION_ID` is rejected with `cd-transaction-required`; this is a
|
||||
bypass guard, not a second lock.
|
||||
|
||||
Daily Cloud Web source validation does not require ignored `dist/` to exist:
|
||||
|
||||
@@ -107,12 +125,32 @@ registry prefix. The script rejects third-party registry hosts and any prefix
|
||||
that names PROD. Use `--base-image` only to override `HWLAB_DEV_BASE_IMAGE` with
|
||||
an approved local image for that run.
|
||||
|
||||
Docker build output is visible by default so a slow layer is observable from the
|
||||
runner log. Use `--quiet-build` only when a caller already captures the full
|
||||
Docker log elsewhere. The artifact report records per-service
|
||||
`dockerBuildDurationMs`, `cloudWebBuildDurationMs`, and `publishDurationMs`.
|
||||
|
||||
An optional dependency base image can be prepared with:
|
||||
|
||||
```sh
|
||||
node scripts/dev-runtime-base-image.mjs --dry-run
|
||||
node scripts/dev-runtime-base-image.mjs
|
||||
```
|
||||
|
||||
The default tag is
|
||||
`127.0.0.1:5000/hwlab/hwlab-node-runtime-base:deps-<package-lock-hash>`.
|
||||
Selecting that image through `HWLAB_DEV_BASE_IMAGE` lets service builds reuse a
|
||||
preinstalled `node_modules` layer instead of repeating `npm ci` for every
|
||||
service.
|
||||
|
||||
## Report
|
||||
|
||||
The script writes `/tmp/hwlab-dev-gate/dev-artifacts.json`. The file is kept in the
|
||||
existing DEV gate report envelope for compatibility with
|
||||
The script writes `/tmp/hwlab-dev-gate/dev-artifacts.json`. The file is kept in
|
||||
the existing DEV gate report envelope for compatibility with
|
||||
`scripts/validate-dev-gate-report.mjs`, while the `artifactPublish` object is
|
||||
the HWLAB#35-specific publish record.
|
||||
the HWLAB#35-specific publish record. This report is evidence, not desired
|
||||
state. A later CD rollout must verify catalog images directly against the
|
||||
registry instead of trusting this file.
|
||||
|
||||
Each service record contains:
|
||||
|
||||
|
||||
@@ -57,10 +57,10 @@ Selection order is intentionally narrow:
|
||||
tag such as `127.0.0.1:5000/hwlab/hwlab-dev-base:node20-bookworm-slim`.
|
||||
4. If neither source is available, the result is a blocker.
|
||||
|
||||
Explicit base image values must match local `node:20-*`, `hwlab-dev-base:*`, or
|
||||
`hwlab-node20-base:*` references. They must not reference UniDesk, Code Queue,
|
||||
`backend-core`, `provider-gateway`, `microservice-proxy`, or an HWLAB runtime
|
||||
service image. A service image such as
|
||||
Explicit base image values must match local `node:20-*`, `hwlab-dev-base:*`,
|
||||
`hwlab-node20-base:*`, or `hwlab-node-runtime-base:*` references. They must not
|
||||
reference UniDesk, Code Queue, `backend-core`, `provider-gateway`,
|
||||
`microservice-proxy`, or an HWLAB runtime service image. A service image such as
|
||||
`127.0.0.1:5000/hwlab/hwlab-cloud-api:<sha>` is an artifact output, not a
|
||||
builder base image.
|
||||
|
||||
@@ -113,6 +113,19 @@ docker tag node:20-bookworm-slim 127.0.0.1:5000/hwlab/hwlab-dev-base:node20-book
|
||||
HWLAB_DEV_BASE_IMAGE=127.0.0.1:5000/hwlab/hwlab-dev-base:node20-bookworm-slim node scripts/preflight-dev-base-image.mjs
|
||||
```
|
||||
|
||||
Optional dependency runtime base provision:
|
||||
|
||||
```sh
|
||||
node scripts/dev-runtime-base-image.mjs --dry-run
|
||||
node scripts/dev-runtime-base-image.mjs
|
||||
HWLAB_DEV_BASE_IMAGE=127.0.0.1:5000/hwlab/hwlab-node-runtime-base:deps-<hash> node scripts/preflight-dev-base-image.mjs
|
||||
```
|
||||
|
||||
This runtime base caches `package.json` / `package-lock.json` dependencies in
|
||||
`/opt/hwlab-node-runtime-base/node_modules`. Service artifact builds copy that
|
||||
directory when present, which avoids repeating the same `npm ci` layer for each
|
||||
service.
|
||||
|
||||
Offline preload path when D601 cannot pull directly:
|
||||
|
||||
```sh
|
||||
|
||||
@@ -43,7 +43,7 @@ truth.
|
||||
| Stage | Current HWLAB state | Required next behavior |
|
||||
| --- | --- | --- |
|
||||
| 1. Long-term reference | This file records DEV deployment, Cloud Web publish, API/edge/health, frp, D601 registry, k3s rollout, rollback, and verification rules. `AGENTS.md` indexes it. | Keep process issues as sources only; update this reference when a deployment rule changes. |
|
||||
| 2. CLI or controlled scripts | `scripts/dev-cd-apply.mjs` is the only formal DEV CD side-effect entrypoint for publish, desired-state refresh, apply, and live verify. Other controlled scripts still cover contract planning, dry-run planning, preflight, single-step internals, k3s read-only visibility, and Cloud Web checks. Rollback is documented as commands/report hints but is not yet one unified CLI command. | Converge check, plan, rollback, status, and verify into HWLAB CLI while preserving the single transaction lock for side effects backed by `deploy/deploy.json`. |
|
||||
| 2. CLI or controlled scripts | `scripts/dev-artifact-publish.mjs` is the formal CI artifact publish entrypoint under CI identity; `scripts/dev-cd-apply.mjs` is the formal DEV CD side-effect entrypoint for desired-state apply and live verify. Other controlled scripts still cover contract planning, dry-run planning, preflight, single-step internals, k3s read-only visibility, and Cloud Web checks. Rollback is documented as commands/report hints but is not yet one unified CLI command. | Converge check, plan, rollback, status, and verify into HWLAB CLI while preserving CI/CD separation and the single CD transaction lock backed by `deploy/deploy.json`. |
|
||||
| 3. UniDesk CI/CD plus images | DEV services are imageized and published to the D601 registry. D601 rollout still includes manual bridge steps for Cloud Web and frp/k3s execution. | Move build, publish, desired-state refresh, rollout, post-deploy validation, and report upload into standard UniDesk CI/CD jobs without replacing HWLAB runtime. |
|
||||
|
||||
## DEV CD Transaction
|
||||
@@ -54,8 +54,10 @@ truth.
|
||||
node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json
|
||||
```
|
||||
|
||||
该命令是唯一允许一次性发布 DEV artifact、从发布报告刷新 desired
|
||||
state、apply 到 `hwlab-dev`、并复验公开 `16666/16667` 的正式写路径。
|
||||
该命令是唯一允许按 `deploy.json` 对 `hwlab-dev` 执行 desired-state apply、并复验
|
||||
公开 `16666/16667` 的正式 CD 写路径。镜像发布属于 CI artifact 职责；过渡期
|
||||
latest-main 兼容路径仍可由该命令调用 artifact publish，但正式 rollout 默认应消费
|
||||
已经发布并经过 registry manifest 验证的 artifact。
|
||||
事务开始时读取 `deploy/deploy.json`，记录 manifest hash 和 target commit。
|
||||
在获取 `Lease/hwlab-dev/hwlab-dev-cd-lock` 或运行任何 publish、runtime Job、
|
||||
apply、rollout 类副作用前，脚本必须先完成 apply-only preflight：强制
|
||||
@@ -90,14 +92,17 @@ checkout promotion parent 或手工拼装发布环境。
|
||||
`origin/main` 控制入口，`target.promotionCommit` 是从 `deploy/deploy.json`
|
||||
解析出的真实 promotion commit，`target.artifactBoundary` 记录最近一次触碰
|
||||
`deploy/deploy.json`、`deploy/artifact-catalog.dev.json`、
|
||||
`deploy/k8s/base/workloads.yaml`、`/tmp/hwlab-dev-gate/dev-artifacts.json` 的
|
||||
first-parent artifact merge commit，以及该 merge commit 的 promotion parent
|
||||
和 artifact/report parent。`artifactCatalog`、`artifactReport`、`deployJson`
|
||||
分别记录 hash、commit、digest 计数和 publish source。若 deploy/catalog/report
|
||||
或 artifact merge parent 不能同时指向同一个 promotion commit，状态降级为
|
||||
`degraded`，真实 `--apply` 在获取 Lease 前以 artifact-boundary blocker 停止。
|
||||
这保证 `deploy/deploy.json` 仍是唯一发布控制源；Lease 只串行化事务，不保存第
|
||||
二套 desired-state。
|
||||
`deploy/k8s/base/workloads.yaml` 的 first-parent desired-state commit，以及该
|
||||
commit 的 promotion parent。`artifactCatalog`、`artifactReport`、`deployJson`
|
||||
分别记录 hash、commit、digest 计数和 publish source，但只有
|
||||
`deploy/deploy.json`、artifact catalog、workload manifest 和 registry manifest
|
||||
共同决定 CD 是否可执行；`/tmp/hwlab-dev-gate/dev-artifacts.json` 只是 CI 审计快照。
|
||||
若 deploy/catalog/workloads 或 desired-state parent 不能同时指向同一个
|
||||
promotion commit，状态降级为 `degraded`，真实 `--apply` 在获取 Lease 前以
|
||||
artifact-boundary blocker 停止。缺失或过期的 CI 产物报告只能产生诊断提示，不得
|
||||
阻塞已经由 desired-state 和 registry 证明一致的 CD rollout。这保证
|
||||
`deploy/deploy.json` 仍是唯一发布控制源；Lease 只串行化事务，不保存第二套
|
||||
desired-state。
|
||||
|
||||
Lease 只是发布平面的互斥锁，不是 desired-state 来源。Lease annotations
|
||||
至少记录 `promotionCommit`、`deployJsonHash`、`ownerTaskId`、
|
||||
@@ -145,7 +150,7 @@ repo-owned 入口，不能绕过它们直接拼接 publish、apply、rollout 或
|
||||
| 读取 DEV CD 状态 | `node scripts/dev-cd-apply.mjs --status` | 不写入、不获取 Lease、不读取 Secret value。 |
|
||||
| 事务 dry-run preflight | `node scripts/dev-cd-apply.mjs --dry-run` | 不 publish、不 apply、不 rollout。 |
|
||||
| 检查 desired-state 收敛 | `node scripts/deploy-desired-state-plan.mjs --promotion-commit <sha> --check` | 只读 source/report。 |
|
||||
| 执行 DEV CD 事务 | `node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json` | 仅 DEV，在 `Lease/hwlab-dev/hwlab-dev-cd-lock` 下 publish/apply/verify。 |
|
||||
| 执行 DEV CD 事务 | `node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json` | 仅 DEV，在 `Lease/hwlab-dev/hwlab-dev-cd-lock` 下 apply/verify；legacy latest-main publish 仅作过渡兼容。 |
|
||||
| 读取 D601 k3s 证据 | `node scripts/d601-k3s-readonly-observability.mjs` | 只读可见性报告；不 rollout、不输出 Secret value。 |
|
||||
|
||||
使用 master-side wrapper 时，稳定形态是：
|
||||
@@ -179,9 +184,10 @@ desired-state 来源关系固定如下：
|
||||
| 来源 | 权威范围 |
|
||||
| --- | --- |
|
||||
| `deploy/deploy.json` | DEV environment、namespace、endpoint、service set、promotion commit 和事务消费的 image tag。 |
|
||||
| `deploy/artifact-catalog.dev.json` | 同一 promotion commit 的不可变 artifact digest/tag 证据。 |
|
||||
| `deploy/artifact-catalog.dev.json` | 同一 promotion commit 的不可变 artifact digest/tag 期望；CD 必须直接向 registry 校验 tag/digest 是否真实存在。 |
|
||||
| `deploy/k8s/base/workloads.yaml` | Kubernetes workload desired state，必须镜像 `deploy.json` 和 artifact catalog 后才能 apply。 |
|
||||
| `/tmp/hwlab-dev-gate/*.json` | 只作为证据快照；不能覆盖 repo desired state。 |
|
||||
| Registry manifest | 镜像事实源；回答目标 tag/digest 是否存在，不能决定哪一组服务应该上线。 |
|
||||
| `/tmp/hwlab-dev-gate/*.json` | 只作为 CI/CD 审计、排障和耗时统计快照；不能覆盖 repo desired state，也不能成为 CD 放行硬依赖。 |
|
||||
| `Lease/hwlab-dev/hwlab-dev-cd-lock` | 只作为事务互斥锁和审计状态；永远不是 desired-state 来源。 |
|
||||
|
||||
只改文档的 commit 如果没有修改 `deploy/deploy.json`、
|
||||
@@ -204,6 +210,75 @@ live health verification 或看板/status 收口权限。Runner 发布 artifact
|
||||
输出必须包含 tag、digest 和 report 证据，并把 desired-state convergence 与
|
||||
live rollout evidence 分开。
|
||||
|
||||
## CI/CD 职责拆分
|
||||
|
||||
长期稳定规则是 CI 负责构建和发布镜像，CD 只消费已经构建好的镜像和
|
||||
repo-owned desired state。CD 入口不得临场重新解释 source、手工拼装 build
|
||||
env、重新构建镜像或把构建失败伪装成 rollout blocker。
|
||||
|
||||
CI/CD 三源分工固定如下：
|
||||
|
||||
| 来源 | 职责 | 不能承担 |
|
||||
| --- | --- | --- |
|
||||
| `deploy/deploy.json` | 唯一发布控制源，声明 promotion commit、service image tag 和 DEV 目标。 | 证明镜像真实存在。 |
|
||||
| Registry manifest | 镜像事实源，证明目标 tag/digest 可被 registry 读取，并与 catalog 记录一致。 | 决定哪一批服务应该作为一个发布集合上线。 |
|
||||
| CI 产物报告 | 自动生成的一次性审计记录，包含构建、推送、digest、耗时和日志摘要。 | 作为手工维护状态表，或成为 CD rollout 的必要本地文件。 |
|
||||
|
||||
`deploy/artifact-catalog.dev.json` 是 `deploy.json` 与 registry 之间的 repo-owned
|
||||
digest 期望表；`deploy/k8s/base/workloads.yaml` 是最终 Kubernetes desired state。
|
||||
CD 在 apply 前必须根据 catalog 的 image tag/digest 直接读取 registry manifest，
|
||||
确认镜像事实存在且 digest 匹配。过期、缺失或路径漂移的
|
||||
`/tmp/hwlab-dev-gate/dev-artifacts.json` 只能帮助定位 CI 问题，不得推翻
|
||||
`deploy.json + catalog + registry` 已经证明一致的发布结论。
|
||||
|
||||
过渡期如果 `scripts/dev-cd-apply.mjs` 发现 `deploy/deploy.json` 已经指向一个
|
||||
通过 desired-state check 的 promotion commit，应走 `promotionSource=deploy-json`
|
||||
路径，只执行 desired-state 校验、runtime 前置、apply、postflight 和 live verify，
|
||||
不得重新 publish 当前 HEAD。未来 CI 常态化后，latest-main target-ref publish
|
||||
路径应降级为受限兼容路径，正式 DEV rollout 默认走已发布 artifact 的
|
||||
deploy-json promotion。
|
||||
|
||||
CI/artifact 层可以使用以下加速面：
|
||||
|
||||
- `scripts/dev-artifact-publish.mjs --build --services <id,...>`：只构建受影响服务，避免 web-only 变更重建所有服务；
|
||||
- `scripts/dev-artifact-publish.mjs --publish --services <id,...>`：只在 artifact 任务边界内发布选定服务并记录 digest；
|
||||
- `scripts/dev-artifact-publish.mjs --publish --concurrency <n>`：并行构建/推送服务镜像，默认 4，最高 8；
|
||||
- `scripts/dev-runtime-base-image.mjs`：按 `package.json` / `package-lock.json`
|
||||
hash 构建 `hwlab-node-runtime-base:deps-<hash>`，让后续服务镜像复用依赖层。
|
||||
|
||||
CD 层不得把上述 CI/artifact 加速面当作 rollout 授权。只有当 desired-state
|
||||
文件已经收敛、registry manifest 直接验证通过，CD 才能拿 Lease 执行 DEV apply。
|
||||
CD apply 内部可以用 `--rollout-concurrency <n>` 或 `HWLAB_DEV_CD_CONCURRENCY`
|
||||
并行执行 registry manifest 验证、DEV allowlist 清理和 post-apply rollout status；
|
||||
并发只加速同一事务内的独立检查，不创建第二个 CD 控制源。
|
||||
|
||||
## Git Repo Owner 权限治理
|
||||
|
||||
GitHub repo owner/admin 权限是治理权限，不是日常构建、合并、发布和清理权限。
|
||||
长期目标是把 owner 从常规交付链路中解耦，避免 owner 账号或 owner-owned
|
||||
工作区成为单点阻塞，也避免 root/owner 文件污染让 runner、CI 或 CD 无法继续。
|
||||
|
||||
稳定分工如下：
|
||||
|
||||
| 权限域 | 可以做什么 | 不应承担什么 |
|
||||
| --- | --- | --- |
|
||||
| Repo owner/admin | 维护 branch protection、required checks、runner/App/token 授权、紧急权限恢复和审计规则。 | 日常手工 build、长期持有脏 worktree、替 runner 修权限污染、绕过 PR 合并主线。 |
|
||||
| CI build identity | 从受保护主线或 PR commit 构建镜像，写 CI artifact、digest report 和受控 PR/commit。 | DEV apply、live rollout、看板收口、读取 Secret value。 |
|
||||
| CD deploy identity | 读取已合并 desired-state 和 registry artifact，获取 DEV Lease，执行 apply/verify。 | 构建镜像、修改 source、改写 `deploy.json` promotion commit。 |
|
||||
| Code Queue runner | 在任务分支完成代码/文档/desired-state PR，运行只读 status/preflight 和授权的 artifact 任务。 | 占用 owner 权限、写 main、污染共享发布 worktree、承担长期 CD lock。 |
|
||||
|
||||
所有自动化应使用可审计、可轮换的 GitHub App 或 fine-grained token；token 只授予
|
||||
所需 repo 和最小 scope。日常主线合并通过 branch protection 和 required CI
|
||||
结果完成，owner/admin 只处理策略变更、紧急解锁和权限恢复。至少保留两个维护者
|
||||
具备 owner/admin 恢复能力，避免单人离线造成 repo 或部署链路停摆。
|
||||
|
||||
共享发布目录只允许作为缓存或 clean mirror，不允许成为 source of truth。任何
|
||||
CI/CD 写操作都应在 ephemeral worktree 或明确 owner 的 clean worktree 中执行；
|
||||
进入写路径前必须检查 `git status --short`、`.git` 可读写权限、工作区 owner/group
|
||||
和 expected remote commit。发现 root-owned 文件、不可快进、未知本地修改或权限
|
||||
不一致时，应结构化报 `environment_blocker`，由权限治理流程修复；不得直接在污染
|
||||
工作区继续 build、apply 或 push。
|
||||
|
||||
## SecretRef Preflight
|
||||
|
||||
apply 前还必须把必要 SecretRef 作为 preflight 条件处理，而不是等待
|
||||
@@ -269,11 +344,11 @@ value。
|
||||
完整 JSON。DEV apply、rollout 和 live verification 仍由 host commander
|
||||
统一执行；runner 默认只使用 status/dry-run 观测面，不竞争 DEV CD lock。
|
||||
|
||||
旧的 side-effect 命令已经降为事务内部步骤。直接手动执行
|
||||
`scripts/dev-artifact-publish.mjs --publish` 或
|
||||
`scripts/dev-deploy-apply.mjs --apply` 且没有 `HWLAB_CD_TRANSACTION_ID` 时，
|
||||
会以 `cd-transaction-required` 拒绝；preflight、build、dry-run、check 和
|
||||
read-only 模式仍可用。
|
||||
旧的 deploy side-effect 命令已经降为事务内部步骤。直接手动执行
|
||||
`scripts/dev-deploy-apply.mjs --apply` 且没有 `HWLAB_CD_TRANSACTION_ID` 时，会以
|
||||
`cd-transaction-required` 拒绝。artifact publish 是 CI 职责，正式入口需要
|
||||
`HWLAB_CI_ARTIFACT_RUN_ID`；过渡期也接受 `HWLAB_CD_TRANSACTION_ID`。preflight、
|
||||
build、dry-run、check 和 read-only 模式仍可用。
|
||||
|
||||
## Artifact Publish
|
||||
|
||||
@@ -283,15 +358,16 @@ DEV artifacts publish to the D601 local/internal registry:
|
||||
127.0.0.1:5000/hwlab/*
|
||||
```
|
||||
|
||||
The single-step publish command is a transaction-internal side-effect step:
|
||||
The single-step publish command is a CI artifact side-effect step:
|
||||
|
||||
```sh
|
||||
node scripts/dev-artifact-publish.mjs --publish
|
||||
HWLAB_CI_ARTIFACT_RUN_ID=<run-id> node scripts/dev-artifact-publish.mjs --publish
|
||||
```
|
||||
|
||||
It only runs in side-effect mode when `HWLAB_CD_TRANSACTION_ID` is set by
|
||||
`scripts/dev-cd-apply.mjs`. Use `node scripts/dev-cd-apply.mjs --apply
|
||||
--confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json` for normal DEV CD.
|
||||
It only runs in side-effect mode when `HWLAB_CI_ARTIFACT_RUN_ID` is set by the
|
||||
CI artifact job, or when the legacy `HWLAB_CD_TRANSACTION_ID` is set by
|
||||
`scripts/dev-cd-apply.mjs` during transition. Normal DEV CD must consume
|
||||
`deploy.json` and registry-verified catalog state; it should not rebuild images.
|
||||
|
||||
For Cloud Web source freshness before the transaction:
|
||||
|
||||
@@ -496,9 +572,10 @@ use the repo-owned scripts below instead of ad hoc shell fragments.
|
||||
| Plan desired image/workload state | `node scripts/deploy-desired-state-plan.mjs --plan --pretty` and `node scripts/deploy-desired-state-plan.mjs --promotion-commit <sha> --check` | Fold into `hwlab deploy plan --env dev --file deploy/deploy.json` with a promotion commit gate. |
|
||||
| Observe DEV CD status | `node scripts/dev-cd-apply.mjs`, `node scripts/dev-cd-apply.mjs --status`, or `node scripts/dev-cd-apply.mjs --dry-run` | Keep read-only for host and runners; no Lease mutation, artifact publish, apply, rollout, report write, or Secret read. |
|
||||
| Preflight artifact publish | `node scripts/dev-artifact-publish.mjs --preflight` | Keep as backend for `hwlab artifact publish --preflight`. |
|
||||
| Prepare dependency runtime base | `node scripts/dev-runtime-base-image.mjs --dry-run` and `node scripts/dev-runtime-base-image.mjs` | Move into a CI base-image job keyed by dependency hash; CD only consumes the resulting image tag. |
|
||||
| Publish/apply/verify DEV | `node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json` | Keep as the single side-effect transaction and run from repo-owned DEV CI/CD. |
|
||||
| Publish DEV images | `node scripts/dev-artifact-publish.mjs --publish` inside `HWLAB_CD_TRANSACTION_ID` | Internal step only; direct side-effect calls are rejected. |
|
||||
| Refresh catalog/desired state | `node scripts/refresh-artifact-catalog.mjs --target-ref origin/main --publish-report /tmp/hwlab-dev-gate/dev-artifacts.json` | Internal transaction step after publish; read/write desired-state only from repo files. |
|
||||
| Publish DEV images | `HWLAB_CI_ARTIFACT_RUN_ID=<run-id> node scripts/dev-artifact-publish.mjs --publish` | CI artifact step only; legacy CD transaction env remains transitional. |
|
||||
| Refresh catalog/desired state | `node scripts/refresh-artifact-catalog.mjs --target-ref origin/main --publish-report /tmp/hwlab-dev-gate/dev-artifacts.json` | CI promotion step after publish; report is input evidence for catalog refresh, not CD rollout state. |
|
||||
| Apply DEV rollout | `node scripts/dev-deploy-apply.mjs --apply --confirm-dev --confirmed-non-production --report /tmp/hwlab-dev-gate/report.json` inside `HWLAB_CD_TRANSACTION_ID` | Internal transaction step; direct side-effect calls are rejected. |
|
||||
| Roll back DEV Deployment | `KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev rollout undo deployment/<name>` | Add `hwlab deploy rollback --env dev --service <id> --to-revision <n>`. |
|
||||
| Verify status | `curl` checks plus `node scripts/d601-k3s-readonly-observability.mjs` | Add `hwlab deploy verify --env dev --report /tmp/hwlab-dev-gate/report.json`. |
|
||||
@@ -512,9 +589,10 @@ converge manual operations into CLI plus `deploy/deploy.json`, then run that
|
||||
surface from UniDesk CI/CD:
|
||||
|
||||
- render and verify frps/frpc from the deploy manifest;
|
||||
- publish images and write digest reports;
|
||||
- refresh catalog/workload desired state from the publish report;
|
||||
- publish images and write digest reports as CI audit evidence;
|
||||
- refresh catalog/workload desired state from a verified CI publish report;
|
||||
- apply DEV with immutable Job replacement policy;
|
||||
- verify catalog tag/digest directly against registry manifests before apply;
|
||||
- verify `16666` Cloud Web and `16667/health/live` API;
|
||||
- invalidate or regenerate active reports after endpoint or artifact changes.
|
||||
|
||||
|
||||
+3
-2
File diff suppressed because one or more lines are too long
@@ -16,7 +16,10 @@ import {
|
||||
resolveDevArtifactServices,
|
||||
serviceInventoryFromServices
|
||||
} from "./src/dev-artifact-services.mjs";
|
||||
import { requireDevCdTransactionForSideEffect } from "./src/dev-cd-transaction-guard.mjs";
|
||||
import {
|
||||
devArtifactProducer,
|
||||
requireDevArtifactSideEffectEnv
|
||||
} from "./src/dev-cd-transaction-guard.mjs";
|
||||
import { runDevBaseImagePreflight } from "./src/dev-base-image-preflight.mjs";
|
||||
import { probeRegistryCapabilities } from "./src/registry-capabilities.mjs";
|
||||
import { inspectCloudWebDistFreshness } from "../web/hwlab-cloud-web/scripts/dist-contract.mjs";
|
||||
@@ -66,7 +69,9 @@ function parseArgs(argv) {
|
||||
baseImage: defaultBaseImage,
|
||||
reportPath: defaultReportPath,
|
||||
services: [...SERVICE_IDS],
|
||||
emitReport: true
|
||||
emitReport: true,
|
||||
quietBuild: false,
|
||||
concurrency: parseConcurrency(process.env.HWLAB_DEV_ARTIFACT_CONCURRENCY, 4)
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
@@ -79,6 +84,8 @@ function parseArgs(argv) {
|
||||
else if (arg === "--report") args.reportPath = readOption(argv, ++index, arg);
|
||||
else if (arg === "--services") args.services = readOption(argv, ++index, arg).split(",").map((item) => item.trim()).filter(Boolean);
|
||||
else if (arg === "--no-report") args.emitReport = false;
|
||||
else if (arg === "--quiet-build") args.quietBuild = true;
|
||||
else if (arg === "--concurrency") args.concurrency = parseConcurrency(readOption(argv, ++index, arg), 4);
|
||||
else if (arg === "--help" || arg === "-h") args.help = true;
|
||||
else throw new Error(`unknown argument ${arg}`);
|
||||
}
|
||||
@@ -94,6 +101,12 @@ function readOption(argv, index, name) {
|
||||
return value;
|
||||
}
|
||||
|
||||
function parseConcurrency(value, fallback) {
|
||||
const parsed = Number.parseInt(String(value ?? ""), 10);
|
||||
if (!Number.isInteger(parsed)) return fallback;
|
||||
return Math.min(Math.max(parsed, 1), 8);
|
||||
}
|
||||
|
||||
function printHelp() {
|
||||
console.log([
|
||||
"usage: node scripts/dev-artifact-publish.mjs [--preflight|--build|--publish]",
|
||||
@@ -105,7 +118,9 @@ function printHelp() {
|
||||
" --base-image IMAGE override HWLAB_DEV_BASE_IMAGE for this run; must already be local",
|
||||
" --services LIST comma-separated service IDs; default: all frozen DEV services",
|
||||
` --report PATH default: ${defaultReportPath}`,
|
||||
" --no-report print JSON without updating the temporary artifact JSON"
|
||||
" --no-report print JSON without updating the temporary artifact JSON",
|
||||
" --quiet-build keep docker build quiet; default keeps build output visible",
|
||||
" --concurrency N parallel service build/push workers; default: 4, max: 8"
|
||||
].join("\n"));
|
||||
}
|
||||
|
||||
@@ -114,7 +129,7 @@ async function readJson(relativePath) {
|
||||
}
|
||||
|
||||
function emit(event, payload = {}) {
|
||||
process.stderr.write(`${JSON.stringify({ event, ...payload })}\n`);
|
||||
process.stderr.write(`${JSON.stringify({ event, at: new Date().toISOString(), ...payload })}\n`);
|
||||
}
|
||||
|
||||
function blocker({ type, scope, summary, next }) {
|
||||
@@ -186,6 +201,7 @@ function commandLine(command, args) {
|
||||
}
|
||||
|
||||
async function run(command, args, options = {}) {
|
||||
const startedAt = Date.now();
|
||||
const child = spawn(command, args, {
|
||||
cwd: options.cwd ?? repoRoot,
|
||||
env: options.env ?? process.env,
|
||||
@@ -210,7 +226,8 @@ async function run(command, args, options = {}) {
|
||||
command: commandLine(command, args),
|
||||
code,
|
||||
stdout,
|
||||
stderr
|
||||
stderr,
|
||||
durationMs: Date.now() - startedAt
|
||||
};
|
||||
}
|
||||
|
||||
@@ -802,6 +819,28 @@ function dockerfile(baseImage, port) {
|
||||
return [
|
||||
`FROM ${baseImage}`,
|
||||
"WORKDIR /app",
|
||||
"ENV CODEX_HOME=/codex-home",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_HOME=/codex-home",
|
||||
"ENV HWLAB_CODE_AGENT_WORKSPACE=/workspace/hwlab",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_WORKSPACE=/workspace/hwlab",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_SANDBOX=workspace-write",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED=1",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR=repo-owned",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_COMMAND=/app/node_modules/.bin/codex",
|
||||
"ENV HWLAB_CODE_AGENT_PROVIDER=codex-stdio",
|
||||
"ENV HWLAB_CODE_AGENT_SKILLS_DIRS=/app/skills:/root/.agents/skills:/home/ubuntu/.agents/skills",
|
||||
"COPY package.json ./package.json",
|
||||
"COPY package-lock.json* ./",
|
||||
"RUN if [ -d /opt/hwlab-node-runtime-base/node_modules ]; then cp -a /opt/hwlab-node-runtime-base/node_modules ./node_modules; elif [ -f package-lock.json ]; then npm ci --omit=dev --ignore-scripts; else npm install --omit=dev --ignore-scripts; fi",
|
||||
"COPY internal ./internal",
|
||||
"COPY cmd ./cmd",
|
||||
"COPY scripts ./scripts",
|
||||
"COPY web ./web",
|
||||
"COPY tools ./tools",
|
||||
"COPY skills ./skills",
|
||||
"COPY deploy ./deploy",
|
||||
"RUN mkdir -p /workspace /codex-home && rm -rf /workspace/hwlab && ln -s /app /workspace/hwlab && chmod -R a+rwX /app /workspace /codex-home && test -x /app/node_modules/.bin/codex && /app/node_modules/.bin/codex --version >/tmp/hwlab-codex-version.txt && (test -x /app/node_modules/@openai/codex-linux-x64/vendor/x86_64-unknown-linux-musl/codex/codex || test -x /app/node_modules/@openai/codex-linux-arm64/vendor/aarch64-unknown-linux-musl/codex/codex)",
|
||||
`RUN node -e "const fs=require('node:fs'); fs.writeFileSync('/usr/local/bin/hwlab-dev-artifact-runtime.mjs', Buffer.from('${runtimeScriptBase64()}', 'base64')); fs.chmodSync('/usr/local/bin/hwlab-dev-artifact-runtime.mjs', 0o755);"`,
|
||||
"ARG HWLAB_ENVIRONMENT",
|
||||
"ARG HWLAB_SERVICE_ID",
|
||||
"ARG HWLAB_ARTIFACT_KIND",
|
||||
@@ -816,16 +855,6 @@ function dockerfile(baseImage, port) {
|
||||
"ARG HWLAB_BUILD_PROVENANCE",
|
||||
"ARG PORT",
|
||||
"ARG HWLAB_PORT",
|
||||
"ENV CODEX_HOME=/codex-home",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_HOME=/codex-home",
|
||||
"ENV HWLAB_CODE_AGENT_WORKSPACE=/workspace/hwlab",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_WORKSPACE=/workspace/hwlab",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_SANDBOX=workspace-write",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED=1",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR=repo-owned",
|
||||
"ENV HWLAB_CODE_AGENT_CODEX_COMMAND=/app/node_modules/.bin/codex",
|
||||
"ENV HWLAB_CODE_AGENT_PROVIDER=codex-stdio",
|
||||
"ENV HWLAB_CODE_AGENT_SKILLS_DIRS=/app/skills:/root/.agents/skills:/home/ubuntu/.agents/skills",
|
||||
"ENV HWLAB_ENVIRONMENT=$HWLAB_ENVIRONMENT",
|
||||
"ENV HWLAB_SERVICE_ID=$HWLAB_SERVICE_ID",
|
||||
"ENV HWLAB_ARTIFACT_KIND=$HWLAB_ARTIFACT_KIND",
|
||||
@@ -840,18 +869,6 @@ function dockerfile(baseImage, port) {
|
||||
"ENV HWLAB_BUILD_PROVENANCE=$HWLAB_BUILD_PROVENANCE",
|
||||
"ENV PORT=$PORT",
|
||||
"ENV HWLAB_PORT=$HWLAB_PORT",
|
||||
"COPY package.json ./package.json",
|
||||
"COPY package-lock.json* ./",
|
||||
"RUN if [ -f package-lock.json ]; then npm ci --omit=dev --ignore-scripts; else npm install --omit=dev --ignore-scripts; fi",
|
||||
"COPY internal ./internal",
|
||||
"COPY cmd ./cmd",
|
||||
"COPY scripts ./scripts",
|
||||
"COPY web ./web",
|
||||
"COPY tools ./tools",
|
||||
"COPY skills ./skills",
|
||||
"COPY deploy ./deploy",
|
||||
"RUN mkdir -p /workspace /codex-home && rm -rf /workspace/hwlab && ln -s /app /workspace/hwlab && chmod -R a+rwX /app /workspace /codex-home && test -x /app/node_modules/.bin/codex && /app/node_modules/.bin/codex --version >/tmp/hwlab-codex-version.txt && (test -x /app/node_modules/@openai/codex-linux-x64/vendor/x86_64-unknown-linux-musl/codex/codex || test -x /app/node_modules/@openai/codex-linux-arm64/vendor/aarch64-unknown-linux-musl/codex/codex)",
|
||||
`RUN node -e "const fs=require('node:fs'); fs.writeFileSync('/usr/local/bin/hwlab-dev-artifact-runtime.mjs', Buffer.from('${runtimeScriptBase64()}', 'base64')); fs.chmodSync('/usr/local/bin/hwlab-dev-artifact-runtime.mjs', 0o755);"`,
|
||||
`EXPOSE ${port}`,
|
||||
"CMD [\"node\", \"/usr/local/bin/hwlab-dev-artifact-runtime.mjs\"]",
|
||||
""
|
||||
@@ -1022,7 +1039,8 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC
|
||||
summary: "cloud web dist build failed before Docker build",
|
||||
next: "Run node web/hwlab-cloud-web/scripts/build.mjs, fix the static asset build, then rerun --publish."
|
||||
}),
|
||||
logTail: tailText(`${build.stdout}\n${build.stderr}`)
|
||||
logTail: tailText(`${build.stdout}\n${build.stderr}`),
|
||||
cloudWebBuildDurationMs: build.durationMs
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1048,7 +1066,8 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC
|
||||
|
||||
service = {
|
||||
...service,
|
||||
distFreshness
|
||||
distFreshness,
|
||||
cloudWebBuildDurationMs: build.durationMs
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1083,11 +1102,9 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC
|
||||
];
|
||||
const argsList = [
|
||||
"build",
|
||||
"--pull=false",
|
||||
"--quiet",
|
||||
"--build-arg",
|
||||
`BASE_IMAGE=${args.baseImage}`
|
||||
"--pull=false"
|
||||
];
|
||||
if (args.quietBuild) argsList.push("--quiet");
|
||||
|
||||
for (const [name, value] of labels) {
|
||||
argsList.push("--label", `${name}=${value}`);
|
||||
@@ -1113,9 +1130,11 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC
|
||||
summary: `docker build failed for ${service.serviceId}`,
|
||||
next: `Inspect the Docker build output for ${service.serviceId}, then fix its copied source or base image.`
|
||||
}),
|
||||
logTail: tailText(`${result.stdout}\n${result.stderr}`)
|
||||
logTail: tailText(`${result.stdout}\n${result.stderr}`),
|
||||
dockerBuildDurationMs: result.durationMs
|
||||
};
|
||||
}
|
||||
const imageInspect = await run("docker", ["image", "inspect", "--format", "{{.Id}}", ref]);
|
||||
|
||||
return {
|
||||
...service,
|
||||
@@ -1124,12 +1143,17 @@ async function buildService({ args, repo, commitId, shortCommit, service, buildC
|
||||
buildCreatedAt,
|
||||
buildSource,
|
||||
status: "built",
|
||||
localImageId: result.stdout.trim(),
|
||||
localImageId: imageInspect.code === 0 ? imageInspect.stdout.trim() : null,
|
||||
dockerBuildDurationMs: result.durationMs,
|
||||
dockerImageInspectDurationMs: imageInspect.durationMs,
|
||||
dockerImageInspectStatus: imageInspect.code === 0 ? "pass" : "failed",
|
||||
dockerBuildLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200),
|
||||
digest: "not_published"
|
||||
};
|
||||
}
|
||||
|
||||
async function runWithInput(command, args, input) {
|
||||
const startedAt = Date.now();
|
||||
const child = spawn(command, args, {
|
||||
cwd: repoRoot,
|
||||
env: process.env,
|
||||
@@ -1155,7 +1179,8 @@ async function runWithInput(command, args, input) {
|
||||
command: commandLine(command, args),
|
||||
code,
|
||||
stdout,
|
||||
stderr
|
||||
stderr,
|
||||
durationMs: Date.now() - startedAt
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1175,7 +1200,8 @@ async function publishService(artifact) {
|
||||
summary: `docker push failed for ${artifact.image}`,
|
||||
next: "Verify the D601 local/internal registry is reachable from the Docker daemon and rerun --publish."
|
||||
}),
|
||||
logTail: tailText(`${result.stdout}\n${result.stderr}`)
|
||||
logTail: tailText(`${result.stdout}\n${result.stderr}`),
|
||||
publishDurationMs: result.durationMs
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1191,7 +1217,8 @@ async function publishService(artifact) {
|
||||
summary: `docker push completed for ${artifact.image} but did not return an immutable digest`,
|
||||
next: "Verify registry digest reporting, then rerun --publish; do not update the catalog with a digest until a sha256 registry digest is observed."
|
||||
}),
|
||||
pushLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200)
|
||||
pushLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200),
|
||||
publishDurationMs: result.durationMs
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1200,7 +1227,8 @@ async function publishService(artifact) {
|
||||
status: "published",
|
||||
digest,
|
||||
repositoryDigest: `${repositoryFromImageRef(artifact.image)}@${digest}`,
|
||||
pushLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200)
|
||||
pushLogTail: tailText(`${result.stdout}\n${result.stderr}`, 1200),
|
||||
publishDurationMs: result.durationMs
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1288,13 +1316,34 @@ function publishPlanFor({ args, commitId, shortCommit, mode, artifacts, fatalBlo
|
||||
repositoryDigest: artifact.repositoryDigest ?? null,
|
||||
digestStatus: /^sha256:[a-f0-9]{64}$/u.test(digest) ? "real" : "placeholder",
|
||||
status: artifact.status,
|
||||
dockerBuildDurationMs: artifact.dockerBuildDurationMs ?? null,
|
||||
cloudWebBuildDurationMs: artifact.cloudWebBuildDurationMs ?? null,
|
||||
publishDurationMs: artifact.publishDurationMs ?? null,
|
||||
notPublishedReason: artifactNotPublishedReason(artifact, mode, fatalBlocked)
|
||||
};
|
||||
})
|
||||
};
|
||||
}
|
||||
|
||||
function createReport({ args, repo, commitId, shortCommit, mode, services, artifacts, blockers, baseImagePreflight, registryCapabilities }) {
|
||||
function sumDurations(artifacts, field) {
|
||||
return artifacts.reduce((sum, artifact) => sum + (Number.isFinite(artifact[field]) ? artifact[field] : 0), 0);
|
||||
}
|
||||
|
||||
async function mapWithConcurrency(items, concurrency, worker) {
|
||||
const results = new Array(items.length);
|
||||
let nextIndex = 0;
|
||||
const workerCount = Math.min(Math.max(concurrency, 1), items.length);
|
||||
await Promise.all(Array.from({ length: workerCount }, async () => {
|
||||
while (nextIndex < items.length) {
|
||||
const index = nextIndex;
|
||||
nextIndex += 1;
|
||||
results[index] = await worker(items[index], index);
|
||||
}
|
||||
}));
|
||||
return results;
|
||||
}
|
||||
|
||||
function createReport({ args, repo, commitId, shortCommit, mode, services, artifacts, blockers, baseImagePreflight, registryCapabilities, producer }) {
|
||||
const status = reportStatus(mode, artifacts, blockers);
|
||||
const fatalBlocked = hasFatalBlocker(blockers);
|
||||
const requiredArtifacts = artifacts.filter((artifact) => artifact.artifactRequired);
|
||||
@@ -1331,6 +1380,8 @@ function createReport({ args, repo, commitId, shortCommit, mode, services, artif
|
||||
"node --check scripts/src/registry-capabilities.mjs",
|
||||
"node --check scripts/preflight-dev-base-image.mjs",
|
||||
"node --check scripts/src/dev-base-image-preflight.mjs",
|
||||
"node --check scripts/dev-runtime-base-image.mjs",
|
||||
"node --test scripts/refresh-artifact-catalog.test.mjs",
|
||||
"node scripts/preflight-dev-base-image.mjs",
|
||||
"node scripts/dev-artifact-publish.mjs --preflight --no-report",
|
||||
"node --check scripts/validate-dev-gate-report.mjs",
|
||||
@@ -1377,8 +1428,12 @@ function createReport({ args, repo, commitId, shortCommit, mode, services, artif
|
||||
repo,
|
||||
sourceCommitId: commitId,
|
||||
registryPrefix: args.registryPrefix,
|
||||
producer,
|
||||
registryCapabilities: summarizeRegistryCapabilities(registryCapabilities),
|
||||
baseImage: args.baseImage ?? null,
|
||||
selectedServiceIds: args.services,
|
||||
quietBuild: args.quietBuild,
|
||||
concurrency: args.concurrency,
|
||||
baseImagePreflight: summarizeBaseImagePreflight(baseImagePreflight),
|
||||
environment: ENVIRONMENT_DEV,
|
||||
buildCreatedAt: artifacts.find((artifact) => artifact.buildCreatedAt)?.buildCreatedAt ?? null,
|
||||
@@ -1389,6 +1444,11 @@ function createReport({ args, repo, commitId, shortCommit, mode, services, artif
|
||||
disabledServiceCount: serviceInventory.disabledServiceCount,
|
||||
builtCount,
|
||||
publishedCount,
|
||||
timings: {
|
||||
dockerBuildDurationMs: sumDurations(artifacts, "dockerBuildDurationMs"),
|
||||
cloudWebBuildDurationMs: sumDurations(artifacts, "cloudWebBuildDurationMs"),
|
||||
publishDurationMs: sumDurations(artifacts, "publishDurationMs")
|
||||
},
|
||||
generatedAt: new Date().toISOString(),
|
||||
services: artifacts.map((artifact) => ({
|
||||
serviceId: artifact.serviceId,
|
||||
@@ -1408,7 +1468,13 @@ function createReport({ args, repo, commitId, shortCommit, mode, services, artif
|
||||
artifactRequired: artifact.artifactRequired,
|
||||
artifactScope: artifact.artifactScope,
|
||||
notPublishedReason: artifactNotPublishedReason(artifact, mode, fatalBlocked),
|
||||
localImageId: artifact.localImageId
|
||||
localImageId: artifact.localImageId,
|
||||
dockerBuildDurationMs: artifact.dockerBuildDurationMs ?? null,
|
||||
cloudWebBuildDurationMs: artifact.cloudWebBuildDurationMs ?? null,
|
||||
publishDurationMs: artifact.publishDurationMs ?? null,
|
||||
dockerBuildLogTail: artifact.dockerBuildLogTail ?? null,
|
||||
logTail: artifact.logTail ?? null,
|
||||
pushLogTail: artifact.pushLogTail ?? null
|
||||
})),
|
||||
blockers
|
||||
},
|
||||
@@ -1429,7 +1495,7 @@ async function main() {
|
||||
return;
|
||||
}
|
||||
if (args.mode === "publish") {
|
||||
const transactionGuard = requireDevCdTransactionForSideEffect({
|
||||
const transactionGuard = requireDevArtifactSideEffectEnv({
|
||||
script: "scripts/dev-artifact-publish.mjs",
|
||||
mode: "--publish"
|
||||
});
|
||||
@@ -1461,6 +1527,7 @@ async function main() {
|
||||
const shortCommit = commitId.slice(0, 7);
|
||||
const buildCreatedAt = new Date().toISOString();
|
||||
const repo = repoLabelFromRemote(remoteUrl);
|
||||
const producer = devArtifactProducer(process.env);
|
||||
const services = await resolveServices(args.services, catalog);
|
||||
|
||||
let blockers = await preflight({ args, services, catalog, deployManifest, baseImagePreflight, registryCapabilities });
|
||||
@@ -1478,12 +1545,18 @@ async function main() {
|
||||
});
|
||||
} else if (args.mode === "build" || args.mode === "publish") {
|
||||
const artifactByServiceId = new Map(artifacts.map((artifact) => [artifact.serviceId, artifact]));
|
||||
for (const service of services.filter((item) => item.artifactRequired)) {
|
||||
await mapWithConcurrency(services.filter((item) => item.artifactRequired), args.concurrency, async (service) => {
|
||||
emit("build_start", { serviceId: service.serviceId, image: imageRef(args.registryPrefix, service.serviceId, shortCommit) });
|
||||
const artifact = await buildService({ args, repo, commitId, shortCommit, service, buildCreatedAt });
|
||||
artifactByServiceId.set(service.serviceId, artifact);
|
||||
if (artifact.blocker) blockers.push(artifact.blocker);
|
||||
emit("build_done", { serviceId: service.serviceId, status: artifact.status, image: artifact.image });
|
||||
emit("build_done", {
|
||||
serviceId: service.serviceId,
|
||||
status: artifact.status,
|
||||
image: artifact.image,
|
||||
dockerBuildDurationMs: artifact.dockerBuildDurationMs ?? null,
|
||||
cloudWebBuildDurationMs: artifact.cloudWebBuildDurationMs ?? null
|
||||
});
|
||||
|
||||
if (args.mode === "publish" && artifact.status === "built") {
|
||||
emit("publish_start", { serviceId: service.serviceId, image: artifact.image });
|
||||
@@ -1494,14 +1567,15 @@ async function main() {
|
||||
serviceId: service.serviceId,
|
||||
status: published.status,
|
||||
image: published.image,
|
||||
digest: published.digest
|
||||
digest: published.digest,
|
||||
publishDurationMs: published.publishDurationMs ?? null
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
artifacts = services.map((service) => artifactByServiceId.get(service.serviceId));
|
||||
}
|
||||
|
||||
const report = createReport({ args, repo, commitId, shortCommit, mode: args.mode, services, artifacts, blockers, baseImagePreflight, registryCapabilities });
|
||||
const report = createReport({ args, repo, commitId, shortCommit, mode: args.mode, services, artifacts, blockers, baseImagePreflight, registryCapabilities, producer });
|
||||
if (args.emitReport) {
|
||||
await writeReport(args.reportPath, report);
|
||||
}
|
||||
@@ -1514,6 +1588,9 @@ async function main() {
|
||||
registryPrefix: args.registryPrefix,
|
||||
registryCapabilities: summarizeRegistryCapabilities(registryCapabilities),
|
||||
baseImagePreflight: summarizeBaseImagePreflight(baseImagePreflight),
|
||||
selectedServiceIds: report.artifactPublish.selectedServiceIds,
|
||||
concurrency: report.artifactPublish.concurrency,
|
||||
timings: report.artifactPublish.timings,
|
||||
publishPlan: report.artifactPublish.publishPlan,
|
||||
services: report.artifactPublish.services.map((service) => ({
|
||||
serviceId: service.serviceId,
|
||||
@@ -1523,6 +1600,11 @@ async function main() {
|
||||
buildCreatedAt: service.buildCreatedAt,
|
||||
digest: service.digest,
|
||||
distFreshness: service.distFreshness,
|
||||
dockerBuildDurationMs: service.dockerBuildDurationMs,
|
||||
cloudWebBuildDurationMs: service.cloudWebBuildDurationMs,
|
||||
publishDurationMs: service.publishDurationMs,
|
||||
logTail: service.logTail,
|
||||
pushLogTail: service.pushLogTail,
|
||||
notPublishedReason: service.notPublishedReason
|
||||
})),
|
||||
blockers
|
||||
|
||||
@@ -0,0 +1,301 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import { createHash } from "node:crypto";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
runDevBaseImagePreflight,
|
||||
summarizeForRuntimeBase
|
||||
} from "./src/dev-base-image-preflight.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const defaultRegistryPrefix =
|
||||
process.env.HWLAB_DEV_REGISTRY_PREFIX || process.env.HWLAB_DEV_REGISTRY || "127.0.0.1:5000/hwlab";
|
||||
const defaultBaseImage = process.env.HWLAB_DEV_RUNTIME_BASE_PARENT || process.env.HWLAB_DEV_BASE_IMAGE || null;
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
registryPrefix: defaultRegistryPrefix,
|
||||
baseImage: defaultBaseImage,
|
||||
tag: null,
|
||||
build: true,
|
||||
push: false,
|
||||
help: false
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--registry-prefix") args.registryPrefix = readOption(argv, ++index, arg);
|
||||
else if (arg === "--base-image") args.baseImage = readOption(argv, ++index, arg);
|
||||
else if (arg === "--tag") args.tag = readOption(argv, ++index, arg);
|
||||
else if (arg === "--dry-run" || arg === "--plan") args.build = false;
|
||||
else if (arg === "--push") args.push = true;
|
||||
else if (arg === "--help" || arg === "-h") args.help = true;
|
||||
else throw new Error(`unknown argument ${arg}`);
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
function readOption(argv, index, name) {
|
||||
const value = argv[index];
|
||||
if (!value || value.startsWith("--")) {
|
||||
throw new Error(`${name} requires a value`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function usage() {
|
||||
return [
|
||||
"usage: node scripts/dev-runtime-base-image.mjs [--dry-run] [--push] [--base-image IMAGE] [--tag IMAGE]",
|
||||
"",
|
||||
"Build an optional D601-local HWLAB Node runtime base image with package dependencies preinstalled.",
|
||||
"",
|
||||
"options:",
|
||||
" --registry-prefix PREFIX default: 127.0.0.1:5000/hwlab",
|
||||
" --base-image IMAGE approved local parent image; default resolved by DEV base preflight",
|
||||
" --tag IMAGE output image tag; default: <registry>/hwlab-node-runtime-base:deps-<hash>",
|
||||
" --dry-run print the build plan without building",
|
||||
" --push push the resulting base image to the local/internal registry"
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function normalizeRegistryPrefix(prefix) {
|
||||
const normalized = prefix.replace(/\/+$/u, "");
|
||||
const [hostPort, ...pathParts] = normalized.split("/");
|
||||
const host = hostPort.split(":")[0].toLowerCase();
|
||||
assert.ok(hostPort && pathParts.length > 0, "registry prefix must include host and namespace");
|
||||
assert.ok(!/prod|production/iu.test(normalized), "registry prefix must not include prod");
|
||||
assert.ok(
|
||||
host === "localhost" ||
|
||||
host === "127.0.0.1" ||
|
||||
/^10\./u.test(host) ||
|
||||
/^192\.168\./u.test(host) ||
|
||||
/^172\.(1[6-9]|2\d|3[0-1])\./u.test(host) ||
|
||||
host.endsWith(".local") ||
|
||||
host.endsWith(".svc") ||
|
||||
host.endsWith(".cluster.local"),
|
||||
"registry prefix must target localhost or a private/internal host"
|
||||
);
|
||||
return normalized;
|
||||
}
|
||||
|
||||
async function fileOrEmpty(relativePath) {
|
||||
try {
|
||||
return await readFile(path.join(repoRoot, relativePath), "utf8");
|
||||
} catch (error) {
|
||||
if (error?.code === "ENOENT") return "";
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
async function dependencyHash() {
|
||||
const packageJson = await fileOrEmpty("package.json");
|
||||
const packageLockJson = await fileOrEmpty("package-lock.json");
|
||||
assert.ok(packageJson.trim(), "package.json is required");
|
||||
const hash = createHash("sha256");
|
||||
hash.update("package.json\0");
|
||||
hash.update(packageJson);
|
||||
hash.update("\0package-lock.json\0");
|
||||
hash.update(packageLockJson);
|
||||
return hash.digest("hex");
|
||||
}
|
||||
|
||||
function dockerfile(parentImage, depsHash) {
|
||||
return [
|
||||
`FROM ${parentImage}`,
|
||||
"WORKDIR /opt/hwlab-node-runtime-base",
|
||||
`LABEL hwlab.pikastech.local/runtime-base-deps-hash=${depsHash}`,
|
||||
"COPY package.json ./package.json",
|
||||
"COPY package-lock.json* ./",
|
||||
"RUN if [ -f package-lock.json ]; then npm ci --omit=dev --ignore-scripts; else npm install --omit=dev --ignore-scripts; fi",
|
||||
""
|
||||
].join("\n");
|
||||
}
|
||||
|
||||
function commandLine(command, args) {
|
||||
return [command, ...args].map((part) => (/\s/u.test(part) ? JSON.stringify(part) : part)).join(" ");
|
||||
}
|
||||
|
||||
function runWithInput(command, args, input) {
|
||||
const startedAt = Date.now();
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(command, args, {
|
||||
cwd: repoRoot,
|
||||
env: process.env,
|
||||
stdio: ["pipe", "pipe", "pipe"]
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdin.end(input);
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
resolve({
|
||||
command: commandLine(command, args),
|
||||
code: 127,
|
||||
stdout,
|
||||
stderr: error.message,
|
||||
durationMs: Date.now() - startedAt
|
||||
});
|
||||
});
|
||||
child.on("close", (code) => {
|
||||
resolve({
|
||||
command: commandLine(command, args),
|
||||
code: code ?? 1,
|
||||
stdout,
|
||||
stderr,
|
||||
durationMs: Date.now() - startedAt
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function run(command, args) {
|
||||
const startedAt = Date.now();
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(command, args, {
|
||||
cwd: repoRoot,
|
||||
env: process.env,
|
||||
stdio: ["ignore", "pipe", "pipe"]
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
resolve({
|
||||
command: commandLine(command, args),
|
||||
code: 127,
|
||||
stdout,
|
||||
stderr: error.message,
|
||||
durationMs: Date.now() - startedAt
|
||||
});
|
||||
});
|
||||
child.on("close", (code) => {
|
||||
resolve({
|
||||
command: commandLine(command, args),
|
||||
code: code ?? 1,
|
||||
stdout,
|
||||
stderr,
|
||||
durationMs: Date.now() - startedAt
|
||||
});
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function tailText(value, maxLength = 2500) {
|
||||
const text = value.trim();
|
||||
if (text.length <= maxLength) return text;
|
||||
return text.slice(text.length - maxLength);
|
||||
}
|
||||
|
||||
function inspectDigest(pushOutput) {
|
||||
const matches = [...pushOutput.matchAll(/digest:\s+(sha256:[a-f0-9]{64})/giu)];
|
||||
return matches.length ? matches[matches.length - 1][1] : null;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
if (args.help) {
|
||||
console.log(usage());
|
||||
return;
|
||||
}
|
||||
|
||||
args.registryPrefix = normalizeRegistryPrefix(args.registryPrefix);
|
||||
const depsHash = await dependencyHash();
|
||||
const image = args.tag ?? `${args.registryPrefix}/hwlab-node-runtime-base:deps-${depsHash.slice(0, 12)}`;
|
||||
const baseImagePreflight = await runDevBaseImagePreflight({
|
||||
env: args.baseImage ? { ...process.env, HWLAB_DEV_BASE_IMAGE: args.baseImage } : process.env
|
||||
});
|
||||
const parentImage = baseImagePreflight.localTag;
|
||||
const plan = {
|
||||
status: args.build ? "planned" : "dry-run",
|
||||
image,
|
||||
parentImage,
|
||||
depsHash,
|
||||
push: args.push,
|
||||
baseImagePreflight: summarizeForRuntimeBase(baseImagePreflight)
|
||||
};
|
||||
|
||||
if (!baseImagePreflight.publishUsable) {
|
||||
console.log(JSON.stringify({
|
||||
...plan,
|
||||
status: "blocked",
|
||||
blockers: baseImagePreflight.blockers,
|
||||
nextSteps: baseImagePreflight.nextSteps
|
||||
}, null, 2));
|
||||
process.exitCode = 2;
|
||||
return;
|
||||
}
|
||||
|
||||
if (!args.build) {
|
||||
console.log(JSON.stringify(plan, null, 2));
|
||||
return;
|
||||
}
|
||||
|
||||
const build = await runWithInput(
|
||||
"docker",
|
||||
["build", "--pull=false", "-t", image, "-f", "-", "."],
|
||||
dockerfile(parentImage, depsHash)
|
||||
);
|
||||
if (build.code !== 0) {
|
||||
console.log(JSON.stringify({
|
||||
...plan,
|
||||
status: "build_failed",
|
||||
command: build.command,
|
||||
durationMs: build.durationMs,
|
||||
logTail: tailText(`${build.stdout}\n${build.stderr}`)
|
||||
}, null, 2));
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const inspect = await run("docker", ["image", "inspect", "--format", "{{.Id}}", image]);
|
||||
let push = null;
|
||||
if (args.push) {
|
||||
const pushResult = await run("docker", ["push", image]);
|
||||
const digest = pushResult.code === 0 ? inspectDigest(`${pushResult.stdout}\n${pushResult.stderr}`) : null;
|
||||
push = {
|
||||
status: pushResult.code === 0 && digest ? "published" : "blocked",
|
||||
command: pushResult.command,
|
||||
code: pushResult.code,
|
||||
durationMs: pushResult.durationMs,
|
||||
digest,
|
||||
logTail: tailText(`${pushResult.stdout}\n${pushResult.stderr}`, 1200)
|
||||
};
|
||||
}
|
||||
|
||||
const status = push ? push.status : "built";
|
||||
console.log(JSON.stringify({
|
||||
...plan,
|
||||
status,
|
||||
imageId: inspect.code === 0 ? inspect.stdout.trim() : null,
|
||||
build: {
|
||||
command: build.command,
|
||||
durationMs: build.durationMs,
|
||||
logTail: tailText(`${build.stdout}\n${build.stderr}`, 1200)
|
||||
},
|
||||
push
|
||||
}, null, 2));
|
||||
process.exitCode = status === "blocked" ? 2 : 0;
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error(JSON.stringify({
|
||||
status: "failed",
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
}, null, 2));
|
||||
process.exitCode = 1;
|
||||
});
|
||||
@@ -77,7 +77,8 @@ function usage() {
|
||||
}
|
||||
|
||||
async function readJson(relativePath) {
|
||||
return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8"));
|
||||
const filePath = path.isAbsolute(relativePath) ? relativePath : path.join(repoRoot, relativePath);
|
||||
return JSON.parse(await readFile(filePath, "utf8"));
|
||||
}
|
||||
|
||||
async function writeJson(relativePath, value) {
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdtemp, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import test from "node:test";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import { SERVICE_IDS } from "../internal/protocol/index.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), "..");
|
||||
|
||||
async function git(args) {
|
||||
const result = await execFileAsync("git", args, {
|
||||
cwd: repoRoot,
|
||||
timeout: 5000,
|
||||
maxBuffer: 1024 * 1024
|
||||
});
|
||||
return result.stdout.trim();
|
||||
}
|
||||
|
||||
function digestFor(index) {
|
||||
return `sha256:${String(index + 1).padStart(64, "0")}`;
|
||||
}
|
||||
|
||||
test("refresh accepts an absolute publish report path", async () => {
|
||||
const commitId = await git(["rev-parse", "HEAD"]);
|
||||
const shortCommitId = await git(["rev-parse", "--short=7", "HEAD"]);
|
||||
const tempDir = await mkdtemp(path.join(os.tmpdir(), "hwlab-refresh-"));
|
||||
const reportPath = path.join(tempDir, "dev-artifacts.json");
|
||||
await writeFile(reportPath, `${JSON.stringify({
|
||||
reportVersion: "v1",
|
||||
taskId: "dev-artifact-publish",
|
||||
commitId: shortCommitId,
|
||||
artifactPublish: {
|
||||
status: "published",
|
||||
mode: "publish",
|
||||
sourceCommitId: commitId,
|
||||
registryPrefix: "127.0.0.1:5000/hwlab",
|
||||
serviceCount: SERVICE_IDS.length,
|
||||
requiredServiceCount: SERVICE_IDS.length,
|
||||
disabledServiceCount: 0,
|
||||
publishedCount: SERVICE_IDS.length,
|
||||
publishPlan: {
|
||||
version: "v2",
|
||||
services: SERVICE_IDS.map((serviceId, index) => ({
|
||||
serviceId,
|
||||
required: true,
|
||||
digest: digestFor(index),
|
||||
image: `127.0.0.1:5000/hwlab/${serviceId}:${shortCommitId}`,
|
||||
imageTag: shortCommitId
|
||||
}))
|
||||
},
|
||||
services: SERVICE_IDS.map((serviceId, index) => ({
|
||||
serviceId,
|
||||
status: "published",
|
||||
artifactRequired: true,
|
||||
image: `127.0.0.1:5000/hwlab/${serviceId}:${shortCommitId}`,
|
||||
imageTag: shortCommitId,
|
||||
digest: digestFor(index),
|
||||
repositoryDigest: `127.0.0.1:5000/hwlab/${serviceId}@${digestFor(index)}`,
|
||||
buildCreatedAt: "2026-05-24T00:00:00.000Z",
|
||||
buildSource: `pikasTech/HWLAB@${commitId}`
|
||||
}))
|
||||
}
|
||||
}, null, 2)}\n`);
|
||||
|
||||
const result = await execFileAsync(process.execPath, [
|
||||
"scripts/refresh-artifact-catalog.mjs",
|
||||
"--target-ref",
|
||||
"HEAD",
|
||||
"--publish-report",
|
||||
reportPath,
|
||||
"--no-write"
|
||||
], {
|
||||
cwd: repoRoot,
|
||||
timeout: 15000,
|
||||
maxBuffer: 10 * 1024 * 1024
|
||||
});
|
||||
const payload = JSON.parse(result.stdout);
|
||||
assert.equal(payload.status, "published");
|
||||
assert.equal(payload.publishedCount, SERVICE_IDS.length);
|
||||
assert.deepEqual(payload.wrote, []);
|
||||
});
|
||||
@@ -21,7 +21,9 @@ const allowedExplicitBaseImages = Object.freeze([
|
||||
"hwlab-dev-base:*",
|
||||
"*/hwlab-dev-base:*",
|
||||
"hwlab-node20-base:*",
|
||||
"*/hwlab-node20-base:*"
|
||||
"*/hwlab-node20-base:*",
|
||||
"hwlab-node-runtime-base:*",
|
||||
"*/hwlab-node-runtime-base:*"
|
||||
]);
|
||||
|
||||
function baseImageRecommendation(envValue = recommendedNode20Image) {
|
||||
@@ -131,7 +133,7 @@ function isNode20BaseRef(imageRef) {
|
||||
|
||||
function isHwlabDevBaseRef(imageRef) {
|
||||
const lowerRef = imageRef.toLowerCase();
|
||||
return /(^|\/)(hwlab-dev-base|hwlab-node20-base)(?=[:@]|$)/u.test(lowerRef);
|
||||
return /(^|\/)(hwlab-dev-base|hwlab-node20-base|hwlab-node-runtime-base)(?=[:@]|$)/u.test(lowerRef);
|
||||
}
|
||||
|
||||
function isTaggedImage(image) {
|
||||
@@ -612,3 +614,17 @@ export function formatPreflightResult(result) {
|
||||
export function exitCodeForPreflight(result) {
|
||||
return result.publishUsable ? 0 : 2;
|
||||
}
|
||||
|
||||
export function summarizeForRuntimeBase(result) {
|
||||
return {
|
||||
status: result.status,
|
||||
publishUsable: result.publishUsable,
|
||||
imageSource: result.imageSource,
|
||||
requestedImage: result.requestedImage,
|
||||
localTag: result.localTag,
|
||||
imageId: result.imageId,
|
||||
blockers: result.blockers,
|
||||
nextSteps: result.nextSteps,
|
||||
containerEngine: result.containerEngine
|
||||
};
|
||||
}
|
||||
|
||||
@@ -513,19 +513,24 @@ async function resolveArtifactBoundary(ctx, control, promotion, deployBefore, ar
|
||||
const reportCommitMatches = commitMatches(reportSource, promotion.commitId);
|
||||
const deployCommitMatches = commitMatches(deployBefore.commitId, promotion.commitId);
|
||||
const isMergeCommit = parents.length >= 2;
|
||||
const status = (
|
||||
const desiredStateStatus = (
|
||||
deployCommitMatches &&
|
||||
catalogCommitMatches &&
|
||||
reportCommitMatches &&
|
||||
(
|
||||
promotionParentMatches ||
|
||||
artifactParentMatches ||
|
||||
commitMatches(mergeCommitId, promotion.commitId)
|
||||
)
|
||||
) ? "pass" : "degraded";
|
||||
const reportStatus = artifactEvidence.report.exists
|
||||
? (reportCommitMatches ? "matches_promotion" : "stale_or_unrelated")
|
||||
: "missing";
|
||||
|
||||
return {
|
||||
status,
|
||||
status: desiredStateStatus,
|
||||
reportStatus,
|
||||
reportIsReleaseGate: false,
|
||||
reportPolicy: "CI artifact reports are audit snapshots; deploy.json, artifact catalog, workloads, and registry manifests are the CD release gate.",
|
||||
controlCommit: {
|
||||
commitId: control.commitId,
|
||||
shortCommitId: control.shortCommitId
|
||||
@@ -557,6 +562,7 @@ async function resolveArtifactBoundary(ctx, control, promotion, deployBefore, ar
|
||||
deployCommitMatches,
|
||||
catalogCommitMatches,
|
||||
reportCommitMatches,
|
||||
reportExists: artifactEvidence.report.exists,
|
||||
desiredStatePaths: artifactDesiredStatePaths
|
||||
}
|
||||
};
|
||||
@@ -2335,7 +2341,7 @@ function buildReport({
|
||||
"docs/dev-deploy-apply.md",
|
||||
"docs/artifact-catalog.md"
|
||||
],
|
||||
summary: "DEV CD publish/apply/verify side effects are serialized by one Kubernetes Lease and use deploy/deploy.json as the apply desired-state source."
|
||||
summary: "DEV CD apply/verify side effects are serialized by one Kubernetes Lease and use deploy/deploy.json as the publish-control desired-state source; CI artifact reports are audit snapshots."
|
||||
},
|
||||
validationCommands: [
|
||||
"node --check scripts/dev-cd-apply.mjs",
|
||||
@@ -2367,8 +2373,8 @@ function buildReport({
|
||||
"DEV CD apply must reject docker-desktop, desktop-control-plane, 127.0.0.1:11700, bare kubectl ambiguity, and second hwlab-dev control-plane signals before mutation.",
|
||||
"DEV CD apply must verify required SecretRef existence and key names without reading or printing Secret values.",
|
||||
"HEAD must match the requested target ref before publish.",
|
||||
"Lease/hwlab-dev-cd-lock must be acquired before publish/apply.",
|
||||
"Legacy publish/apply side-effect scripts must run with HWLAB_CD_TRANSACTION_ID.",
|
||||
"Lease/hwlab-dev-cd-lock must be acquired before DEV apply/verify side effects.",
|
||||
"Artifact publish must run under CI artifact identity, with HWLAB_CD_TRANSACTION_ID accepted only for the legacy transition path.",
|
||||
"DEV DB role/database provisioning and runtime migration must run through repo-owned commands before workload apply.",
|
||||
"deploy/deploy.json, artifact catalog, and workloads must converge before apply.",
|
||||
"When deploy/deploy.json already points to a published promotion commit, HEAD may be a later control commit and the transaction must consume that promotion without republishing HEAD.",
|
||||
|
||||
@@ -587,6 +587,56 @@ test("dev-cd status accepts a published deploy-json promotion under a later cont
|
||||
assertNoReadOnlySideEffects(commandLog);
|
||||
});
|
||||
|
||||
test("dev-cd status treats stale CI artifact report as audit-only when desired state matches", async () => {
|
||||
const repoRoot = await makeRepo({ commitId: "abc1234" });
|
||||
await writeFile(
|
||||
artifactReportFixturePath,
|
||||
`${JSON.stringify({
|
||||
status: "published",
|
||||
commitId: "9999999",
|
||||
artifactPublish: {
|
||||
status: "published",
|
||||
sourceCommitId: "9999999",
|
||||
serviceCount: 0,
|
||||
requiredServiceCount: 0,
|
||||
publishedCount: 0,
|
||||
services: []
|
||||
}
|
||||
}, null, 2)}\n`
|
||||
);
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
commandLog,
|
||||
targetCommitId: laterControlCommit,
|
||||
headCommitId: laterControlCommit,
|
||||
extraGitRefs: {
|
||||
abc1234: publishedCommit
|
||||
},
|
||||
latestArtifactMergeCommit: artifactMergeCommit,
|
||||
commitParents: {
|
||||
[artifactMergeCommit]: [publishedCommit, artifactReportCommit],
|
||||
[artifactReportCommit]: [publishedCommit]
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const status = JSON.parse(output);
|
||||
assert.equal(code, 0);
|
||||
assert.equal(status.status, "pass");
|
||||
assert.equal(status.target.artifactBoundary.status, "pass");
|
||||
assert.equal(status.target.artifactBoundary.reportStatus, "stale_or_unrelated");
|
||||
assert.equal(status.target.artifactBoundary.reportIsReleaseGate, false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
|
||||
assertNoReadOnlySideEffects(commandLog);
|
||||
});
|
||||
|
||||
test("dev-cd status exposes moving origin/main when checkout is behind a deploy-json promotion control ref", async () => {
|
||||
const repoRoot = await makeRepo({ commitId: "abc1234" });
|
||||
const commandLog = [];
|
||||
@@ -603,8 +653,8 @@ test("dev-cd status exposes moving origin/main when checkout is behind a deploy-
|
||||
},
|
||||
latestArtifactMergeCommit: artifactMergeCommit,
|
||||
commitParents: {
|
||||
[artifactMergeCommit]: [publishedCommit, artifactReportCommit],
|
||||
[artifactReportCommit]: [publishedCommit]
|
||||
[artifactMergeCommit]: [laterControlCommit, artifactReportCommit],
|
||||
[artifactReportCommit]: [laterControlCommit]
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
@@ -666,7 +716,7 @@ test("apply stops before lock acquisition when checkout is behind deploy-json pr
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
|
||||
});
|
||||
|
||||
test("dev-cd status degrades artifact merge provenance when catalog/report do not match deploy-json promotion", async () => {
|
||||
test("dev-cd status degrades artifact merge provenance when catalog does not match deploy-json promotion", async () => {
|
||||
const repoRoot = await makeRepo({ commitId: "abc1234" });
|
||||
await writeFile(
|
||||
path.join(repoRoot, "deploy/artifact-catalog.dev.json"),
|
||||
@@ -751,8 +801,8 @@ test("apply stops before lock acquisition when artifact merge provenance mismatc
|
||||
},
|
||||
latestArtifactMergeCommit: artifactMergeCommit,
|
||||
commitParents: {
|
||||
[artifactMergeCommit]: [publishedCommit, artifactReportCommit],
|
||||
[artifactReportCommit]: [publishedCommit]
|
||||
[artifactMergeCommit]: [laterControlCommit, artifactReportCommit],
|
||||
[artifactReportCommit]: [laterControlCommit]
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
|
||||
@@ -5,11 +5,45 @@ const transactionEnvNames = [
|
||||
"HWLAB_CD_TRANSACTION_OWNER",
|
||||
"HWLAB_CD_LOCK_NAME"
|
||||
];
|
||||
const ciArtifactEnvNames = [
|
||||
"HWLAB_CI_ARTIFACT_RUN_ID",
|
||||
"HWLAB_CI_ARTIFACT_RUN_OWNER"
|
||||
];
|
||||
|
||||
export function hasDevCdTransactionEnv(env = process.env) {
|
||||
return typeof env.HWLAB_CD_TRANSACTION_ID === "string" && env.HWLAB_CD_TRANSACTION_ID.trim().length > 0;
|
||||
}
|
||||
|
||||
export function hasDevCiArtifactEnv(env = process.env) {
|
||||
return typeof env.HWLAB_CI_ARTIFACT_RUN_ID === "string" && env.HWLAB_CI_ARTIFACT_RUN_ID.trim().length > 0;
|
||||
}
|
||||
|
||||
export function hasDevArtifactSideEffectEnv(env = process.env) {
|
||||
return hasDevCdTransactionEnv(env) || hasDevCiArtifactEnv(env);
|
||||
}
|
||||
|
||||
export function devArtifactProducer(env = process.env) {
|
||||
if (hasDevCiArtifactEnv(env)) {
|
||||
return {
|
||||
kind: "ci-artifact",
|
||||
runId: env.HWLAB_CI_ARTIFACT_RUN_ID,
|
||||
owner: env.HWLAB_CI_ARTIFACT_RUN_OWNER ?? null
|
||||
};
|
||||
}
|
||||
if (hasDevCdTransactionEnv(env)) {
|
||||
return {
|
||||
kind: "cd-transaction-legacy",
|
||||
runId: env.HWLAB_CD_TRANSACTION_ID,
|
||||
owner: env.HWLAB_CD_TRANSACTION_OWNER ?? null
|
||||
};
|
||||
}
|
||||
return {
|
||||
kind: "missing",
|
||||
runId: null,
|
||||
owner: null
|
||||
};
|
||||
}
|
||||
|
||||
export function devCdTransactionGuardFailure({
|
||||
script,
|
||||
mode,
|
||||
@@ -32,7 +66,25 @@ export function devCdTransactionGuardFailure({
|
||||
};
|
||||
}
|
||||
|
||||
export function devArtifactSideEffectGuardFailure({
|
||||
script,
|
||||
mode,
|
||||
requiredEnv = "HWLAB_CI_ARTIFACT_RUN_ID or HWLAB_CD_TRANSACTION_ID"
|
||||
}) {
|
||||
return {
|
||||
...devCdTransactionGuardFailure({ script, mode, requiredEnv }),
|
||||
acceptedEnv: [...ciArtifactEnvNames, ...transactionEnvNames],
|
||||
entrypoint: `HWLAB_CI_ARTIFACT_RUN_ID=<run-id> node scripts/dev-artifact-publish.mjs --publish --report ${tempReportPath("dev-artifacts.json")}`,
|
||||
summary: `${script} ${mode} is a DEV artifact side-effect step and must run inside a CI artifact run or the legacy DEV CD transaction.`
|
||||
};
|
||||
}
|
||||
|
||||
export function requireDevCdTransactionForSideEffect({ env = process.env, script, mode }) {
|
||||
if (hasDevCdTransactionEnv(env)) return null;
|
||||
return devCdTransactionGuardFailure({ script, mode });
|
||||
}
|
||||
|
||||
export function requireDevArtifactSideEffectEnv({ env = process.env, script, mode }) {
|
||||
if (hasDevArtifactSideEffectEnv(env)) return null;
|
||||
return devArtifactSideEffectGuardFailure({ script, mode });
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
#!/usr/bin/env node
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import { access, readFile, writeFile } from "node:fs/promises";
|
||||
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import { request as httpRequest } from "node:http";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
@@ -88,6 +88,8 @@ const forbiddenActions = [
|
||||
"force-push"
|
||||
];
|
||||
const runtimeIdentityEnvNames = ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"];
|
||||
const digestPattern = /^sha256:[a-f0-9]{64}$/u;
|
||||
const defaultCdConcurrency = parseBoundedInteger(process.env.HWLAB_DEV_CD_CONCURRENCY, 4, 1, 8);
|
||||
|
||||
export function parseArgs(argv) {
|
||||
const flags = new Set();
|
||||
@@ -95,6 +97,7 @@ export function parseArgs(argv) {
|
||||
let kubeconfig = null;
|
||||
let kubeconfigSpecified = false;
|
||||
let reportPath = defaultReportPath;
|
||||
let rolloutConcurrency = defaultCdConcurrency;
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
@@ -141,6 +144,22 @@ export function parseArgs(argv) {
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (arg === "--rollout-concurrency") {
|
||||
flags.add("--rollout-concurrency");
|
||||
const next = argv[index + 1];
|
||||
if (typeof next !== "string" || next.startsWith("--")) {
|
||||
errors.push("--rollout-concurrency requires a numeric value");
|
||||
} else {
|
||||
rolloutConcurrency = parseBoundedInteger(next, defaultCdConcurrency, 1, 8);
|
||||
index += 1;
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (arg.startsWith("--rollout-concurrency=")) {
|
||||
flags.add("--rollout-concurrency");
|
||||
rolloutConcurrency = parseBoundedInteger(arg.slice("--rollout-concurrency=".length), defaultCdConcurrency, 1, 8);
|
||||
continue;
|
||||
}
|
||||
flags.add(arg);
|
||||
}
|
||||
|
||||
@@ -151,6 +170,7 @@ export function parseArgs(argv) {
|
||||
skipLiveProbe: flags.has("--skip-live-probe"),
|
||||
writeReport: flags.has("--report-output"),
|
||||
reportPath,
|
||||
rolloutConcurrency,
|
||||
kubeconfig,
|
||||
kubeconfigSpecified,
|
||||
errors,
|
||||
@@ -158,6 +178,12 @@ export function parseArgs(argv) {
|
||||
};
|
||||
}
|
||||
|
||||
function parseBoundedInteger(value, fallback, min, max) {
|
||||
const parsed = Number.parseInt(String(value ?? ""), 10);
|
||||
if (!Number.isInteger(parsed)) return fallback;
|
||||
return Math.min(Math.max(parsed, min), max);
|
||||
}
|
||||
|
||||
function addBlocker(blockers, type, scope, summary) {
|
||||
const key = `${type}::${scope}`;
|
||||
if (!blockers.some((blocker) => `${blocker.type}::${blocker.scope}` === key)) {
|
||||
@@ -169,6 +195,21 @@ function oneLine(value) {
|
||||
return String(value).replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
async function mapWithConcurrency(items, concurrency, worker) {
|
||||
if (items.length === 0) return [];
|
||||
const results = new Array(items.length);
|
||||
let nextIndex = 0;
|
||||
const workerCount = Math.min(Math.max(concurrency, 1), items.length);
|
||||
await Promise.all(Array.from({ length: workerCount }, async () => {
|
||||
while (nextIndex < items.length) {
|
||||
const index = nextIndex;
|
||||
nextIndex += 1;
|
||||
results[index] = await worker(items[index], index);
|
||||
}
|
||||
}));
|
||||
return results;
|
||||
}
|
||||
|
||||
async function readJson(relativePath, blockers) {
|
||||
try {
|
||||
return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8"));
|
||||
@@ -675,6 +716,146 @@ function parseImageTag(image) {
|
||||
return image.slice(colonIndex + 1);
|
||||
}
|
||||
|
||||
function parseTaggedImageReference(image) {
|
||||
if (typeof image !== "string" || image.includes("@")) return null;
|
||||
const firstSlash = image.indexOf("/");
|
||||
const lastSlash = image.lastIndexOf("/");
|
||||
const colonIndex = image.lastIndexOf(":");
|
||||
if (firstSlash <= 0 || colonIndex <= lastSlash) return null;
|
||||
return {
|
||||
image,
|
||||
registry: image.slice(0, firstSlash),
|
||||
repository: image.slice(firstSlash + 1, colonIndex),
|
||||
tag: image.slice(colonIndex + 1)
|
||||
};
|
||||
}
|
||||
|
||||
function registryManifestUrlForImage(image) {
|
||||
const parsed = parseTaggedImageReference(image);
|
||||
if (!parsed) return null;
|
||||
const repositoryPath = parsed.repository.split("/").map((part) => encodeURIComponent(part)).join("/");
|
||||
return {
|
||||
...parsed,
|
||||
url: `http://${parsed.registry}/v2/${repositoryPath}/manifests/${encodeURIComponent(parsed.tag)}`
|
||||
};
|
||||
}
|
||||
|
||||
function registryAcceptHeader() {
|
||||
return [
|
||||
"application/vnd.docker.distribution.manifest.v2+json",
|
||||
"application/vnd.oci.image.manifest.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.list.v2+json",
|
||||
"application/vnd.oci.image.index.v1+json",
|
||||
"application/vnd.docker.distribution.manifest.v1+json"
|
||||
].join(", ");
|
||||
}
|
||||
|
||||
function httpRequestText(url, { method = "GET", headers = {}, timeoutMs = 15000 } = {}) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = httpRequest(url, { method, headers, timeout: timeoutMs }, (response) => {
|
||||
response.setEncoding("utf8");
|
||||
let body = "";
|
||||
response.on("data", (chunk) => {
|
||||
body += chunk;
|
||||
if (body.length > 4096) {
|
||||
request.destroy(new Error("registry manifest response exceeded 4096 bytes during CD verification"));
|
||||
}
|
||||
});
|
||||
response.on("end", () => resolve({
|
||||
statusCode: response.statusCode ?? 0,
|
||||
headers: response.headers,
|
||||
body
|
||||
}));
|
||||
});
|
||||
request.on("timeout", () => {
|
||||
request.destroy(new Error(`timeout after ${timeoutMs}ms`));
|
||||
});
|
||||
request.on("error", reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function verifyCatalogRegistryManifest(service, blockers) {
|
||||
const image = service.image;
|
||||
const expectedDigest = service.digest;
|
||||
const parsed = registryManifestUrlForImage(image);
|
||||
const base = {
|
||||
serviceId: service.serviceId,
|
||||
image,
|
||||
imageTag: service.imageTag ?? parseImageTag(image),
|
||||
expectedDigest,
|
||||
url: parsed?.url ?? null
|
||||
};
|
||||
|
||||
if (!parsed) {
|
||||
const reason = `${service.serviceId} image is not a tagged registry image`;
|
||||
addBlocker(blockers, "contract_blocker", `registry-manifest-${service.serviceId}`, reason);
|
||||
return { ...base, status: "blocked", reason };
|
||||
}
|
||||
if (!digestPattern.test(expectedDigest ?? "")) {
|
||||
const reason = `${service.serviceId} catalog digest is not an immutable sha256 digest`;
|
||||
addBlocker(blockers, "contract_blocker", `registry-manifest-${service.serviceId}`, reason);
|
||||
return { ...base, status: "blocked", reason };
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await httpRequestText(parsed.url, {
|
||||
method: "HEAD",
|
||||
headers: {
|
||||
Accept: registryAcceptHeader()
|
||||
},
|
||||
timeoutMs: 15000
|
||||
});
|
||||
const observedDigest = String(response.headers["docker-content-digest"] ?? "").trim();
|
||||
const okStatus = response.statusCode >= 200 && response.statusCode < 300;
|
||||
const digestMatches = observedDigest === expectedDigest;
|
||||
if (!okStatus || !digestMatches) {
|
||||
const reason = !okStatus
|
||||
? `${service.serviceId} registry manifest returned HTTP ${response.statusCode}`
|
||||
: `${service.serviceId} registry digest mismatch: expected ${expectedDigest}, observed ${observedDigest || "missing"}`;
|
||||
addBlocker(blockers, "environment_blocker", `registry-manifest-${service.serviceId}`, reason);
|
||||
return {
|
||||
...base,
|
||||
status: "blocked",
|
||||
reason,
|
||||
httpStatus: response.statusCode,
|
||||
observedDigest,
|
||||
digestMatches
|
||||
};
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
status: "pass",
|
||||
httpStatus: response.statusCode,
|
||||
observedDigest,
|
||||
digestMatches: true
|
||||
};
|
||||
} catch (error) {
|
||||
const reason = `${service.serviceId} registry manifest read failed: ${oneLine(error.message)}`;
|
||||
addBlocker(blockers, "environment_blocker", `registry-manifest-${service.serviceId}`, reason);
|
||||
return {
|
||||
...base,
|
||||
status: "blocked",
|
||||
reason
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
async function verifyCatalogRegistryManifests(catalog, blockers, concurrency = defaultCdConcurrency) {
|
||||
const services = (catalog?.services ?? []).filter((service) => service.artifactRequired !== false);
|
||||
const results = await mapWithConcurrency(services, concurrency, async (service) =>
|
||||
verifyCatalogRegistryManifest(service, blockers)
|
||||
);
|
||||
return {
|
||||
status: results.every((result) => result.status === "pass") ? "pass" : "blocked",
|
||||
source: "registry-manifest",
|
||||
releaseGate: true,
|
||||
serviceCount: results.length,
|
||||
concurrency,
|
||||
results
|
||||
};
|
||||
}
|
||||
|
||||
function expectedRuntimeIdentityEnv(artifact) {
|
||||
return {
|
||||
HWLAB_COMMIT_ID: artifact.sourceCommitId,
|
||||
@@ -1675,9 +1856,9 @@ async function buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads,
|
||||
return cleanups;
|
||||
}
|
||||
|
||||
async function executeDevTemplateJobReplacements(kubectl, replacements, blockers) {
|
||||
async function executeDevTemplateJobReplacements(kubectl, replacements, blockers, concurrency = defaultCdConcurrency) {
|
||||
const planned = replacements.filter((replacement) => replacement.replace && replacement.result === "planned");
|
||||
for (const replacement of planned) {
|
||||
await mapWithConcurrency(planned, concurrency, async (replacement) => {
|
||||
const commandArgs = [
|
||||
"-n",
|
||||
replacement.namespace,
|
||||
@@ -1694,17 +1875,17 @@ async function executeDevTemplateJobReplacements(kubectl, replacements, blockers
|
||||
replacement.result = "delete_failed";
|
||||
replacement.reason = `Failed to delete live suspended template Job before apply: ${oneLine(commandOutput(result))}`;
|
||||
addBlocker(blockers, "environment_blocker", `template-job-replace-${replacement.jobName}`, replacement.reason);
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
replacement.result = "deleted_pending_recreate";
|
||||
replacement.reason = "Live suspended template Job was deleted; kubectl apply must recreate it from the desired manifest.";
|
||||
}
|
||||
});
|
||||
return planned.length;
|
||||
}
|
||||
|
||||
async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, blockers) {
|
||||
async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, blockers, concurrency = defaultCdConcurrency) {
|
||||
const planned = cleanups.filter((cleanup) => cleanup.cleanup && cleanup.result === "planned");
|
||||
for (const cleanup of planned) {
|
||||
await mapWithConcurrency(planned, concurrency, async (cleanup) => {
|
||||
const commandArgs = [
|
||||
"-n",
|
||||
cleanup.namespace,
|
||||
@@ -1721,14 +1902,107 @@ async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, bl
|
||||
cleanup.result = "delete_failed";
|
||||
cleanup.reason = `Failed to delete stale legacy simulator Deployment before apply: ${oneLine(commandOutput(result))}`;
|
||||
addBlocker(blockers, "environment_blocker", `legacy-simulator-deployment-cleanup-${cleanup.deploymentName}`, cleanup.reason);
|
||||
continue;
|
||||
return;
|
||||
}
|
||||
cleanup.result = "deleted_pending_apply";
|
||||
cleanup.reason = "Live stale legacy simulator Deployment was deleted; kubectl apply must leave the desired indexed StatefulSet in place.";
|
||||
}
|
||||
});
|
||||
return planned.length;
|
||||
}
|
||||
|
||||
function rolloutApiName(kind) {
|
||||
if (kind === "Deployment") return "deployment";
|
||||
if (kind === "StatefulSet") return "statefulset";
|
||||
return null;
|
||||
}
|
||||
|
||||
function rolloutTargetsFromWorkloads(workloads) {
|
||||
return listItems(workloads)
|
||||
.map((item) => {
|
||||
const apiName = rolloutApiName(item?.kind);
|
||||
if (!apiName) return null;
|
||||
return {
|
||||
kind: item.kind,
|
||||
apiName,
|
||||
name: item?.metadata?.name ?? "unknown",
|
||||
namespace: item?.metadata?.namespace ?? namespace,
|
||||
replicas: replicaPlan(item),
|
||||
serviceIds: uniqueStrings(containersFor(item).map((container) => serviceIdFor(item, container)))
|
||||
};
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
async function verifyRolloutTarget(kubectl, target, blockers) {
|
||||
const commandArgs = [
|
||||
"-n",
|
||||
target.namespace,
|
||||
"rollout",
|
||||
"status",
|
||||
`${target.apiName}/${target.name}`,
|
||||
"--timeout=180s"
|
||||
];
|
||||
const base = {
|
||||
...target,
|
||||
command: kubectlCommand(kubectl, commandArgs)
|
||||
};
|
||||
if (kubectl.status !== "ready") {
|
||||
addBlocker(blockers, "environment_blocker", `rollout-status-${target.name}`, kubectl.reason);
|
||||
return {
|
||||
...base,
|
||||
status: "blocked",
|
||||
reason: kubectl.reason
|
||||
};
|
||||
}
|
||||
const result = await kubectlResult(kubectl, commandArgs, 190000);
|
||||
if (!result.ok) {
|
||||
const reason = oneLine(commandOutput(result));
|
||||
addBlocker(blockers, "runtime_blocker", `rollout-status-${target.name}`, reason);
|
||||
return {
|
||||
...base,
|
||||
status: "blocked",
|
||||
reason,
|
||||
stdout: result.redactedStdout ?? result.stdout,
|
||||
stderr: result.redactedStderr ?? result.stderr
|
||||
};
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
status: "pass",
|
||||
stdout: result.redactedStdout ?? result.stdout,
|
||||
stderr: result.redactedStderr ?? result.stderr
|
||||
};
|
||||
}
|
||||
|
||||
async function verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep) {
|
||||
const targets = rolloutTargetsFromWorkloads(workloads);
|
||||
if (!args.apply) {
|
||||
return {
|
||||
status: "not_run",
|
||||
reason: "dry-run mode",
|
||||
concurrency: args.rolloutConcurrency,
|
||||
targets
|
||||
};
|
||||
}
|
||||
if (applyStep.status !== "pass") {
|
||||
return {
|
||||
status: "not_run",
|
||||
reason: "apply did not complete",
|
||||
concurrency: args.rolloutConcurrency,
|
||||
targets
|
||||
};
|
||||
}
|
||||
const results = await mapWithConcurrency(targets, args.rolloutConcurrency, async (target) =>
|
||||
verifyRolloutTarget(kubectl, target, blockers)
|
||||
);
|
||||
return {
|
||||
status: results.every((result) => result.status === "pass") ? "pass" : "blocked",
|
||||
concurrency: args.rolloutConcurrency,
|
||||
targetCount: targets.length,
|
||||
results
|
||||
};
|
||||
}
|
||||
|
||||
function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) {
|
||||
const plannedNames = replacementsNeeded.map((replacement) => replacement.jobName);
|
||||
if (result.ok || plannedNames.length === 0) return false;
|
||||
@@ -1758,8 +2032,8 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements, le
|
||||
const replacementsNeeded = templateJobReplacements.filter((replacement) => replacement.replace);
|
||||
|
||||
if (args.apply) {
|
||||
const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers);
|
||||
const cleanupCount = await executeDevLegacySimulatorDeploymentCleanups(kubectl, legacySimulatorDeploymentCleanups, blockers);
|
||||
const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers, args.rolloutConcurrency);
|
||||
const cleanupCount = await executeDevLegacySimulatorDeploymentCleanups(kubectl, legacySimulatorDeploymentCleanups, blockers, args.rolloutConcurrency);
|
||||
mutationAttempted = replaceCount > 0 || cleanupCount > 0;
|
||||
if (blockers.length > 0) {
|
||||
return {
|
||||
@@ -1858,6 +2132,7 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
const commitId = resolveApplySourceCommit(deploy, catalog, gitHeadCommitId);
|
||||
|
||||
const artifactEvidence = validateDeployAndCatalog(deploy, catalog, commitId, blockers);
|
||||
const registryManifests = await verifyCatalogRegistryManifests(catalog, blockers, args.rolloutConcurrency);
|
||||
const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers);
|
||||
const cloudApiDb = await checkCloudApiDb(deploy, workloads, services, blockers);
|
||||
const codeAgentProviderDesiredState = inspectCodeAgentProviderDesiredState(deploy, workloads);
|
||||
@@ -1886,33 +2161,41 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
templateJobReplacements,
|
||||
legacySimulatorDeploymentCleanups
|
||||
);
|
||||
const cloudWebRolloutAfterApply = args.apply && applyStep.status === "pass"
|
||||
? await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
|
||||
let rolloutStatusAfterApply;
|
||||
let cloudWebRolloutAfterApply;
|
||||
let codeAgentProviderLiveAfterApply;
|
||||
if (args.apply && applyStep.status === "pass") {
|
||||
[rolloutStatusAfterApply, cloudWebRolloutAfterApply, codeAgentProviderLiveAfterApply] = await Promise.all([
|
||||
verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep),
|
||||
observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
|
||||
observationPhase: "after_apply",
|
||||
blockRuntimeEnvDrift: true
|
||||
})
|
||||
: skippedDeploymentRolloutObservation(
|
||||
kubectl,
|
||||
deploy,
|
||||
catalog,
|
||||
"hwlab-cloud-web",
|
||||
"after_apply",
|
||||
args.apply ? "apply did not complete successfully; post-apply rollout env verification was not run" : "dry-run mode; no post-apply rollout env verification"
|
||||
);
|
||||
const cloudWebRollout = args.apply ? cloudWebRolloutAfterApply : cloudWebRolloutBeforeApply;
|
||||
const codeAgentProviderLiveAfterApply = args.apply && applyStep.status === "pass"
|
||||
? await observeCodeAgentProviderLiveDeployment(kubectl, blockers, {
|
||||
}),
|
||||
observeCodeAgentProviderLiveDeployment(kubectl, blockers, {
|
||||
phase: "after_apply",
|
||||
blockOnMismatch: true
|
||||
})
|
||||
: {
|
||||
phase: "after_apply",
|
||||
status: "not_run",
|
||||
reason: args.apply ? "apply did not complete" : "dry-run mode",
|
||||
secretValuesRead: false,
|
||||
kubernetesSecretDataRead: false,
|
||||
valuesRedacted: true
|
||||
};
|
||||
]);
|
||||
} else {
|
||||
rolloutStatusAfterApply = await verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep);
|
||||
cloudWebRolloutAfterApply = skippedDeploymentRolloutObservation(
|
||||
kubectl,
|
||||
deploy,
|
||||
catalog,
|
||||
"hwlab-cloud-web",
|
||||
"after_apply",
|
||||
args.apply ? "apply did not complete successfully; post-apply rollout env verification was not run" : "dry-run mode; no post-apply rollout env verification"
|
||||
);
|
||||
codeAgentProviderLiveAfterApply = {
|
||||
phase: "after_apply",
|
||||
status: "not_run",
|
||||
reason: args.apply ? "apply did not complete" : "dry-run mode",
|
||||
secretValuesRead: false,
|
||||
kubernetesSecretDataRead: false,
|
||||
valuesRedacted: true
|
||||
};
|
||||
}
|
||||
const cloudWebRollout = args.apply ? cloudWebRolloutAfterApply : cloudWebRolloutBeforeApply;
|
||||
const status = blockers.length > 0 ? "blocked" : "pass";
|
||||
const artifactPlan = buildArtifactPlan(deploy, catalog, commitId, artifactEvidence);
|
||||
const workloadPlan = buildWorkloadPlan(workloads);
|
||||
@@ -1964,6 +2247,7 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
evidence: [
|
||||
`artifact services checked: ${artifactEvidence.length}`,
|
||||
`expected artifact commit: ${artifactPlan.expectedArtifactCommit}`,
|
||||
`registry manifests: ${registryManifests.status} (${registryManifests.serviceCount} services)`,
|
||||
`namespace: ${namespace}`,
|
||||
`workloads planned: ${workloadPlan.length}`,
|
||||
`kubectl executor: ${kubectl.executor ?? "missing"}`,
|
||||
@@ -1972,6 +2256,7 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
`live health: ${liveProbe.status}`,
|
||||
`code agent provider desired-state: ${codeAgentProviderDesiredState.status}`,
|
||||
`code agent provider live env before apply: ${codeAgentProviderLiveBeforeApply.status}`,
|
||||
`rollout status concurrency: ${args.rolloutConcurrency}`,
|
||||
`template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}`,
|
||||
`legacy simulator deployment cleanups: ${legacySimulatorDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}`
|
||||
],
|
||||
@@ -1980,7 +2265,7 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
devPreconditions: {
|
||||
status,
|
||||
requirements: [
|
||||
"DEV artifact catalog must contain CI publish evidence and registry digests",
|
||||
"DEV artifact catalog must contain CI publish expectations and registry digests; CD must verify those tag/digest pairs directly against registry manifests",
|
||||
"kubectl must be available for D601 hwlab-dev and must not target PROD",
|
||||
"hwlab-cloud-api /health/live must report serviceId, dev environment, and DB env readiness without exposing secret values",
|
||||
"hwlab-cloud-api must declare and preserve OPENAI_API_KEY from hwlab-code-agent-provider/openai-api-key plus the DEV Code Agent egress proxy env without reading Secret values",
|
||||
@@ -2009,6 +2294,7 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
cloudWebRollout,
|
||||
cloudWebRolloutBeforeApply,
|
||||
cloudWebRolloutAfterApply,
|
||||
rolloutStatusAfterApply,
|
||||
codeAgentProvider: {
|
||||
desiredState: codeAgentProviderDesiredState,
|
||||
liveBeforeApply: codeAgentProviderLiveBeforeApply,
|
||||
@@ -2024,6 +2310,7 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
rollbackHint: buildRollbackHint(kubectl, workloads),
|
||||
remainingBlockers,
|
||||
artifactEvidence,
|
||||
registryManifests,
|
||||
cloudApiDb,
|
||||
k8sManifest,
|
||||
clusterObservation,
|
||||
|
||||
@@ -15,8 +15,10 @@ import {
|
||||
resolveDevKubeconfigSelection
|
||||
} from "./dev-deploy-apply.mjs";
|
||||
import {
|
||||
devArtifactProducer,
|
||||
devCdTransactionGuardFailure,
|
||||
hasDevCdTransactionEnv,
|
||||
requireDevArtifactSideEffectEnv,
|
||||
requireDevCdTransactionForSideEffect
|
||||
} from "./dev-cd-transaction-guard.mjs";
|
||||
|
||||
@@ -141,6 +143,32 @@ test("DEV CD transaction guard rejects direct legacy side-effect calls", () => {
|
||||
);
|
||||
});
|
||||
|
||||
test("DEV artifact publish guard accepts CI artifact identity without CD lock", () => {
|
||||
const ciEnv = {
|
||||
HWLAB_CI_ARTIFACT_RUN_ID: "ci-artifact-123",
|
||||
HWLAB_CI_ARTIFACT_RUN_OWNER: "tekton"
|
||||
};
|
||||
assert.equal(requireDevArtifactSideEffectEnv({
|
||||
env: ciEnv,
|
||||
script: "scripts/dev-artifact-publish.mjs",
|
||||
mode: "--publish"
|
||||
}), null);
|
||||
assert.deepEqual(devArtifactProducer(ciEnv), {
|
||||
kind: "ci-artifact",
|
||||
runId: "ci-artifact-123",
|
||||
owner: "tekton"
|
||||
});
|
||||
|
||||
const failure = requireDevArtifactSideEffectEnv({
|
||||
env: {},
|
||||
script: "scripts/dev-artifact-publish.mjs",
|
||||
mode: "--publish"
|
||||
});
|
||||
assert.equal(failure.error, "cd-transaction-required");
|
||||
assert.match(failure.requiredEnv, /HWLAB_CI_ARTIFACT_RUN_ID/u);
|
||||
assert.match(failure.entrypoint, /HWLAB_CI_ARTIFACT_RUN_ID/u);
|
||||
});
|
||||
|
||||
test("matching allowlisted suspended template Job image does not replace", () => {
|
||||
const image = "127.0.0.1:5000/hwlab/hwlab-cli:73b379f";
|
||||
const decision = decideDevTemplateJobReplacement({
|
||||
|
||||
Reference in New Issue
Block a user