diff --git a/deploy/README.md b/deploy/README.md index 62418206..ce8d8c63 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -1,16 +1,16 @@ # Deploy Contract -`deploy.schema.json` defines the future `deploy/deploy.json` manifest shape. +`deploy.schema.json` defines the future `deploy/deploy.yaml` manifest shape. The MVP acceptance environment is DEV only. Public endpoint source fields live in -`deploy/deploy.json`: frontend `http://74.48.78.17:16666` and API/edge +`deploy/deploy.yaml`: frontend `http://74.48.78.17:16666` and API/edge `http://74.48.78.17:16667`. PROD profile fields may be represented in schema for future compatibility, but PROD deployment is not part of MVP acceptance. ## L5 DEV Skeleton -- `deploy/deploy.json` is the DEV-only deploy manifest. -- `deploy/deploy.json` is the single source for `health.path`, +- `deploy/deploy.yaml` is the DEV-only deploy manifest. +- `deploy/deploy.yaml` is the single source for `health.path`, `publicEndpoints`, `frp.proxies`, and `k3s.serviceMappings`. - `deploy/k8s/base` contains parseable k3s resource skeletons for the frozen HWLAB service IDs rendered from the manifest contract. @@ -67,9 +67,9 @@ node scripts/deploy-desired-state-plan.mjs --target-tag --pretty node scripts/deploy-desired-state-plan.mjs --promotion-commit --check ``` -The desired-state plan reads `deploy/deploy.json`, +The desired-state plan reads `deploy/deploy.yaml`, `deploy/artifact-catalog.dev.json`, `deploy/k8s/base/workloads.yaml`, and the -optional artifact report snapshot. `deploy/deploy.json` is the human-authored +optional artifact report snapshot. `deploy/deploy.yaml` is the human-authored runtime config truth source. Published image identity lives in `G14-gitops:deploy/artifact-catalog.dev.json`; the source branch catalog is only a seed contract for local checks. GitOps render overlays catalog image fields and @@ -101,7 +101,7 @@ Next DEV deploy smoke commands, for a separately authorized deployment task: ```sh npm run check -node -e "JSON.parse(require('node:fs').readFileSync('deploy/deploy.json','utf8'))" +node -e "JSON.parse(require('node:fs').readFileSync('deploy/deploy.yaml','utf8'))" KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl apply --dry-run=server -k deploy/k8s/dev curl -fsS http://74.48.78.17:16667/health/live node scripts/dev-edge-health-smoke.mjs --live --report /tmp/hwlab-dev-gate/report.json diff --git a/deploy/artifact-catalog.dev.json b/deploy/artifact-catalog.dev.json index 9c28e6c7..f43c30e4 100644 --- a/deploy/artifact-catalog.dev.json +++ b/deploy/artifact-catalog.dev.json @@ -244,8 +244,8 @@ ], "serviceInventory": { "version": "v2", - "serviceCount": 8, - "requiredServiceCount": 8, + "serviceCount": 5, + "requiredServiceCount": 5, "disabledServiceCount": 0, "requiredServiceIds": [ "hwlab-cloud-api", diff --git a/deploy/deploy.json b/deploy/deploy.json index f8cb8749..d0b3cb8c 100644 --- a/deploy/deploy.json +++ b/deploy/deploy.json @@ -136,6 +136,97 @@ "hwlab-edge-proxy": "deploy/runtime/boot/hwlab-edge-proxy.sh", "hwlab-agent-skills": "deploy/runtime/boot/hwlab-agent-skills.sh" }, + "serviceDeclarations": { + "hwlab-cloud-api": { + "runtimeKind": "bun-command", + "entrypoint": "cmd/hwlab-cloud-api/main.ts", + "artifactKind": "bun-command", + "healthPath": "/health/live", + "healthPort": 6667, + "componentPaths": [ + "cmd/hwlab-cloud-api/", + "cmd/hwlab-codex-api-responses-forwarder/", + "cmd/hwlab-deepseek-responses-bridge/", + "deploy/runtime/boot/hwlab-codex-api-responses-forwarder.sh", + "deploy/runtime/boot/hwlab-deepseek-responses-bridge.sh", + "internal/cloud/", + "internal/db/", + "internal/audit/", + "internal/agent/agentrun-dispatch.mjs", + "internal/agent/prompts/", + "skills/hwlab-agent-runtime/" + ], + "env": {}, + "observable": true + }, + "hwlab-cloud-web": { + "runtimeKind": "cloud-web", + "entrypoint": "web/hwlab-cloud-web/index.html", + "artifactKind": "cloud-web", + "healthPath": "/health/live", + "healthPort": 8080, + "componentPaths": [ + "web/hwlab-cloud-web/", + "internal/dev-entrypoint/cloud-web-runtime.mjs", + "internal/dev-entrypoint/cloud-web-proxy.mjs", + "internal/dev-entrypoint/cloud-web-routes.mjs", + "tools/src/hwlab-cli/trace-renderer.ts" + ], + "env": {}, + "observable": true + }, + "hwlab-gateway": { + "runtimeKind": "bun-command", + "entrypoint": "cmd/hwlab-gateway/main.ts", + "artifactKind": "bun-command", + "healthPath": "/health/live", + "healthPort": 7001, + "componentPaths": ["cmd/hwlab-gateway/", "internal/sim/"], + "env": {}, + "observable": false + }, + "hwlab-edge-proxy": { + "runtimeKind": "bun-command", + "entrypoint": "cmd/hwlab-edge-proxy/main.ts", + "artifactKind": "bun-command", + "healthPath": "/health", + "healthPort": 6667, + "componentPaths": ["cmd/hwlab-edge-proxy/", "internal/dev-entrypoint/http.mjs"], + "env": {}, + "observable": true + }, + "hwlab-agent-skills": { + "runtimeKind": "skills-bundle", + "entrypoint": "skills/hwlab-agent-runtime/SKILL.md", + "artifactKind": "skills-bundle", + "healthPath": "/health/live", + "healthPort": 7430, + "componentPaths": ["skills/"], + "env": {}, + "observable": true + } + }, + "envRecipe": { + "osPackages": ["ca-certificates", "git"], + "bunVersion": "1.3.13", + "launcherPath": "deploy/runtime/launcher/hwlab-env-reuse-launcher.ts", + "runtimeNodeModulesPath": "/opt/hwlab-env/node_modules", + "additionalEnvPaths": [ + "internal/dev-entrypoint/artifact-runtime.mjs", + "scripts/artifact-publish.mjs", + "scripts/g14-artifact-publish.mjs", + "scripts/src/dev-artifact-services.mjs" + ], + "hwpodAliases": [ + { "command": "hwpod", "script": "tools/hwpod-cli.ts" }, + { "command": "hwpod-ctl", "script": "tools/hwpod-ctl.ts" }, + { "command": "hwpod-compiler", "script": "tools/hwpod-compiler-cli.ts" } + ] + }, + "bootConfig": { + "template": "default", + "overrides": {} + }, "services": [ { "serviceId": "hwlab-cloud-api", diff --git a/deploy/deploy.schema.json b/deploy/deploy.schema.json index e7591b8e..9021dada 100644 --- a/deploy/deploy.schema.json +++ b/deploy/deploy.schema.json @@ -48,6 +48,9 @@ }, "additionalProperties": false }, + "lanes": { + "$ref": "#/$defs/lanes" + }, "services": { "type": "array", "minItems": 1, @@ -303,6 +306,138 @@ }, "additionalProperties": false }, + "lanes": { + "type": "object", + "properties": { + "v02": { + "$ref": "#/$defs/v02Lane" + } + }, + "additionalProperties": false + }, + "v02Lane": { + "type": "object", + "required": ["name", "sourceBranch", "gitopsBranch", "namespace", "envReuseServices", "bootScripts", "serviceDeclarations", "envRecipe", "bootConfig"], + "properties": { + "name": { "type": "string", "const": "v0.2" }, + "sourceBranch": { "type": "string", "const": "v0.2" }, + "gitopsBranch": { "type": "string", "const": "v0.2-gitops" }, + "namespace": { "type": "string", "const": "hwlab-v02" }, + "endpoint": { "type": "string" }, + "publicEndpoints": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "artifactCatalog": { "type": "string" }, + "runtimePath": { "type": "string" }, + "imageTagMode": { "type": "string", "enum": ["short", "full"] }, + "sourceRepo": { "type": "string" }, + "envReuseServices": { + "type": "array", + "items": { "$ref": "#/$defs/serviceId" }, + "uniqueItems": true + }, + "bootScripts": { + "type": "object", + "minProperties": 1, + "additionalProperties": { "$ref": "#/$defs/repoPath" } + }, + "serviceDeclarations": { + "type": "object", + "minProperties": 1, + "additionalProperties": { "$ref": "#/$defs/serviceDeclaration" } + }, + "envRecipe": { "$ref": "#/$defs/envRecipe" }, + "bootConfig": { "$ref": "#/$defs/bootConfig" }, + "services": { + "type": "array", + "items": { "$ref": "#/$defs/laneService" } + } + }, + "additionalProperties": false + }, + "serviceDeclaration": { + "type": "object", + "required": ["runtimeKind", "entrypoint", "componentPaths"], + "properties": { + "runtimeKind": { "type": "string", "enum": ["bun-command", "cloud-web", "skills-bundle", "node-command"] }, + "entrypoint": { "$ref": "#/$defs/repoPath" }, + "artifactKind": { "type": "string", "enum": ["bun-command", "cloud-web", "skills-bundle", "node-command"] }, + "healthPath": { "type": "string", "pattern": "^/" }, + "healthPort": { "type": "integer", "minimum": 1, "maximum": 65535 }, + "componentPaths": { + "type": "array", + "minItems": 1, + "items": { "$ref": "#/$defs/repoPath" } + }, + "env": { + "type": "object", + "additionalProperties": { "type": "string" } + }, + "observable": { "type": "boolean" } + }, + "additionalProperties": false + }, + "envRecipe": { + "type": "object", + "required": ["osPackages", "bunVersion", "launcherPath", "runtimeNodeModulesPath", "additionalEnvPaths", "hwpodAliases"], + "properties": { + "osPackages": { + "type": "array", + "minItems": 1, + "items": { "type": "string", "minLength": 1 }, + "uniqueItems": true + }, + "bunVersion": { "type": "string", "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+" }, + "launcherPath": { "$ref": "#/$defs/repoPath" }, + "runtimeNodeModulesPath": { "type": "string", "pattern": "^/" }, + "additionalEnvPaths": { + "type": "array", + "items": { "$ref": "#/$defs/repoPath" }, + "uniqueItems": true + }, + "hwpodAliases": { + "type": "array", + "items": { "$ref": "#/$defs/hwpodAlias" } + } + }, + "additionalProperties": false + }, + "hwpodAlias": { + "type": "object", + "required": ["command", "script"], + "properties": { + "command": { "type": "string", "minLength": 1 }, + "script": { "$ref": "#/$defs/repoPath" } + }, + "additionalProperties": false + }, + "bootConfig": { + "type": "object", + "required": ["template"], + "properties": { + "template": { "type": "string", "enum": ["default"] }, + "overrides": { "type": "object" } + }, + "additionalProperties": false + }, + "laneService": { + "type": "object", + "required": ["serviceId"], + "properties": { + "serviceId": { "$ref": "#/$defs/serviceId" }, + "env": { + "type": "object", + "additionalProperties": { "type": "string" } + } + }, + "additionalProperties": false + }, + "repoPath": { + "type": "string", + "minLength": 1, + "not": { "pattern": "(^/|(^|/)\\.\\.(/|$))" } + }, "service": { "type": "object", "required": ["serviceId", "namespace", "healthPath", "profile"], diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml new file mode 100644 index 00000000..cb9ffaf5 --- /dev/null +++ b/deploy/deploy.yaml @@ -0,0 +1,292 @@ +manifestVersion: v1 +environment: dev +namespace: hwlab-dev +endpoint: http://74.48.78.17:16667 +health: + method: GET + path: /health/live + responseFormat: json +publicEndpoints: + frontend: + url: http://74.48.78.17:16666 + protocol: http + host: 74.48.78.17 + port: 16666 + serviceId: hwlab-cloud-web + frpProxy: hwlab-dev-cloud-web + api: + url: http://74.48.78.17:16667 + protocol: http + host: 74.48.78.17 + port: 16667 + serviceId: hwlab-edge-proxy + frpProxy: hwlab-dev-edge-proxy +frp: + server: + address: 74.48.78.17 + bindPort: 7000 + proxies: + - name: hwlab-dev-cloud-web + endpointId: frontend + type: tcp + localServiceId: hwlab-cloud-web + localHost: hwlab-cloud-web.hwlab-dev.svc.cluster.local + localPort: 8080 + remotePort: 16666 + - name: hwlab-dev-edge-proxy + endpointId: api + type: tcp + localServiceId: hwlab-edge-proxy + localHost: hwlab-edge-proxy.hwlab-dev.svc.cluster.local + localPort: 6667 + remotePort: 16667 +k3s: + serviceMappings: + - serviceId: hwlab-cloud-api + name: hwlab-cloud-api + namespace: hwlab-dev + port: 6667 + targetPort: http + - serviceId: hwlab-cloud-web + name: hwlab-cloud-web + namespace: hwlab-dev + port: 8080 + targetPort: http + - serviceId: hwlab-gateway + name: hwlab-gateway + namespace: hwlab-dev + port: 7001 + targetPort: http + - serviceId: hwlab-agent-skills + name: hwlab-agent-skills + namespace: hwlab-dev + port: 7430 + targetPort: http + - serviceId: hwlab-edge-proxy + name: hwlab-edge-proxy + namespace: hwlab-dev + port: 6667 + targetPort: http +profiles: + dev: + name: dev + enabled: true + namespace: hwlab-dev + endpoint: http://74.48.78.17:16667 + notes: DEV-only k3s deployment skeleton. D601 reaches the master edge through frp reverse tunnel. + prod: + name: prod + enabled: false + namespace: hwlab-prod + notes: Reserved placeholder only. PROD deployment and acceptance are explicitly out of scope. +lanes: + v02: + name: v0.2 + sourceBranch: v0.2 + gitopsBranch: v0.2-gitops + namespace: hwlab-v02 + endpoint: https://hwlab.74-48-78-17.nip.io + publicEndpoints: + frontend: https://hwlab.74-48-78-17.nip.io + api: https://hwlab.74-48-78-17.nip.io + legacyFrontend: http://74.48.78.17:19666 + legacyApi: http://74.48.78.17:19667 + artifactCatalog: deploy/artifact-catalog.v02.json + runtimePath: deploy/gitops/g14/runtime-v02 + imageTagMode: full + envReuseServices: + - hwlab-cloud-api + - hwlab-cloud-web + - hwlab-gateway + - hwlab-edge-proxy + - hwlab-agent-skills + bootScripts: + hwlab-cloud-api: deploy/runtime/boot/hwlab-cloud-api.sh + hwlab-cloud-web: deploy/runtime/boot/hwlab-cloud-web.sh + hwlab-gateway: deploy/runtime/boot/hwlab-gateway.sh + hwlab-edge-proxy: deploy/runtime/boot/hwlab-edge-proxy.sh + hwlab-agent-skills: deploy/runtime/boot/hwlab-agent-skills.sh + serviceDeclarations: + hwlab-cloud-api: + runtimeKind: bun-command + entrypoint: cmd/hwlab-cloud-api/main.ts + artifactKind: bun-command + healthPath: /health/live + healthPort: 6667 + componentPaths: + - cmd/hwlab-cloud-api/ + - cmd/hwlab-codex-api-responses-forwarder/ + - cmd/hwlab-deepseek-responses-bridge/ + - deploy/runtime/boot/hwlab-codex-api-responses-forwarder.sh + - deploy/runtime/boot/hwlab-deepseek-responses-bridge.sh + - internal/cloud/ + - internal/db/ + - internal/audit/ + - internal/agent/agentrun-dispatch.mjs + - internal/agent/prompts/ + - skills/hwlab-agent-runtime/ + env: {} + observable: true + hwlab-cloud-web: + runtimeKind: cloud-web + entrypoint: web/hwlab-cloud-web/index.html + artifactKind: cloud-web + healthPath: /health/live + healthPort: 8080 + componentPaths: + - web/hwlab-cloud-web/ + - internal/dev-entrypoint/cloud-web-runtime.mjs + - internal/dev-entrypoint/cloud-web-proxy.mjs + - internal/dev-entrypoint/cloud-web-routes.mjs + - tools/src/hwlab-cli/trace-renderer.ts + env: {} + observable: true + hwlab-gateway: + runtimeKind: bun-command + entrypoint: cmd/hwlab-gateway/main.ts + artifactKind: bun-command + healthPath: /health/live + healthPort: 7001 + componentPaths: + - cmd/hwlab-gateway/ + - internal/sim/ + env: {} + observable: false + hwlab-edge-proxy: + runtimeKind: bun-command + entrypoint: cmd/hwlab-edge-proxy/main.ts + artifactKind: bun-command + healthPath: /health + healthPort: 6667 + componentPaths: + - cmd/hwlab-edge-proxy/ + - internal/dev-entrypoint/http.mjs + env: {} + observable: true + hwlab-agent-skills: + runtimeKind: skills-bundle + entrypoint: skills/hwlab-agent-runtime/SKILL.md + artifactKind: skills-bundle + healthPath: /health/live + healthPort: 7430 + componentPaths: + - skills/ + env: {} + observable: true + envRecipe: + osPackages: + - ca-certificates + - git + bunVersion: 1.3.13 + launcherPath: deploy/runtime/launcher/hwlab-env-reuse-launcher.ts + runtimeNodeModulesPath: /opt/hwlab-env/node_modules + additionalEnvPaths: + - internal/dev-entrypoint/artifact-runtime.mjs + - scripts/artifact-publish.mjs + - scripts/g14-artifact-publish.mjs + - scripts/src/dev-artifact-services.mjs + hwpodAliases: + - command: hwpod + script: tools/hwpod-cli.ts + - command: hwpod-ctl + script: tools/hwpod-ctl.ts + - command: hwpod-compiler + script: tools/hwpod-compiler-cli.ts + bootConfig: + template: default + overrides: {} + services: + - serviceId: hwlab-cloud-api + env: + HWLAB_CLOUD_DB_URL: secretRef:hwlab-cloud-api-v02-db/database-url + HWLAB_CLOUD_DB_CONTRACT: v02-redacted-presence-only + HWLAB_ACCESS_CONTROL_REQUIRED: "1" + HWLAB_BOOTSTRAP_ADMIN_ID: usr_v02_admin + HWLAB_BOOTSTRAP_ADMIN_USERNAME: admin + HWLAB_BOOTSTRAP_ADMIN_DISPLAY_NAME: HWLAB v0.2 Admin + HWLAB_BOOTSTRAP_ADMIN_PASSWORD_HASH: secretRef:hwlab-v02-bootstrap-admin/password-hash + HWLAB_BOOTSTRAP_ADMIN_API_KEY_ID: key_master_server_admin + HWLAB_BOOTSTRAP_ADMIN_API_KEY: secretRef:hwlab-v02-master-server-admin-api-key/api-key + HWLAB_KEYCLOAK_ISSUER: https://auth.74-48-78-17.nip.io/realms/hwlab + HWLAB_KEYCLOAK_CLIENT_ID: hwlab-cloud-web + HWLAB_KEYCLOAK_CLIENT_SECRET: secretRef:hwlab-cloud-web-client/client-secret + HWLAB_CODE_AGENT_ADAPTER: agentrun-v01 + AGENTRUN_MGR_URL: http://agentrun-mgr.agentrun-v01.svc.cluster.local:8080 + HWLAB_CODE_AGENT_AGENTRUN_PROVIDER_ID: G14 + HWLAB_CODE_AGENT_AGENTRUN_RUNNER_NAMESPACE: agentrun-v01 + HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE: agentrun-v01 + HWLAB_CODE_AGENT_AGENTRUN_REPO_URL: http://git-mirror-http.devops-infra.svc.cluster.local/pikasTech/HWLAB.git +services: + - serviceId: hwlab-cloud-api + namespace: hwlab-dev + healthPath: /health/live + profile: dev + replicas: 1 + env: + HWLAB_ENVIRONMENT: dev + HWLAB_PUBLIC_ENDPOINT: http://74.48.78.17:16667 + HWLAB_CLOUD_API_PORT: "6667" + HWLAB_RUNTIME_SUBSTITUTE_FORBIDDEN: unidesk-backend,provider-gateway,microservice-proxy + HWLAB_CLOUD_DB_URL: secretRef:hwlab-cloud-api-dev-db/database-url + HWLAB_CLOUD_DB_SSL_MODE: disable + HWLAB_CLOUD_DB_CONTRACT: dev-redacted-presence-only + HWLAB_CLOUD_RUNTIME_ADAPTER: postgres + HWLAB_CLOUD_RUNTIME_DURABLE: "true" + HWLAB_M3_IO_CONTROL_ENABLED: "false" + HWLAB_CODE_AGENT_PROVIDER: codex-stdio + HWLAB_CODE_AGENT_MODEL: gpt-5.5 + HWLAB_CODE_AGENT_TIMEOUT_MS: "1200000" + HWLAB_CODE_AGENT_OPENAI_BASE_URL: http://127.0.0.1:49280/responses + HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED: "1" + HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR: repo-owned + HWLAB_CODE_AGENT_WORKSPACE: /workspace/hwlab + HWLAB_CODE_AGENT_CODEX_WORKSPACE: /workspace/hwlab + HWLAB_CODE_AGENT_CODEX_SANDBOX: danger-full-access + HWLAB_CODE_AGENT_CODEX_COMMAND: /app/node_modules/.bin/codex + CODEX_HOME: /codex-home + NO_PROXY: hyueapi.com,.hyueapi.com,hyui.com,.hyui.com,127.0.0.1,localhost,::1,api.minimaxi.com,.minimaxi.com + no_proxy: hyueapi.com,.hyueapi.com,hyui.com,.hyui.com,127.0.0.1,localhost,::1,api.minimaxi.com,.minimaxi.com + OPENAI_API_KEY: secretRef:hwlab-code-agent-provider/openai-api-key + HWLAB_CODE_AGENT_SKILLS_DIRS: /app/skills:/data/user-skills + HWLAB_PREINSTALLED_SKILLS_DIR: /app/skills + HWLAB_USER_SKILLS_DIR: /data/user-skills + HWLAB_CODE_AGENT_DEFAULT_PROVIDER_PROFILE: deepseek + HWLAB_CODE_AGENT_DEEPSEEK_MODEL: deepseek-chat + HWLAB_CODE_AGENT_DEEPSEEK_BASE_URL: http://hwlab-deepseek-proxy.hwlab-dev.svc.cluster.local:4000/v1/responses + HWLAB_CODE_AGENT_CODEX_API_MODEL: gpt-5.5 + HWLAB_CODE_AGENT_CODEX_API_BASE_URL: http://127.0.0.1:49280/responses + HWLAB_CODE_AGENT_CODEX_API_UPSTREAM_BASE_URL: https://hyueapi.com + HWLAB_CODE_AGENT_CODEX_API_FORWARDER_PORT: "49280" + - serviceId: hwlab-cloud-web + namespace: hwlab-dev + healthPath: /health/live + profile: dev + replicas: 1 + env: + HWLAB_ENVIRONMENT: dev + HWLAB_API_BASE_URL: http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667 + HWLAB_CLOUD_WEB_PROXY_TIMEOUT_MS: "1260000" + - serviceId: hwlab-gateway + namespace: hwlab-dev + healthPath: /health/live + profile: dev + replicas: 0 + env: + HWLAB_GATEWAY_MODE: hardware-boundary + HWLAB_ENVIRONMENT: dev + - serviceId: hwlab-edge-proxy + namespace: hwlab-dev + healthPath: /health/live + profile: dev + replicas: 1 + env: + HWLAB_EDGE_LISTEN: 0.0.0.0:6667 + HWLAB_EDGE_UPSTREAM: http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667 + HWLAB_EDGE_PROXY_TIMEOUT_MS: "1260000" + - serviceId: hwlab-agent-skills + namespace: hwlab-dev + healthPath: /health/live + profile: dev + replicas: 1 + env: {} +createdAt: 2026-05-21T00:00:00Z diff --git a/deploy/frp/README.md b/deploy/frp/README.md index bcf7e0bb..78ae5684 100644 --- a/deploy/frp/README.md +++ b/deploy/frp/README.md @@ -9,7 +9,7 @@ The DEV reverse link is D601 to master edge through frp. - `frps.dev.toml` reserves D601 `16666` / `16667`, G14 DEV `17666` / `17667`, and G14 PROD `18666` / `18667` as TCP `remotePort` values; do not also bind them as `vhostHTTPPort`, because that collides with the DEV TCP proxy. -- `deploy/deploy.json` is the source for the public endpoints and FRP proxy +- `deploy/deploy.yaml` is the source for the public endpoints and FRP proxy mapping. Run `node scripts/deploy-contract-plan.mjs --check` to detect drift such as public `6666`/`6667` reuse before any future apply command. - No secret values are stored in this repository. Operators must provide auth diff --git a/deploy/master-edge/README.md b/deploy/master-edge/README.md index be09ec44..5e2e7e68 100644 --- a/deploy/master-edge/README.md +++ b/deploy/master-edge/README.md @@ -21,6 +21,6 @@ Health ownership: PROD is intentionally gated off for this task. -`deploy/deploy.json` is the source for the public endpoints, FRP proxy ports, +`deploy/deploy.yaml` is the source for the public endpoints, FRP proxy ports, k3s service mappings, and health path. The dry-run renderer validates these files only and does not perform a real edge apply or restart. diff --git a/docs/reference/g14-gitops-cicd.md b/docs/reference/g14-gitops-cicd.md index 69bc344a..41ccec74 100644 --- a/docs/reference/g14-gitops-cicd.md +++ b/docs/reference/g14-gitops-cicd.md @@ -28,7 +28,7 @@ G14 是 HWLAB 当前 DEV/PROD 原生 k8s 与 GitOps 运行面目标。G14 CI/CD - 架构迁移时,过时的自检、预检、guard、gate 优先删除,不在旧门禁上叠加例外或复杂度;新门禁只允许覆盖明确高价值风险,必须保持最小、低噪声、易迁移。 - 直接声明 Tekton Pipeline、最小原语校验 task 和 PipelineRun 样板;不再读取 `CI.json` 或生成 `ci-json` step。 -- 读取 `deploy/deploy.json` 与 `deploy/k8s/*`,生成 Argo CD 可消费的 G14 runtime Kustomize path。 +- 读取 `deploy/deploy.yaml` 与 `deploy/k8s/*`,生成 Argo CD 可消费的 G14 runtime Kustomize path。 - G14 Tekton 的镜像构建发布入口必须是 `scripts/g14-artifact-publish.mjs`;它只是集群内 Task 的 build/push helper,不做 rollout、不写 D601、不获取 legacy DEV CD Lease。旧 `scripts/dev-artifact-publish.mjs` 入口已删除;`dev-cd-apply`、`ci-publish` 和旧 `main` JS CD 入口禁止出现在 G14 Pipeline 生成脚本和 G14 验收证据中。 - `g14-contract-check` 在 fresh source clone 内取当前 `HEAD`,把 GitOps 产物渲染到 `mktemp -d` 临时目录,并立刻用同一个 `--source-revision` 执行 `--check`;它校验 render 代码、原生 Tekton 产物生成合同和 forbidden-fragment 护栏,不依赖 source branch 预先存在 `deploy/gitops/g14/source.json`。面向已发布 runtime 的 `source.json` 只存在于 `G14-gitops` 生成分支,作为 promotion evidence。 - 生成 `hwlab-g14-branch-poller` CronJob:它使用 G14 集群内的 Git SSH Secret 轮询 `G14` 分支,按 source commit 创建确定命名的 Tekton PipelineRun。 @@ -73,17 +73,17 @@ npm run g14:gitops:render -- --source-revision ## Monorepo 组件计划与兼容 render -HWLAB 是 monorepo,G14 CI/CD 加速必须按组件输入判断构建和滚动,并直接依赖内建 component model、`deploy/deploy.json` 与 per-service artifact catalog;`CI.json` 已删除,不再作为任何 planner 输入。 +HWLAB 是 monorepo,G14 CI/CD 加速必须按组件输入判断构建和滚动;v0.2 env-reuse 的组件边界与环境配方直接来自 `deploy/deploy.yaml`,运行态镜像身份来自 per-service artifact catalog;`CI.json` 已删除,不再作为任何 planner 输入。 -- `scripts/g14-ci-plan.mjs` 是只读 planner,默认读取当前 workspace 的 `deploy/deploy.json`、`deploy/artifact-catalog.dev.json` 和 `scripts/src/g14-ci-plan-lib.mjs` 内建 component model,输出 `affectedServices`、`reusedServices`、`componentCommitId`、`componentInputHash`、`dockerfileHash`、`baseImageDigest`、`buildArgsHash` 和原因;它不得修改 deploy、catalog 或 GitOps 文件。Tekton `prepare-source` 会先从 `G14-gitops` 注入上一轮发布态 catalog,source 分支里的 catalog 只作为 seed contract。 -- 服务清单兼容顺序固定为:显式 `--services`、`deploy.services[]`、`deploy.k3s.serviceMappings[]`、`internal/protocol.SERVICE_IDS`。因此旧 `deploy/deploy.json` 形态和当前完整 `deploy.services[]` 形态都必须能被 planner 识别。 -- 组件边界固定由 `scripts/src/g14-ci-plan-lib.mjs` 的内建 service-path model 定义;如需新增或调整 `componentPaths`、`sharedPaths`、`runtimeDeps` 或 `buildSystemPaths`,直接修改 planner 库和对应测试,不再额外维护 repo-local commands/forbidden skeleton。 +- `scripts/g14-ci-plan.mjs` 是只读 planner,默认读取当前 workspace 的 `deploy/deploy.yaml`、`deploy/artifact-catalog.dev.json` 和 `deploy/deploy.yaml` 内的 lane service declarations/env recipe,输出 `affectedServices`、`reusedServices`、`componentCommitId`、`componentInputHash`、`dockerfileHash`、`baseImageDigest`、`buildArgsHash` 和原因;它不得修改 deploy、catalog 或 GitOps 文件。Tekton `prepare-source` 会先从 `G14-gitops` 注入上一轮发布态 catalog,source 分支里的 catalog 只作为 seed contract。 +- v0.2 服务清单来自显式 `--services` 或 `deploy.lanes.v02.envReuseServices`;旧 `deploy.services[]`、`deploy.k3s.serviceMappings[]` 和 `internal/protocol.SERVICE_IDS` 推导路径不再作为 planner 输入。 +- v0.2 env-reuse 组件边界和环境镜像配方以 `deploy/deploy.yaml` 的 `lanes.v02.serviceDeclarations`、`lanes.v02.envRecipe` 和 `lanes.v02.bootConfig` 为单一配置点;新增服务或调整 `componentPaths`、`runtimeKind`、`entrypoint`、Bun 版本、系统包、launcher 路径和 HWPOD alias 时先改 deploy 配置与 schema/测试,不再把这些属性硬编码进 planner 或 artifact publish helper。 - `hwpod` 是 runner 内的稳定 HWPOD task 入口,由 `tools/hwpod-cli.ts`、`tools/hwpod-compiler-cli.ts`、`tools/hwpod-ctl.ts`、`tools/hwpod-node.ts` 和 `skills/hwpod-cli/`、`skills/hwpod-ctl/` 组成。修改这些路径时,G14/v0.2 CI 必须至少触发携带 `/usr/local/bin/hwpod` 的 runtime skills/env-reuse 重新装配或 rollout,不能全量复用旧 artifact 后只报告 PipelineRun 成功。 - `scripts/g14-artifact-publish.mjs` 默认启用组件级 lazy build:先运行 planner,再只构建/推送 `affectedServices`,`reusedServices` 从 `deploy/artifact-catalog.dev.json` 或 lane catalog 复用已有 sha256 digest。非 env-reuse 服务复用前必须满足 artifact provenance 自证:catalog 有可验证 digest、catalog 的 `sourceCommitId` 在当前 repo 可解析、用该 `sourceCommitId` 的 source tree 重新计算出的 `componentInputHash` 等于 catalog 记录、该 hash 再等于本轮 planner 计算值,且 `dockerfileHash`/`buildArgsHash` 没有不一致。catalog 缺 digest、缺 provenance、source tree 无法解析、catalog hash 与 catalog source tree 不一致、或 catalog hash 与本轮 input 不一致时,planner 必须把该服务列为 affected 并重新发布,不能用旧 guard 阻塞,也不能退回 Docker 或 legacy full-build 路线。 - Artifact catalog 的 per-service provenance 是镜像 digest 的身份证明,不是本轮 planner 状态缓存。reuse 路径只能保留旧 artifact 自身的 `sourceCommitId`、`componentCommitId`、`componentInputHash`、`dockerfileHash`、`baseImage*` 和 `buildArgsHash`;禁止把当前 planner 的 component/build 元数据写入复用的旧 digest,否则下一轮 planner 会把旧镜像误判成已包含新输入,造成 CI/CD false-green。 -- `scripts/g14-artifact-publish.mjs` 的 publish report 必须携带 planner 的 per-service 元数据;`scripts/refresh-artifact-catalog.mjs` 默认只预览,只有 G14 Tekton promotion 显式传 `--write` 时,才把生成的 `commitId`、`image`、`imageTag`、`digest`、`publishState` 和 component provenance 字段写进当前 workspace 的 `deploy/artifact-catalog.dev.json`,随后只提交到 `G14-gitops`。`deploy/deploy.json` 是人写的 runtime config 真相源,不得被 promotion、refresh 脚本或人工发布流程回写镜像身份字段。 -- `scripts/g14-gitops-render.mjs` 只支持混合 desired state:workload 的 container image、`HWLAB_IMAGE`、`HWLAB_IMAGE_TAG` 和 pod template `source-commit` 来自 `deploy/artifact-catalog.dev.json` 的 per-service artifact identity;普通 env、replica、healthPath、profile 等配置来自 `deploy/deploy.json`。全局 GitOps metadata 仍记录本次 source commit。`--legacy-source-images` 与 `HWLAB_G14_USE_DEPLOY_IMAGES=0` 已废弃,不得把所有 workload image 回退渲染为同一个 source commit tag。 -- G14 Tekton promotion 在推送 `G14-gitops` 前,必须用 publish report 显式 `--write` 刷新 workspace 内的 `deploy/artifact-catalog.dev.json`,然后把刷新后的 catalog 和 rendered GitOps desired state 一起提交到 `G14-gitops`。promotion 不得自动修改或推送 `G14` source branch,也不得自动修改 `deploy/deploy.json` 或 `deploy/k8s/base/workloads.yaml`,这样人写配置不会和 CI 生成身份反复冲突。 +- `scripts/g14-artifact-publish.mjs` 的 publish report 必须携带 planner 的 per-service 元数据;`scripts/refresh-artifact-catalog.mjs` 默认只预览,只有 G14 Tekton promotion 显式传 `--write` 时,才把生成的 `commitId`、`image`、`imageTag`、`digest`、`publishState` 和 component provenance 字段写进当前 workspace 的 `deploy/artifact-catalog.dev.json`,随后只提交到 `G14-gitops`。`deploy/deploy.yaml` 是人写的 runtime config 真相源,不得被 promotion、refresh 脚本或人工发布流程回写镜像身份字段。 +- `scripts/g14-gitops-render.mjs` 只支持混合 desired state:workload 的 container image、`HWLAB_IMAGE`、`HWLAB_IMAGE_TAG` 和 pod template `source-commit` 来自 `deploy/artifact-catalog.dev.json` 的 per-service artifact identity;普通 env、replica、healthPath、profile 等配置来自 `deploy/deploy.yaml`。全局 GitOps metadata 仍记录本次 source commit。`--legacy-source-images` 与 `HWLAB_G14_USE_DEPLOY_IMAGES=0` 已废弃,不得把所有 workload image 回退渲染为同一个 source commit tag。 +- G14 Tekton promotion 在推送 `G14-gitops` 前,必须用 publish report 显式 `--write` 刷新 workspace 内的 `deploy/artifact-catalog.dev.json`,然后把刷新后的 catalog 和 rendered GitOps desired state 一起提交到 `G14-gitops`。promotion 不得自动修改或推送 `G14` source branch,也不得自动修改 `deploy/deploy.yaml` 或 `deploy/k8s/base/workloads.yaml`,这样人写配置不会和 CI 生成身份反复冲突。 - Tekton 并发化只能以 planner 输出作为输入;每个 service 都有独立 TaskRun,changed service 启动 BuildKit,unchanged service 只写 reuse result 并复用 catalog digest,且不能改 pod template,避免无意义 rollout。没有完整 per-service desired state 证据时,必须修复 planner/catalog 证据,不能使用 `--full-build`、`--legacy-source-images`、DIND 或 Docker fallback 回退。 ## 加速判定与当前瓶颈 diff --git a/docs/reference/m3-loop-rollout-runbook.md b/docs/reference/m3-loop-rollout-runbook.md index 54140f87..ceaa3932 100644 --- a/docs/reference/m3-loop-rollout-runbook.md +++ b/docs/reference/m3-loop-rollout-runbook.md @@ -18,7 +18,7 @@ front-end acceptance path. - Historical public `:6666` and `:6667` endpoints are not current acceptance targets; internal k3s services may still use `6667`. - Runtime intent and artifact identity are reviewed through - `deploy/deploy.json` and `deploy/artifact-catalog.dev.json`. + `deploy/deploy.yaml` and `deploy/artifact-catalog.dev.json`. - Suspended template Job replacement is tracked by `pikasTech/HWLAB#63` and must not be confused with M3 loop evidence. diff --git a/docs/reference/spec-hwpod-harness.md b/docs/reference/spec-hwpod-harness.md index 199e4f34..bbf24206 100644 --- a/docs/reference/spec-hwpod-harness.md +++ b/docs/reference/spec-hwpod-harness.md @@ -295,6 +295,22 @@ CaseRun 的 trace 处理必须遵循 [spec-v02-code-agent-trace.md](spec-v02-cod - `case run`、异步 worker 完成收口和 `case run result` 刷新归档后,默认必须把当前 `runs///` 目录自动 `git add`、`git commit` 并 `git push origin HEAD` 到 case registry repo;只允许提交当前 run 目录,不得顺带提交 registry 里其他历史未跟踪 run。`registrySync` 必须写入 result/summary,说明 `pushed`、`unchanged` 或失败阶段。`--no-case-repo-record` 是完全不记录 registry 的诊断出口;`--no-case-repo-git-sync` / `--no-registry-git-sync` 只用于特殊本地调试,不能作为正常 CaseRun 产物收口方式。 - HWLAB repo 的 harness、`hwpod-cli`、`hwpod-ctl`、`hwpod-compiler-cli` 或文档改进可以按风险进入 `v0.2` 分支;单纯文档和轻量 CLI/helper 变更可直接提交,业务代码、运行面或发布链路变更走 PR 工作流。 +### CaseRun registry 聚合阅读入口 + +`hwlab-cli case aggregate --run-id --case-repo /root/hwlab-case-registry` 是已完成 CaseRun 的无服务二次整理入口。它只读取 case registry repo 里已有的 `runs///` 产物,不启动新的 CaseRun、不访问硬件、不触发 CI/CD、不追加自动评价或 pass/fail 判定;默认输出 `runs///aggregate.md`,并只把这个 markdown 文件 `git add`、`git commit`、`git push` 回 case registry repo。若 registry 里存在其他并行 dirty 产物,`case aggregate` 不得顺带提交它们。 + +`aggregate.md` 是该 run 的主阅读入口,必须按固定顺序聚合运行环境信息、HWPOD 信息、Code Agent 信息、输入 Prompt、低噪声 Trace、final response 和最后 diff。Trace 正文优先使用 `agent-messages.json` 里的共享 Web/CLI renderer rows;普通 message row 直接展示正文,不做折叠,只有工具调用 row 使用 `
/`,且 `` 本身就是折叠按钮,文案写成 `已运行 `。缺少 rows 时才嵌入已有 `agent-trace.md`。聚合文件可以列原始产物索引和 trace/result/inspect 命令作为审计线索,但不能把 `summary.md`、`result.json`、`final-response.md` 或原 trace 文件继续声明为对外主阅读路径。 + +验收已完成 case04 时,使用现有 registry 产物回放,不重新启动 CaseRun: + +```bash +hwlab-cli case aggregate d601-f103-v2-arm2d-integration \ + --case-repo /root/hwlab-case-registry \ + --run-id +``` + +通过证据至少包括 CLI JSON 输出中的 `action=case.aggregate`、`autoEvaluation=false`、`aggregate.rel=runs///aggregate.md`、`registrySync.status`,以及打开 `aggregate.md` 后能在单文件内读到上述七类内容。 + 当前边界:CaseRun 仍是无服务化短连接 CLI 编排,不负责排队、并发、评分、自动合并、长期 run retention 或 epoch 汇总;这些能力属于后续强化学习 Harness 层。CaseRun 也不负责代替 HWLAB Code Agent 完成研发任务,后续 agent-task CaseRun 必须把 prompt/session/trace/diff/evidence 作为一等输出。AgentRun 私有 GitHub repo git transport 必须具备有界失败和可观测性后,才能把单次 CaseRun 扩展为批量 runner 或 epoch 系统。 ## 验收标准 @@ -310,5 +326,6 @@ CaseRun 的 trace 处理必须遵循 [spec-v02-code-agent-trace.md](spec-v02-cod 7. 长任务验收必须覆盖 `hwlab-cli case run start ` 立即返回,以及 `status/result/logs ` 在 worker 运行中能短查询到阶段、耗时、stdout/stderr byte count、trace/session/conversation/thread 信息和下一条轮询命令;不得把同步命令长时间无输出视为有效 CaseRun 操作体验。 8. `d601-f103-v2-compile` 的真实流程只覆盖当前编排链路和 compile-only smoke;操作员直接运行 `hwpod-cli build`、直接调用 Keil 或只检查 node 侧 job,不等同于 CaseRun 流程跑完。 9. agent-task CaseRun 的真实流程必须证明 `case run` 已创建或调用 Code Agent / AgentRun session,任务 prompt 已进入 agent 上下文,CaseRun 已采集隔离 subject worktree diff,并已继续执行编译、下载或 I/O smoke 等后续动作;当前版本只记录这些事实,不自动判断 agent 是否完成任务。 -10. Windows subject worktree 文本编辑必须能在 CRLF 文件中通过 `workspace.insert-after` 或 `workspace.replace` 完成一行源码修改,返回 before/after SHA 与 diff 摘要,并保留原文件 CRLF;`workspace.apply-patch` 在 context 不匹配时必须返回可诊断 payload,而不是只给 `context not found`。 -11. Keil `debug.download` 自动装配必须输出结构化 `cmd.run` argv 步骤;带空格的 `probeName` 在 plan 中必须是单独 argv 元素。`io.uart.read` 必须通过节点本地 `serial-monitor` 读取正在监控的 COM/baud;未监控或工具失败时必须返回包含 `ioProbe`/COM 口/baud/monitor status/启动建议的结构化 blocker details。 +10. 已完成 run 的 registry 聚合验收必须通过 `hwlab-cli case aggregate --run-id ` 输出单个 `aggregate.md`,并证明该文件包含运行环境、HWPOD、Code Agent、Prompt、低噪声 Trace、final response 和最后 diff;Trace 中 message 直接展示,工具调用用 `已运行 ` summary 折叠。该命令不得启动新 CaseRun,也不得提交当前聚合 markdown 以外的并行 registry dirty 文件。 +11. Windows subject worktree 文本编辑必须能在 CRLF 文件中通过 `workspace.insert-after` 或 `workspace.replace` 完成一行源码修改,返回 before/after SHA 与 diff 摘要,并保留原文件 CRLF;`workspace.apply-patch` 在 context 不匹配时必须返回可诊断 payload,而不是只给 `context not found`。 +12. Keil `debug.download` 自动装配必须输出结构化 `cmd.run` argv 步骤;带空格的 `probeName` 在 plan 中必须是单独 argv 元素。`io.uart.read` 必须通过节点本地 `serial-monitor` 读取正在监控的 COM/baud;未监控或工具失败时必须返回包含 `ioProbe`/COM 口/baud/monitor status/启动建议的结构化 blocker details。 diff --git a/docs/reference/spec-v02-cicd.md b/docs/reference/spec-v02-cicd.md index c64ca52d..47a7c27e 100644 --- a/docs/reference/spec-v02-cicd.md +++ b/docs/reference/spec-v02-cicd.md @@ -93,7 +93,7 @@ devops-infra git mirror 仍是 PipelineRun 和 Argo CD 的集群内读写源。` `v0.2` source branch 可以包含: - 源码、测试、文档、人写配置和模板。 -- `deploy/deploy.json` 或等价 lane 配置。 +- `deploy/deploy.yaml` 或等价 lane 配置。 - k8s 模板、render 脚本、CI/CD helper 和 catalog schema。 `v0.2` source branch 不得跟踪: @@ -447,7 +447,7 @@ kubectl get pod -n "$ns" -o name | grep -E "registry|hwlab-registry" || true ## Env 容器复用与三变量启动 -`v0.2` code-only fast lane 的 runtime desired state 由三类输入组成:可复用 env image digest、自动推导的 code boot metadata、以及既有 service runtime config。开发者仍按当前 DEV/OPS 流程提交 `v0.2` source commit 和维护 `deploy/deploy.json`;发布入口不得要求人工填写 repo、commitId 或 boot script 路径。 +`v0.2` code-only fast lane 的 runtime desired state 由三类输入组成:可复用 env image digest、自动推导的 code boot metadata、以及既有 service runtime config。开发者仍按当前 DEV/OPS 流程提交 `v0.2` source commit 和维护 `deploy/deploy.yaml`;发布入口不得要求人工填写 repo、commitId 或 boot script 路径。 code boot metadata 固定映射到三个启动环境变量: @@ -455,7 +455,7 @@ code boot metadata 固定映射到三个启动环境变量: | --- | --- | --- | | `HWLAB_BOOT_REPO` | `v0.2` lane 的 canonical GitHub source repo 配置 | 必须是 canonical GitHub URL;用于身份记录,运行时读取由 resolver 自动分流到 mirror/cache。 | | `HWLAB_BOOT_COMMIT` | UniDesk `trigger-current` 解析到的 `origin/v0.2` 完整 source commit SHA,也就是 PipelineRun revision | 必须是完整 40 位 commit SHA;禁止 branch、tag、`latest` 或人工覆盖。 | -| `HWLAB_BOOT_SH` | service model 或 `deploy/deploy.json` 中 serviceId 到 boot script 的映射,默认形态为 `deploy/runtime/boot/.sh` | 必须是 repo 内相对路径;禁止绝对路径、`..` 越界和从 env image 中隐式寻找旧脚本。 | +| `HWLAB_BOOT_SH` | service model 或 `deploy/deploy.yaml` 中 serviceId 到 boot script 的映射,默认形态为 `deploy/runtime/boot/.sh` | 必须是 repo 内相对路径;禁止绝对路径、`..` 越界和从 env image 中隐式寻找旧脚本。 | CI/CD 必须把三变量同时写入 `deploy/artifact-catalog.v02.json`、rendered workload Pod template env/annotation 和 runtime health identity。三变量是由 lane 自动推导的发布事实,不是人工 OPS 参数;如果自动推导缺失或无法证明 `HWLAB_BOOT_COMMIT` 属于 `v0.2` 允许 ancestry,本轮 promotion 必须失败。 @@ -482,7 +482,7 @@ registry 与 git mirror/relay 分属不同基础设施边界。registry 保持 - runtime manifest 必须使用 digest pin 作为部署身份。 - catalog 必须记录 lane/profile、source branch、GitOps branch、source commitId、serviceId、image tag、digest、component identity 和 publish/reuse 状态;启用 env 容器复用时还必须记录 `runtimeMode=env-reuse-git-mirror-checkout`、`environmentImage`、`environmentDigest`、`environmentInputHash`、`bootRepo`、`bootCommit`、`bootSh`、`codeInputHash` 和三变量写入证据。 - 同一 source commit 对同一 service 应生成同一镜像;lane 差异放在 manifest、env、SecretRef、namespace、FRP 和 DB 配置中,不 bake 进镜像。 -- `deploy/deploy.json` 只承载人写 runtime intent,不承载 digest、publish state 或 reuse evidence。 +- `deploy/deploy.yaml` 只承载人写 runtime intent,不承载 digest、publish state 或 reuse evidence。 ## Kubernetes 与 Argo 边界 @@ -513,7 +513,7 @@ registry 与 git mirror/relay 分属不同基础设施边界。registry 保持 - 自动 CD、手动 trigger 和 once 补偿都必须复用 UniDesk `trigger-current`、定点 `status` 和 `git-mirror flush`;不得绕到裸 `kubectl`、Argo、Tekton 或手写 GitHub API。 - 旧 DEV/D601/main gate、fallback、legacy mode 和双路径兼容不得进入 `v0.2` 调用链。 - env 容器复用 fast lane 中,`HWLAB_BOOT_REPO`、`HWLAB_BOOT_COMMIT` 和 `HWLAB_BOOT_SH` 必须由 CI/CD 自动推导并写入 GitOps desired state,不得作为人工发布参数或 runtime 临时 patch。 -- `deploy/deploy.json` 只保存人写运行意图,不得被 auto-CD、promotion 或 flush 回写 `commitId`、service `image`、`HWLAB_COMMIT_ID`、`HWLAB_IMAGE` 或 `HWLAB_IMAGE_TAG` 等发布产物字段。 +- `deploy/deploy.yaml` 只保存人写运行意图,不得被 auto-CD、promotion 或 flush 回写 `commitId`、service `image`、`HWLAB_COMMIT_ID`、`HWLAB_IMAGE` 或 `HWLAB_IMAGE_TAG` 等发布产物字段。 - git mirror/relay 必须来自独立 `devops-infra` 服务;`hwlab-v02` runtime namespace 不部署 mirror、不持有 GitHub deploy key、不在 mirror miss 时直连 GitHub fallback。 - GitOps promotion 必须写入 `devops-infra` 本地 mirror/relay 的 `v0.2-gitops`;除 mirror/relay flush 外,不得在 CI 关键路径直接 push GitHub canonical remote。 - mirror/relay write 必须只允许 allowlist refs,拒绝 non-fast-forward,拒绝越界 changed paths,并在 receive 成功前完成 object closure 校验。 @@ -556,7 +556,7 @@ registry 与 git mirror/relay 分属不同基础设施边界。registry 保持 - `http://74.48.78.17:19667/health/live` 返回 `v0.2` runtime health,payload 中的 namespace、revision 或 runtime identity 能与 `hwlab-v02`/`v0.2` 对齐。 - `trigger-current` 的异步 job 或 `--wait` 输出必须能显示 trigger 阶段进度;若 control-plane refresh、mirror pre-sync 或 delete/create PipelineRun 卡住,日志必须能定位阶段名和状态。 - 自动 CD 回写 PR 或关联 issue 的 rollout 评论必须包含 source commit、PipelineRun、GitOps revision、planArtifacts build/reuse 摘要、Argo/runtime/public probe 状态和耗时;该评论不能替代需要用户原入口验收的问题关闭证据。 -- 自动 CD 过程中 source branch 的 `deploy/deploy.json` 不得出现 `commitId`、service `image`、`HWLAB_COMMIT_ID`、`HWLAB_IMAGE` 或 `HWLAB_IMAGE_TAG` 回写。 +- 自动 CD 过程中 source branch 的 `deploy/deploy.yaml` 不得出现 `commitId`、service `image`、`HWLAB_COMMIT_ID`、`HWLAB_IMAGE` 或 `HWLAB_IMAGE_TAG` 回写。 - no-op env-reuse fast lane 必须输出完整证据链:`g14-ci-plan` 全复用、`skipped-runtime-unchanged`、`gitops-commit`/`gitops-push` skipped、`runtime-ready` skipped。 不能只用 PipelineRun `Completed` 或总耗时证明没有退化。 - 真实 rollout 的 `runtime-ready` 必须输出 `started`、周期性 `progress` 和最终 `argo-refresh`/`argo-sync-health`/`workload-ready` 类事件;缺少这些事件时应按可见性回归处理。 @@ -604,7 +604,7 @@ GitOps branch 已更新、source branch render 通过、PipelineRun 名称存在 ## T10 -阅读 docs/reference/spec-v02-cicd.md,然后用 cli 手动测试以下内容:用一个低风险 base=`v0.2` PR 验证 UniDesk v02 PR monitor 的端到端链路,确认 merge 后自动触发对应 merge head 的 `hwlab-v02-ci-poll-`,定点 status 通过,`pendingFlush=true` 时自动 flush,PR 或关联 issue 评论包含 source commit、PipelineRun、GitOps revision、build/reuse 摘要、Argo/runtime/public probe 状态和耗时,且 `deploy/deploy.json` 没有发布产物回写。 +阅读 docs/reference/spec-v02-cicd.md,然后用 cli 手动测试以下内容:用一个低风险 base=`v0.2` PR 验证 UniDesk v02 PR monitor 的端到端链路,确认 merge 后自动触发对应 merge head 的 `hwlab-v02-ci-poll-`,定点 status 通过,`pendingFlush=true` 时自动 flush,PR 或关联 issue 评论包含 source commit、PipelineRun、GitOps revision、build/reuse 摘要、Argo/runtime/public probe 状态和耗时,且 `deploy/deploy.yaml` 没有发布产物回写。 ## 规格的实现情况 diff --git a/docs/reference/spec-v02-services.md b/docs/reference/spec-v02-services.md index edd4a818..0ae6c1a8 100644 --- a/docs/reference/spec-v02-services.md +++ b/docs/reference/spec-v02-services.md @@ -121,7 +121,7 @@ hwlab-agent-worker ## T1 -阅读 docs/reference/spec-v02-services.md,然后用 CLI 手动测试以下内容:列出 `deploy/deploy.json`、`deploy/deploy.schema.json`、`deploy/artifact-catalog.dev.json` 和 `deploy/gitops/g14/runtime-v02`,确认 runtime service set 只包含 `hwlab-cloud-api`、`hwlab-cloud-web`、`hwlab-gateway`、`hwlab-edge-proxy` 和 `hwlab-agent-skills`。 +阅读 docs/reference/spec-v02-services.md,然后用 CLI 手动测试以下内容:列出 `deploy/deploy.yaml`、`deploy/deploy.schema.json`、`deploy/artifact-catalog.dev.json` 和 `deploy/gitops/g14/runtime-v02`,确认 runtime service set 只包含 `hwlab-cloud-api`、`hwlab-cloud-web`、`hwlab-gateway`、`hwlab-edge-proxy` 和 `hwlab-agent-skills`。 ## T2 diff --git a/internal/cloud/code-agent-agentrun-adapter.ts b/internal/cloud/code-agent-agentrun-adapter.ts index 948ca6ee..ffdce70f 100644 --- a/internal/cloud/code-agent-agentrun-adapter.ts +++ b/internal/cloud/code-agent-agentrun-adapter.ts @@ -288,7 +288,7 @@ export async function submitAgentRunChatTurn({ params = {}, options = {}, traceI }); let runnerJob = null; try { - const runnerJobInput = buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, toolCapabilities, backendProfile }); + const runnerJobInput = buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, toolCapabilities, backendProfile, params }); runnerJob = await agentRunJson(fetchImpl, managerUrl, `/api/v1/runs/${encodeURIComponent(reusable.mapping.runId)}/runner-jobs`, { method: "POST", body: runnerJobInput, @@ -386,7 +386,7 @@ export async function submitAgentRunChatTurn({ params = {}, options = {}, traceI message: "AgentRun command " + commandId + " created; hwlab-cloud-api will start a runner Job explicitly without relying on scheduler automation.", runId, commandId, backendProfile, waitingFor: "agentrun-runner-job-create", valuesPrinted: false, }); - const runnerJobInput = buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, toolCapabilities, backendProfile }); + const runnerJobInput = buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, toolCapabilities, backendProfile, params }); try { runnerJob = await agentRunJson(fetchImpl, managerUrl, "/api/v1/runs/" + encodeURIComponent(runId) + "/runner-jobs", { method: "POST", body: runnerJobInput, timeoutMs }); } catch (error) { @@ -856,6 +856,14 @@ function rejectRemovedResourceWorkspaceFiles(value) { throw adapterError("legacy_workspace_files_removed", "workspaceFiles/resourceWorkspaceFiles are removed; use AgentRun ResourceBundleRef kind=gitbundle with bundles[]"); } +function adapterError(code, message, details = {}) { + return Object.assign(new Error(message), { + code, + statusCode: 400, + details + }); +} + function buildAgentRunCommandInput({ params, traceId, backendProfile, sessionId }) { const prompt = String(params.message ?? params.prompt ?? "").trim(); const threadId = safeOpaqueId(params.threadId); @@ -911,8 +919,8 @@ async function resolveOwnerApiKey({ params, options, now }) { if (!key) return ""; return text(key.displaySecret ?? ""); } -function buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, toolCapabilities = null, backendProfile }) { - const baseTransient = buildAgentRunTransientEnv(env, { providerProfile: backendProfile, parentTraceId: traceId }); +function buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, toolCapabilities = null, backendProfile, params = {} }) { + const baseTransient = buildAgentRunTransientEnv(env, { providerProfile: backendProfile, parentTraceId: traceId, hwpodSpecContent: params.hwpodSpecContent }); const hwpodAllowed = toolCapabilityAllowed(toolCapabilities, "hwpod"); const transientEnv = ownerApiKey && hwpodAllowed ? baseTransient.concat([{ name: "HWLAB_API_KEY", value: ownerApiKey, sensitive: true }]) @@ -934,6 +942,7 @@ function buildAgentRunRunnerJobInput({ env, traceId, commandId, ownerApiKey, too function buildAgentRunTransientEnv(env = process.env, options = {}) { const providerProfile = firstNonEmpty(options.providerProfile); const parentTraceId = firstNonEmpty(options.parentTraceId); + const hwpodSpecContent = hwpodSpecContentForEnv(options.hwpodSpecContent); const entries = []; for (const name of [ "HWLAB_RUNTIME_API_URL", @@ -949,9 +958,18 @@ function buildAgentRunTransientEnv(env = process.env, options = {}) { } if (providerProfile) entries.push({ name: "HWLAB_CODE_AGENT_PROVIDER_PROFILE", value: providerProfile, sensitive: false }); if (parentTraceId) entries.push({ name: "HWLAB_CODE_AGENT_PARENT_TRACE_ID", value: parentTraceId, sensitive: false }); + if (hwpodSpecContent) entries.push({ name: "HWPOD_SPEC_CONTENT", value: `base64:${Buffer.from(hwpodSpecContent, "utf8").toString("base64")}`, sensitive: false }); return entries; } +function hwpodSpecContentForEnv(value) { + const content = typeof value === "string" ? value.trim() : ""; + if (!content) return ""; + if (Buffer.byteLength(content, "utf8") > 64 * 1024) throw adapterError("hwpod_spec_too_large", "hwpodSpecContent must be 64 KiB or smaller"); + if (!/^apiVersion:/mu.test(content) || !/\nkind:\s*Hwpod\b/mu.test(content)) throw adapterError("invalid_hwpod_spec_content", "hwpodSpecContent must be a Hwpod YAML document"); + return content; +} + function agentRunToolCredentials(env = process.env, toolCapabilities = null) { const namespace = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_TOOL_SECRET_NAMESPACE, env.HWLAB_CODE_AGENT_AGENTRUN_SECRET_NAMESPACE, DEFAULT_RUNNER_NAMESPACE); const githubSecretName = firstNonEmpty(env.HWLAB_CODE_AGENT_AGENTRUN_GITHUB_TOOL_SECRET_NAME, "agentrun-v01-tool-github-pr"); diff --git a/internal/cloud/server-agent-chat.test.ts b/internal/cloud/server-agent-chat.test.ts index af3558d7..a8495ab6 100644 --- a/internal/cloud/server-agent-chat.test.ts +++ b/internal/cloud/server-agent-chat.test.ts @@ -369,17 +369,6 @@ test("cloud api /v1/agent/chat delegates v0.2 turns to AgentRun v0.1 over adapte if (request.method === "POST" && url.pathname === "/api/v1/runs/run_hwlab_adapter/runner-jobs") { const secondRunnerJob = body.commandId === "cmd_hwlab_adapter_second"; assert.ok(body.commandId === "cmd_hwlab_adapter" || secondRunnerJob); - assert.deepEqual(body.transientEnv.map((entry) => entry.name), [ - "HWLAB_RUNTIME_API_URL", - "HWLAB_RUNTIME_WEB_URL", - "HWLAB_RUNTIME_NAMESPACE", - "HWLAB_RUNTIME_LANE", - "HWLAB_RUNTIME_ENDPOINT_LOCKED", - "HWLAB_CODE_AGENT_ASSEMBLED_RUNTIME", - "UNIDESK_MAIN_SERVER_IP", - "HWLAB_CODE_AGENT_PROVIDER_PROFILE", - "HWLAB_CODE_AGENT_PARENT_TRACE_ID" - ]); const transientEnv = Object.fromEntries(body.transientEnv.map((entry) => [entry.name, entry.value])); assert.equal(transientEnv.HWLAB_RUNTIME_API_URL, "http://hwlab-cloud-api.hwlab-v02.svc.cluster.local:6667"); assert.equal(transientEnv.HWLAB_RUNTIME_WEB_URL, "http://hwlab-cloud-web.hwlab-v02.svc.cluster.local:8080"); diff --git a/package-lock.json b/package-lock.json index 52dc1e4b..3d4eaffc 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,8 @@ "@openai/codex": "^0.128.0", "fzstd": "0.1.1", "pg": "^8.21.0", - "playwright": "1.59.1" + "playwright": "1.59.1", + "yaml": "^2.8.3" } }, "node_modules/@openai/codex": { @@ -330,6 +331,20 @@ "engines": { "node": ">=0.4" } + }, + "node_modules/yaml": { + "version": "2.8.3", + "resolved": "https://registry.npmjs.org/yaml/-/yaml-2.8.3.tgz", + "integrity": "sha512-AvbaCLOO2Otw/lW5bmh9d/WEdcDFdQp2Z2ZUH3pX9U2ihyUY0nvLv7J6TrWowklRGPYbB/IuIMfYgxaCPg5Bpg==", + "bin": { + "yaml": "bin.mjs" + }, + "engines": { + "node": ">= 14.6" + }, + "funding": { + "url": "https://github.com/sponsors/eemeli" + } } } } diff --git a/package.json b/package.json index 03d13d62..7636d963 100644 --- a/package.json +++ b/package.json @@ -45,6 +45,7 @@ "@openai/codex": "^0.128.0", "fzstd": "0.1.1", "pg": "^8.21.0", - "playwright": "1.59.1" + "playwright": "1.59.1", + "yaml": "^2.8.3" } } diff --git a/scripts/artifact-publish.mjs b/scripts/artifact-publish.mjs index 8113bae9..a8b25b5b 100644 --- a/scripts/artifact-publish.mjs +++ b/scripts/artifact-publish.mjs @@ -22,6 +22,7 @@ import { } from "./src/dev-artifact-services.mjs"; import { probeRegistryCapabilities } from "./src/registry-capabilities.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./src/report-paths.mjs"; +import { readStructuredFile } from "./src/structured-config.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const cliEntrypoint = process.env.HWLAB_ARTIFACT_PUBLISH_ENTRYPOINT || "scripts/artifact-publish.mjs"; @@ -35,7 +36,7 @@ const v02EnvReuseRuntimeMode = "env-reuse-git-mirror-checkout"; const defaultBuildkitCommand = process.env.HWLAB_BUILDKIT_BUILDCTL || "buildctl-daemonless.sh"; const defaultBuildkitAddr = process.env.HWLAB_BUILDKIT_ADDR || null; const defaultCatalogPath = process.env.HWLAB_ARTIFACT_CATALOG_PATH || "deploy/artifact-catalog.dev.json"; -const defaultDeployPath = process.env.HWLAB_DEPLOY_MANIFEST_PATH || "deploy/deploy.json"; +const defaultDeployPath = process.env.HWLAB_DEPLOY_MANIFEST_PATH || "deploy/deploy.yaml"; const mutableTags = new Set(["latest", "dev", "main", "master", "prod", "production"]); const shaDigestPattern = /^sha256:[a-f0-9]{64}$/u; @@ -66,18 +67,6 @@ const servicePorts = new Map([ ["hwlab-agent-skills", 7430] ]); -const v02RuntimeServiceIds = Object.freeze([ - "hwlab-cloud-api", - "hwlab-cloud-web", - "hwlab-gateway", - "hwlab-edge-proxy", - "hwlab-agent-skills" -]); - -function serviceIdsForLane(lane) { - return lane === "v02" ? v02RuntimeServiceIds : SERVICE_IDS; -} - function parseArgs(argv) { const args = { mode: "preflight", @@ -109,7 +98,7 @@ function parseArgs(argv) { else if (arg === "--publish") args.mode = "publish"; else if (arg === "--lane") args.lane = readOption(argv, ++index, arg); else if (arg === "--catalog-path") args.catalogPath = readOption(argv, ++index, arg); - else if (arg === "--deploy-json") args.deployPath = readOption(argv, ++index, arg); + else if (arg === "--deploy-config") args.deployPath = readOption(argv, ++index, arg); else if (arg === "--image-tag-mode") args.imageTagMode = readOption(argv, ++index, arg); else if (arg === "--registry-prefix") args.registryPrefix = readOption(argv, ++index, arg); else if (arg === "--base-image") args.baseImage = readOption(argv, ++index, arg); @@ -133,7 +122,7 @@ function parseArgs(argv) { assert.ok(["g14", "v02"].includes(args.lane), `unknown artifact lane ${args.lane}`); assert.ok(["short", "full"].includes(args.imageTagMode), `unknown image tag mode ${args.imageTagMode}`); - if (!args.servicesExplicit) args.services = [...serviceIdsForLane(args.lane)]; + if (!args.servicesExplicit && args.lane !== "v02") args.services = [...SERVICE_IDS]; return args; } @@ -210,11 +199,11 @@ function printHelp() { "options:", " --lane LANE g14 or v02; default: g14", ` --catalog-path PATH default: ${defaultCatalogPath}`, - ` --deploy-json PATH default: ${defaultDeployPath}`, + ` --deploy-config PATH default: ${defaultDeployPath}`, " --image-tag-mode MODE short or full; v02 uses full source commit IDs", " --registry-prefix PREFIX default: 127.0.0.1:5000/hwlab", " --base-image IMAGE override HWLAB_DEV_BASE_IMAGE for this run; must already be local", - " --services LIST comma-separated service IDs; default: all frozen DEV services", + " --services LIST comma-separated service IDs; v02 default comes from deploy.lanes.v02.envReuseServices", " --affected-only default and only supported scope: build/publish affected services and reuse unchanged catalog digests", " --buildkit-command PATH buildctl-daemonless.sh/buildctl command for buildkit backend", " --buildkit-addr ADDR optional buildctl --addr endpoint for a sidecar BuildKit daemon", @@ -228,13 +217,13 @@ function printHelp() { ].join("\n")); } -async function readJson(relativePath) { - return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8")); +async function readConfig(relativePath) { + return readStructuredFile(repoRoot, relativePath); } -async function readJsonIfPresent(relativePath, fallback = null) { +async function readConfigIfPresent(relativePath, fallback = null) { try { - return await readJson(relativePath); + return await readConfig(relativePath); } catch (error) { if (error?.code === "ENOENT") return fallback; throw error; @@ -593,6 +582,11 @@ function assertV02Contracts(catalog, deployManifest) { assert.ok(lane && typeof lane === "object", "deploy.lanes.v02 must exist for v02 artifact publish"); assert.equal(lane.namespace, "hwlab-v02", "deploy.lanes.v02.namespace must be hwlab-v02"); assert.ok(isHttpEndpoint(lane.endpoint), "deploy.lanes.v02.endpoint must be a non-empty http(s) URL"); + const declaredServiceIds = new Set(v02ServiceIdsFromDeclarations(deployManifest)); + assert.ok( + v02ServiceIdsFromDeploy(deployManifest).every((serviceId) => declaredServiceIds.has(serviceId)), + "deploy.lanes.v02.serviceDeclarations must cover envReuseServices" + ); assert.equal(catalog.environment, "v02", "catalog.environment must be v02"); assert.equal(catalog.profile, "v02", "catalog.profile must be v02"); assert.equal(catalog.namespace, "hwlab-v02", "catalog.namespace must be hwlab-v02"); @@ -600,7 +594,7 @@ function assertV02Contracts(catalog, deployManifest) { assert.deepEqual(catalog.allowedProfiles, ["v02"], "catalog.allowedProfiles must only allow v02"); assert.deepEqual(catalog.forbiddenProfiles, ["dev", "prod"], "catalog.forbiddenProfiles must forbid dev/prod"); const catalogIds = (catalog.services ?? []).map((service) => service.serviceId); - assert.deepEqual(catalogIds, serviceIdsForLane("v02"), "v02 catalog services must match retained service IDs"); + assert.deepEqual(catalogIds, v02ServiceIdsFromDeploy(deployManifest), "v02 catalog services must match deploy.lanes.v02.envReuseServices"); for (const service of catalog.services ?? []) { assert.equal(service.profile, "v02", `${service.serviceId}.profile must be v02`); assert.equal(service.namespace, "hwlab-v02", `${service.serviceId}.namespace must be hwlab-v02`); @@ -642,7 +636,7 @@ function artifactCatalogSkeleton({ args, deployManifest, commitId }) { }, allowedProfiles: [artifactEnvironment(args)], forbiddenProfiles: args.lane === "v02" ? ["dev", "prod"] : ["prod"], - services: serviceIdsForLane(args.lane).map((serviceId) => artifactCatalogSkeletonService({ args, deployManifest, serviceId, tag, commitId })) + services: serviceIdsForArtifactLane(args, deployManifest).map((serviceId) => artifactCatalogSkeletonService({ args, deployManifest, serviceId, tag, commitId })) }; } @@ -688,20 +682,55 @@ function artifactCatalogSkeletonService({ args, deployManifest, serviceId, tag, function envReuseServiceIdsForLane(deployManifest, lane) { if (lane !== "v02") return new Set(); const configured = deployManifest?.lanes?.v02?.envReuseServices; - const allowed = new Set(serviceIdsForLane(lane)); - return new Set((Array.isArray(configured) ? configured : []).filter((serviceId) => allowed.has(serviceId))); + return new Set(normalizeServiceIdList(configured)); +} + +function serviceIdsForArtifactLane(args, deployManifest) { + if (args.lane !== "v02") return [...SERVICE_IDS]; + return v02ServiceIdsFromDeploy(deployManifest); +} + +function applyDeployDefaultServices(args, deployManifest) { + if (args.servicesExplicit) return; + args.services = serviceIdsForArtifactLane(args, deployManifest); +} + +function v02ServiceIdsFromDeploy(deployManifest) { + const serviceIds = normalizeServiceIdList(deployManifest?.lanes?.v02?.envReuseServices); + assert.ok(serviceIds.length > 0, "deploy.lanes.v02.envReuseServices must not be empty"); + return serviceIds; +} + +function v02ServiceIdsFromDeclarations(deployManifest) { + return normalizeServiceIdList(Object.keys(deployManifest?.lanes?.v02?.serviceDeclarations ?? {})); +} + +function normalizeServiceIdList(value) { + return uniquePreserveOrder((Array.isArray(value) ? value : []).map((item) => String(item ?? "").trim()).filter(Boolean)); +} + +function uniquePreserveOrder(values) { + const seen = new Set(); + const result = []; + for (const value of values) { + if (seen.has(value)) continue; + seen.add(value); + result.push(value); + } + return result; } function bootShForService(deployManifest, serviceId) { - const value = deployManifest?.lanes?.v02?.bootScripts?.[serviceId] || `deploy/runtime/boot/${serviceId}.sh`; + const value = deployManifest?.lanes?.v02?.bootScripts?.[serviceId]; + assert.ok(value, `deploy.lanes.v02.bootScripts.${serviceId} is required`); const text = String(value).replaceAll("\\", "/").replace(/^\.\//u, ""); assert.ok(text && !text.startsWith("/") && !text.split("/").includes(".."), `${serviceId} boot script must be repo-relative`); return text; } -function normalizeCatalogForLane(catalog, lane) { +function normalizeCatalogForLane(catalog, lane, deployManifest) { if (!catalog || lane !== "v02") return catalog; - const allowed = new Set(serviceIdsForLane(lane)); + const allowed = new Set(v02ServiceIdsFromDeploy(deployManifest)); return { ...catalog, services: (catalog.services ?? []).filter((service) => allowed.has(service.serviceId)) @@ -826,28 +855,69 @@ function dockerfile(baseImage, port, labels = []) { ].join("\n"); } -function envReuseDockerfile(baseImage, labels = []) { - const osPackageScript = "set -eu; apt-get update; apt-get install -y --no-install-recommends ca-certificates git; rm -rf /var/lib/apt/lists/*"; - const dependencyScript = "set -eu; if [ -f package-lock.json ]; then npm ci --omit=dev --include=optional --ignore-scripts; else npm install --omit=dev --include=optional --ignore-scripts; fi; if command -v bun >/dev/null 2>&1; then bun_path=\"$(command -v bun)\"; if [ \"$bun_path\" != \"/usr/local/bin/bun\" ]; then ln -sf \"$bun_path\" /usr/local/bin/bun; fi; fi; if [ ! -x /usr/local/bin/bun ]; then bun_pkg=\"\"; case \"$(uname -m)\" in x86_64|amd64) bun_pkg=\"@oven/bun-linux-x64@1.3.13\" ;; aarch64|arm64) bun_pkg=\"@oven/bun-linux-aarch64@1.3.13\" ;; *) echo \"unsupported bun architecture: $(uname -m)\" >&2; false ;; esac; npm install --no-save --omit=dev --include=optional --ignore-scripts \"$bun_pkg\"; bun_bin=\"node_modules/${bun_pkg%@*}/bin/bun\"; test -x \"$bun_bin\"; ln -sf \"/opt/hwlab-env/$bun_bin\" /usr/local/bin/bun; fi"; - const readinessScript = "set -eu; command -v node >/dev/null; command -v npm >/dev/null; command -v git >/dev/null; command -v sh >/dev/null; test -d /opt/hwlab-env/node_modules; test -x /usr/local/bin/bun; /usr/local/bin/bun --version >/tmp/hwlab-env-bun-version.txt; printf '%s\\n' '#!/usr/bin/env sh' 'exec node /workspace/hwlab-boot/repo/scripts/run-bun.mjs /workspace/hwlab-boot/repo/tools/hwpod-cli.ts \"$@\"' > /usr/local/bin/hwpod; chmod 755 /usr/local/bin/hwpod; printf '%s\\n' '#!/usr/bin/env sh' 'exec node /workspace/hwlab-boot/repo/scripts/run-bun.mjs /workspace/hwlab-boot/repo/tools/hwpod-ctl.ts \"$@\"' > /usr/local/bin/hwpod-ctl; chmod 755 /usr/local/bin/hwpod-ctl; printf '%s\\n' '#!/usr/bin/env sh' 'exec node /workspace/hwlab-boot/repo/scripts/run-bun.mjs /workspace/hwlab-boot/repo/tools/hwpod-compiler-cli.ts \"$@\"' > /usr/local/bin/hwpod-compiler; chmod 755 /usr/local/bin/hwpod-compiler"; +function envReuseDockerfile(baseImage, labels = [], recipe) { + const normalizedRecipe = normalizeEnvReuseRecipe(recipe); + const osPackageScript = `set -eu; apt-get update; apt-get install -y --no-install-recommends ${normalizedRecipe.osPackages.map(shellQuote).join(" ")}; rm -rf /var/lib/apt/lists/*`; + const dependencyScript = `set -eu; if [ -f package-lock.json ]; then npm ci --omit=dev --include=optional --ignore-scripts; else npm install --omit=dev --include=optional --ignore-scripts; fi; if command -v bun >/dev/null 2>&1; then bun_path=\"$(command -v bun)\"; if [ \"$bun_path\" != \"/usr/local/bin/bun\" ]; then ln -sf \"$bun_path\" /usr/local/bin/bun; fi; fi; if [ ! -x /usr/local/bin/bun ]; then bun_pkg=\"\"; case \"$(uname -m)\" in x86_64|amd64) bun_pkg=\"@oven/bun-linux-x64@${normalizedRecipe.bunVersion}\" ;; aarch64|arm64) bun_pkg=\"@oven/bun-linux-aarch64@${normalizedRecipe.bunVersion}\" ;; *) echo \"unsupported bun architecture: $(uname -m)\" >&2; false ;; esac; npm install --no-save --omit=dev --include=optional --ignore-scripts \"$bun_pkg\"; bun_bin=\"node_modules/\${bun_pkg%@*}/bin/bun\"; test -x \"$bun_bin\"; ln -sf \"${normalizedRecipe.runtimeNodeModulesPath}/$bun_bin\" /usr/local/bin/bun; fi`; + const aliasScript = normalizedRecipe.hwpodAliases.map((alias) => `printf '%s\\n' '#!/usr/bin/env sh' 'exec node /workspace/hwlab-boot/repo/scripts/run-bun.mjs /workspace/hwlab-boot/repo/${alias.script} \"$@\"' > /usr/local/bin/${alias.command}; chmod 755 /usr/local/bin/${alias.command}`).join("; "); + const readinessScript = `set -eu; command -v node >/dev/null; command -v npm >/dev/null; command -v git >/dev/null; command -v sh >/dev/null; test -d ${shellQuote(normalizedRecipe.runtimeNodeModulesPath)}; test -x /usr/local/bin/bun; /usr/local/bin/bun --version >/tmp/hwlab-env-bun-version.txt; ${aliasScript}`; return [ `FROM ${baseImage}`, - "WORKDIR /opt/hwlab-env", + `WORKDIR ${normalizedRecipe.runtimeNodeModulesPath.replace(/\/node_modules$/u, "")}`, ...labels.map(([name, value]) => `LABEL ${name}=${dockerfileQuote(value)}`), "COPY package.json ./package.json", "COPY package-lock.json* ./", `RUN ${osPackageScript}`, `RUN ${dependencyScript}`, - "COPY deploy/runtime/launcher/hwlab-env-reuse-launcher.ts /usr/local/bin/hwlab-env-reuse-launcher.ts", + `COPY ${normalizedRecipe.launcherPath} /usr/local/bin/hwlab-env-reuse-launcher.ts`, `RUN ${readinessScript}`, "ENV HWLAB_RUNTIME_MODE=env-reuse-git-mirror-checkout", - "ENV HWLAB_RUNTIME_NODE_MODULES=/opt/hwlab-env/node_modules", + `ENV HWLAB_RUNTIME_NODE_MODULES=${normalizedRecipe.runtimeNodeModulesPath}`, "ENV PATH=/usr/local/bin:$PATH", "CMD [\"/usr/local/bin/bun\", \"/usr/local/bin/hwlab-env-reuse-launcher.ts\"]", "" ].join("\n"); } +function normalizeEnvReuseRecipe(recipe) { + assert.ok(recipe && typeof recipe === "object", "envRecipe must be provided by deploy.lanes.v02.envRecipe"); + const normalized = { + osPackages: normalizeStringList(recipe.osPackages), + bunVersion: String(recipe.bunVersion ?? "").trim(), + launcherPath: normalizeRepoPath(recipe.launcherPath), + runtimeNodeModulesPath: normalizeAbsolutePath(recipe.runtimeNodeModulesPath), + hwpodAliases: normalizeHwpodAliases(recipe.hwpodAliases) + }; + assert.ok(normalized.osPackages.length > 0, "envRecipe.osPackages must not be empty"); + assert.match(normalized.bunVersion, /^\d+\.\d+\.\d+(?:[-+][0-9A-Za-z.-]+)?$/u, "envRecipe.bunVersion must be a semver-like version"); + assert.ok(normalized.launcherPath && !normalized.launcherPath.split("/").includes(".."), "envRecipe.launcherPath must be repo-relative"); + assert.ok(normalized.runtimeNodeModulesPath, "envRecipe.runtimeNodeModulesPath must be absolute"); + assert.ok(normalized.hwpodAliases.length > 0, "envRecipe.hwpodAliases must not be empty"); + return normalized; +} + +function normalizeStringList(value) { + const source = Array.isArray(value) ? value : []; + return source.map((item) => String(item ?? "").trim()).filter(Boolean); +} + +function normalizeRepoPath(value) { + return String(value ?? "").replaceAll("\\", "/").replace(/^\.\//u, "").replace(/^\/+/, "").trim(); +} + +function normalizeAbsolutePath(value) { + const text = String(value ?? "").trim(); + return text.startsWith("/") && !text.includes("..") ? text : null; +} + +function normalizeHwpodAliases(value) { + const source = Array.isArray(value) ? value : []; + return source.map((item) => ({ + command: String(item?.command ?? "").trim(), + script: normalizeRepoPath(item?.script) + })).filter((item) => item.command && item.script); +} + function imageRef(registryPrefix, serviceId, tag) { return `${registryPrefix}/${serviceId}:${tag}`; } @@ -1071,7 +1141,7 @@ async function buildEnvReuseServiceWithBuildkit({ args, service, ref, buildCreat const buildkitAddrArgs = args.buildkitAddr ? ["--addr", args.buildkitAddr] : []; const startedAt = Date.now(); try { - await writeFile(dockerfilePath, envReuseDockerfile(args.baseImage, labels)); + await writeFile(dockerfilePath, envReuseDockerfile(args.baseImage, labels, service.envReuseRecipe)); const argsList = [ ...buildkitAddrArgs, "build", @@ -2076,14 +2146,15 @@ async function main() { args.baseImage = baseImagePreflight.localTag; } const [deployManifest, commitId, remoteUrl] = await Promise.all([ - readJson(args.deployPath), + readConfig(args.deployPath), gitValue(["rev-parse", "HEAD"]), gitValue(["remote", "get-url", "origin"]).catch(() => "git@github.com:pikasTech/HWLAB.git") ]); - let catalog = await readJsonIfPresent(args.catalogPath, null); + applyDeployDefaultServices(args, deployManifest); + let catalog = await readConfigIfPresent(args.catalogPath, null); const catalogWasMissing = !catalog; if (!catalog && args.lane === "v02") catalog = artifactCatalogSkeleton({ args, deployManifest, commitId }); - catalog = normalizeCatalogForLane(catalog, args.lane); + catalog = normalizeCatalogForLane(catalog, args.lane, deployManifest); assert.ok(catalog, `${args.catalogPath} is required for ${args.lane} artifact publish`); const shortCommit = imageTagForCommit(args, commitId); let effectiveCatalogPath = args.catalogPath; @@ -2098,11 +2169,11 @@ async function main() { repoRoot, lane: args.lane, targetRef: "HEAD", - deployJsonPath: args.deployPath, + deployConfigPath: args.deployPath, artifactCatalogPath: effectiveCatalogPath, registryPrefix: args.registryPrefix, baseImage: args.baseImage ?? undefined, - services: args.services + ...(args.servicesExplicit ? { services: args.services } : {}) }); const services = enrichServicesWithCiPlan(await resolveServices(args.services, catalog), ciPlan); const catalogByServiceId = new Map((catalog?.services ?? []).map((service) => [service.serviceId, service])); diff --git a/scripts/deploy-desired-state-plan.test.mjs b/scripts/deploy-desired-state-plan.test.mjs index be85e2c1..572a36d3 100644 --- a/scripts/deploy-desired-state-plan.test.mjs +++ b/scripts/deploy-desired-state-plan.test.mjs @@ -7,6 +7,7 @@ import test from "node:test"; import { promisify } from "node:util"; import { buildDesiredStatePlan } from "./src/deploy-desired-state-plan.mjs"; +import { writeStructuredFile } from "./src/structured-config.mjs"; const execFileAsync = promisify(execFile); @@ -316,7 +317,7 @@ async function makeFixture({ } ] }; - await writeFile(path.join(root, "deploy/deploy.json"), `${JSON.stringify(deploy, null, 2)}\n`); + await writeStructuredFile(root, "deploy/deploy.yaml", deploy); await writeFile(path.join(root, "deploy/artifact-catalog.dev.json"), `${JSON.stringify(catalog, null, 2)}\n`); await writeFile(path.join(root, "deploy/k8s/base/workloads.yaml"), `${JSON.stringify(workloads, null, 2)}\n`); return root; @@ -463,7 +464,7 @@ test("blocks when cloud-api DB SSL mode drifts back to require", async () => { assert.ok(plan.diagnostics.some((diagnostic) => diagnostic.code === "cloud_api_db_contract_mismatch")); }); -test("blocks when generated runtime identity leaks into deploy.json", async () => { +test("blocks when generated runtime identity leaks into deploy.yaml", async () => { const repoRoot = await makeFixture({ serviceId: "hwlab-cloud-web", deployEnvMirrors: true, @@ -474,7 +475,7 @@ test("blocks when generated runtime identity leaks into deploy.json", async () = assert.equal(plan.summary.blockers, 3); assert.ok(plan.diagnostics.some((diagnostic) => diagnostic.code === "generated_artifact_identity" && - diagnostic.path === "deploy/deploy.json.services.hwlab-cloud-web.env.HWLAB_COMMIT_ID" + diagnostic.path === "deploy/deploy.yaml.services.hwlab-cloud-web.env.HWLAB_COMMIT_ID" )); assert.ok(plan.diagnostics.some((diagnostic) => diagnostic.code === "generated_artifact_identity" && @@ -541,7 +542,7 @@ test("target check accepts accepted first-parent main on the checked-out merge c await git(repoRoot, ["checkout", "-b", "desired-state-refresh"]); for (const relativePath of [ - "deploy/deploy.json", + "deploy/deploy.yaml", "deploy/artifact-catalog.dev.json", "deploy/k8s/base/workloads.yaml" ]) { @@ -600,7 +601,7 @@ test("ignores generated workload mirror drift because render owns runtime identi assert.deepEqual(plan.diagnostics, []); }); -test("blocks on skills commit mirror leak in deploy.json", async () => { +test("blocks on skills commit mirror leak in deploy.yaml", async () => { const repoRoot = await makeFixture({ deployEnvMirrors: true, deploySkillsCommitId: "badcafe" }); const plan = await buildDesiredStatePlan({ repoRoot }); assert.equal(plan.status, "blocked"); diff --git a/scripts/dev-m3-hardware-loop-smoke.mjs b/scripts/dev-m3-hardware-loop-smoke.mjs index c26ed29e..9fad2b57 100644 --- a/scripts/dev-m3-hardware-loop-smoke.mjs +++ b/scripts/dev-m3-hardware-loop-smoke.mjs @@ -12,6 +12,7 @@ import { fileURLToPath, pathToFileURL } from "node:url"; import { activeReportLifecycle } from "../internal/dev-report-lifecycle.mjs"; import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./src/report-paths.mjs"; +import { readStructuredFile } from "./src/structured-config.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const defaultReportPath = tempReportPath("dev-m3-hardware-loop.json"); @@ -382,9 +383,9 @@ function serviceByName(services) { async function collectReadOnlySupplementalEvidence() { const [deploy, workloads, services] = await Promise.all([ - readFile(path.join(repoRoot, "deploy/deploy.json"), "utf8").then(JSON.parse), - readFile(path.join(repoRoot, deployWorkloadsPath), "utf8").then(JSON.parse), - readFile(path.join(repoRoot, deployServicesPath), "utf8").then(JSON.parse) + readStructuredFile(repoRoot, "deploy/deploy.yaml"), + readStructuredFile(repoRoot, deployWorkloadsPath), + readStructuredFile(repoRoot, deployServicesPath) ]); const targets = ["hwlab-box-simu", "hwlab-gateway-simu", "hwlab-patch-panel"]; const observed = listItems(workloads) @@ -468,7 +469,7 @@ async function collectReadOnlySupplementalEvidence() { id: "deploy-skeleton-m3-cardinality", status: manifestReadyForM3 ? "manifest-ready" : "gap", blockerClass: manifestReadyForM3 ? null : "runtime_blocker", - source: ["deploy/deploy.json", deployWorkloadsPath, deployServicesPath], + source: ["deploy/deploy.yaml", deployWorkloadsPath, deployServicesPath], summary: manifestReadyForM3 ? "Checked-in DEV deploy skeleton declares distinct indexed M3 identities for two box simulators, two gateway simulators, one patch panel, and the M3 DO1 -> DI1 endpoint map." : "Read-only static comparison found the checked-in DEV deploy skeleton does not declare distinct indexed M3 simulator identities and patch-panel M3 wiring; proving live DEV still requires a separate deploy/runtime task or authorized runtime observation.", diff --git a/scripts/g14-ci-plan.mjs b/scripts/g14-ci-plan.mjs index 79694b0b..0044da43 100644 --- a/scripts/g14-ci-plan.mjs +++ b/scripts/g14-ci-plan.mjs @@ -17,7 +17,7 @@ function parseArgs(argv) { const arg = argv[index]; if (arg === "--base-ref") parsed.baseRef = readOption(argv, ++index, arg); else if (arg === "--target-ref") parsed.targetRef = readOption(argv, ++index, arg); - else if (arg === "--deploy-json") parsed.deployJsonPath = readOption(argv, ++index, arg); + else if (arg === "--deploy-config") parsed.deployConfigPath = readOption(argv, ++index, arg); else if (arg === "--artifact-catalog") parsed.artifactCatalogPath = readOption(argv, ++index, arg); else if (arg === "--lane") parsed.lane = readOption(argv, ++index, arg); else if (arg === "--registry-prefix") parsed.registryPrefix = readOption(argv, ++index, arg); @@ -38,13 +38,13 @@ function readOption(argv, index, name) { function usage() { return [ - "usage: node scripts/g14-ci-plan.mjs [--base-ref REF] [--target-ref REF] [--pretty]", + "usage: node scripts/g14-ci-plan.mjs [--lane v02] [--base-ref REF] [--target-ref REF] [--pretty]", "", "Plan G14 monorepo image builds without mutating CI/CD state.", - "The planner reads current deploy/deploy.json and the built-in G14 CI component model by default.", + "For v02 env reuse, service component boundaries and env recipe must come from deploy/deploy.yaml.", "", "examples:", - " node scripts/g14-ci-plan.mjs --base-ref HEAD~1 --target-ref HEAD --pretty", + " node scripts/g14-ci-plan.mjs --lane v02 --base-ref HEAD~1 --target-ref HEAD --pretty", " node scripts/g14-ci-plan.mjs --services hwlab-cloud-api,hwlab-cloud-web --pretty" ].join("\n"); } diff --git a/scripts/g14-ci-plan.test.mjs b/scripts/g14-ci-plan.test.mjs index f220fca4..8965dbda 100644 --- a/scripts/g14-ci-plan.test.mjs +++ b/scripts/g14-ci-plan.test.mjs @@ -10,9 +10,11 @@ import { classifyGlobalChange, componentModelsForServices, createG14CiPlan, + envReuseRecipeForLane, matchingPaths, packageRuntimeFieldsChanged } from "./src/g14-ci-plan-lib.mjs"; +import { readStructuredFile, writeStructuredFile } from "./src/structured-config.mjs"; const execFileAsync = promisify(execFile); @@ -30,12 +32,12 @@ test("G14 CI planner maps bridge changes to cloud-api only", async () => { await git(repo, ["add", "."]); await git(repo, ["commit", "-m", "change bridge"]); - const plan = await createG14CiPlan({ repoRoot: repo, baseRef: "HEAD~1", targetRef: "HEAD" }); + const plan = await planV02CoreServices(repo, { baseRef: "HEAD~1", targetRef: "HEAD" }); assert.deepEqual(plan.affectedServices, ["hwlab-cloud-api"]); assert.equal(plan.reusedServices.includes("hwlab-cloud-web"), true); const cloudApi = plan.services.find((service) => service.serviceId === "hwlab-cloud-api"); assert.equal(cloudApi.affected, true); - assert.deepEqual(cloudApi.reason, ["component-path-changed"]); + assert.deepEqual(cloudApi.reason, ["code-input-changed", "component-path-changed"]); }); test("G14 CI planner treats docs-only changes as no image build", async () => { @@ -45,7 +47,7 @@ test("G14 CI planner treats docs-only changes as no image build", async () => { await git(repo, ["add", "."]); await git(repo, ["commit", "-m", "change docs"]); - const plan = await createG14CiPlan({ repoRoot: repo, baseRef: "HEAD~1", targetRef: "HEAD" }); + const plan = await planV02CoreServices(repo, { baseRef: "HEAD~1", targetRef: "HEAD" }); assert.deepEqual(plan.affectedServices, []); assert.equal(plan.imageBuildRequired, false); assert.equal(plan.changedPathSummary.docsOnly, true); @@ -58,10 +60,11 @@ test("planner rebuilds when catalog digest is missing instead of blocking reuse" await git(repo, ["add", "."]); await git(repo, ["commit", "-m", "change docs"]); - const plan = await createG14CiPlan({ repoRoot: repo, baseRef: "HEAD~1", targetRef: "HEAD" }); + const plan = await planV02CoreServices(repo, { baseRef: "HEAD~1", targetRef: "HEAD" }); assert.deepEqual(plan.affectedServices, ["hwlab-cloud-api", "hwlab-cloud-web"]); assert.equal(plan.imageBuildRequired, true); - assert.equal(plan.services[0].reason.includes("catalog-digest-missing"), true); + assert.equal(plan.services.every((service) => service.envChanged === true), true); + assert.equal(plan.services.every((service) => service.reuse.status === "not-reused"), true); }); test("v02 planner rolls cloud-api service code changes without rebuilding service image", async () => { @@ -140,8 +143,9 @@ test("artifact reuse paths preserve catalog provenance instead of current planne assert.doesNotMatch(gitopsRender, /"component-input-hash": planned\.componentInputHash \|\| service\.componentInputHash \|\| ""/u); }); -test("component model uses built-in service paths", () => { - const models = componentModelsForServices(["hwlab-cloud-api"]); +test("component model uses service paths declared in deploy json", () => { + const deploy = createDeployFixture(); + const models = componentModelsForServices(["hwlab-cloud-api"], deploy.lanes.v02); assert.deepEqual(models[0].componentPaths, [ "cmd/hwlab-cloud-api/", "cmd/hwlab-codex-api-responses-forwarder/", @@ -159,6 +163,50 @@ test("component model uses built-in service paths", () => { assert.deepEqual(matchingPaths(["cmd/hwlab-cloud-api/main.ts"], models[0].componentPaths), ["cmd/hwlab-cloud-api/main.ts"]); }); +test("v02 planner rejects missing service declarations instead of falling back to an internal model", async () => { + const repo = await createFixtureRepo(); + const deployPath = path.join(repo, "deploy/deploy.yaml"); + const deploy = await readStructuredFile(repo, deployPath); + delete deploy.lanes.v02.serviceDeclarations; + await writeStructuredFile(repo, deployPath, deploy); + await git(repo, ["add", "deploy/deploy.yaml"]); + await git(repo, ["commit", "-m", "remove v02 declarations"]); + + await assert.rejects( + () => createG14CiPlan({ + repoRoot: repo, + lane: "v02", + baseRef: "HEAD~1", + targetRef: "HEAD", + artifactCatalogPath: "deploy/artifact-catalog.v02.json", + services: ["hwlab-cloud-api"] + }), + /deploy\.lanes\.v02\.serviceDeclarations is required/u + ); +}); + +test("v02 planner rejects missing boot script declarations instead of guessing paths", async () => { + const repo = await createFixtureRepo(); + const deployPath = path.join(repo, "deploy/deploy.yaml"); + const deploy = await readStructuredFile(repo, deployPath); + delete deploy.lanes.v02.bootScripts["hwlab-cloud-api"]; + await writeStructuredFile(repo, deployPath, deploy); + await git(repo, ["add", "deploy/deploy.yaml"]); + await git(repo, ["commit", "-m", "remove v02 cloud-api boot script"]); + + await assert.rejects( + () => createG14CiPlan({ + repoRoot: repo, + lane: "v02", + baseRef: "HEAD~1", + targetRef: "HEAD", + artifactCatalogPath: "deploy/artifact-catalog.v02.json", + services: ["hwlab-cloud-api"] + }), + /deploy\.lanes\.v02\.bootScripts\.hwlab-cloud-api is required/u + ); +}); + test("v02 planner keeps hwpod-spec changes out of runtime service env reuse", async () => { const repo = await createFixtureRepo(); await writeFile(path.join(repo, ".hwlab/hwpod-spec.yaml"), "kind: Hwpod\nmetadata:\n name: changed-preinstalled\nspec:\n nodeBinding:\n nodeId: node-changed\n workspace:\n path: F:\\\\Work\\\\D601-HWLAB\n targetDevice:\n board: D601-F103-V2\n debugProbe:\n type: daplink\n ioProbe:\n uart:\n port: COM9\n", "utf8"); @@ -232,11 +280,11 @@ test("planner scopes cloud-web runtime changes to cloud-web", async () => { await git(repo, ["add", "."]); await git(repo, ["commit", "-m", "change cloud web runtime"]); - const plan = await createG14CiPlan({ repoRoot: repo, baseRef: "HEAD~1", targetRef: "HEAD" }); + const plan = await planV02CoreServices(repo, { baseRef: "HEAD~1", targetRef: "HEAD" }); assert.deepEqual(plan.affectedServices, ["hwlab-cloud-web"]); assert.equal(plan.services.some((service) => service.serviceId === "hwlab-agent-worker"), false); const cloudWeb = plan.services.find((service) => service.serviceId === "hwlab-cloud-web"); - assert.deepEqual(cloudWeb.reason, ["component-path-changed"]); + assert.deepEqual(cloudWeb.reason, ["code-input-changed", "component-path-changed"]); }); test("v02 planner treats shared trace renderer changes as cloud-web code rollout inputs", async () => { @@ -312,7 +360,7 @@ test("planner ignores package script cleanup for runtime images", async () => { await git(repo, ["commit", "-m", "change package scripts"]); assert.equal(await packageRuntimeFieldsChanged(repo, "HEAD~1", "HEAD"), false); - const plan = await createG14CiPlan({ repoRoot: repo, baseRef: "HEAD~1", targetRef: "HEAD" }); + const plan = await planV02CoreServices(repo, { baseRef: "HEAD~1", targetRef: "HEAD" }); assert.deepEqual(plan.affectedServices, []); assert.deepEqual(plan.buildServices, []); assert.equal(plan.imageBuildRequired, false); @@ -340,24 +388,11 @@ test("planner treats dependency changes as runtime image inputs", async () => { await git(repo, ["commit", "-m", "change dependencies"]); assert.equal(await packageRuntimeFieldsChanged(repo, "HEAD~1", "HEAD"), true); - const plan = await createG14CiPlan({ repoRoot: repo, baseRef: "HEAD~1", targetRef: "HEAD" }); + const plan = await planV02CoreServices(repo, { baseRef: "HEAD~1", targetRef: "HEAD" }); assert.deepEqual(plan.affectedServices, ["hwlab-cloud-api", "hwlab-cloud-web"]); assert.equal(plan.imageBuildRequired, true); }); -test("planner remains compatible with deploy.k3s.serviceMappings shape", async () => { - const repo = await createFixtureRepo({ deployServices: false, k3sServiceMappings: true }); - await writeFile(path.join(repo, "cmd/hwlab-deepseek-responses-bridge/main.ts"), "console.log('bridge v2');\n"); - await git(repo, ["add", "."]); - await git(repo, ["commit", "-m", "change bridge"]); - - const plan = await createG14CiPlan({ repoRoot: repo, baseRef: "HEAD~1", targetRef: "HEAD" }); - assert.equal(plan.compatibility.currentRenderCompatible, true); - assert.equal(plan.compatibility.serviceIdSource, "deploy.k3s.serviceMappings"); - assert.deepEqual(plan.affectedServices, ["hwlab-cloud-api"]); - assert.equal(plan.inputs.serviceCount, 2); -}); - test("global change classifier distinguishes gitops-only and test-only", () => { assert.equal(classifyGlobalChange(["deploy/gitops/g14/runtime-dev/workloads.yaml"]).gitopsOnly, true); assert.equal(classifyGlobalChange(["cmd/hwlab-cloud-api/runtime-options.test.ts"]).testOnly, true); @@ -556,10 +591,27 @@ test("v02 planner scopes hwpod skill bundle changes to the skills service", asyn assert.deepEqual(skillsService.reason, ["code-input-changed", "component-path-changed"]); }); +test("v02 planner reads env reuse declarations and recipe from deploy json", async () => { + const deploy = createDeployFixture(); + deploy.lanes.v02.serviceDeclarations["hwlab-cloud-web"].componentPaths.push("custom/web-plugin/"); + deploy.lanes.v02.envRecipe.bunVersion = "1.3.14"; + deploy.lanes.v02.envRecipe.additionalEnvPaths.push("custom/env-tool.mjs"); + + const models = componentModelsForServices(["hwlab-cloud-web"], deploy.lanes.v02); + assert.equal(models[0].runtimeKind, "cloud-web"); + assert.equal(models[0].entrypoint, "web/hwlab-cloud-web/index.html"); + assert.equal(models[0].componentPaths.includes("custom/web-plugin/"), true); + assert.equal(models[0].buildSystemPaths.includes("custom/env-tool.mjs"), true); + + const recipe = envReuseRecipeForLane(deploy, "v02"); + assert.equal(recipe.bunVersion, "1.3.14"); + assert.equal(recipe.launcherInputPaths.includes("deploy/runtime/launcher/hwlab-env-reuse-launcher.ts"), true); + assert.equal(recipe.launcherInputPaths.includes("custom/env-tool.mjs"), true); +}); + async function createFixtureRepo(options = {}) { - const deployServices = options.deployServices !== false; - const k3sServiceMappings = options.k3sServiceMappings === true; const serviceIds = options.serviceIds ?? ["hwlab-cloud-api", "hwlab-cloud-web"]; + const laneServiceIds = options.laneServiceIds ?? V02_RUNTIME_SERVICE_IDS; const repo = await mkdtemp(path.join(os.tmpdir(), "hwlab-g14-ci-plan-")); await mkdir(path.join(repo, "cmd/hwlab-cloud-api"), { recursive: true }); await mkdir(path.join(repo, "cmd/hwlab-codex-api-responses-forwarder"), { recursive: true }); @@ -578,7 +630,7 @@ async function createFixtureRepo(options = {}) { await mkdir(path.join(repo, "deploy/runtime/boot"), { recursive: true }); await mkdir(path.join(repo, "deploy/runtime/launcher"), { recursive: true }); await mkdir(path.join(repo, "deploy"), { recursive: true }); - await writeFile(path.join(repo, "deploy/deploy.json"), JSON.stringify(createDeployFixture({ deployServices, k3sServiceMappings, serviceIds }), null, 2)); + await writeStructuredFile(repo, "deploy/deploy.yaml", createDeployFixture({ serviceIds: laneServiceIds })); const catalogMode = options.catalog ?? "ready"; await writeFile(path.join(repo, "package.json"), JSON.stringify({ type: "module" }, null, 2)); await writeFile(path.join(repo, "package-lock.json"), JSON.stringify({ lockfileVersion: 3 }, null, 2)); @@ -642,43 +694,136 @@ async function refreshFixtureCatalogForHead(repo) { const services = ["hwlab-cloud-api", "hwlab-cloud-web"]; const plan = await createG14CiPlan({ repoRoot: repo, + lane: "v02", baseRef: "HEAD", targetRef: "HEAD", artifactCatalogPath: "deploy/nonexistent-artifact-catalog.json", services }); - const componentInputHashes = Object.fromEntries(plan.services.map((service) => [service.serviceId, service.componentInputHash])); - const catalog = createCatalogFixture("ready", { sourceCommitId, componentInputHashes }); + const catalog = createCatalogFixture("ready", catalogOptionsFromPlan(plan, { sourceCommitId, serviceIds: services })); await writeFile(path.join(repo, "deploy/artifact-catalog.dev.json"), JSON.stringify(catalog, null, 2)); await writeFile(path.join(repo, "deploy/artifact-catalog.v02.json"), JSON.stringify(catalog, null, 2)); await git(repo, ["add", "deploy/artifact-catalog.dev.json", "deploy/artifact-catalog.v02.json"]); await git(repo, ["commit", "-m", "refresh fixture artifact catalogs"]); } -function createDeployFixture({ deployServices, k3sServiceMappings, serviceIds = ["hwlab-cloud-api", "hwlab-cloud-web"] }) { +function planV02CoreServices(repo, options) { + return createG14CiPlan({ + repoRoot: repo, + lane: "v02", + artifactCatalogPath: "deploy/artifact-catalog.v02.json", + services: ["hwlab-cloud-api", "hwlab-cloud-web"], + ...options + }); +} + +function createDeployFixture({ serviceIds = V02_RUNTIME_SERVICE_IDS } = {}) { const deploy = { lanes: { v02: { sourceRepo: "git@github.com:pikasTech/HWLAB.git", - envReuseServices: [...V02_RUNTIME_SERVICE_IDS], + envReuseServices: [...serviceIds], bootScripts: { "hwlab-cloud-api": "deploy/runtime/boot/hwlab-cloud-api.sh", "hwlab-cloud-web": "deploy/runtime/boot/hwlab-cloud-web.sh", "hwlab-gateway": "deploy/runtime/boot/hwlab-gateway.sh", "hwlab-edge-proxy": "deploy/runtime/boot/hwlab-edge-proxy.sh", "hwlab-agent-skills": "deploy/runtime/boot/hwlab-agent-skills.sh" + }, + serviceDeclarations: { + "hwlab-cloud-api": { + runtimeKind: "bun-command", + entrypoint: "cmd/hwlab-cloud-api/main.ts", + artifactKind: "bun-command", + healthPath: "/health/live", + healthPort: 6667, + componentPaths: [ + "cmd/hwlab-cloud-api/", + "cmd/hwlab-codex-api-responses-forwarder/", + "cmd/hwlab-deepseek-responses-bridge/", + "deploy/runtime/boot/hwlab-codex-api-responses-forwarder.sh", + "deploy/runtime/boot/hwlab-deepseek-responses-bridge.sh", + "internal/cloud/", + "internal/db/", + "internal/audit/", + "internal/agent/agentrun-dispatch.mjs", + "internal/agent/prompts/", + "skills/hwlab-agent-runtime/" + ], + env: {}, + observable: true + }, + "hwlab-cloud-web": { + runtimeKind: "cloud-web", + entrypoint: "web/hwlab-cloud-web/index.html", + artifactKind: "cloud-web", + healthPath: "/health/live", + healthPort: 8080, + componentPaths: [ + "web/hwlab-cloud-web/", + "internal/dev-entrypoint/cloud-web-runtime.mjs", + "internal/dev-entrypoint/cloud-web-proxy.mjs", + "internal/dev-entrypoint/cloud-web-routes.mjs", + "tools/src/hwlab-cli/trace-renderer.ts" + ], + env: {}, + observable: true + }, + "hwlab-gateway": { + runtimeKind: "bun-command", + entrypoint: "cmd/hwlab-gateway/main.ts", + artifactKind: "bun-command", + healthPath: "/health/live", + healthPort: 7001, + componentPaths: ["cmd/hwlab-gateway/", "internal/sim/"], + env: {}, + observable: false + }, + "hwlab-edge-proxy": { + runtimeKind: "bun-command", + entrypoint: "cmd/hwlab-edge-proxy/main.ts", + artifactKind: "bun-command", + healthPath: "/health", + healthPort: 6667, + componentPaths: ["cmd/hwlab-edge-proxy/", "internal/dev-entrypoint/http.mjs"], + env: {}, + observable: true + }, + "hwlab-agent-skills": { + runtimeKind: "skills-bundle", + entrypoint: "skills/hwlab-agent-runtime/SKILL.md", + artifactKind: "skills-bundle", + healthPath: "/health/live", + healthPort: 7430, + componentPaths: ["skills/"], + env: {}, + observable: true + } + }, + envRecipe: { + osPackages: ["ca-certificates", "git"], + bunVersion: "1.3.13", + launcherPath: "deploy/runtime/launcher/hwlab-env-reuse-launcher.ts", + runtimeNodeModulesPath: "/opt/hwlab-env/node_modules", + additionalEnvPaths: [ + "internal/dev-entrypoint/artifact-runtime.mjs", + "scripts/artifact-publish.mjs", + "scripts/g14-artifact-publish.mjs", + "scripts/src/dev-artifact-services.mjs" + ], + hwpodAliases: [ + { command: "hwpod", script: "tools/hwpod-cli.ts" }, + { command: "hwpod-ctl", script: "tools/hwpod-ctl.ts" }, + { command: "hwpod-compiler", script: "tools/hwpod-compiler-cli.ts" } + ] + }, + bootConfig: { + template: "default", + overrides: {} } } } }; - if (deployServices) { - deploy.services = serviceIds.map((serviceId) => ({ serviceId })); - } - if (k3sServiceMappings) { - deploy.k3s = { - serviceMappings: serviceIds.map((serviceId) => ({ serviceId, name: serviceId })) - }; - } return deploy; } diff --git a/scripts/g14-gitops-render.mjs b/scripts/g14-gitops-render.mjs index b97eedf0..81a64de9 100644 --- a/scripts/g14-gitops-render.mjs +++ b/scripts/g14-gitops-render.mjs @@ -7,6 +7,8 @@ import path from "node:path"; import { promisify } from "node:util"; import { fileURLToPath } from "node:url"; +import { readStructuredFile } from "./src/structured-config.mjs"; + const execFileAsync = promisify(execFile); const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const defaultOutDir = "deploy/gitops/g14"; @@ -37,21 +39,9 @@ const defaultServiceIds = Object.freeze([ "hwlab-edge-proxy", "hwlab-agent-skills" ]); -const v02RuntimeServiceIds = Object.freeze([ - "hwlab-cloud-api", - "hwlab-cloud-web", - "hwlab-gateway", - "hwlab-edge-proxy", - "hwlab-agent-skills" -]); -const v02ObservableServices = Object.freeze([ - { serviceId: "hwlab-cloud-api", port: 6667, healthPath: "/health/live" }, - { serviceId: "hwlab-cloud-web", port: 8080, healthPath: "/health/live" }, - { serviceId: "hwlab-edge-proxy", port: 6667, healthPath: "/health" }, - { serviceId: "hwlab-agent-skills", port: 7430, healthPath: "/health/live" }, +const v02ExtraObservableServices = Object.freeze([ { serviceId: "hwlab-deepseek-proxy", port: 4000, healthPath: "/health/live" } ]); -const v02ObservableServiceIds = new Set(v02ObservableServices.map((item) => item.serviceId)); const v02RemovedServiceIds = new Set([ "hwlab-agent-mgr", "hwlab-agent-worker", @@ -275,7 +265,7 @@ function usage() { return [ "usage: node scripts/g14-gitops-render.mjs [options]", "", - "Render G14-only Kubernetes-native CI/CD manifests from built-in Tekton CI primitives, deploy/deploy.json, and deploy/k8s/*.", + "Render G14-only Kubernetes-native CI/CD manifests from built-in Tekton CI primitives, deploy/deploy.yaml, and deploy/k8s/*.", "", "options:", " --lane LANE g14 or v02; default: g14", @@ -304,7 +294,7 @@ function generatedPath(filePath) { } async function readJson(relativePath) { - return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8")); + return readStructuredFile(repoRoot, relativePath); } async function readJsonIfPresent(relativePath, fallback = null) { @@ -441,7 +431,8 @@ function v02EnvReuseEnabled(catalog, serviceId, envReuseServiceIds = null) { } function bootShForService(deployService, serviceId) { - const value = deployService?.bootSh || deployService?.bootScript || `deploy/runtime/boot/${serviceId}.sh`; + assert.ok(deployService?.bootSh, `deploy.lanes.v02.bootScripts.${serviceId} is required`); + const value = deployService.bootSh; const text = String(value).replaceAll("\\", "/").replace(/^\.\//u, ""); assert.ok(text && !text.startsWith("/") && !text.split("/").includes(".."), `${serviceId} boot script must be repo-relative`); return text; @@ -450,7 +441,7 @@ function bootShForService(deployService, serviceId) { function envReuseServiceIdsForLane(deploy, lane) { if (lane !== "v02") return new Set(); const configured = deploy?.lanes?.v02?.envReuseServices; - const allowed = new Set(serviceIdsForLane(lane)); + const allowed = new Set(serviceIdsForLane(lane, deploy)); return new Set((Array.isArray(configured) ? configured : []).filter((serviceId) => allowed.has(serviceId))); } @@ -575,20 +566,56 @@ function artifactEndpoint(args) { return args.lane === "v02" ? defaultV02RuntimeEndpoint : defaultRuntimeEndpoint; } -function serviceIdsForLane(lane) { - return lane === "v02" ? v02RuntimeServiceIds : defaultServiceIds; +function serviceIdsForLane(lane, deploy = null) { + return lane === "v02" ? v02ServiceIdsFromDeploy(deploy) : defaultServiceIds; } -function serviceIdsForProfile(profile) { - return profile === "v02" ? v02RuntimeServiceIds : defaultServiceIds; +function serviceIdsForProfile(profile, deploy = null) { + return profile === "v02" ? v02ServiceIdsFromDeploy(deploy) : defaultServiceIds; } -function v02ObservableService(serviceId) { - return v02ObservableServices.find((item) => item.serviceId === serviceId) ?? null; +function v02ServiceIdsFromDeploy(deploy) { + const serviceIds = uniquePreserveOrder((Array.isArray(deploy?.lanes?.v02?.envReuseServices) ? deploy.lanes.v02.envReuseServices : []) + .map((item) => String(item ?? "").trim()) + .filter(Boolean)); + assert.ok(serviceIds.length > 0, "deploy.lanes.v02.envReuseServices must not be empty"); + return serviceIds; } -function servicesParamForLane(lane) { - return serviceIdsForLane(lane).join(","); +function v02ObservableServicesForDeploy(deploy) { + const declarations = deploy?.lanes?.v02?.serviceDeclarations ?? {}; + const serviceIds = new Set(v02ServiceIdsFromDeploy(deploy)); + const declared = Object.entries(declarations) + .filter(([serviceId, declaration]) => serviceIds.has(serviceId) && declaration?.observable !== false && Number.isInteger(declaration?.healthPort)) + .map(([serviceId, declaration]) => ({ + serviceId, + port: declaration.healthPort, + healthPath: declaration.healthPath || "/health/live" + })); + return [...declared, ...v02ExtraObservableServices]; +} + +function v02ObservableService(deploy, serviceId) { + return v02ObservableServicesForDeploy(deploy).find((item) => item.serviceId === serviceId) ?? null; +} + +function v02ObservableServiceIdsForDeploy(deploy) { + return new Set(v02ObservableServicesForDeploy(deploy).map((item) => item.serviceId)); +} + +function servicesParamForLane(lane, deploy = null) { + return serviceIdsForLane(lane, deploy).join(","); +} + +function uniquePreserveOrder(values) { + const seen = new Set(); + const result = []; + for (const value of values) { + if (seen.has(value)) continue; + seen.add(value); + result.push(value); + } + return result; } function isV02RemovedServiceId(serviceId) { @@ -616,7 +643,7 @@ function artifactCatalogSkeleton({ args, deploy, source }) { }, allowedProfiles: [environment], forbiddenProfiles: args.lane === "v02" ? ["dev", "prod"] : ["prod"], - services: serviceIdsForLane(args.lane).map((serviceId) => artifactCatalogSkeletonService({ args, deploy, source, serviceId, environment, namespace, tag })) + services: serviceIdsForLane(args.lane, deploy).map((serviceId) => artifactCatalogSkeletonService({ args, deploy, source, serviceId, environment, namespace, tag })) }; } @@ -817,7 +844,7 @@ function removeV02RepoOwnedCodeAgentPodConfig(podSpec) { } function deployServicesForProfile(deploy, profile) { - const allowed = new Set(serviceIdsForProfile(profile)); + const allowed = new Set(serviceIdsForProfile(profile, deploy)); const services = new Map((deploy.services ?? []) .filter((service) => allowed.has(service.serviceId)) .map((service) => [service.serviceId, cloneJson(service)])); @@ -850,8 +877,8 @@ function v02MetricsSidecarAnnotations(metricsSidecarSha256) { return metricsSidecarSha256 ? { "hwlab.pikastech.local/metrics-sidecar-sha256": metricsSidecarSha256 } : {}; } -function v02MetricsSidecarContainer({ serviceId, namespace, gitopsTarget }) { - const service = v02ObservableService(serviceId); +function v02MetricsSidecarContainer({ deploy, serviceId, namespace, gitopsTarget }) { + const service = v02ObservableService(deploy, serviceId); assert.ok(service, `unknown v0.2 observable service ${serviceId}`); const extraMetricsEnv = serviceId === "hwlab-cloud-api" ? [ @@ -904,6 +931,7 @@ function transformWorkloads({ workloads, deploy, catalog, source, sourceBranch = const gitopsTarget = gitopsTargetForProfile(profile); const digestPin = profile === "v02"; const envReuseServiceIds = envReuseServiceIdsForLane(deploy, profile); + const v02ObservableServiceIds = profile === "v02" ? v02ObservableServiceIdsForDeploy(deploy) : new Set(); const stableRuntimeLabels = { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/environment": profileLabel, @@ -948,7 +976,7 @@ function transformWorkloads({ workloads, deploy, catalog, source, sourceBranch = label(item.metadata, v02MetricsLabels(templateServiceId)); label(podTemplate.metadata, v02MetricsLabels(templateServiceId)); annotate(podTemplate.metadata, v02MetricsSidecarAnnotations(metricsSidecarSha256)); - upsertV02MetricsSidecar(podTemplate.spec, { serviceId: templateServiceId, namespace, gitopsTarget }); + upsertV02MetricsSidecar(podTemplate.spec, { deploy, serviceId: templateServiceId, namespace, gitopsTarget }); } if ((item.kind === "Deployment" || item.kind === "StatefulSet") && item.spec?.selector?.matchLabels) { item.spec.selector.matchLabels = { @@ -1097,7 +1125,8 @@ function transformListNamespace(value, namespace, labels, annotations) { return result; } -function transformServices({ services, namespace, labels, annotations, profile = "dev" }) { +function transformServices({ services, namespace, labels, annotations, profile = "dev", deploy = null }) { + const v02ObservableServiceIds = profile === "v02" ? v02ObservableServiceIdsForDeploy(deploy) : new Set(); const result = transformListNamespace(services, namespace, labels, annotations); if (result.kind === "List") { result.items = (result.items ?? []).filter((item) => !(profile === "v02" && isV02RemovedServiceId(serviceIdForWorkload(item, null)))); @@ -1111,13 +1140,13 @@ function transformServices({ services, namespace, labels, annotations, profile = return result; } -function observabilityManifest({ namespace, labels, annotations, metricsSidecarScript }) { +function observabilityManifest({ deploy, namespace, labels, annotations, metricsSidecarScript }) { const baseLabels = { ...labels, "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/monitoring": "enabled" }; - const serviceMonitors = v02ObservableServices.map((service) => ({ + const serviceMonitors = v02ObservableServicesForDeploy(deploy).map((service) => ({ apiVersion: "monitoring.coreos.com/v1", kind: "ServiceMonitor", metadata: { @@ -1463,7 +1492,7 @@ function prepareSourceScript() { " git_timed catalog-fetch 90 git fetch --depth 1 gitops-catalog \"$(params.gitops-branch)\" >/dev/null || true", " if git cat-file -e FETCH_HEAD:\"$catalog_path\" 2>/dev/null; then", " git show FETCH_HEAD:\"$catalog_path\" > /tmp/hwlab-gitops-artifact-catalog.json", - " if node -e 'const fs=require(\"fs\"); const ids=(doc)=>(doc.services||[]).map((s)=>s.serviceId); const selected=(process.argv[3]||\"\").split(\",\").filter(Boolean); const gitops=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const deploy=JSON.parse(fs.readFileSync(process.argv[2],\"utf8\")); const expected=selected.length ? selected : ids(deploy); if (JSON.stringify(ids(gitops)) !== JSON.stringify(expected)) process.exit(42);' /tmp/hwlab-gitops-artifact-catalog.json deploy/deploy.json \"$(params.services)\"; then", + " if node --input-type=module -e 'import fs from \"node:fs\"; import { readStructuredFile } from \"./scripts/src/structured-config.mjs\"; const ids=(doc)=>(doc.services||[]).map((s)=>s.serviceId); const selected=(process.argv[3]||\"\").split(\",\").filter(Boolean); const gitops=JSON.parse(fs.readFileSync(process.argv[1],\"utf8\")); const deploy=await readStructuredFile(process.cwd(),process.argv[2]); const expected=selected.length ? selected : ids(deploy); if (JSON.stringify(ids(gitops)) !== JSON.stringify(expected)) process.exit(42);' /tmp/hwlab-gitops-artifact-catalog.json deploy/deploy.yaml \"$(params.services)\"; then", " cp /tmp/hwlab-gitops-artifact-catalog.json \"$catalog_path\"", " echo '{\"event\":\"gitops-artifact-catalog\",\"phase\":\"prepare-source\",\"status\":\"loaded\",\"branch\":\"'\"$(params.gitops-branch)\"'\"}'", " else", @@ -1476,11 +1505,11 @@ function prepareSourceScript() { " echo '{\"event\":\"gitops-artifact-catalog\",\"phase\":\"prepare-source\",\"status\":\"seed\",\"reason\":\"gitops-branch-missing\"}'", "fi", "if [ ! -s \"$catalog_path\" ] && [ \"$(params.lane)\" = \"v02\" ]; then", - " node - \"$catalog_path\" \"$(params.revision)\" \"$(params.registry-prefix)\" \"$(params.image-tag-mode)\" \"$(params.services)\" <<'NODE'", - "const fs = require('node:fs');", - "const path = require('node:path');", + " node --input-type=module - \"$catalog_path\" \"$(params.revision)\" \"$(params.registry-prefix)\" \"$(params.image-tag-mode)\" \"$(params.services)\" <<'NODE'", + "import fs from 'node:fs';", + "import { readStructuredFile } from './scripts/src/structured-config.mjs';", "const [catalogPath, revision, registryPrefix, imageTagMode, selectedServices] = process.argv.slice(2);", - "const deploy = JSON.parse(fs.readFileSync('deploy/deploy.json', 'utf8'));", + "const deploy = await readStructuredFile(process.cwd(), 'deploy/deploy.yaml');", "const tag = imageTagMode === 'full' ? revision : revision.slice(0, 7);", "const namespace = 'hwlab-v02';", "const selected = (selectedServices || '').split(',').filter(Boolean);", @@ -1551,7 +1580,7 @@ echo '{"event":"ci-base-image","phase":"plan-artifacts","image":"${ciToolsRunner for tool in node git; do command -v "$tool" >/dev/null 2>&1 || { echo '{"event":"ci-base-image","phase":"plan-artifacts","ok":false,"reason":"missing-tool","tool":"'"$tool"'"}'; exit 31; }; done cd /workspace/source/repo test "$(git rev-parse HEAD)" = "$(params.revision)" -node scripts/g14-ci-plan.mjs --lane "$(params.lane)" --target-ref HEAD --deploy-json deploy/deploy.json --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" > /workspace/source/ci-plan.json +node scripts/g14-ci-plan.mjs --lane "$(params.lane)" --target-ref HEAD --deploy-config deploy/deploy.yaml --artifact-catalog "$(params.catalog-path)" --registry-prefix "$(params.registry-prefix)" --services "$(params.services)" > /workspace/source/ci-plan.json node - <<'NODE' const fs = require("node:fs"); const plan = JSON.parse(fs.readFileSync("/workspace/source/ci-plan.json", "utf8")); @@ -1615,7 +1644,7 @@ cd /workspace/source/repo test "$(git rev-parse HEAD)" = "$(params.revision)" export HWLAB_ARTIFACT_LANE="$(params.lane)" export HWLAB_ARTIFACT_CATALOG_PATH="$(params.catalog-path)" -export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.json" +export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.yaml" export HWLAB_ARTIFACT_IMAGE_TAG_MODE="$(params.image-tag-mode)" mkdir -p /workspace/source/service-results if [ -s /workspace/source/affected-services.json ] && ! node -e 'const fs=require("node:fs"); const p=JSON.parse(fs.readFileSync("/workspace/source/affected-services.json","utf8")); const service=process.argv[1]; const item=(p.entries||[]).find((entry)=>entry.serviceId===service); process.exit(item && item.affected ? 0 : 1);' "$(params.service-id)"; then @@ -1694,7 +1723,7 @@ if [ "$buildkit_ready" != "1" ]; then echo '{"event":"buildkit-sidecar-not-ready","serviceId":"'"$(params.service-id)"'","addr":"'"$HWLAB_BUILDKIT_ADDR"'"}' >&2 exit 1 fi -node scripts/g14-artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-json deploy/deploy.json --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report "/workspace/source/service-results/$(params.service-id).json" --tekton-results-dir /tekton/results --quiet-build --concurrency 1 --services "$(params.service-id)" --buildkit-command /workspace/buildkit-bin/buildctl --buildkit-addr "$HWLAB_BUILDKIT_ADDR" +node scripts/g14-artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report "/workspace/source/service-results/$(params.service-id).json" --tekton-results-dir /tekton/results --quiet-build --concurrency 1 --services "$(params.service-id)" --buildkit-command /workspace/buildkit-bin/buildctl --buildkit-addr "$HWLAB_BUILDKIT_ADDR" `; } @@ -1712,10 +1741,10 @@ export HWLAB_DEV_REGISTRY_K8S_NATIVE=1 if [ -n "$(params.base-image)" ]; then export HWLAB_DEV_BASE_IMAGE="$(params.base-image)"; fi export HWLAB_ARTIFACT_LANE="$(params.lane)" export HWLAB_ARTIFACT_CATALOG_PATH="$(params.catalog-path)" -export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.json" +export HWLAB_DEPLOY_MANIFEST_PATH="deploy/deploy.yaml" export HWLAB_ARTIFACT_IMAGE_TAG_MODE="$(params.image-tag-mode)" -node scripts/g14-artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-json deploy/deploy.json --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report /workspace/source/dev-artifacts.json --quiet-build --concurrency 1 --services "$(params.services)" --external-service-report-dir /workspace/source/service-results -node scripts/refresh-artifact-catalog.mjs --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-json deploy/deploy.json --image-tag-mode "$(params.image-tag-mode)" --target-ref HEAD --publish-report /workspace/source/dev-artifacts.json --no-write +node scripts/g14-artifact-publish.mjs --publish --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --registry-prefix "$(params.registry-prefix)" --report /workspace/source/dev-artifacts.json --quiet-build --concurrency 1 --services "$(params.services)" --external-service-report-dir /workspace/source/service-results +node scripts/refresh-artifact-catalog.mjs --lane "$(params.lane)" --catalog-path "$(params.catalog-path)" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --target-ref HEAD --publish-report /workspace/source/dev-artifacts.json --no-write `; } @@ -1862,7 +1891,7 @@ NODE } if [ -s /workspace/source/dev-artifacts.json ]; then - node scripts/refresh-artifact-catalog.mjs --lane "$lane" --catalog-path "$catalog_path" --deploy-json deploy/deploy.json --image-tag-mode "$(params.image-tag-mode)" --target-ref "$(params.revision)" --publish-report /workspace/source/dev-artifacts.json --write + node scripts/refresh-artifact-catalog.mjs --lane "$lane" --catalog-path "$catalog_path" --deploy-config deploy/deploy.yaml --image-tag-mode "$(params.image-tag-mode)" --target-ref "$(params.revision)" --publish-report /workspace/source/dev-artifacts.json --write fi gitops_render_started_ms="$(ci_now_ms)" node scripts/g14-gitops-render.mjs --lane "$lane" --catalog-path "$catalog_path" --image-tag-mode "$(params.image-tag-mode)" --source-revision "$(params.revision)" --source-repo "$(params.git-url)" --source-branch "$(params.source-branch)" --gitops-branch "$(params.gitops-branch)" --registry-prefix "$(params.registry-prefix)" --use-deploy-images @@ -2961,8 +2990,8 @@ function runtimeReadyTask({ profile = "dev" } = {}) { }; } -function imagePublishTaskSet(args = { lane: "g14" }) { - const serviceIds = serviceIdsForLane(args.lane); +function imagePublishTaskSet(args = { lane: "g14" }, deploy = null) { + const serviceIds = serviceIdsForLane(args.lane, deploy); const buildTasks = serviceIds.map((serviceId) => perServiceBuildTask(serviceId, { runAfter: ["plan-artifacts"] })); @@ -2973,7 +3002,7 @@ function imagePublishTaskSet(args = { lane: "g14" }) { ]; } -function tektonPipeline(args = { lane: "g14" }) { +function tektonPipeline(args = { lane: "g14" }, deploy = null) { const settings = ciLaneSettings(args); const runtimePath = `deploy/gitops/g14/${runtimePathForProfile(settings.profile)}`; return { @@ -3005,7 +3034,7 @@ function tektonPipeline(args = { lane: "g14" }) { { name: "runtime-path", type: "string", default: runtimePath }, { name: "revision", type: "string", description: "Full git commit SHA; the only source truth for image tags." }, { name: "registry-prefix", type: "string", default: defaultRegistryPrefix }, - { name: "services", type: "string", default: servicesParamForLane(args.lane) }, + { name: "services", type: "string", default: servicesParamForLane(args.lane, deploy) }, { name: "base-image", type: "string", default: defaultDevBaseImage } ], workspaces: [ @@ -3049,7 +3078,7 @@ function tektonPipeline(args = { lane: "g14" }) { ] }, ...primitiveValidationTasks.map(primitiveValidationTask), - ...imagePublishTaskSet(args), + ...imagePublishTaskSet(args, deploy), { name: "gitops-promote", runAfter: ["collect-artifacts"], @@ -3203,7 +3232,7 @@ function tektonPollerCronJob(args) { { name: "SOURCE_BRANCH", value: args.sourceBranch }, { name: "GITOPS_BRANCH", value: args.gitopsBranch }, { name: "REGISTRY_PREFIX", value: args.registryPrefix }, - { name: "SERVICES", value: servicesParamForLane(args.lane) }, + { name: "SERVICES", value: servicesParamForLane(args.lane, deploy) }, { name: "BASE_IMAGE", value: defaultDevBaseImage } ], volumeMounts: [{ name: "git-ssh", mountPath: "/workspace/git-ssh", readOnly: true }], @@ -4204,7 +4233,7 @@ function deepSeekProxyManifest({ profile = "dev", source, sourceBranch = "G14", { name: "moonbridge-config", mountPath: "/config", readOnly: true }, { name: "moonbridge-data", mountPath: "/data" } ] - }, ...(profile === "v02" ? [v02MetricsSidecarContainer({ serviceId: "hwlab-deepseek-proxy", namespace, gitopsTarget: gitopsTargetForProfile(profile) })] : [])], + }, ...(profile === "v02" ? [v02MetricsSidecarContainer({ deploy, serviceId: "hwlab-deepseek-proxy", namespace, gitopsTarget: gitopsTargetForProfile(profile) })] : [])], volumes: [ { name: "moonbridge-scripts", configMap: { name: "hwlab-deepseek-proxy-config" } }, { name: "moonbridge-config", emptyDir: {} }, @@ -4736,11 +4765,11 @@ This directory is generated by \`scripts/g14-gitops-render.mjs --lane v02\` from } return `# HWLAB G14 GitOps CI/CD -This directory is generated from built-in Tekton-native CI primitive declarations plus human-authored \`deploy/deploy.json\`, CI-authored \`deploy/artifact-catalog.dev.json\`, and \`deploy/k8s/*\` by \`scripts/g14-gitops-render.mjs\`. +This directory is generated from built-in Tekton-native CI primitive declarations plus human-authored \`deploy/deploy.yaml\`, CI-authored \`deploy/artifact-catalog.dev.json\`, and \`deploy/k8s/*\` by \`scripts/g14-gitops-render.mjs\`. - Target: G14 k3s only. - CI: Tekton Pipeline \`hwlab-ci/hwlab-g14-ci-image-publish\`. -- Artifact contract: runtime images come from \`deploy/artifact-catalog.dev.json\`; \`deploy/deploy.json\` only carries human-authored config such as env and topology. +- Artifact contract: runtime images come from \`deploy/artifact-catalog.dev.json\`; \`deploy/deploy.yaml\` only carries human-authored config such as env and topology. - Branch split: source branch \`${args.sourceBranch}\` remains the watched code branch; generated desired state is promoted to \`${args.gitopsBranch}\`. - CD: Argo CD Applications \`argocd/hwlab-g14-dev\` and \`argocd/hwlab-g14-prod\` consume \`${args.gitopsBranch}:deploy/gitops/g14/runtime-dev\` and \`runtime-prod\`. - Public preview: FRP exposes G14 DEV web \`${args.webEndpoint}\` / edge \`${args.runtimeEndpoint}\` through \`hwlab-dev/hwlab-g14-frpc\`, and G14 PROD web \`${args.prodWebEndpoint}\` / edge \`${args.prodRuntimeEndpoint}\` through \`hwlab-prod/hwlab-g14-prod-frpc\`; D601 keeps \`:16666/:16667\`. @@ -4753,7 +4782,7 @@ This directory is generated from built-in Tekton-native CI primitive declaration async function plannedFiles(args) { const [deploy, namespaceTemplate, config, services, health, workloads] = await Promise.all([ - readJson("deploy/deploy.json"), + readJson("deploy/deploy.yaml"), readJson("deploy/k8s/base/namespace.yaml"), readJson("deploy/k8s/base/code-agent-codex-config.yaml"), readJson("deploy/k8s/base/services.yaml"), @@ -4805,7 +4834,7 @@ async function plannedFiles(args) { if (args.useDeployImages) { sourceDescriptor.renderMode = { imageSource: args.catalogPath, - configSource: "deploy/deploy.json", + configSource: "deploy/deploy.yaml", mixedDesiredState: true }; } @@ -4814,7 +4843,7 @@ async function plannedFiles(args) { putJson("registry/registry.yaml", registryManifest()); if (args.lane === "v02") putJson("devops-infra/git-mirror.yaml", devopsInfraGitMirrorManifest()); putJson(`${settings.tektonDir}/rbac.yaml`, tektonRbac(args)); - putJson(`${settings.tektonDir}/pipeline.yaml`, tektonPipeline(args)); + putJson(`${settings.tektonDir}/pipeline.yaml`, tektonPipeline(args, deploy)); if (args.lane !== "v02") { putJson(`${settings.tektonDir}/poller.yaml`, tektonPollerCronJob(args)); putJson(`${settings.tektonDir}/control-plane-reconciler.yaml`, tektonControlPlaneReconcilerCronJob(args)); @@ -4836,13 +4865,13 @@ async function plannedFiles(args) { putJson(`${runtimePath}/kustomization.yaml`, runtimeKustomization({ profile, includeDeviceAgent })); putJson(`${runtimePath}/namespace.yaml`, runtimeNamespace); if (profile !== "v02") putJson(`${runtimePath}/code-agent-codex-config.yaml`, transformListNamespace(config, namespace, profileLabels, annotations)); - putJson(`${runtimePath}/services.yaml`, transformServices({ services, namespace, labels: profileLabels, annotations, profile })); + putJson(`${runtimePath}/services.yaml`, transformServices({ services, namespace, labels: profileLabels, annotations, profile, deploy })); putJson(`${runtimePath}/health-contract.yaml`, transformHealthContract(health, namespace, profileLabels, annotations, endpoints.runtimeEndpoint, endpoints.webEndpoint, profile)); putJson(`${runtimePath}/workloads.yaml`, transformWorkloads({ workloads, deploy, catalog: artifactCatalog, source, sourceBranch: args.sourceBranch, registryPrefix: args.registryPrefix, runtimeEndpoint: endpoints.runtimeEndpoint, webEndpoint: endpoints.webEndpoint, profile, useDeployImages: args.useDeployImages, metricsSidecarSha256 })); putJson(`${runtimePath}/deepseek-proxy.yaml`, deepSeekProxyManifest({ profile, source, sourceBranch: args.sourceBranch, sourceRepo: args.sourceRepo, deploy, registryPrefix: args.registryPrefix, catalog: artifactCatalog, useDeployImages: args.useDeployImages, metricsSidecarSha256 })); if (profile === "v02") putJson(`${runtimePath}/postgres.yaml`, v02PostgresManifest({ migrationSql, source })); if (profile === "v02") putJson(`${runtimePath}/openfga.yaml`, v02OpenFgaManifest({ source })); - if (profile === "v02") putJson(`${runtimePath}/observability.yaml`, observabilityManifest({ namespace, labels: profileLabels, annotations, metricsSidecarScript })); + if (profile === "v02") putJson(`${runtimePath}/observability.yaml`, observabilityManifest({ deploy, namespace, labels: profileLabels, annotations, metricsSidecarScript })); if (includeDeviceAgent) putJson(`${runtimePath}/device-agent-71-freq.yaml`, deviceAgent71FreqManifest({ profile, source, registryPrefix: args.registryPrefix, catalog: artifactCatalog, useDeployImages: args.useDeployImages })); putJson(`${runtimePath}/g14-frpc.yaml`, g14FrpcManifest({ profile, source })); } diff --git a/scripts/refresh-artifact-catalog.mjs b/scripts/refresh-artifact-catalog.mjs index b6a6bb64..5741a63a 100644 --- a/scripts/refresh-artifact-catalog.mjs +++ b/scripts/refresh-artifact-catalog.mjs @@ -12,12 +12,13 @@ import { serviceInventoryFromServices } from "./src/dev-artifact-services.mjs"; import { tempReportPath } from "./src/report-paths.mjs"; +import { readStructuredFile } from "./src/structured-config.mjs"; const execFileAsync = promisify(execFile); const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const defaultLane = process.env.HWLAB_ARTIFACT_LANE || process.env.HWLAB_GITOPS_LANE || "g14"; const defaultCatalogPath = process.env.HWLAB_ARTIFACT_CATALOG_PATH || "deploy/artifact-catalog.dev.json"; -const defaultDeployPath = process.env.HWLAB_DEPLOY_MANIFEST_PATH || "deploy/deploy.json"; +const defaultDeployPath = process.env.HWLAB_DEPLOY_MANIFEST_PATH || "deploy/deploy.yaml"; const defaultPublishReportPath = tempReportPath("dev-artifacts.json"); const defaultRegistryPrefix = process.env.HWLAB_DEV_REGISTRY_PREFIX || process.env.HWLAB_DEV_REGISTRY || "127.0.0.1:5000/hwlab"; const digestPattern = /^sha256:[a-f0-9]{64}$/; @@ -26,16 +27,27 @@ const commitPattern = /^[a-f0-9]{7,40}$/; const publishReadyStatuses = new Set(["published", "reused"]); const v02EnvReuseRuntimeMode = "env-reuse-git-mirror-checkout"; -const v02RuntimeServiceIds = Object.freeze([ - "hwlab-cloud-api", - "hwlab-cloud-web", - "hwlab-gateway", - "hwlab-edge-proxy", - "hwlab-agent-skills" -]); +function serviceIdsForLane(lane, deploy = null) { + return lane === "v02" ? v02ServiceIdsFromDeploy(deploy) : SERVICE_IDS; +} -function serviceIdsForLane(lane) { - return lane === "v02" ? v02RuntimeServiceIds : SERVICE_IDS; +function v02ServiceIdsFromDeploy(deploy) { + const serviceIds = uniquePreserveOrder((Array.isArray(deploy?.lanes?.v02?.envReuseServices) ? deploy.lanes.v02.envReuseServices : []) + .map((item) => String(item ?? "").trim()) + .filter(Boolean)); + assert.ok(serviceIds.length > 0, "deploy.lanes.v02.envReuseServices must not be empty"); + return serviceIds; +} + +function uniquePreserveOrder(values) { + const seen = new Set(); + const result = []; + for (const value of values) { + if (seen.has(value)) continue; + seen.add(value); + result.push(value); + } + return result; } function parseArgs(argv) { @@ -58,7 +70,7 @@ function parseArgs(argv) { args.lane = readOption(argv, ++index, arg); } else if (arg === "--catalog-path") { args.catalogPath = readOption(argv, ++index, arg); - } else if (arg === "--deploy-json") { + } else if (arg === "--deploy-config") { args.deployPath = readOption(argv, ++index, arg); } else if (arg === "--image-tag-mode") { args.imageTagMode = readOption(argv, ++index, arg); @@ -101,7 +113,7 @@ function usage() { "usage: node scripts/refresh-artifact-catalog.mjs --target-ref REF (--blocked|--publish-report PATH) [--write|--no-write]", "", "Preview or explicitly write DEV artifact catalog identity without faking digest evidence.", - "Default mode is read-only. --write is reserved for G14 Tekton GitOps promotion and never writes deploy/deploy.json.", + "Default mode is read-only. --write is reserved for G14 Tekton GitOps promotion and never writes deploy/deploy.yaml.", "", "examples:", " node scripts/refresh-artifact-catalog.mjs --target-ref HEAD --blocked --no-write", @@ -110,14 +122,13 @@ function usage() { ].join("\n"); } -async function readJson(relativePath) { - const filePath = path.isAbsolute(relativePath) ? relativePath : path.join(repoRoot, relativePath); - return JSON.parse(await readFile(filePath, "utf8")); +async function readConfig(relativePath) { + return readStructuredFile(repoRoot, relativePath); } -async function readJsonIfPresent(relativePath, fallback = null) { +async function readConfigIfPresent(relativePath, fallback = null) { try { - return await readJson(relativePath); + return await readConfig(relativePath); } catch (error) { if (error?.code === "ENOENT") return fallback; throw error; @@ -212,7 +223,7 @@ function artifactCatalogSkeleton({ args, deploy, target, registryPrefix = defaul }, allowedProfiles: [environment], forbiddenProfiles: args.lane === "v02" ? ["dev", "prod"] : ["prod"], - services: serviceIdsForLane(args.lane).map((serviceId) => catalogSkeletonService({ args, serviceId, target, registryPrefix, environment, namespace })) + services: serviceIdsForLane(args.lane, deploy).map((serviceId) => catalogSkeletonService({ args, serviceId, target, registryPrefix, environment, namespace })) }; } @@ -291,7 +302,9 @@ function assertV02DeployAndCatalog(deploy, catalog) { assert.ok(lane && typeof lane === "object", "deploy.lanes.v02 must exist for v02 catalog refresh"); assert.equal(lane.namespace, "hwlab-v02", "deploy.lanes.v02.namespace must be hwlab-v02"); assert.ok(isHttpEndpoint(lane.endpoint), "deploy.lanes.v02.endpoint must be a non-empty http(s) URL"); - assert.deepEqual((deploy.services ?? []).map((service) => service.serviceId), SERVICE_IDS, "deploy services must cover frozen service IDs in order"); + const serviceIds = serviceIdsForLane("v02", deploy); + assert.ok(serviceIds.every((serviceId) => lane.serviceDeclarations?.[serviceId]), "deploy.lanes.v02.serviceDeclarations must cover envReuseServices"); + assert.ok(serviceIds.every((serviceId) => lane.bootScripts?.[serviceId]), "deploy.lanes.v02.bootScripts must cover envReuseServices"); assert.equal(catalog.environment, "v02", "catalog environment must be v02"); assert.equal(catalog.profile, "v02", "catalog profile must be v02"); @@ -299,12 +312,12 @@ function assertV02DeployAndCatalog(deploy, catalog) { if (catalog.endpoint != null) assert.ok(isHttpEndpoint(catalog.endpoint), "catalog endpoint must be a non-empty http(s) URL"); assert.deepEqual(catalog.allowedProfiles, ["v02"], "catalog allowedProfiles must only allow v02"); assert.deepEqual(catalog.forbiddenProfiles, ["dev", "prod"], "catalog forbiddenProfiles must forbid dev/prod"); - assert.deepEqual((catalog.services ?? []).map((service) => service.serviceId), serviceIdsForLane("v02"), "catalog services must cover retained v0.2 service IDs in order"); + assert.deepEqual((catalog.services ?? []).map((service) => service.serviceId), serviceIds, "catalog services must cover deploy.lanes.v02.envReuseServices in order"); } -function normalizeCatalogForLane(catalog, lane) { +function normalizeCatalogForLane(catalog, lane, deploy) { if (!catalog || lane !== "v02") return catalog; - const allowed = new Set(serviceIdsForLane(lane)); + const allowed = new Set(serviceIdsForLane(lane, deploy)); return { ...catalog, services: (catalog.services ?? []).filter((service) => allowed.has(service.serviceId)) @@ -529,16 +542,16 @@ async function main() { const target = await resolveTarget(args.targetRef, args.imageTagMode); const [deploy, publishReport] = await Promise.all([ - readJson(args.deployPath), - args.publishReportPath ? readJson(args.publishReportPath) : Promise.resolve(null) + readConfig(args.deployPath), + args.publishReportPath ? readConfig(args.publishReportPath) : Promise.resolve(null) ]); - let catalog = await readJsonIfPresent(args.catalogPath, null); + let catalog = await readConfigIfPresent(args.catalogPath, null); if (!catalog && args.lane === "v02") catalog = artifactCatalogSkeleton({ args, deploy, target, registryPrefix: publishReport?.artifactPublish?.registryPrefix ?? defaultRegistryPrefix }); - catalog = normalizeCatalogForLane(catalog, args.lane); + catalog = normalizeCatalogForLane(catalog, args.lane, deploy); assert.ok(catalog, `${args.catalogPath} is required for ${args.lane} catalog refresh`); assertLaneDeployAndCatalog(args, deploy, catalog); - const serviceIds = serviceIdsForLane(args.lane); + const serviceIds = serviceIdsForLane(args.lane, deploy); const publishRecords = publishReport ? publishRecordsFromReport(publishReport, target, serviceIds) : null; const serviceInventory = publishReport?.artifactPublish?.serviceInventory ?? serviceInventoryFromServices( await resolveDevArtifactServices(repoRoot, serviceIds, SERVICE_IDS) @@ -572,7 +585,7 @@ async function main() { endpoint: catalog.endpoint ?? null, imageTagMode: args.imageTagMode, wrote: args.write ? [args.catalogPath] : [], - deployTruthPolicy: "deploy.json is human-authored runtime config; artifact promotion must not rewrite it", + deployTruthPolicy: "deploy.yaml is human-authored runtime config; artifact promotion must not rewrite it", ciPublished: Boolean(publishRecords), registryVerified: Boolean(publishRecords), publishedCount, diff --git a/scripts/refresh-artifact-catalog.test.mjs b/scripts/refresh-artifact-catalog.test.mjs index 95eb9949..f4c53f68 100644 --- a/scripts/refresh-artifact-catalog.test.mjs +++ b/scripts/refresh-artifact-catalog.test.mjs @@ -7,6 +7,7 @@ import test from "node:test"; import { promisify } from "node:util"; import { SERVICE_IDS } from "../internal/protocol/index.mjs"; +import { writeStructuredFile } from "./src/structured-config.mjs"; const execFileAsync = promisify(execFile); const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), ".."); @@ -154,7 +155,7 @@ test("refresh is read-only by default", async () => { const payload = JSON.parse(result.stdout); assert.equal(payload.status, "published"); assert.deepEqual(payload.wrote, []); - assert.equal(payload.deployTruthPolicy, "deploy.json is human-authored runtime config; artifact promotion must not rewrite it"); + assert.equal(payload.deployTruthPolicy, "deploy.yaml is human-authored runtime config; artifact promotion must not rewrite it"); }); test("refresh keeps reused service image tags from the publish report", async () => { @@ -327,9 +328,9 @@ test("v02 refresh accepts current runtime service full image tags", async () => test("v02 refresh follows the current deploy endpoint instead of the retired 19667 freeze", async () => { const commitId = await git(["rev-parse", "HEAD"]); const tempDir = await mkdtemp(path.join(os.tmpdir(), "hwlab-refresh-v02-https-endpoint-")); - const deployPath = path.join(tempDir, "deploy.json"); + const deployPath = path.join(tempDir, "deploy.yaml"); const catalogPath = path.join(tempDir, "artifact-catalog.v02.json"); - await writeFile(deployPath, `${JSON.stringify({ + await writeStructuredFile(tempDir, deployPath, { environment: "dev", namespace: "hwlab-dev", endpoint: "http://74.48.78.17:16667", @@ -341,7 +342,7 @@ test("v02 refresh follows the current deploy endpoint instead of the retired 196 } }, services: SERVICE_IDS.map((serviceId) => ({ serviceId })) - }, null, 2)}\n`); + }); await writeFile(catalogPath, `${JSON.stringify({ catalogVersion: "v1", kind: "hwlab-artifact-catalog", @@ -374,7 +375,7 @@ test("v02 refresh follows the current deploy endpoint instead of the retired 196 "full", "--target-ref", "HEAD", - "--deploy-json", + "--deploy-config", deployPath, "--catalog-path", catalogPath, diff --git a/scripts/src/artifact-runtime-readiness-guard.mjs b/scripts/src/artifact-runtime-readiness-guard.mjs index 7ed5a78b..76ed4604 100644 --- a/scripts/src/artifact-runtime-readiness-guard.mjs +++ b/scripts/src/artifact-runtime-readiness-guard.mjs @@ -8,6 +8,7 @@ import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs"; import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT } from "../../internal/protocol/index.mjs"; import { buildDesiredStatePlan } from "./deploy-desired-state-plan.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs"; +import { readStructuredFile } from "./structured-config.mjs"; const defaultRepoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const cloudWebSourceRoot = "web/hwlab-cloud-web/src"; @@ -154,7 +155,7 @@ export function evaluateArtifactRuntimeReadiness({ pass: desiredState.status !== "blocked" && desiredState.status !== "missing", type: "contract_blocker", summary: `Desired-state planner status=${desiredState.status}; convergence=${desiredState.targetConvergence.state}.`, - nextTask: "Repair deploy/deploy.json, deploy/artifact-catalog.dev.json, and deploy/k8s/base/workloads.yaml before any apply." + nextTask: "Repair deploy/deploy.yaml, deploy/artifact-catalog.dev.json, and deploy/k8s/base/workloads.yaml before any apply." }); addCheck(checks, { id: "desired-state-target-match", @@ -1246,8 +1247,7 @@ function nextCommands({ canPublish, canApply, blockedReasons }) { } async function readJson(repoRoot, relativePath) { - const raw = await readFile(path.join(repoRoot, relativePath), "utf8"); - return JSON.parse(raw); + return readStructuredFile(repoRoot, relativePath); } async function readOptionalJson(repoRoot, relativePath) { diff --git a/scripts/src/deploy-config.mjs b/scripts/src/deploy-config.mjs new file mode 100644 index 00000000..dd2eb2ea --- /dev/null +++ b/scripts/src/deploy-config.mjs @@ -0,0 +1,29 @@ +import { + parseStructuredConfig, + readStructuredFile, + readStructuredFileIfPresent, + stringifyStructuredConfig, + writeStructuredFile +} from "./structured-config.mjs"; + +export const DEFAULT_DEPLOY_CONFIG_PATH = "deploy/deploy.yaml"; + +export { + parseStructuredConfig, + readStructuredFile, + readStructuredFileIfPresent, + stringifyStructuredConfig, + writeStructuredFile +}; + +export async function readDeployConfig(repoRoot, relativePath = DEFAULT_DEPLOY_CONFIG_PATH) { + return readStructuredFile(repoRoot, relativePath); +} + +export async function readDeployConfigIfPresent(repoRoot, relativePath = DEFAULT_DEPLOY_CONFIG_PATH, fallback = null) { + return readStructuredFileIfPresent(repoRoot, relativePath, fallback); +} + +export async function writeDeployConfig(repoRoot, relativePath = DEFAULT_DEPLOY_CONFIG_PATH, value) { + return writeStructuredFile(repoRoot, relativePath, value); +} diff --git a/scripts/src/deploy-contract-plan.mjs b/scripts/src/deploy-contract-plan.mjs index 022f9aca..f2dd3b78 100644 --- a/scripts/src/deploy-contract-plan.mjs +++ b/scripts/src/deploy-contract-plan.mjs @@ -7,6 +7,7 @@ import { codeAgentSecretRefPlaceholder } from "../../internal/cloud/code-agent-contract.ts"; import { HWLAB_M3_IO_API_BASE_URL_ENV } from "../../skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs"; +import { readStructuredFile } from "./structured-config.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const expectedPorts = Object.freeze({ frontend: 16666, api: 16667 }); @@ -30,7 +31,7 @@ function parseArgs(argv) { } async function readJson(relativePath) { - return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8")); + return readStructuredFile(repoRoot, relativePath); } function stripTomlComment(line) { @@ -736,7 +737,7 @@ function validateCodeAgentProxyTimeoutArtifacts(ctx, workloads) { async function buildPlan() { const ctx = { checks: 0, diagnostics: [] }; const [manifest, services, workloads, healthContract, masterEdge, frpc, frps] = await Promise.all([ - readJson("deploy/deploy.json"), + readJson("deploy/deploy.yaml"), readJson("deploy/k8s/base/services.yaml"), readJson("deploy/k8s/base/workloads.yaml"), readJson("deploy/k8s/dev/health-contract.yaml"), @@ -753,7 +754,7 @@ async function buildPlan() { kind: "hwlab-deploy-contract-plan", mode: "dry-run", status: ctx.diagnostics.length === 0 ? "pass" : "blocked", - source: "deploy/deploy.json", + source: "deploy/deploy.yaml", dryRunOnly: true, safety: { frpApply: false, diff --git a/scripts/src/deploy-desired-state-plan.mjs b/scripts/src/deploy-desired-state-plan.mjs index ba53b028..54e36d91 100644 --- a/scripts/src/deploy-desired-state-plan.mjs +++ b/scripts/src/deploy-desired-state-plan.mjs @@ -10,11 +10,12 @@ import { } from "../../internal/cloud/code-agent-contract.ts"; import { DEV_DB_ENV_CONTRACT } from "../../internal/cloud/db-contract.ts"; import { tempReportPath } from "./report-paths.mjs"; +import { readStructuredFile } from "./structured-config.mjs"; const execFileAsync = promisify(execFile); const defaultRepoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); -const deployPath = "deploy/deploy.json"; +const deployPath = "deploy/deploy.yaml"; const catalogPath = "deploy/artifact-catalog.dev.json"; const workloadsPath = "deploy/k8s/base/workloads.yaml"; const artifactReportPath = tempReportPath("dev-artifacts.json"); @@ -99,8 +100,7 @@ function usage() { } async function readJson(repoRoot, relativePath) { - const raw = await readFile(path.join(repoRoot, relativePath), "utf8"); - return JSON.parse(raw); + return readStructuredFile(repoRoot, relativePath); } async function readOptionalJson(repoRoot, relativePath) { @@ -753,7 +753,7 @@ export async function buildDesiredStatePlan(options = {}) { }); if (Object.hasOwn(deploy, "commitId")) { - addMismatch(ctx, "generated_artifact_identity", `${deployPath}.commitId`, "deploy.json is human-authored config; generated commit identity must stay in artifact catalog", "absent", deploy.commitId); + addMismatch(ctx, "generated_artifact_identity", `${deployPath}.commitId`, "deploy.yaml is human-authored config; generated commit identity must stay in artifact catalog", "absent", deploy.commitId); } if (!commitPattern.test(String(catalog.commitId ?? ""))) { addMismatch(ctx, "invalid_commit", `${catalogPath}.commitId`, "artifact catalog commitId must be a short or full lowercase Git SHA", "7-40 lowercase hex", catalog.commitId); @@ -949,7 +949,7 @@ export async function buildDesiredStatePlan(options = {}) { prod: false, devLiveClaim: false }, - checkSemantics: "--check exits non-zero for missing/invalid desired-state sources, generated artifact identity leaking into deploy.json, invalid target tags, partial target drift, or catalog identity mismatch with an explicit --target-ref, --target-tag, or --promotion-commit. When the checked target ref is the current checked-out merge commit, the comparable current-main target is its accepted first parent because a commit cannot embed its own content hash. Without --check, a uniform older catalog under --target-ref/--target-tag is a read-only promotion plan.", + checkSemantics: "--check exits non-zero for missing/invalid desired-state sources, generated artifact identity leaking into deploy.yaml, invalid target tags, partial target drift, or catalog identity mismatch with an explicit --target-ref, --target-tag, or --promotion-commit. When the checked target ref is the current checked-out merge commit, the comparable current-main target is its accepted first parent because a commit cannot embed its own content hash. Without --check, a uniform older catalog under --target-ref/--target-tag is a read-only promotion plan.", summary: { desiredCommitId, desiredImageTag, @@ -974,7 +974,7 @@ export async function buildDesiredStatePlan(options = {}) { humanAuthoredTruth: [deployPath, workloadsPath], artifactIdentityTruth: [catalogPath], nonAuthoritativeEvidence: [artifactReportPath], - note: "This planner is read-only. deploy.json is human-authored config; artifact-catalog carries generated image identity and is written by G14 Tekton promotion only to G14-gitops. It does not prove image existence, registry reachability, a real DEV apply, or M3 DEV-LIVE hardware-loop evidence." + note: "This planner is read-only. deploy.yaml is human-authored config; artifact-catalog carries generated image identity and is written by G14 Tekton promotion only to G14-gitops. It does not prove image existence, registry reachability, a real DEV apply, or M3 DEV-LIVE hardware-loop evidence." }, cloudApiDb: cloudApiDb ?? { status: "blocked", diff --git a/scripts/src/dev-edge-health-smoke-lib.mjs b/scripts/src/dev-edge-health-smoke-lib.mjs index d52e2592..478ed000 100644 --- a/scripts/src/dev-edge-health-smoke-lib.mjs +++ b/scripts/src/dev-edge-health-smoke-lib.mjs @@ -16,6 +16,7 @@ import { import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs"; import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs"; +import { readStructuredFile } from "./structured-config.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const defaultReportPath = tempReportPath("dev-edge-health.json"); @@ -270,7 +271,7 @@ function requireDevEndpoint() { async function inspectContracts() { const frpc = await readText("deploy/frp/frpc.dev.toml"); const frps = await readText("deploy/frp/frps.dev.toml"); - const deploy = await readJson("deploy/deploy.json"); + const deploy = await readJson("deploy/deploy.yaml"); const master = await readJson("deploy/master-edge/health-contract.json"); const frpsUsesVhostEdgePort = /vhostHTTPPort\s*=\s*(16667|6667)/u.test(frps); @@ -1189,7 +1190,7 @@ function parseJsonOrNull(text) { } async function readJson(relativePath) { - return JSON.parse(await readText(relativePath)); + return readStructuredFile(repoRoot, relativePath); } async function readText(relativePath) { diff --git a/scripts/src/dev-gate-preflight.mjs b/scripts/src/dev-gate-preflight.mjs index 4079fa96..16297308 100644 --- a/scripts/src/dev-gate-preflight.mjs +++ b/scripts/src/dev-gate-preflight.mjs @@ -18,6 +18,7 @@ import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs"; import { DEV_ENDPOINT, ENVIRONMENT_DEV, SERVICE_IDS } from "../../internal/protocol/index.mjs"; import { probeRegistryCapabilities } from "./registry-capabilities.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs"; +import { readStructuredFile } from "./structured-config.mjs"; const execFileAsync = promisify(execFile); const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); @@ -145,8 +146,7 @@ async function commandExists(command) { } async function readJson(relativePath) { - const raw = await readFile(path.join(repoRoot, relativePath), "utf8"); - return JSON.parse(raw); + return readStructuredFile(repoRoot, relativePath); } async function readOptionalJson(relativePath) { @@ -430,7 +430,7 @@ async function artifactIdentityFor({ deploy, catalog, artifactReport, targetComm }, targetCoverage, deployManifest: { - path: "deploy/deploy.json", + path: "deploy/deploy.yaml", identitySource: "human-authored-config", artifactIdentity: false, matchesSource: true, @@ -546,7 +546,7 @@ function assertFrpStatic(frpc, frps, masterEdge) { async function loadContracts() { return Promise.all([ - readJson("deploy/deploy.json"), + readJson("deploy/deploy.yaml"), readJson("deploy/artifact-catalog.dev.json"), readJson("deploy/k8s/base/namespace.yaml"), readJson("deploy/k8s/base/workloads.yaml"), @@ -871,7 +871,7 @@ function validateCloudApiDbContract(reporter, deploy, workloads, services) { type: "contract_blocker", scope: "cloud-api-db-env-contract", summary: `cloud-api DEV DB contract is incomplete; blocked by ${missing.join(", ")}.`, - nextTask: "Repair deploy/deploy.json and deploy/k8s/base/workloads.yaml so they expose the required DB env names, redacted Secret reference, and HWLAB_CLOUD_DB_SSL_MODE=disable. Do not make cloud-api-db a hard readiness gate unless this repo also owns its Service/Endpoint rollout contract." + nextTask: "Repair deploy/deploy.yaml and deploy/k8s/base/workloads.yaml so they expose the required DB env names, redacted Secret reference, and HWLAB_CLOUD_DB_SSL_MODE=disable. Do not make cloud-api-db a hard readiness gate unless this repo also owns its Service/Endpoint rollout contract." }); } @@ -1317,7 +1317,7 @@ function validateCodeAgentProviderContract(reporter, deploy, workloads) { type: "agent_blocker", scope: "code-agent-provider-env-contract", summary: `cloud-api DEV Code Agent provider contract is incomplete; missing ${missing.join(", ")}.`, - nextTask: "Repair deploy/deploy.json and deploy/k8s/base/workloads.yaml so hwlab-cloud-api has OPENAI_API_KEY from hwlab-code-agent-provider/openai-api-key and HWLAB_CODE_AGENT_OPENAI_BASE_URL through DEV egress/proxy; do not print the Secret value." + nextTask: "Repair deploy/deploy.yaml and deploy/k8s/base/workloads.yaml so hwlab-cloud-api has OPENAI_API_KEY from hwlab-code-agent-provider/openai-api-key and HWLAB_CODE_AGENT_OPENAI_BASE_URL through DEV egress/proxy; do not print the Secret value." }); } diff --git a/scripts/src/g14-ci-plan-lib.mjs b/scripts/src/g14-ci-plan-lib.mjs index 4b04b793..939ea86d 100644 --- a/scripts/src/g14-ci-plan-lib.mjs +++ b/scripts/src/g14-ci-plan-lib.mjs @@ -1,28 +1,15 @@ import { createHash } from "node:crypto"; import { execFile } from "node:child_process"; -import { readFile } from "node:fs/promises"; import path from "node:path"; import { promisify } from "node:util"; -import { SERVICE_IDS } from "../../internal/protocol/index.mjs"; +import { readStructuredFileIfPresent } from "./structured-config.mjs"; const execFileAsync = promisify(execFile); export const G14_CI_PLAN_VERSION = "v1"; export const V02_ENV_REUSE_RUNTIME_MODE = "env-reuse-git-mirror-checkout"; -export const DEFAULT_BUILD_SYSTEM_PATHS = Object.freeze([ - "internal/dev-entrypoint/artifact-runtime.mjs", - "scripts/artifact-publish.mjs", - "scripts/g14-artifact-publish.mjs", - "scripts/src/dev-artifact-services.mjs" -]); - -export const DEFAULT_ENV_REUSE_LAUNCHER_PATHS = Object.freeze([ - "deploy/runtime/launcher/", - "scripts/artifact-publish.mjs" -]); - export const DEFAULT_RUNTIME_DEP_PATHS = Object.freeze([ "package.json", "package-lock.json" @@ -56,65 +43,35 @@ export const DEFAULT_GITOPS_ONLY_PATHS = Object.freeze([ "scripts/g14-gitops-render.mjs" ]); -const serviceSpecificPaths = Object.freeze({ - "hwlab-cloud-api": [ - "cmd/hwlab-cloud-api/", - "cmd/hwlab-codex-api-responses-forwarder/", - "cmd/hwlab-deepseek-responses-bridge/", - "deploy/runtime/boot/hwlab-codex-api-responses-forwarder.sh", - "deploy/runtime/boot/hwlab-deepseek-responses-bridge.sh", - "internal/cloud/", - "internal/db/", - "internal/audit/", - "internal/agent/agentrun-dispatch.mjs", - "internal/agent/prompts/", - "skills/hwlab-agent-runtime/" - ], - "hwlab-cloud-web": [ - "web/hwlab-cloud-web/", - "internal/dev-entrypoint/cloud-web-runtime.mjs", - "internal/dev-entrypoint/cloud-web-proxy.mjs", - "internal/dev-entrypoint/cloud-web-routes.mjs", - "tools/src/hwlab-cli/trace-renderer.ts" - ], - "hwlab-gateway": ["cmd/hwlab-gateway/", "internal/sim/"], - "hwlab-edge-proxy": ["cmd/hwlab-edge-proxy/", "internal/dev-entrypoint/http.mjs"], - "hwlab-agent-skills": ["skills/"] -}); - -const bunCommandServices = new Set([ - "hwlab-cloud-api", - "hwlab-codex-api-responses-forwarder", - "hwlab-deepseek-responses-bridge", - "hwlab-gateway", - "hwlab-edge-proxy" -]); - export async function createG14CiPlan(options = {}) { const repoRoot = options.repoRoot ?? process.cwd(); - const deployJsonPath = options.deployJsonPath ?? "deploy/deploy.json"; + const deployConfigPath = options.deployConfigPath ?? "deploy/deploy.yaml"; const targetRef = options.targetRef ?? "HEAD"; const baseRef = options.baseRef ?? await defaultBaseRef(repoRoot, targetRef); - const lane = options.lane ?? (options.artifactCatalogPath?.includes(".v02.") ? "v02" : "g14"); + const lane = options.lane ?? "v02"; + if (lane !== "v02") throw new Error(`unsupported CI plan lane ${lane}; v02 env reuse is configured from deploy/deploy.yaml`); const artifactCatalogPath = options.artifactCatalogPath ?? (lane === "v02" ? "deploy/artifact-catalog.v02.json" : "deploy/artifact-catalog.dev.json"); const registryPrefix = options.registryPrefix ?? process.env.HWLAB_DEV_REGISTRY_PREFIX ?? process.env.HWLAB_DEV_REGISTRY ?? "127.0.0.1:5000/hwlab"; const baseImage = options.baseImage ?? process.env.HWLAB_DEV_BASE_IMAGE ?? "127.0.0.1:5000/hwlab/hwlab-node20-base:20-bookworm-slim"; const [deployJson, artifactCatalog] = await Promise.all([ - readJsonIfPresent(repoRoot, deployJsonPath, {}), - readJsonIfPresent(repoRoot, artifactCatalogPath, null) + readStructuredFileIfPresent(repoRoot, deployConfigPath, {}), + readStructuredFileIfPresent(repoRoot, artifactCatalogPath, null) ]); const sourceCommitId = await gitValue(repoRoot, ["rev-parse", targetRef]); const shortCommitId = await gitValue(repoRoot, ["rev-parse", "--short=7", targetRef]); const changedPaths = await changedPathsBetween(repoRoot, baseRef, targetRef); const normalizedChangedPaths = changedPaths.map(normalizeRepoPath).filter(Boolean); const semanticRuntimeDepPaths = await runtimeDependencyChangedPaths(repoRoot, baseRef, targetRef, normalizedChangedPaths); - const serviceIdResolution = resolveServiceIds({ deployJson, options }); - const componentModels = componentModelsForServices(serviceIdResolution.serviceIds); + const laneConfig = laneDeployConfig(deployJson, lane); + assertV02EnvReuseConfig(laneConfig, lane); + const serviceIdResolution = resolveServiceIds({ deployJson, options, laneConfig, lane }); + const envReuseRecipe = envReuseRecipeForLane(deployJson, lane); + const componentModels = componentModelsForServices(serviceIdResolution.serviceIds, laneConfig); const envReuseServices = enabledEnvReuseServices(deployJson, lane, serviceIdResolution.serviceIds); const globalChange = classifyGlobalChange(normalizedChangedPaths); const dockerfileHash = await hashGitPaths(repoRoot, targetRef, [ - ...DEFAULT_BUILD_SYSTEM_PATHS, + ...envReuseRecipe.additionalEnvPaths, ...DEFAULT_RUNTIME_DEP_PATHS ], { semanticPackageJson: true }); const catalogByServiceId = new Map((artifactCatalog?.services ?? []).map((service) => [service.serviceId, service])); @@ -130,7 +87,7 @@ export async function createG14CiPlan(options = {}) { const bootSh = envReuse ? bootShForService(deployJson, model.serviceId) : null; const envInputPaths = envReuse ? uniqueSorted([ ...model.runtimeDeps, - ...DEFAULT_ENV_REUSE_LAUNCHER_PATHS.map(normalizeRepoPath) + ...envReuseRecipe.launcherInputPaths ]) : []; const codeInputPaths = envReuse ? uniqueSorted([ ...model.componentPaths, @@ -215,10 +172,14 @@ export async function createG14CiPlan(options = {}) { serviceId: model.serviceId, runtimeKind: model.runtimeKind, entrypoint: model.entrypoint, + artifactKind: model.artifactKind, + healthPath: model.healthPath, + healthPort: model.healthPort, componentPaths: model.componentPaths, sharedPaths: model.sharedPaths, runtimeDeps: model.runtimeDeps, buildSystemPaths: model.buildSystemPaths, + envReuseRecipe: envReuse ? envReuseRecipe.publicRecipe : null, componentCommitId, componentInputHash, catalogSourceCommitId: catalogSourceCommitId || null, @@ -263,7 +224,7 @@ export async function createG14CiPlan(options = {}) { baseRef, targetRef, compatibility: { - deployJson: deployJsonPath, + deployConfig: deployConfigPath, artifactCatalog: artifactCatalog ? artifactCatalogPath : null, lane, ciContractSource: "scripts/g14-gitops-render.mjs", @@ -271,6 +232,8 @@ export async function createG14CiPlan(options = {}) { currentRenderCompatible: true, plannerMutatesDeployJson: false, serviceIdSource: serviceIdResolution.source, + serviceDeclarationSource: lane === "v02" ? `deploy.lanes.${lane}.serviceDeclarations` : "not-applicable", + envRecipeSource: lane === "v02" ? `deploy.lanes.${lane}.envRecipe` : "not-applicable", v02EnvReuseServiceIds: [...envReuseServices], defaultComponentLazyBuild: true, envReuseCodeOnlyFastLane: true, @@ -302,20 +265,123 @@ export async function createG14CiPlan(options = {}) { }; } -export function componentModelsForServices(serviceIds) { +export function componentModelsForServices(serviceIds, laneConfig = null) { return serviceIds.map((serviceId) => { + const declaration = serviceDeclarationFor(laneConfig, serviceId); return { serviceId, - runtimeKind: runtimeKindForService(serviceId), - entrypoint: entrypointForService(serviceId), - componentPaths: uniqueSorted((serviceSpecificPaths[serviceId] ?? [`cmd/${serviceId}/`]).map(normalizeRepoPath)), + runtimeKind: declaration.runtimeKind, + entrypoint: declaration.entrypoint, + artifactKind: declaration.artifactKind, + healthPath: declaration.healthPath, + healthPort: declaration.healthPort, + componentPaths: uniqueSorted(declaration.componentPaths.map(normalizeRepoPath)), sharedPaths: uniqueSorted(DEFAULT_SHARED_RUNTIME_PATHS.map(normalizeRepoPath)), runtimeDeps: uniqueSorted(DEFAULT_RUNTIME_DEP_PATHS.map(normalizeRepoPath)), - buildSystemPaths: uniqueSorted(DEFAULT_BUILD_SYSTEM_PATHS.map(normalizeRepoPath)) + buildSystemPaths: uniqueSorted(envReuseRecipeForLaneConfig(laneConfig).additionalEnvPaths) }; }); } +function serviceDeclarationFor(laneConfig, serviceId) { + const configured = laneConfig?.serviceDeclarations?.[serviceId]; + const runtimeKind = requireDeclarationString(configured?.runtimeKind, `serviceDeclarations.${serviceId}.runtimeKind`); + const entrypoint = requireDeclarationString(configured?.entrypoint, `serviceDeclarations.${serviceId}.entrypoint`); + const artifactKind = normalizeDeclarationString(configured?.artifactKind) || runtimeKind; + if (!Array.isArray(configured?.componentPaths) || configured.componentPaths.length === 0) { + throw new Error(`deploy.lanes.v02.serviceDeclarations.${serviceId}.componentPaths is required`); + } + return { + runtimeKind, + entrypoint, + artifactKind, + healthPath: normalizeDeclarationString(configured?.healthPath) || "/health/live", + healthPort: Number.isInteger(configured?.healthPort) ? configured.healthPort : null, + componentPaths: configured.componentPaths + }; +} + +function requireDeclarationString(value, label) { + const text = normalizeDeclarationString(value); + if (!text) throw new Error(`deploy.lanes.v02.${label} is required`); + return text; +} + +function normalizeDeclarationString(value) { + const text = String(value ?? "").trim(); + return text || null; +} + +function laneDeployConfig(deployJson, lane) { + return deployJson?.lanes?.[lane] ?? null; +} + +export function envReuseRecipeForLane(deployJson, lane = "v02") { + return envReuseRecipeForLaneConfig(laneDeployConfig(deployJson, lane)); +} + +function envReuseRecipeForLaneConfig(laneConfig) { + const configured = laneConfig?.envRecipe; + if (!configured) throw new Error("deploy.lanes.v02.envRecipe is required"); + const osPackages = normalizeStringList(configured.osPackages, []); + const bunVersion = requireDeclarationString(configured.bunVersion, "envRecipe.bunVersion"); + const launcherPath = normalizeRepoPath(requireDeclarationString(configured.launcherPath, "envRecipe.launcherPath")); + const runtimeNodeModulesPath = normalizeAbsolutePath(configured.runtimeNodeModulesPath); + if (!runtimeNodeModulesPath) throw new Error("deploy.lanes.v02.envRecipe.runtimeNodeModulesPath must be absolute"); + const additionalEnvPaths = uniqueSorted(normalizeStringList(configured.additionalEnvPaths, []).map(normalizeRepoPath)); + const launcherInputPaths = uniqueSorted([ + launcherPath, + ...additionalEnvPaths + ]); + const hwpodAliases = normalizeHwpodAliases(configured.hwpodAliases); + if (osPackages.length === 0) throw new Error("deploy.lanes.v02.envRecipe.osPackages is required"); + if (additionalEnvPaths.length === 0) throw new Error("deploy.lanes.v02.envRecipe.additionalEnvPaths is required"); + if (hwpodAliases.length === 0) throw new Error("deploy.lanes.v02.envRecipe.hwpodAliases is required"); + return { + osPackages, + bunVersion, + launcherPath, + runtimeNodeModulesPath, + additionalEnvPaths, + launcherInputPaths, + hwpodAliases, + publicRecipe: { + osPackages, + bunVersion, + launcherPath, + runtimeNodeModulesPath, + additionalEnvPaths, + hwpodAliases + } + }; +} + +function normalizeStringList(value, fallback) { + const list = Array.isArray(value) ? value : fallback; + return list.map((item) => String(item ?? "").trim()).filter(Boolean); +} + +function normalizeAbsolutePath(value) { + const text = String(value ?? "").trim(); + return text.startsWith("/") && !text.includes("..") ? text : null; +} + +function normalizeHwpodAliases(value) { + const list = Array.isArray(value) ? value : []; + return list.map((item) => ({ + command: normalizeDeclarationString(item?.command), + script: normalizeRepoPath(item?.script) + })).filter((item) => item.command && item.script); +} + +function assertV02EnvReuseConfig(laneConfig, lane) { + if (lane !== "v02") return; + if (!laneConfig?.serviceDeclarations || Object.keys(laneConfig.serviceDeclarations).length === 0) { + throw new Error("deploy.lanes.v02.serviceDeclarations is required for v02 env reuse planning"); + } + if (!laneConfig?.envRecipe) throw new Error("deploy.lanes.v02.envRecipe is required for v02 env reuse planning"); +} + export async function runtimeDependencyChangedPaths(repoRoot, baseRef, targetRef, changedPaths) { const result = new Set(); for (const filePath of changedPaths) { @@ -366,7 +432,8 @@ export function enabledEnvReuseServices(deployJson, lane, serviceIds) { export function bootShForService(deployJson, serviceId) { const configured = deployJson?.lanes?.v02?.bootScripts?.[serviceId]; - const bootSh = normalizeRepoPath(configured || `deploy/runtime/boot/${serviceId}.sh`); + if (!configured) throw new Error(`deploy.lanes.v02.bootScripts.${serviceId} is required`); + const bootSh = normalizeRepoPath(configured); if (!bootSh || bootSh.startsWith("/") || bootSh.split("/").includes("..")) { throw new Error(`invalid v02 boot script for ${serviceId}: ${configured}`); } @@ -469,46 +536,28 @@ export async function lastCommitForPaths(repoRoot, targetRef, paths) { return value.trim() || null; } -function resolveServiceIds({ deployJson, options }) { +function resolveServiceIds({ options, laneConfig, lane }) { + const allowedServiceIds = knownServiceIds(laneConfig, lane); if (Array.isArray(options.services) && options.services.length > 0) { const serviceIds = uniqueSorted(options.services.map(String)); - const allowedServiceIds = knownServiceIds(deployJson); const unknownServiceIds = serviceIds.filter((serviceId) => !allowedServiceIds.has(serviceId)); if (unknownServiceIds.length > 0) { throw new Error(`unknown service IDs for G14 CI plan: ${unknownServiceIds.join(", ")}`); } return { source: "cli-services", serviceIds }; } - const deployServices = Array.isArray(deployJson?.services) ? deployJson.services.map((service) => service?.serviceId).filter(Boolean) : []; - if (deployServices.length > 0) return { source: "deploy.services", serviceIds: uniquePreserveOrder(deployServices) }; - const mappingServices = Array.isArray(deployJson?.k3s?.serviceMappings) - ? deployJson.k3s.serviceMappings.map((service) => service?.serviceId).filter(Boolean) - : []; - if (mappingServices.length > 0) return { source: "deploy.k3s.serviceMappings", serviceIds: uniquePreserveOrder(mappingServices) }; - return { source: "internal.protocol.SERVICE_IDS", serviceIds: [...SERVICE_IDS] }; + const configured = Array.isArray(laneConfig?.envReuseServices) ? laneConfig.envReuseServices.filter(Boolean) : []; + if (configured.length === 0) throw new Error("deploy.lanes.v02.envReuseServices is required for v02 CI planning"); + return { source: "deploy.lanes.v02.envReuseServices", serviceIds: uniquePreserveOrder(configured) }; } -function knownServiceIds(deployJson) { - const values = new Set(SERVICE_IDS); - for (const service of deployJson?.services ?? []) { - if (service?.serviceId) values.add(service.serviceId); - } - for (const service of deployJson?.k3s?.serviceMappings ?? []) { - if (service?.serviceId) values.add(service.serviceId); - } +function knownServiceIds(laneConfig, lane) { + if (lane !== "v02") return new Set(); + const values = new Set(Object.keys(laneConfig?.serviceDeclarations ?? {})); + for (const serviceId of laneConfig?.envReuseServices ?? []) values.add(serviceId); return values; } -async function readJsonIfPresent(repoRoot, relativePath, fallback) { - try { - const filePath = path.isAbsolute(relativePath) ? relativePath : path.join(repoRoot, relativePath); - return JSON.parse(await readFile(filePath, "utf8")); - } catch (error) { - if (error?.code === "ENOENT") return fallback; - throw error; - } -} - async function defaultBaseRef(repoRoot, targetRef) { const parent = await gitValue(repoRoot, ["rev-parse", `${targetRef}^`]).catch(() => ""); return parent || targetRef; @@ -650,20 +699,6 @@ function environmentDigestFromCatalog(catalogRecord) { return catalogRecord?.environmentDigest ?? (catalogRecord?.environmentImage ? catalogRecord?.digest : null) ?? null; } -function runtimeKindForService(serviceId) { - if (bunCommandServices.has(serviceId)) return "bun-command"; - if (serviceId === "hwlab-cloud-web") return "cloud-web"; - if (serviceId === "hwlab-agent-skills") return "skills-bundle"; - return "node-command"; -} - -function entrypointForService(serviceId) { - if (bunCommandServices.has(serviceId)) return `cmd/${serviceId}/main.ts`; - if (serviceId === "hwlab-cloud-web") return "web/hwlab-cloud-web/index.html"; - if (serviceId === "hwlab-agent-skills") return "skills/hwlab-agent-runtime/SKILL.md"; - return `cmd/${serviceId}/main.mjs`; -} - function digestFromImageReference(image) { const match = String(image ?? "").match(/@(sha256:[a-f0-9]{64})$/u); return match ? match[1] : null; diff --git a/scripts/src/rpt004-mvp-e2e-harness.mjs b/scripts/src/rpt004-mvp-e2e-harness.mjs index f158c955..0aadd5f1 100644 --- a/scripts/src/rpt004-mvp-e2e-harness.mjs +++ b/scripts/src/rpt004-mvp-e2e-harness.mjs @@ -26,6 +26,7 @@ import { HWLAB_M3_IO_SKILL_NAME } from "../../skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs"; +import { readStructuredFile } from "./structured-config.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const defaultJsonReportPath = tempReportPath("rpt-004-mvp-e2e.json"); @@ -734,7 +735,7 @@ async function buildExpectedArtifactState({ root, expectedCommit, fsJson }) { const readJson = typeof fsJson === "function" ? (relativePath) => fsJson(relativePath) : (relativePath) => readJsonOptional(root, relativePath); - const deploy = await readJson("deploy/deploy.json"); + const deploy = await readJson("deploy/deploy.yaml"); const catalog = await readJson("deploy/artifact-catalog.dev.json"); const artifactReport = await readJson(defaultArtifactReportPath); const deployCommit = sanitizeCommit(deploy?.commitId); @@ -1568,7 +1569,7 @@ function runGit(root, args) { async function readJsonOptional(root, relativePath) { try { - return JSON.parse(await readFile(path.join(root, relativePath), "utf8")); + return await readStructuredFile(root, relativePath); } catch { return null; } diff --git a/scripts/src/rpt004-mvp-e2e-harness.test.mjs b/scripts/src/rpt004-mvp-e2e-harness.test.mjs index 736360cd..7a319f36 100644 --- a/scripts/src/rpt004-mvp-e2e-harness.test.mjs +++ b/scripts/src/rpt004-mvp-e2e-harness.test.mjs @@ -576,7 +576,7 @@ function fixtureArtifactJson() { } }; return async (relativePath) => { - if (relativePath === "deploy/deploy.json") return deploy; + if (relativePath === "deploy/deploy.yaml") return deploy; if (relativePath === "deploy/artifact-catalog.dev.json") return catalog; if (relativePath === "/tmp/hwlab-dev-gate/dev-artifacts.json") return artifactReport; return null; diff --git a/scripts/src/structured-config.mjs b/scripts/src/structured-config.mjs new file mode 100644 index 00000000..0ffdff6c --- /dev/null +++ b/scripts/src/structured-config.mjs @@ -0,0 +1,45 @@ +import { readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; + +import * as YAML from "yaml"; + +export function structuredConfigFormatForPath(relativePath) { + const extension = path.extname(relativePath).toLowerCase(); + if (extension === ".json") return "json"; + if (extension === ".yaml" || extension === ".yml") return "yaml"; + throw new Error(`unsupported structured config extension for ${relativePath}`); +} + +export function parseStructuredConfig(text, relativePath = "") { + const format = structuredConfigFormatForPath(relativePath); + if (format === "json") return JSON.parse(text); + return YAML.parse(text); +} + +export function stringifyStructuredConfig(value, relativePath = "") { + const format = structuredConfigFormatForPath(relativePath); + if (format === "json") return `${JSON.stringify(value, null, 2)}\n`; + const text = YAML.stringify(value, { indent: 2, lineWidth: 0 }); + return text.endsWith("\n") ? text : `${text}\n`; +} + +export async function readStructuredFile(repoRoot, relativePath) { + const filePath = path.isAbsolute(relativePath) ? relativePath : path.join(repoRoot, relativePath); + const raw = await readFile(filePath, "utf8"); + return parseStructuredConfig(raw, relativePath); +} + +export async function readStructuredFileIfPresent(repoRoot, relativePath, fallback = null) { + try { + return await readStructuredFile(repoRoot, relativePath); + } catch (error) { + if (error?.code === "ENOENT") return fallback; + throw error; + } +} + +export async function writeStructuredFile(repoRoot, relativePath, value) { + const filePath = path.isAbsolute(relativePath) ? relativePath : path.join(repoRoot, relativePath); + const raw = stringifyStructuredConfig(value, relativePath); + await writeFile(filePath, raw, "utf8"); +} diff --git a/scripts/validate-artifact-catalog.mjs b/scripts/validate-artifact-catalog.mjs index 2750c53f..b9aad65c 100644 --- a/scripts/validate-artifact-catalog.mjs +++ b/scripts/validate-artifact-catalog.mjs @@ -5,11 +5,12 @@ import path from "node:path"; import { fileURLToPath } from "node:url"; import { DEV_ENDPOINT, ENVIRONMENT_DEV, SERVICE_IDS } from "../internal/protocol/index.mjs"; +import { readStructuredFile } from "./src/structured-config.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const catalogPath = "deploy/artifact-catalog.dev.json"; -const deployPath = "deploy/deploy.json"; +const deployPath = "deploy/deploy.yaml"; const healthContractPath = "deploy/k8s/dev/health-contract.yaml"; const commitPattern = /^[a-f0-9]{7,40}$/; const digestPattern = /^sha256:[a-f0-9]{64}$/; @@ -61,8 +62,7 @@ const forbiddenDeployArtifactEnv = new Set([ ]); async function readJSON(relativePath) { - const raw = await readFile(path.join(repoRoot, relativePath), "utf8"); - return JSON.parse(raw); + return readStructuredFile(repoRoot, relativePath); } function assertUnique(name, values) { diff --git a/scripts/validate-contract.mjs b/scripts/validate-contract.mjs index 58a6a4d8..865621c1 100644 --- a/scripts/validate-contract.mjs +++ b/scripts/validate-contract.mjs @@ -14,11 +14,11 @@ import { import { HWLAB_M3_IO_API_BASE_URL_ENV } from "../skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs"; import { ENVIRONMENT_DEV, - SERVICE_IDS, TABLES, validateRequest, validateResponse } from "../internal/protocol/index.mjs"; +import { readStructuredFile } from "./src/structured-config.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); @@ -53,8 +53,7 @@ async function parseJSONSchemaFiles(dir) { } async function parseDeployManifest(relativePath) { - const raw = await readFile(path.join(repoRoot, relativePath), "utf8"); - const doc = JSON.parse(raw); + const doc = await readStructuredFile(repoRoot, relativePath); assert.equal(doc.manifestVersion, "v1", `${relativePath} manifestVersion`); assert.equal(doc.environment, ENVIRONMENT_DEV, `${relativePath} environment`); assert.equal(Object.hasOwn(doc, "commitId"), false, `${relativePath} commitId must stay in artifact catalog`); @@ -63,8 +62,7 @@ async function parseDeployManifest(relativePath) { } async function parseK8sList(relativePath) { - const raw = await readFile(path.join(repoRoot, relativePath), "utf8"); - const doc = JSON.parse(raw); + const doc = await readStructuredFile(repoRoot, relativePath); assert.equal(doc.kind, "List", `${relativePath} kind`); assert.ok(Array.isArray(doc.items), `${relativePath} items`); return doc; @@ -75,14 +73,10 @@ function assertUnique(name, values) { } function assertCommonSchema(commonSchema) { - const schemaServiceIds = commonSchema.$defs.serviceId.enum; - assert.deepEqual(schemaServiceIds, SERVICE_IDS, "common service ids must match runtime constants"); assert.deepEqual(commonSchema.$defs.environment.enum, [ENVIRONMENT_DEV]); } function assertDeploySchema(deploySchema) { - const serviceIds = deploySchema.$defs.serviceId.enum; - assert.deepEqual(serviceIds, SERVICE_IDS, "deploy service ids must match runtime constants"); assert.equal(deploySchema.properties.endpoint.const, "http://74.48.78.17:16667"); assert.ok(deploySchema.required.includes("health"), "deploy schema requires health source"); assert.ok(deploySchema.required.includes("publicEndpoints"), "deploy schema requires public endpoint source"); @@ -151,14 +145,13 @@ function assertEnvelopeValidation() { const schemas = await parseJSONSchemaFiles("protocol/schemas"); const deploySchemas = await parseJSONSchemaFiles("deploy"); -const deployManifest = await parseDeployManifest("deploy/deploy.json"); +const deployManifest = await parseDeployManifest("deploy/deploy.yaml"); const k8sServices = await parseK8sList("deploy/k8s/base/services.yaml"); const k8sWorkloads = await parseK8sList("deploy/k8s/base/workloads.yaml"); const docsByName = new Map([...schemas, ...deploySchemas].map((item) => [path.basename(item.relativePath), item.doc])); assert.ok(schemas.length >= 13, "expected protocol schemas"); assert.ok(TABLES.includes("patch_panel_status"), "frozen tables must include patch_panel_status"); -assertUnique("service ids", SERVICE_IDS); assertUnique("tables", TABLES); assertCommonSchema(docsByName.get("common.json")); assertDeploySchema(docsByName.get("deploy.schema.json")); @@ -263,7 +256,7 @@ assertCodeAgentProviderWorkloadContract(cloudApiWorkloadEnv); assertDbForbiddenInvalidHostContract(); assertValidationDoesNotExposeSecretValues(cloudApi.env, cloudApiWorkloadEnv); -console.log(`validated ${schemas.length + deploySchemas.length} JSON files and ${SERVICE_IDS.length} service ids`); +console.log(`validated ${schemas.length + deploySchemas.length} JSON files`); function listItems(document) { return document?.kind === "List" ? document.items ?? [] : [document].filter(Boolean); diff --git a/scripts/validate-dev-m3-cardinality.mjs b/scripts/validate-dev-m3-cardinality.mjs index 16e6959d..e9ee260d 100644 --- a/scripts/validate-dev-m3-cardinality.mjs +++ b/scripts/validate-dev-m3-cardinality.mjs @@ -4,6 +4,8 @@ import { readFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath, pathToFileURL } from "node:url"; +import { readStructuredFile } from "./src/structured-config.mjs"; + const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const namespace = "hwlab-dev"; const m3EndpointMap = Object.freeze({ @@ -110,8 +112,7 @@ const expectedInstanceServices = Object.freeze([ ]); async function readJson(relativePath) { - const raw = await readFile(path.join(repoRoot, relativePath), "utf8"); - return JSON.parse(raw); + return readStructuredFile(repoRoot, relativePath); } function listItems(document) { @@ -216,14 +217,14 @@ function assertDevK8sInstanceServices(services) { } function assertDeployManifestCardinality(deploy) { - assert.equal(deploy.environment, "dev", "deploy/deploy.json environment"); - assert.equal(deploy.namespace, namespace, "deploy/deploy.json namespace"); - assert.equal(deploy.profiles?.prod?.enabled, false, "deploy/deploy.json prod disabled"); + assert.equal(deploy.environment, "dev", "deploy/deploy.yaml environment"); + assert.equal(deploy.namespace, namespace, "deploy/deploy.yaml namespace"); + assert.equal(deploy.profiles?.prod?.enabled, false, "deploy/deploy.yaml prod disabled"); const byId = new Map((deploy.services ?? []).map((service) => [service.serviceId, service])); for (const [serviceId, requirement] of Object.entries(expected)) { const service = byId.get(serviceId); - assert.ok(service, `deploy/deploy.json missing ${serviceId}`); + assert.ok(service, `deploy/deploy.yaml missing ${serviceId}`); assert.equal(service.namespace, namespace, `${serviceId} deploy namespace`); assert.equal(service.profile, "dev", `${serviceId} deploy profile`); assert.equal(service.replicas, requirement.replicas, `${serviceId} deploy replicas`); @@ -250,7 +251,7 @@ function assertDeployManifestInstanceMappings(deploy) { const byName = new Map(mappings.map((mapping) => [mapping.name, mapping])); for (const expectedService of expectedInstanceServices) { const mapping = byName.get(expectedService.name); - assert.ok(mapping, `deploy/deploy.json missing k3s serviceMapping ${expectedService.name}`); + assert.ok(mapping, `deploy/deploy.yaml missing k3s serviceMapping ${expectedService.name}`); assert.equal(mapping.serviceId, expectedService.serviceId, `${expectedService.name} serviceId`); assert.equal(mapping.namespace, namespace, `${expectedService.name} namespace`); assert.equal(mapping.port, expectedService.port, `${expectedService.name} port`); @@ -303,7 +304,7 @@ function assertM3RouteConfig(value, label) { export async function readDevM3CardinalityInputs() { const [deploy, workloads, services, topology] = await Promise.all([ - readJson("deploy/deploy.json"), + readJson("deploy/deploy.yaml"), readJson("deploy/k8s/base/workloads.yaml"), readJson("deploy/k8s/base/services.yaml"), readJson("fixtures/mvp/m3-hardware-loop/topology.json") diff --git a/scripts/validate-m3-rollout-runbook.mjs b/scripts/validate-m3-rollout-runbook.mjs index acfe3bc0..37a5ec38 100644 --- a/scripts/validate-m3-rollout-runbook.mjs +++ b/scripts/validate-m3-rollout-runbook.mjs @@ -37,7 +37,7 @@ async function main() { assertIncludes(doc, "2 x hwlab-gateway-simu", "m3 runbook"); assertIncludes(doc, "1 x hwlab-patch-panel", "m3 runbook"); assertIncludes(doc, "DO1 -> patch-panel -> DI1", "m3 runbook"); - assertIncludes(doc, "deploy/deploy.json", "m3 runbook"); + assertIncludes(doc, "deploy/deploy.yaml", "m3 runbook"); assertIncludes(doc, "deploy/artifact-catalog.dev.json", "m3 runbook"); assertIncludes(doc, "pikasTech/HWLAB#63", "m3 runbook"); assertIncludes(doc, "Do not promote SOURCE / LOCAL / DRY-RUN / fixture output to `DEV-LIVE`.", "m3 runbook"); diff --git a/tools/hwlab-cli/caserun.test.ts b/tools/hwlab-cli/caserun.test.ts index 52d0f0c1..4cf96bbf 100644 --- a/tools/hwlab-cli/caserun.test.ts +++ b/tools/hwlab-cli/caserun.test.ts @@ -483,6 +483,8 @@ test("case run orchestrates agent prompt, diff capture, and compile evidence wit assert.equal(requests.some((item) => item.url === "http://web.test/v1/agent/chat"), true); const agentChatRequest = requests.find((item) => item.url === "http://web.test/v1/agent/chat"); assert.equal(Object.hasOwn(agentChatRequest.body, "workspaceFiles"), false); + assert.equal(typeof agentChatRequest.body.hwpodSpecContent, "string"); + assert.match(agentChatRequest.body.hwpodSpecContent, /HWLAB-CASE-F103.*caserun-run-agent-flow/u); assert.doesNotMatch(agentChatRequest.body.message, /path: "F:\\Work\\HWLAB-CASE-F103\\\.worktree\\caserun-run-agent-flow"/u); assert.equal(requests.some((item) => item.url === "http://web.test/v1/agent/chat/result/" + result.payload.agent.traceId), true); assert.equal(requests.some((item) => item.url === "http://web.test/v1/agent/chat/trace/" + result.payload.agent.traceId), false); @@ -638,6 +640,121 @@ test("case run status reports latest run stage when async control stage is stale } }); +test("case aggregate writes one registry markdown entry without grading or broad registry commits", async () => { + const root = await mkdtempCaseRoot(); + const caseRepo = path.join(root, "hwlab-case-registry"); + const cwd = path.join(root, "hwlab"); + const caseId = "d601-f103-v2-arm2d-integration"; + const runId = "case04-completed-run"; + const registryRunDir = path.join(caseRepo, "runs", caseId, runId); + try { + await mkdir(path.join(caseRepo, ".git"), { recursive: true }); + await writeFile(path.join(caseRepo, ".git", "HEAD"), "ref: refs/heads/main\n", "utf8"); + await mkdir(path.join(registryRunDir, ".hwlab"), { recursive: true }); + await writeJsonFile(path.join(registryRunDir, "run.json"), { + caseId, + runId, + runDir: "/root/hwlab-v02/.state/hwlab-cli/caserun/case04-completed-run", + caseRepo, + apiUrl: "http://74.48.78.17:19667", + compileOnly: true, + createdAt: "2026-06-08T00:00:00.000Z", + completedAt: "2026-06-08T00:10:00.000Z", + subject: { repoLocalPath: SUBJECT_REPO_LOCAL_PATH, commitId: SUBJECT_COMMIT_ID, subdir: "projects/01_baseline", worktreePath: "F:\\Work\\HWLAB-CASE-F103\\.worktree\\caserun-case04" }, + agent: { providerProfile: "dsflash-go", projectId: "prj_hwpod_workbench", conversationId: "cnv_case04", sessionId: "ses_case04", threadId: "thread-case04", traceId: "trc_case04", traceLookup: { commands: { trace: "hwlab-cli client agent trace trc_case04 --render web", result: "hwlab-cli client agent result trc_case04", inspect: "hwlab-cli client agent inspect --trace-id trc_case04" } } }, + _control: { status: "completed", completedAt: "2026-06-08T00:10:00.000Z" } + }); + await writeJsonFile(path.join(registryRunDir, "evidence.json"), { + contractVersion: "hwpod-case-run-evidence-v1", + caseId, + runId, + compileOnly: true, + autoEvaluation: false, + status: "recorded", + subject: { repoLocalPath: SUBJECT_REPO_LOCAL_PATH, commitId: SUBJECT_COMMIT_ID, subdir: "projects/01_baseline", worktreePath: "F:\\Work\\HWLAB-CASE-F103\\.worktree\\caserun-case04" }, + agent: { providerProfile: "dsflash-go", projectId: "prj_hwpod_workbench", conversationId: "cnv_case04", sessionId: "ses_case04", threadId: "thread-case04", traceId: "trc_case04" }, + traceLookup: { strategy: "id_plus_existing_cli", commands: { trace: "hwlab-cli client agent trace trc_case04 --render web", result: "hwlab-cli client agent result trc_case04", inspect: "hwlab-cli client agent inspect --trace-id trc_case04" } }, + agentTrace: { traceId: "trc_case04", source: "hwlab-cli.client.agent.trace", sourceEventCount: 3, renderedRowCount: 3, hwpodCommandCount: 2, hwpodBuildCommandCount: 1, agentRun: { runId: "run_agent_case04", commandId: "cmd_case04" } }, + agentDiff: { statusShort: " M projects/01_baseline/User/main.c\n", diffStat: " main.c | 3 +++\n", diffPatchSha256: "diff-sha", sourceRootChangedAfterPrepare: false, sourceRootChangedAfterAgent: true }, + hwpod: { source: "case-run-runner-post-agent-compile-check", exitCode: 0 }, + keilJob: { jobId: "job-case04", status: "completed" }, + artifacts: [], + decisions: { runnerPostAgentCompileCheck: "recorded" } + }); + await writeJsonFile(path.join(registryRunDir, "agent-messages.json"), { + renderer: "tools/src/hwlab-cli/trace-renderer:traceDisplayRows", + sourceEventCount: 3, + renderedRowCount: 3, + rows: [ + { rowId: "row-1", seq: 1, header: "请求接受", body: "CaseRun started" }, + { rowId: "row-2", seq: 2, header: "ok commandExecution", body: "hwpod build --spec .hwlab/hwpod-spec.yaml\nexitCode=0" }, + { rowId: "row-3", seq: 3, header: "助手最终消息", terminal: true, bodyFormat: "markdown", body: "Build completed" } + ] + }); + await writeJsonFile(path.join(registryRunDir, "artifact-manifest.json"), { + contractVersion: "hwpod-case-run-artifact-manifest-v1", + caseId, + runId, + traceLookup: { commands: { trace: "hwlab-cli client agent trace trc_case04 --render web", result: "hwlab-cli client agent result trc_case04", inspect: "hwlab-cli client agent inspect --trace-id trc_case04" } }, + subject: { repoLocalPath: SUBJECT_REPO_LOCAL_PATH, commitId: SUBJECT_COMMIT_ID, subdir: "projects/01_baseline", worktreePath: "F:\\Work\\HWLAB-CASE-F103\\.worktree\\caserun-case04" }, + agent: { providerProfile: "dsflash-go", projectId: "prj_hwpod_workbench", conversationId: "cnv_case04", sessionId: "ses_case04", threadId: "thread-case04", traceId: "trc_case04" }, + trace: { traceId: "trc_case04", source: "hwlab-cli.client.agent.trace", hwpodCommandCount: 2, hwpodBuildCommandCount: 1, agentRun: { runId: "run_agent_case04", commandId: "cmd_case04" } }, + diff: { path: "agent-diff.patch", sha256: "diff-sha", statusShort: " M projects/01_baseline/User/main.c\n", diffStat: " main.c | 3 +++\n", sourceRootChangedAfterPrepare: false, sourceRootChangedAfterAgent: true }, + readableAgent: { renderer: "tools/src/hwlab-cli/trace-renderer:traceDisplayRows", finalResponse: { present: true, text: "Build completed" }, traceRender: { sourceEventCount: 3, renderedRowCount: 3 } }, + keilJob: { jobId: "job-case04", status: "completed" }, + files: [{ path: "evidence.json", bytes: 10, sha256: "evidence-sha" }] + }); + await writeFile(path.join(registryRunDir, "agent-prompt.md"), "Implement ARM-2D integration\n", "utf8"); + await writeFile(path.join(registryRunDir, "final-response.md"), "# CaseRun Final Response\n\nBuild completed\n", "utf8"); + await writeFile(path.join(registryRunDir, "agent-diff.patch"), "diff --git a/main.c b/main.c\n+ d601_arm2d_demo_show();\n", "utf8"); + await writeFile(path.join(registryRunDir, ".hwlab", "hwpod-spec.yaml"), hwpodSpecText(), "utf8"); + await mkdir(cwd, { recursive: true }); + + const gitCommands: string[][] = []; + const result = await runHwlabCli(["case", "aggregate", caseId, "--case-repo", caseRepo, "--run-id", runId], { + cwd, + now: () => "2026-06-08T00:11:00.000Z", + runProcess: async (command) => { + gitCommands.push(command); + const rel = `runs/${caseId}/${runId}/aggregate.md`; + if (command[1] === "status") return { command, exitCode: 0, stdout: `?? ${rel}\n M runs/${caseId}/${runId}/result.json\n`, stderr: "" }; + if (command[1] === "add") return { command, exitCode: 0, stdout: "", stderr: "" }; + if (command[1] === "diff") return { command, exitCode: 1, stdout: "", stderr: "" }; + if (command[1] === "commit") return { command, exitCode: 0, stdout: "[main abcdef1] data: aggregate caserun case04-completed-run\n", stderr: "" }; + if (command[1] === "push") return { command, exitCode: 0, stdout: "", stderr: "To github.com:pikasTech/hwlab-case-registry.git\n" }; + return { command, exitCode: 0, stdout: "", stderr: "" }; + } + }); + + assert.equal(result.exitCode, 0); + assert.equal(result.payload.action, "case.aggregate"); + assert.equal(result.payload.autoEvaluation, false); + assert.equal(result.payload.aggregate.registrySync.status, "pushed"); + assert.equal(result.payload.aggregate.registrySync.path, `runs/${caseId}/${runId}/aggregate.md`); + assert.deepEqual(gitCommands.find((command) => command[1] === "add")?.slice(-2), ["--", `runs/${caseId}/${runId}/aggregate.md`]); + assert.deepEqual(gitCommands.find((command) => command[1] === "commit")?.slice(-2), ["--", `runs/${caseId}/${runId}/aggregate.md`]); + const aggregate = await readFile(path.join(registryRunDir, "aggregate.md"), "utf8"); + assert.match(aggregate, /## 运行环境信息/u); + assert.match(aggregate, /## HWPOD 信息/u); + assert.match(aggregate, /## Code Agent 信息/u); + assert.match(aggregate, /## 输入 Prompt/u); + assert.match(aggregate, /## 低噪声 Trace/u); + assert.doesNotMatch(aggregate, /• 2\. ok commandExecution/u); + assert.match(aggregate, /
\n已运行 hwpod build --spec \.hwlab\/hwpod-spec\.yaml<\/summary>/u); + assert.match(aggregate, /已运行 hwpod build --spec \.hwlab\/hwpod-spec\.yaml<\/summary>[\s\S]*```text\nhwpod build --spec \.hwlab\/hwpod-spec\.yaml\nexitCode=0[\s\S]*<\/details>\n\nBuild completed\n\n## Final Response/u); + assert.doesNotMatch(aggregate, /.*助手最终消息/u); + assert.match(aggregate, /## Final Response/u); + assert.match(aggregate, /## 最后 Diff/u); + assert.match(aggregate, /Implement ARM-2D integration/u); + assert.match(aggregate, /Build completed/u); + assert.match(aggregate, /d601_arm2d_demo_show/u); + assert.match(aggregate, /autoEvaluation: false/u); + assert.doesNotMatch(aggregate, /pass\/fail|score|打分/u); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); + test("case run status keeps worker control stage ahead of prepared worktree evidence", async () => { const root = await mkdtempCaseRoot(); const cwd = path.join(root, "hwlab"); diff --git a/tools/hwpod-harness.test.ts b/tools/hwpod-harness.test.ts index 489c744a..48c9c2fc 100644 --- a/tools/hwpod-harness.test.ts +++ b/tools/hwpod-harness.test.ts @@ -44,6 +44,45 @@ test("hwpod-ctl initializes, validates, edits, and binds workspace-local hwpod-s await rm(root, { recursive: true, force: true }); } }); + +test("hwpod cli materializes workspace spec from env", async () => { + const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-hwpod-env-spec-")); + const specPath = path.join(root, ".hwlab", "hwpod-spec.yaml"); + const content = [ + "apiVersion: hwlab.dev/v0alpha1", + "kind: Hwpod", + "metadata:", + " uid: D601-F103-V2", + " name: d601-f103-v2", + "spec:", + " targetDevice:", + " board: D601-F103-V2", + " mcu: STM32F103", + " workspace:", + " path: F:\\Work\\HWLAB-CASE-F103\\.worktree\\caserun-env-spec", + " debugProbe:", + " type: daplink", + " ioProbe:", + " uart:", + " id: uart/1", + " nodeBinding:", + " nodeId: node-d601-f103-v2", + "" + ].join("\n"); + try { + const encoded = `base64:${Buffer.from(content, "utf8").toString("base64")}`; + const validate = await runHwpodCtl(["spec", "validate", "--spec", specPath], { now: () => NOW, env: { HWPOD_SPEC_CONTENT: encoded } }); + assert.equal(validate.exitCode, 0); + assert.equal(validate.payload.document.metadata.name, "d601-f103-v2"); + assert.match(await readFile(specPath, "utf8"), /caserun-env-spec/u); + + const plan = await runHwpodCli(["workspace", "ls", ".", "--spec", specPath, "--dry-run"], { now: () => NOW, env: { HWPOD_SPEC_CONTENT: encoded } }); + assert.equal(plan.exitCode, 0); + assert.equal(plan.payload.plan.resourceHints.workspacePath, "F:\\Work\\HWLAB-CASE-F103\\.worktree\\caserun-env-spec"); + } finally { + await rm(root, { recursive: true, force: true }); + } +}); test("hwpod-compiler-cli compiles workspace-local spec into node ops", async () => { const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-hwpod-compiler-")); const specPath = path.join(root, ".hwlab", "hwpod-spec.yaml"); diff --git a/tools/src/hwlab-caserun-lib.ts b/tools/src/hwlab-caserun-lib.ts index 8398bbd7..27bb695d 100644 --- a/tools/src/hwlab-caserun-lib.ts +++ b/tools/src/hwlab-caserun-lib.ts @@ -1,10 +1,10 @@ import { spawn } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { closeSync, openSync } from "node:fs"; -import { copyFile, mkdir, readFile, stat, writeFile } from "node:fs/promises"; +import { copyFile, mkdir, readFile, readdir, stat, writeFile } from "node:fs/promises"; import path from "node:path"; -import { traceDisplayRows, traceNoiseEventCount, type TraceEventRow } from "./hwlab-cli/trace-renderer.ts"; +import { renderTraceRowsMarkdown, traceDisplayRows, traceNoiseEventCount, type TraceEventRow } from "./hwlab-cli/trace-renderer.ts"; import { readHwpodSpec } from "./hwpod-harness-lib.ts"; const DEFAULT_API_URL = "http://74.48.78.17:19667"; @@ -206,6 +206,7 @@ type ArchivedAgentTraceSnapshot = { export async function caseCommand(context: CaseContext) { const subcommand = context.rest[0] || "help"; if (["help", "--help", "-h"].includes(subcommand) || context.parsed.help === true) return caseHelp(); + if (subcommand === "aggregate") return aggregateCaseRun(context); if (subcommand === "prepare") return prepareCaseRun(context); if (subcommand === "build") return buildCaseRun(context); if (subcommand === "collect") return collectCaseRun(context); @@ -239,6 +240,7 @@ export function caseHelp() { `case prepare CASE_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--run-id RUN_ID]`, `case build CASE_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--run-dir DIR]`, `case collect CASE_ID --case-repo ${STANDARD_CASE_REPO_PATH} --run-dir DIR`, + `case aggregate CASE_ID --run-id RUN_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--output aggregate.md]`, `case run CASE_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--provider-profile PROFILE]`, `case run start CASE_ID --case-repo ${STANDARD_CASE_REPO_PATH} [--run-id RUN_ID]`, `case run status RUN_ID [--state-dir .state/hwlab-cli/caserun]`, @@ -248,12 +250,33 @@ export function caseHelp() { notes: [ "case build remains a compile-only HWPOD smoke stage.", "case run prepares the subject worktree, submits a prompt to HWLAB Code Agent, records agent provenance and workspace diff, then runs compile validation.", + "case aggregate is a service-free post-processing command: it reads an existing case registry run and writes one aggregate markdown entry without pass/fail grading.", "case run start/status/result/logs is the cli-spec async control surface for long CaseRun flows; start returns immediately and status/result/logs are short polling commands.", "This version records raw stage evidence only; it does not auto-grade agent output, diff contents, or Keil evidence." ] }); } +export async function aggregateCaseRun(context: CaseContext) { + const caseRepo = await resolveCaseRepo(context); + const aggregate = await writeCaseAggregateFromRegistry(context, caseRepo, { sync: true }); + return ok("case.aggregate", { + caseId: aggregate.caseId, + runId: aggregate.runId, + caseRepo, + caseRepoRunDir: aggregate.caseRepoRunDir, + aggregate: { + path: aggregate.path, + rel: aggregate.rel, + bytes: aggregate.bytes, + sha256: aggregate.sha256, + registrySync: aggregate.registrySync + }, + aggregationOnly: true, + autoEvaluation: false + }, "completed"); +} + export async function runCaseRun(context: CaseContext) { const prepared = await prepareCaseRun(context, "run"); const agentStage = await runAgentTaskStage(context, prepared.run); @@ -726,7 +749,8 @@ async function runAgentTaskStage(context: CaseContext, run: PreparedCaseRun) { const threadId = session.threadId; const traceId = text(context.parsed.traceId) || `trc_case_${slug(run.caseId)}_${randomUUID().replace(/-/gu, "")}`; run = await updateRun(context, run, { stage: "agent-submitting", agent: agentFailureStage({ stageStatus: "session_created", baseUrl, projectId, providerProfile, conversationId, sessionId, threadId, traceId, promptPath, promptSha256, sessionResponse: compactObject(session.body) }) }); - const accepted = await submitAgentTask(context, { baseUrl, projectId, providerProfile, conversationId, sessionId, threadId, traceId, prompt }); + const hwpodSpecContent = await readFile(run.specPath, "utf8"); + const accepted = await submitAgentTask(context, { baseUrl, projectId, providerProfile, conversationId, sessionId, threadId, traceId, prompt, hwpodSpecContent }); if (!accepted.ok) { const agent = agentFailureStage({ stageStatus: "submit_failed", baseUrl, projectId, providerProfile, conversationId, sessionId, threadId, traceId, promptPath, promptSha256, accepted: accepted.body, error: accepted.error, sessionResponse: session.body }); const nextRun = await updateRun(context, run, { agent }); @@ -857,7 +881,7 @@ async function createAgentSession(context: CaseContext, input: { baseUrl: string }; } -async function submitAgentTask(context: CaseContext, input: { baseUrl: string; projectId: string; providerProfile: string; conversationId: string; sessionId: string; threadId: string; traceId: string; prompt: string }) { +async function submitAgentTask(context: CaseContext, input: { baseUrl: string; projectId: string; providerProfile: string; conversationId: string; sessionId: string; threadId: string; traceId: string; prompt: string; hwpodSpecContent?: string }) { return caseFetchJson(context, `${input.baseUrl}/v1/agent/chat`, { method: "POST", headers: { ...caseAgentHeaders(context), "x-trace-id": input.traceId, prefer: "respond-async", "x-hwlab-short-connection": "1" }, @@ -868,6 +892,7 @@ async function submitAgentTask(context: CaseContext, input: { baseUrl: string; p threadId: input.threadId, traceId: input.traceId, providerProfile: input.providerProfile, + hwpodSpecContent: input.hwpodSpecContent, shortConnection: true, projectId: input.projectId, updatedByClient: "hwlab-cli.case-run" @@ -1745,6 +1770,223 @@ function markdownTableText(value: unknown, maxBytes = 200) { return clipText(value, maxBytes).replace(/\|/gu, "\\|").replace(/\r?\n/gu, " ").trim(); } +async function writeCaseAggregateFromRegistry(context: CaseContext, caseRepo: string, options: { sync?: boolean } = {}) { + const caseId = requiredText(context.parsed.caseId ?? context.rest[1], "caseId"); + const runId = await resolveAggregateRunId(caseRepo, caseId, text(context.parsed.runId ?? context.rest[2])); + const runRel = artifactRel(path.join("runs", caseId, runId)); + const caseRepoRunDir = path.join(caseRepo, runRel); + const evidence = await readJsonIfExists(path.join(caseRepoRunDir, "evidence.json")); + const run = await readJsonIfExists(path.join(caseRepoRunDir, "run.json")); + const manifest = await readJsonIfExists(path.join(caseRepoRunDir, "artifact-manifest.json")); + if (!evidence && !run && !manifest) throw cliError("case_registry_run_not_found", "case registry run artifacts were not found", { caseRepoRunDir, caseId, runId }); + const outputRel = aggregateOutputRel(context); + const outputPath = path.join(caseRepoRunDir, outputRel); + const markdown = await renderCaseAggregateMarkdown({ caseRepoRunDir, caseId, runId, evidence, run, manifest }); + await mkdir(path.dirname(outputPath), { recursive: true }); + await writeFile(outputPath, markdown, "utf8"); + const file = await registryAggregateFileRecord(outputPath); + const registrySync = options.sync === true ? await syncCaseRegistryRel(context, caseRepo, artifactRel(path.join(runRel, outputRel)), `data: aggregate caserun ${runId}`) : null; + return { caseId, runId, caseRepoRunDir, path: outputPath, rel: artifactRel(path.join(runRel, outputRel)), bytes: file.bytes, sha256: file.sha256, registrySync }; +} + +async function resolveAggregateRunId(caseRepo: string, caseId: string, explicit: string) { + if (explicit) return explicit; + const runRoot = path.join(caseRepo, "runs", caseId); + let entries: string[] = []; + try { + entries = await readdir(runRoot); + } catch { + throw cliError("case_registry_runs_not_found", "case registry has no run directory for this case", { caseId, runRoot }); + } + const runs = entries.filter((entry) => !entry.startsWith(".")).sort(); + if (runs.length === 0) throw cliError("case_registry_runs_empty", "case registry has no runs for this case", { caseId, runRoot }); + return runs.at(-1) ?? ""; +} + +function aggregateOutputRel(context: CaseContext) { + const value = text(context.parsed.output ?? context.parsed.outputPath ?? context.parsed.aggregatePath) || "aggregate.md"; + const rel = artifactRel(value).replace(/^\/+/, ""); + if (!rel || rel.includes("../") || rel === ".." || path.isAbsolute(value)) throw cliError("invalid_aggregate_output_path", "aggregate output must be a relative path inside the registry run directory", { output: value }); + if (!rel.endsWith(".md")) throw cliError("invalid_aggregate_output_ext", "aggregate output must be a markdown file", { output: value }); + return rel; +} + +async function renderCaseAggregateMarkdown(input: { caseRepoRunDir: string; caseId: string; runId: string; evidence: any; run: any; manifest: any }) { + const prompt = await readTextIfExists(path.join(input.caseRepoRunDir, "agent-prompt.md")); + const finalResponse = await readTextIfExists(path.join(input.caseRepoRunDir, "final-response.md")); + const diffPatch = await readTextIfExists(path.join(input.caseRepoRunDir, "agent-diff.patch")); + const traceMarkdown = await readTextIfExists(path.join(input.caseRepoRunDir, "agent-trace.md")); + const specText = await readTextIfExists(path.join(input.caseRepoRunDir, ".hwlab", "hwpod-spec.yaml")); + const messages = await readJsonIfExists(path.join(input.caseRepoRunDir, "agent-messages.json")); + const evidence = input.evidence ?? {}; + const run = input.run ?? {}; + const manifest = input.manifest ?? {}; + const traceLookup = manifest.traceLookup ?? evidence.traceLookup ?? run.agent?.traceLookup; + const commands = traceLookup?.commands ?? {}; + const agent = manifest.agent ?? manifest.trace ?? evidence.agent ?? run.agent ?? {}; + const subject = manifest.subject ?? evidence.subject ?? run.subject ?? {}; + const trace = manifest.trace ?? evidence.agentTrace ?? run.agentTrace ?? {}; + const diff = manifest.diff ?? evidence.agentDiff ?? run.agentDiff ?? {}; + const keilJob = manifest.keilJob ?? evidence.keilJob ?? null; + const agentStage = manifest.agentStage ?? agentStageSummary(evidence); + const traceBody = aggregateTraceBody(messages, traceMarkdown); + return `${[ + `# HWPOD CaseRun Aggregate: ${input.caseId}`, + ``, + `- caseId: ${input.caseId}`, + `- runId: ${input.runId}`, + `- generatedFrom: case registry artifacts`, + `- aggregationOnly: true`, + `- autoEvaluation: false`, + `- primaryEntry: aggregate.md`, + ``, + `## 运行环境信息`, + ``, + aggregateBullets([ + ["apiUrl", run.apiUrl ?? evidence.apiUrl], + ["compileOnly", evidence.compileOnly ?? run.compileOnly], + ["caseRepoRunDir", input.caseRepoRunDir], + ["sourceRunDir", manifest.sourceRunDir ?? run.runDir], + ["createdAt", run.createdAt], + ["completedAt", run.completedAt ?? run._control?.completedAt], + ["runnerPostAgentCompileCheck", manifest.decisions?.runnerPostAgentCompileCheck ?? evidence.decisions?.runnerPostAgentCompileCheck ?? "recorded"] + ]), + ``, + `## HWPOD 信息`, + ``, + aggregateBullets([ + ["subjectRepoLocalPath", subject.repoLocalPath], + ["subjectCommitId", subject.commitId], + ["subjectSubdir", subject.subdir], + ["subjectWorktreePath", subject.worktreePath], + ["sourceRootBaselineStatus", subject.sourceRootBaseline?.statusShort], + ["sourceRootAfterPrepareStatus", subject.sourceRootAfterPrepare?.statusShort], + ["keilJobId", keilJob?.jobId], + ["keilStatus", keilJob?.status], + ["hwpodExitCode", evidence.hwpod?.exitCode] + ]), + specText ? aggregateDetails("Run-local HWPOD spec", "yaml", specText) : `_Run-local HWPOD spec artifact is missing._`, + ``, + `## Code Agent 信息`, + ``, + aggregateBullets([ + ["providerProfile", agent.providerProfile], + ["projectId", agent.projectId], + ["conversationId", agent.conversationId], + ["sessionId", agent.sessionId], + ["threadId", agent.threadId], + ["traceId", agent.traceId ?? trace.traceId], + ["agentRunId", trace.agentRun?.runId], + ["agentCommandId", trace.agentRun?.commandId], + ["traceSource", trace.source], + ["traceCommand", commands.trace], + ["resultCommand", commands.result], + ["inspectCommand", commands.inspect] + ]), + ``, + `## 输入 Prompt`, + ``, + prompt ? aggregateFence("markdown", prompt) : `_agent-prompt.md artifact is missing._`, + ``, + `## 低噪声 Trace`, + ``, + `- renderer: ${manifest.readableAgent?.renderer ?? messages?.renderer ?? CASE_TRACE_RENDERER}`, + `- sourceEventCount: ${manifest.readableAgent?.traceRender?.sourceEventCount ?? messages?.sourceEventCount ?? trace.sourceEventCount ?? ""}`, + `- renderedRowCount: ${manifest.readableAgent?.traceRender?.renderedRowCount ?? messages?.renderedRowCount ?? trace.renderedRowCount ?? ""}`, + `- hwpodCommandCount: ${trace.hwpodCommandCount ?? evidence.agentTrace?.hwpodCommandCount ?? ""}`, + `- hwpodBuildCommandCount: ${trace.hwpodBuildCommandCount ?? evidence.agentTrace?.hwpodBuildCommandCount ?? ""}`, + ``, + traceBody, + ``, + `## Final Response`, + ``, + finalResponse ? finalResponse.trimEnd() : `_final-response.md artifact is missing._`, + ``, + `## 最后 Diff`, + ``, + aggregateBullets([ + ["statusShort", diff.statusShort], + ["diffStat", diff.diffStat], + ["diffSha256", diff.sha256 ?? diff.diffPatchSha256], + ["sourceRootChangedAfterPrepare", diff.sourceRootChangedAfterPrepare], + ["sourceRootChangedAfterAgent", diff.sourceRootChangedAfterAgent] + ]), + diffPatch ? aggregateFence("diff", diffPatch) : `_agent-diff.patch artifact is missing._`, + ``, + `## 原始产物索引`, + ``, + aggregateArtifactIndex(manifest), + ``, + `## 说明`, + ``, + `本文件只做已完成 CaseRun registry 产物的二次整理聚合,不新增自动结论、门禁或等级。` + ].join("\n")}\n`; +} + +function aggregateTraceBody(messages: any, traceMarkdown: string) { + const rows = arrayOfObjects(messages?.rows); + if (rows.length > 0) { + return renderTraceRowsMarkdown(rows.map((row) => traceEventRowFromAggregate(row))); + } + if (traceMarkdown) { + return renderTraceRowsMarkdown([{ rowId: "trace-markdown", seq: null, tone: "source", header: "Rendered trace markdown", body: traceMarkdown, bodyFormat: "markdown" }]); + } + return `_No readable trace artifact was found._`; +} + +function traceEventRowFromAggregate(row: Record): TraceEventRow { + const bodyFormat = text(row.bodyFormat); + return { + rowId: text(row.rowId) || text(row.header) || "trace-row", + seq: numberOption(row.seq) ?? null, + tone: traceRowTone(row.tone), + header: text(row.header) || text(row.rowId) || "trace row", + body: row.body === undefined || row.body === null ? null : String(row.body), + terminal: row.terminal === true ? true : undefined, + bodyFormat: bodyFormat === "markdown" || bodyFormat === "text" ? bodyFormat : undefined + }; +} + +function aggregateBullets(items: Array<[string, unknown]>) { + const lines = items.map(([key, value]) => `- ${key}: ${aggregateInlineValue(value)}`).filter((line) => !line.endsWith(": ")); + return lines.length > 0 ? lines.join("\n") : `_No data recorded._`; +} + +function aggregateInlineValue(value: unknown) { + if (value === undefined || value === null || value === "") return ""; + if (typeof value === "boolean" || typeof value === "number") return String(value); + if (typeof value === "object") return `\`${markdownInline(JSON.stringify(value))}\``; + return markdownInline(String(value)); +} + +function markdownInline(value: string) { + return value.replace(/\r?\n/gu, " ").trim(); +} + +function aggregateDetails(summary: string, language: string, body: string) { + return [`
`, `${escapeHtml(summary)}`, ``, aggregateFence(language, body), ``, `
`].join("\n"); +} + +function escapeHtml(value: string) { + return String(value).replace(/&/gu, "&").replace(//gu, ">"); +} + +function aggregateFence(language: string, body: string) { + const fence = body.includes("```") ? "````" : "```"; + return `${fence}${language}\n${body.trimEnd()}\n${fence}`; +} + +function aggregateArtifactIndex(manifest: any) { + const files = arrayOfObjects(manifest?.files); + if (files.length === 0) return `_artifact-manifest.json is missing or has no files list._`; + return [`| Path | Bytes | SHA-256 |`, `|---|---:|---|`, ...files.map((file) => `| ${markdownTableText(file.path, 220)} | ${file.bytes ?? ""} | ${markdownTableText(file.sha256, 80)} |`)].join("\n"); +} + +async function registryAggregateFileRecord(file: string) { + const content = await readFile(file); + return { bytes: content.length, sha256: sha256Buffer(content) }; +} + async function archiveCaseRunArtifacts(context: CaseContext, run: PreparedCaseRun, evidence: any, options: { sync?: boolean } = {}): Promise { const runRel = caseRegistryRunRel(run); const caseRepoRunDir = path.join(run.caseRepo, runRel); @@ -1759,6 +2001,7 @@ async function archiveCaseRunArtifacts(context: CaseContext, run: PreparedCaseRu ["evidence.json", path.join(run.runDir, "evidence.json")], ["summary.md", undefined] ]); + if (await fileExists(path.join(caseRepoRunDir, "aggregate.md"))) materialized.set("aggregate.md", undefined); const readableAgent = await writeReadableAgentArtifacts(caseRepoRunDir, run, evidence, agentTraceSnapshot); for (const rel of readableAgent.generatedFiles) materialized.set(rel, undefined); const skippedFiles: string[] = []; @@ -1781,22 +2024,25 @@ async function archiveCaseRunArtifacts(context: CaseContext, run: PreparedCaseRu } async function syncCaseRegistryRun(context: CaseContext, run: PreparedCaseRun, runRel: string) { + return syncCaseRegistryRel(context, run.caseRepo, runRel, `data: archive caserun ${run.runId}`); +} + +async function syncCaseRegistryRel(context: CaseContext, caseRepo: string, relInput: string, commitMessage: string) { if (context.parsed.noCaseRepoGitSync === true || context.parsed.noRegistryGitSync === true) { return { status: "skipped", reason: "disabled_by_cli_flag" }; } - const rel = artifactRel(runRel); - const before = await gitProcess(context, run.caseRepo, ["status", "--short", "--", rel]); + const rel = artifactRel(relInput); + const before = await gitProcess(context, caseRepo, ["status", "--short", "--", rel]); if (before.exitCode !== 0) return { status: "failed", stage: "status_before", command: before.command, exitCode: before.exitCode, stderr: clipText(before.stderr) }; if (!text(before.stdout)) return { status: "unchanged", path: rel }; - const add = await gitProcess(context, run.caseRepo, ["add", "--", rel]); + const add = await gitProcess(context, caseRepo, ["add", "--", rel]); if (add.exitCode !== 0) return { status: "failed", stage: "add", command: add.command, exitCode: add.exitCode, stderr: clipText(add.stderr) }; - const staged = await gitProcess(context, run.caseRepo, ["diff", "--cached", "--quiet", "--", rel]); + const staged = await gitProcess(context, caseRepo, ["diff", "--cached", "--quiet", "--", rel]); if (staged.exitCode === 0) return { status: "unchanged_after_add", path: rel }; if (staged.exitCode !== 1) return { status: "failed", stage: "diff_cached", command: staged.command, exitCode: staged.exitCode, stderr: clipText(staged.stderr) }; - const commitMessage = `data: archive caserun ${run.runId}`; - const commit = await gitProcess(context, run.caseRepo, ["commit", "-m", commitMessage, "--", rel]); + const commit = await gitProcess(context, caseRepo, ["commit", "-m", commitMessage, "--", rel]); if (commit.exitCode !== 0) return { status: "failed", stage: "commit", command: commit.command, exitCode: commit.exitCode, stderr: clipText(commit.stderr), stdout: clipText(commit.stdout) }; - const push = await gitProcess(context, run.caseRepo, ["push", "origin", "HEAD"]); + const push = await gitProcess(context, caseRepo, ["push", "origin", "HEAD"]); if (push.exitCode !== 0) return { status: "failed", stage: "push", command: push.command, exitCode: push.exitCode, stderr: clipText(push.stderr), stdout: clipText(push.stdout) }; const commitSha = text(commit.stdout.match(/\[\S+\s+([0-9a-f]{7,40})\]/iu)?.[1]); return clean({ status: "pushed", path: rel, commitMessage, commitSha, commitStdout: clipText(commit.stdout, 1000), pushStdout: clipText(push.stdout, 1000), pushStderr: clipText(push.stderr, 1000) }); @@ -2134,6 +2380,7 @@ function caseRunArtifactManifest(context: CaseContext, run: PreparedCaseRun, evi traceRender: readableAgent.traceRender, finalResponse: readableAgent.finalResponse }), + aggregate: aggregateManifestEntry(files), prompt: clean({ path: "agent-prompt.md", sha256: evidence.agent?.promptSha256 ?? run.agent?.promptSha256 }), diff: clean({ path: "agent-diff.patch", @@ -2160,6 +2407,12 @@ function caseRunArtifactManifest(context: CaseContext, run: PreparedCaseRun, evi }); } +function aggregateManifestEntry(files: CaseRegistryArchiveFile[]) { + const file = files.find((entry) => entry.path === "aggregate.md"); + if (!file) return undefined; + return { path: file.path, bytes: file.bytes, sha256: file.sha256, primaryEntry: true, aggregationOnly: true, autoEvaluation: false }; +} + function caseRunArtifactTrace(run: PreparedCaseRun, evidence: any) { return clean({ traceId: evidence.agent?.traceId ?? run.agent?.traceId ?? evidence.agentTrace?.traceId ?? run.agentTrace?.traceId, @@ -2319,6 +2572,10 @@ async function fileSize(file: string) { try { return (await stat(file)).size; } catch { return 0; } } +async function fileExists(file: string) { + try { return (await stat(file)).isFile(); } catch { return false; } +} + async function readTail(file: string, maxBytes: number) { try { const content = await readFile(file); diff --git a/tools/src/hwlab-cli/trace-renderer.ts b/tools/src/hwlab-cli/trace-renderer.ts index 9ad0e242..36df1c29 100644 --- a/tools/src/hwlab-cli/trace-renderer.ts +++ b/tools/src/hwlab-cli/trace-renderer.ts @@ -79,6 +79,43 @@ export function traceDisplayRows(trace: Record = {}, events: Tr return rows.length > 0 ? rows : events.map((event) => traceDisplayRow(trace, event)); } +export function renderTraceRowsMarkdown(rows: TraceEventRow[] = []): string { + if (!Array.isArray(rows) || rows.length === 0) return `_No rendered trace rows were returned._`; + return rows.map((row, index) => renderTraceRowMarkdown(row, index + 1)).join("\n\n"); +} + +export function renderTraceRowMarkdown(row: TraceEventRow, index = 1): string { + if (!isToolTraceRow(row)) return renderTraceMessageRowMarkdown(row); + const body = renderTraceRowDetailBody(row) || row.header || `trace row ${index}`; + return [ + `
`, + `${escapeHtml(`已运行 ${traceToolSummary(row)}`)}`, + ``, + traceMarkdownFence(row.bodyFormat === "markdown" ? "markdown" : "text", body), + ``, + `
` + ].join("\n"); +} + +function renderTraceMessageRowMarkdown(row: TraceEventRow): string { + const body = row.body?.trimEnd(); + if (body) return body; + return row.header || `_No readable trace row body._`; +} + +function isToolTraceRow(row: TraceEventRow): boolean { + return row.rowId.startsWith("tool:") || /commandExecution/u.test(row.header); +} + +function traceToolSummary(row: TraceEventRow): string { + const firstLine = traceMarkdownPreviewLines(row.body).find((line) => !/^(stdout|stderr|exitCode|rowId|terminal):/iu.test(line.trim())); + return compactTraceOneLine(stripTraceToolOutputFromSummary(firstLine || row.header || "tool call"), 320); +} + +function stripTraceToolOutputFromSummary(value: string): string { + return value.replace(/\s+(stdout:|stderr:|exitCode=).*$/isu, "").trim(); +} + interface AssistantRowState { rowIndex: number; comparableText: string; @@ -429,6 +466,32 @@ function traceDisplayBody(event: TraceEvent): string | null { return lines.length > 0 ? lines.join("\n").slice(0, 1400) : null; } +function traceMarkdownPreviewLines(body: string | null | undefined): string[] { + return String(body ?? "") + .replace(/\r\n|\r/gu, "\n") + .trim() + .split("\n") + .map((line) => line.trimEnd()) + .filter((line) => Boolean(line.trim())); +} + +function renderTraceRowDetailBody(row: TraceEventRow): string { + return [ + row.body?.trimEnd() ?? "", + row.rowId ? `rowId: ${row.rowId}` : "", + row.terminal === true ? "terminal: true" : "" + ].filter(Boolean).join("\n"); +} + +function escapeHtml(value: string): string { + return String(value).replace(/&/gu, "&").replace(//gu, ">"); +} + +function traceMarkdownFence(language: string, body: string): string { + const fence = body.includes("```") ? "````" : "```"; + return `${fence}${language}\n${body.trimEnd()}\n${fence}`; +} + function traceStatusToken(event: TraceEvent): string { const status = String(event.status ?? ""); if (status === "completed" || status === "succeeded") return "ok"; diff --git a/tools/src/hwpod-harness-lib.ts b/tools/src/hwpod-harness-lib.ts index 9f13a33d..ee3ff240 100644 --- a/tools/src/hwpod-harness-lib.ts +++ b/tools/src/hwpod-harness-lib.ts @@ -9,6 +9,7 @@ import { HWPOD_NODE_OPS, HWPOD_NODE_OPS_CONTRACT_VERSION } from "./hwpod-node-op import { resolveRuntimeEndpoint, runtimeEndpointVisibility } from "./runtime-endpoint-resolver.ts"; export const DEFAULT_HWPOD_SPEC_PATH = ".hwlab/hwpod-spec.yaml"; +const HWPOD_SPEC_ENV_NAMES = ["HWPOD_SPEC_CONTENT"]; const COMPILER_NAME = "hwpod-compiler-cli"; const CTL_NAME = "hwpod-ctl"; @@ -57,9 +58,11 @@ async function readCliStdinForCommand(argv: string[]): Promise string } = {}) { + const env = options.env ?? process.env; const now = options.now ?? (() => new Date().toISOString()); try { const parsed = parseOptions(argv); + await ensureSpecFromEnv(specPathFrom(parsed), env); const command = parsed._[0] || "help"; if (["help", "--help", "-h"].includes(command)) return result(0, compilerHelp(), now); if (command !== "compile") throw cliError("unsupported_compiler_command", `unsupported hwpod-compiler-cli command: ${command}`); @@ -75,15 +78,20 @@ export async function runHwpodCompilerCli(argv: string[], options: { env?: EnvLi } export async function runHwpodCtl(argv: string[], options: { env?: EnvLike; now?: () => string } = {}) { + const env = options.env ?? process.env; const now = options.now ?? (() => new Date().toISOString()); try { const parsed = parseOptions(argv); const group = parsed._[0] || "help"; if (["help", "--help", "-h"].includes(group)) return result(0, ctlHelp(), now); - if (group === "bind") return result(0, await bindSpec(parsed, now), now); + if (group === "bind") { + await ensureSpecFromEnv(specPathFrom(parsed), env); + return result(0, await bindSpec(parsed, now), now); + } if (group !== "spec") throw cliError("unsupported_ctl_command", `unsupported hwpod-ctl command: ${group}`); const subcommand = parsed._[1] || "validate"; if (subcommand === "init") return result(0, await initSpec(parsed, now), now); + await ensureSpecFromEnv(specPathFrom(parsed), env); if (subcommand === "validate") return result(0, await validateSpec(parsed), now); if (subcommand === "show") return result(0, await showSpec(parsed), now); if (subcommand === "set") return result(0, await setSpec(parsed, now), now); @@ -99,6 +107,7 @@ export async function runHwpodCli(argv: string[], options: { env?: EnvLike; fetc const now = options.now ?? (() => new Date().toISOString()); try { const parsed = parseOptions(argv); + await ensureSpecFromEnv(specPathFrom(parsed), env); const command = parsed._[0] || "help"; if (["help", "--help", "-h"].includes(command)) return result(0, cliHelp(), now); if (parsed.help === true || parsed.h === true) return result(0, hwpodCliCommandHelp(command, parsed), now); @@ -126,6 +135,37 @@ export async function readHwpodSpec(specPath = DEFAULT_HWPOD_SPEC_PATH) { return normalizeHwpodSpec(parseSimpleYaml(text, specPath), specPath); } +async function ensureSpecFromEnv(specPath = DEFAULT_HWPOD_SPEC_PATH, env: EnvLike = process.env) { + for (const name of HWPOD_SPEC_ENV_NAMES) { + const raw = text(env[name]); + if (!raw) continue; + try { + await readFile(specPath, "utf8"); + return { source: "existing", specPath }; + } catch (error) { + if (error?.code !== "ENOENT") throw error; + } + const content = decodeHwpodSpecEnv(raw); + await mkdir(path.dirname(specPath), { recursive: true }); + await writeFile(specPath, content.endsWith("\n") ? content : `${content}\n`, "utf8"); + return { source: name, specPath }; + } + return null; +} + +function decodeHwpodSpecEnv(value: string) { + if (/^base64:/iu.test(value)) return Buffer.from(value.replace(/^base64:/iu, ""), "base64").toString("utf8"); + if (!/[\r\n]/u.test(value) && /^[A-Za-z0-9+/=_-]+$/u.test(value)) { + try { + const decoded = Buffer.from(value, "base64").toString("utf8"); + if (/^apiVersion:/mu.test(decoded) && /\nkind:\s*Hwpod\b/mu.test(decoded)) return decoded; + } catch { + // Plain YAML without newlines is allowed to fall through unchanged. + } + } + return value; +} + export function compileHwpodNodeOpsPlan({ document, specPath = DEFAULT_HWPOD_SPEC_PATH, intent, args = {}, now = () => new Date().toISOString() }: any) { const normalizedIntent = normalizeIntent(intent); const nodeId = document.spec.nodeBinding.nodeId;