From 9d143874f8ef6a51e7598227658a0ca2e1917dd4 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Sun, 24 May 2026 20:26:21 +0800 Subject: [PATCH] docs: add DEV runtime hotfix runbook and audit Refs #462\nRefs #465\nRefs #7\nRefs #239 --- AGENTS.md | 2 + docs/reference/dev-runtime-boundary.md | 14 + docs/reference/dev-runtime-hotfix-runbook.md | 106 ++++ package.json | 5 +- scripts/dev-runtime-hotfix-audit.mjs | 12 + scripts/src/dev-runtime-hotfix-audit.mjs | 565 ++++++++++++++++++ scripts/src/dev-runtime-hotfix-audit.test.mjs | 158 +++++ 7 files changed, 860 insertions(+), 2 deletions(-) create mode 100644 docs/reference/dev-runtime-hotfix-runbook.md create mode 100644 scripts/dev-runtime-hotfix-audit.mjs create mode 100644 scripts/src/dev-runtime-hotfix-audit.mjs create mode 100644 scripts/src/dev-runtime-hotfix-audit.test.mjs diff --git a/AGENTS.md b/AGENTS.md index f8f13d2b..98eb2080 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -37,6 +37,7 @@ HWLAB 是硬件实验室运行面和控制面项目。本文是 agent、指挥 - DEV 运行态、端口、k3s 和 DB DNS 边界:[docs/reference/dev-runtime-boundary.md](docs/reference/dev-runtime-boundary.md) - 部署正规化、`deploy.json` DEV CD 路径、SecretRef preflight、runner/host 边界和镜像发布:[docs/reference/deployment-publish.md](docs/reference/deployment-publish.md) - Code Agent 对话就绪与真实回复判定:[docs/reference/code-agent-chat-readiness.md](docs/reference/code-agent-chat-readiness.md) +- DEV runtime hotfix runbook 与只读审计:[docs/reference/dev-runtime-hotfix-runbook.md](docs/reference/dev-runtime-hotfix-runbook.md) - Gateway 主动出站 demo、poll/result 和本地 smoke:[docs/reference/gateway-outbound-demo.md](docs/reference/gateway-outbound-demo.md) - MVP E2E 验收测试与带编号测试报告 issue 规则:[docs/reference/MVP-e2e-acceptance.md](docs/reference/MVP-e2e-acceptance.md) - 指挥官协作、PR 和 runner 交接:[docs/reference/commander-collaboration.md](docs/reference/commander-collaboration.md) @@ -64,6 +65,7 @@ HWLAB 是硬件实验室运行面和控制面项目。本文是 agent、指挥 - CI/CD job 查询:`node tools/hwlab-cli/bin/hwlab-cli.mjs cicd status|logs|report ` - runner GitHub 可见性预检:`npm run runner:issue-visibility:preflight` - D601 k3s 只读观测:`npm run d601:k3s:readonly` +- DEV runtime hotfix 只读审计计划:`npm run dev-runtime:hotfix-audit` - Gateway 主动出站本地 smoke:`npm run gateway:demo:smoke`;经本地 edge-proxy 验证用 `npm run gateway:demo:edge-smoke`。 ## D601 k3s 只读观测 diff --git a/docs/reference/dev-runtime-boundary.md b/docs/reference/dev-runtime-boundary.md index 9c4b7505..57e90e74 100644 --- a/docs/reference/dev-runtime-boundary.md +++ b/docs/reference/dev-runtime-boundary.md @@ -163,6 +163,20 @@ contracts, docs, and tests so future rollouts reproduce the same env and safety boundaries. The runner must not re-execute the live hotfix, restart services, read Secrets, mutate PROD, or claim live evidence from source-only changes. +DEV pod/Deployment file overrides, ConfigMap mounts, and pod-local edits are +governed by [dev-runtime-hotfix-runbook.md](dev-runtime-hotfix-runbook.md). The +stable audit entry is: + +```sh +node scripts/dev-runtime-hotfix-audit.mjs --pretty +node scripts/dev-runtime-hotfix-audit.mjs --collect-readonly --pretty +``` + +The default mode only prints a read-only plan. `--collect-readonly` may run +`kubectl get` and `kubectl exec` checks with +`KUBECONFIG=/etc/rancher/k3s/k3s.yaml`; it must not apply, patch, rollout, +restart, delete, create, read Secret resources, or print secret values. + ## Durable Runtime Readiness Contract DB live readiness and durable runtime readiness are separate gates. A diff --git a/docs/reference/dev-runtime-hotfix-runbook.md b/docs/reference/dev-runtime-hotfix-runbook.md new file mode 100644 index 00000000..5d3b70d6 --- /dev/null +++ b/docs/reference/dev-runtime-hotfix-runbook.md @@ -0,0 +1,106 @@ +# HWLAB DEV Runtime Hotfix Runbook + +本文定义 `hwlab-dev` 运行态热修的最低边界、审计字段、只读确认和回滚口径。它只适用于 DEV 能力边界或阻塞排障,不是发布验收手册,也不把运行态覆盖当作源码真相。 + +本规则承接 [pikasTech/HWLAB#462](https://github.com/pikasTech/HWLAB/issues/462) 和 [pikasTech/HWLAB#465](https://github.com/pikasTech/HWLAB/issues/465),服务 [pikasTech/HWLAB#7](https://github.com/pikasTech/HWLAB/issues/7) / [pikasTech/HWLAB#239](https://github.com/pikasTech/HWLAB/issues/239) 的“用户工作台 + Code Agent 真实可用”主线。它不改变 `DC-DCSN-P0-2026-003` / [pikasTech/HWLAB#78](https://github.com/pikasTech/HWLAB/issues/78) 的 M3 上位约束:hotfix runbook、审计输出、SOURCE、LOCAL、DRY-RUN 或只读观察都不能被包装成 M3 DEV-LIVE 通过。 + +## 允许条件 + +DEV runtime hotfix 只能在以下条件同时满足时使用: + +- 目标是 `hwlab-dev`,不是 PROD、真实用户生产面或真实硬件生产控制面。 +- 目的是打通能力边界、定位阻塞层或恢复用户工作台关键路径;不能为了绕过源码、测试、artifact 或 CD 流程而热修。 +- 操作前已经明确该任务不会抢占 M3 虚拟硬件可信闭环主线,也不会把热修证据升级成 M3 验收。 +- 操作后必须创建或关联源码化 follow-up。#465 的持久化拆分是固定分工:#460 负责 Code Agent -> PC gateway `shell.exec` 能力路由,#461 负责 hotfix runner 源码化,#462 负责本 runbook 与只读审计,#463 负责 JSON-RPC `serviceId` 结构化诊断,#464 负责 DEV-LIVE smoke/harness。 + +## D601 k3s 前置 + +D601 裸 `kubectl` 不可信。所有只读观测、热修、回滚和复核都必须显式指定原生 k3s kubeconfig,并先确认节点: + +```sh +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +kubectl get nodes -o jsonpath='{.items[*].metadata.name}' +``` + +结果必须包含 `d601`。出现 `docker-desktop`、`desktop-control-plane`、`127.0.0.1:11700` 或第二套 `hwlab-dev` 控制面时,停止 hotfix、回滚和验收判断。此边界与 [dev-runtime-boundary.md](dev-runtime-boundary.md) 和 [deployment-publish.md](deployment-publish.md) 保持一致。 + +## 必须记录 + +每次 DEV hotfix 至少记录以下字段,写入 PR/issue/comment,不写入仓库 `reports/**`: + +| 字段 | 说明 | +| --- | --- | +| Deployment | namespace、name、generation、template annotation、容器 image。 | +| Pod | 新旧 pod 名、phase、node、用于验证的 running pod。 | +| ConfigMap | 名称、key 名、用途;不得把 data 全量贴入 issue。 | +| annotation | 记录热修来源和目的的 Deployment template annotation。 | +| mount/subPath | volume 名、ConfigMap 名、mountPath、subPath、readOnly。 | +| traceId / operationId | 能力验证请求的 trace 与 operation 标识。 | +| 验证命令 | `grep` marker、`node --check`、API 返回字段等只读或最小验证命令。 | +| 回滚方式 | 移除 volume/mount/annotation 的方法,或用源码 artifact/CD 覆盖的方法。 | +| 源码化 follow-up | 关联 issue/PR,说明热修不会作为长期 runtime truth。 | + +#465 的典型对象是: + +| 项 | 值 | +| --- | --- | +| Deployment | `hwlab-dev/hwlab-cloud-api` | +| ConfigMap | `hwlab-cloud-api-code-agent-hotfix` | +| 覆盖文件 | `/app/internal/cloud/code-agent-chat.mjs` | +| subPath | `code-agent-chat.mjs` | +| marker | `runtime-hotfix-pc-gateway-shell` | +| annotation | `hwlab.pikastech.local/pc-gateway-shell-hotfix` | +| 能力路径 | `/v1/agent/chat -> /v1/rpc/hardware.invoke.shell -> PC gateway -> Windows cmd` | + +## 只读审计入口 + +仓库内只读入口是: + +```sh +node scripts/dev-runtime-hotfix-audit.mjs --pretty +node scripts/dev-runtime-hotfix-audit.mjs --collect-readonly --pretty +``` + +默认 `--plan` 模式只输出 compact JSON 计划,不执行 `kubectl`。`--collect-readonly` 只允许 `kubectl get` 和 `kubectl exec` 形态的读检查,用于判断当前 DEV 是否仍被 hotfix 覆盖;它不会执行 `apply`、`patch`、`rollout`、`restart`、`delete`、`create`、`set image`,也不会读取 Secret resources 或 secret values。 + +输出分类固定包含以下语义: + +| 分类 | 含义 | +| --- | --- | +| `hotfix-configmap-present` | 目标 ConfigMap 仍存在。 | +| `deployment-mounts-hotfix` | Deployment template 仍通过 volumeMount/subPath 覆盖目标文件。 | +| `pod-loads-hotfix-marker` | running pod 内目标文件仍包含 hotfix marker。 | +| `source-artifact-expected` | 正式收口应回到 source/PR/artifact/CD。 | +| `rollback-required` | 当前仍检测到热修残留或覆盖,不能把运行态当源码真相。 | +| `no-hotfix-detected` | 只读审计未发现该 hotfix ConfigMap 或 mount 覆盖。 | +| `unknown-needs-manual-readonly-check` | 无法确认节点、对象或 pod 状态,需要人工继续只读核对。 | + +## 回滚口径 + +优先回滚方式是用源码化 artifact/CD 覆盖 runtime hotfix:#460/#461 合并并发布后,Deployment 应消费正式镜像,不再通过 ConfigMap 覆盖 `/app/internal/cloud/code-agent-chat.mjs`。只读确认口径是: + +```sh +export KUBECONFIG=/etc/rancher/k3s/k3s.yaml +kubectl get nodes -o jsonpath='{.items[*].metadata.name}' +kubectl -n hwlab-dev get deployment hwlab-cloud-api -o json +kubectl -n hwlab-dev get configmap hwlab-cloud-api-code-agent-hotfix -o json +node scripts/dev-runtime-hotfix-audit.mjs --collect-readonly --pretty +``` + +如果必须直接移除运行态覆盖,授权操作者应先从 `kubectl -n hwlab-dev get deployment hwlab-cloud-api -o json` 中定位精确 indexes,再移除: + +- 指向 hotfix ConfigMap 的 `spec.template.spec.volumes[]`; +- 指向 `/app/internal/cloud/code-agent-chat.mjs` 的 `containers[].volumeMounts[]`; +- `hwlab.pikastech.local/pc-gateway-shell-hotfix` 等 hotfix annotation; +- 不再需要的 `hwlab-cloud-api-code-agent-hotfix` ConfigMap。 + +直接 `kubectl patch`、`kubectl delete configmap`、`kubectl rollout status` 或等价写操作必须由明确授权的 DEV 操作者执行,并且仍要使用 `KUBECONFIG=/etc/rancher/k3s/k3s.yaml` 与节点 `d601` 确认。本 runbook 的审计脚本不执行这些写操作。回滚后再次运行 `scripts/dev-runtime-hotfix-audit.mjs --collect-readonly`,期望分类包含 `no-hotfix-detected`,且不包含 `deployment-mounts-hotfix`、`pod-loads-hotfix-marker` 或 `rollback-required`。 + +## 禁止误用 + +- 不得把 ConfigMap/pod 覆盖当作 source truth;正式修复必须回到 source、PR、artifact 和 repo-owned CD。 +- 不得把 #462 runbook 或只读审计包装成 #460/#461 源码化完成。 +- 不得把 #462 runbook 或只读审计包装成 #464 DEV-LIVE smoke/harness 完成。 +- 不得把热修期间的 stdout、trace 或 operation 扩大为 M3 trusted evidence;M3 判定仍按 [m3-loop-rollout-runbook.md](m3-loop-rollout-runbook.md)。 +- 不得读取或打印 Secret value、kubeconfig token、DB URL 密码、provider token。 +- 不得新增或修改仓库 `reports/**`;过程证据写入 PR/issue/comment,临时 JSON 只能放 `/tmp`、`.state` 或 CI artifact。 diff --git a/package.json b/package.json index a24828e0..17636dca 100644 --- a/package.json +++ b/package.json @@ -4,8 +4,8 @@ "private": true, "type": "module", "scripts": { - "validate": "node scripts/repo-reports-guard.mjs && node scripts/validate-contract.mjs && node scripts/deploy-contract-plan.mjs --check && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-provisioning.mjs --check && node scripts/dev-runtime-migration.mjs --check && node scripts/dev-runtime-postflight.mjs --check && node scripts/rpt004-mvp-e2e-harness.mjs --check --no-write && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --test scripts/artifact-runtime-readiness-guard.test.mjs", - "check": "node scripts/repo-reports-guard.mjs && node --check scripts/repo-reports-guard.mjs && node --check scripts/src/report-paths.mjs && node --check internal/protocol/index.mjs && node --check internal/build-metadata.mjs && node --check internal/agent/index.mjs && node --check internal/agent/runtime.mjs && node --check internal/audit/index.mjs && node --check internal/db/runtime-store.mjs && node --check internal/db/runtime-store.test.mjs && node --check internal/mvp-gate/summary.mjs && node --check internal/cloud/db-contract.mjs && node --check internal/dev-entrypoint/cloud-web-routes.mjs && node --check internal/dev-entrypoint/cloud-web-proxy.mjs && node --check internal/dev-entrypoint/cloud-web-proxy.test.mjs && node --check internal/dev-entrypoint/http.mjs && node --check internal/dev-entrypoint/http.test.mjs && node --check internal/cloud/code-agent-contract.mjs && node --check internal/cloud/code-agent-session-registry.mjs && node --check internal/cloud/code-agent-session-registry.test.mjs && node --check internal/cloud/code-agent-trace-store.mjs && node --check internal/cloud/codex-stdio-session.mjs && node --check internal/cloud/json-rpc.mjs && node --check internal/cloud/code-agent-chat.mjs && node --check internal/cloud/m3-io-control.mjs && node --check internal/cloud/server.mjs && node --check internal/sim/model.mjs && node --check internal/sim/model.test.mjs && node --check internal/sim/http.mjs && node --check internal/sim/l2-runtime.mjs && node --check internal/patchpanel/model.mjs && node --check internal/patchpanel/runtime.mjs && node --check cmd/hwlab-cloud-api/main.mjs && node --check cmd/hwlab-cloud-api/provision.mjs && node --check cmd/hwlab-cloud-api/migrate.mjs && node --check cmd/hwlab-edge-proxy/main.mjs && node --check cmd/hwlab-agent-mgr/main.mjs && node --check cmd/hwlab-agent-worker/main.mjs && node --check cmd/hwlab-box-simu/main.mjs && node --check cmd/hwlab-gateway/main.mjs && node --check cmd/hwlab-gateway-simu/main.mjs && node --check scripts/cloud-api-runtime-smoke.mjs && node --check scripts/code-agent-chat-smoke.mjs && node --check scripts/dev-edge-health-smoke.mjs && node --check scripts/src/dev-edge-health-smoke-lib.mjs && node --check scripts/validate-contract.mjs && node --check scripts/deploy-contract-plan.test.mjs && node --check scripts/deploy-desired-state-plan.mjs && node --check scripts/src/deploy-desired-state-plan.mjs && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --check scripts/artifact-runtime-readiness-guard.test.mjs && node --check scripts/report-lifecycle.mjs && node --check scripts/report-lifecycle.test.mjs && node --check scripts/validate-dev-gate-report.mjs && node --check scripts/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.test.mjs && node --check scripts/dev-m3-hardware-loop-smoke.test.mjs && node --check scripts/m3-io-control-e2e.mjs && node --check scripts/src/m3-io-control-e2e.mjs && node --check scripts/m3-io-control-e2e.test.mjs && node --check scripts/dev-cloud-workbench-smoke.test.mjs && node --check scripts/dev-cloud-workbench-layout-smoke.mjs && node --check scripts/rpt004-mvp-e2e-harness.mjs && node --check scripts/src/rpt004-mvp-e2e-harness.mjs && node --check scripts/src/rpt004-mvp-e2e-harness.test.mjs && node --check scripts/validate-dev-m3-cardinality.mjs && node --check scripts/validate-dev-m3-cardinality.test.mjs && node --check scripts/validate-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.test.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/dev-runtime-base-image.mjs && node --check scripts/src/dev-artifact-services.mjs && node --check scripts/src/registry-capabilities.mjs && node --check scripts/preflight-dev-base-image.mjs && node --check scripts/src/dev-base-image-preflight.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.test.mjs && node --check scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs && node --check scripts/d601-k3s-readonly-observability.mjs && node --check scripts/src/d601-k3s-readonly-observability.mjs && node --check scripts/dev-runtime-provisioning.mjs && node --check scripts/src/dev-runtime-provisioning.mjs && node --check scripts/src/dev-runtime-provisioning.test.mjs && node --check scripts/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.test.mjs && node --check scripts/dev-runtime-postflight.mjs && node --check scripts/src/dev-runtime-postflight.mjs && node --check scripts/src/dev-runtime-postflight.test.mjs && node --check scripts/l2-runtime-contract-smoke.mjs && node --check scripts/patch-panel-runtime-smoke.mjs && node --check scripts/export-web-gate-summary.mjs && node --check scripts/l6-cli-web-smoke.mjs && node --check skills/hwlab-agent-runtime/scripts/hwlab-agent-runtime-cli.mjs && node --check skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs && node --check skills/hwlab-agent-runtime/scripts/m3-io-skill-client.test.mjs && node --check tools/hwlab-cli/bin/hwlab-cli.mjs && node --check tools/hwlab-cli/lib/cli.mjs && node --check tools/hwlab-cli/lib/cicd-jobs.mjs && node --check tools/hwlab-cli/lib/cli.test.mjs && node --check web/hwlab-cloud-web/app.mjs && node --check web/hwlab-cloud-web/code-agent-facts.mjs && node --check web/hwlab-cloud-web/code-agent-facts.test.mjs && node --check web/hwlab-cloud-web/code-agent-status.mjs && node --check web/hwlab-cloud-web/code-agent-status.test.mjs && node --check web/hwlab-cloud-web/code-agent-m3-evidence.mjs && node --check web/hwlab-cloud-web/code-agent-m3-evidence.test.mjs && node --check web/hwlab-cloud-web/live-status.mjs && node --check web/hwlab-cloud-web/wiring-status.mjs && node --check web/hwlab-cloud-web/gate-summary.mjs && node --check web/hwlab-cloud-web/scripts/check.mjs && node --check web/hwlab-cloud-web/scripts/live-status-contract.test.mjs && node --check web/hwlab-cloud-web/scripts/build.mjs && node --check web/hwlab-cloud-web/scripts/dist-contract.mjs && node scripts/validate-contract.mjs && node scripts/validate-dev-gate-report.mjs && node scripts/report-lifecycle.test.mjs && node scripts/validate-dev-m3-cardinality.mjs && node scripts/validate-artifact-catalog.mjs && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-provisioning.mjs --check && node scripts/dev-runtime-migration.mjs --check && node cmd/hwlab-cloud-api/provision.mjs --check && node cmd/hwlab-cloud-api/migrate.mjs --check && node scripts/dev-runtime-postflight.mjs --check && node scripts/rpt004-mvp-e2e-harness.mjs --check --no-write && node scripts/dev-evidence-blocker-aggregator.mjs --check && node scripts/l2-runtime-contract-smoke.mjs && node scripts/l6-cli-web-smoke.mjs && node web/hwlab-cloud-web/scripts/check.mjs && node scripts/code-agent-chat-smoke.mjs && node --test web/hwlab-cloud-web/code-agent-facts.test.mjs web/hwlab-cloud-web/code-agent-status.test.mjs web/hwlab-cloud-web/wiring-status.test.mjs web/hwlab-cloud-web/code-agent-m3-evidence.test.mjs scripts/artifact-runtime-readiness-guard.test.mjs scripts/deploy-contract-plan.test.mjs scripts/dev-m3-hardware-loop-smoke.test.mjs scripts/validate-dev-m3-cardinality.test.mjs scripts/dev-cloud-workbench-smoke.test.mjs web/hwlab-cloud-web/scripts/live-status-contract.test.mjs scripts/deploy-desired-state-plan.test.mjs scripts/refresh-artifact-catalog.test.mjs scripts/src/dev-cd-apply.test.mjs scripts/src/dev-deploy-apply.test.mjs scripts/src/dev-runtime-provisioning.test.mjs scripts/src/dev-runtime-migration.test.mjs scripts/src/dev-runtime-postflight.test.mjs scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs scripts/src/dev-evidence-blocker-aggregator.test.mjs skills/hwlab-agent-runtime/scripts/m3-io-skill-client.test.mjs tools/hwlab-cli/lib/cli.test.mjs internal/agent/index.test.mjs internal/audit/index.test.mjs internal/db/schema.test.mjs internal/db/runtime-store.test.mjs internal/cloud/json-rpc.test.mjs internal/cloud/m3-io-control.test.mjs internal/cloud/code-agent-session-registry.test.mjs internal/cloud/server.test.mjs internal/dev-entrypoint/cloud-web-proxy.test.mjs internal/dev-entrypoint/http.test.mjs internal/sim/model.test.mjs internal/patchpanel/model.test.mjs internal/patchpanel/runtime.test.mjs && node scripts/cloud-api-runtime-smoke.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", + "validate": "node scripts/repo-reports-guard.mjs && node scripts/validate-contract.mjs && node scripts/deploy-contract-plan.mjs --check && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-provisioning.mjs --check && node scripts/dev-runtime-migration.mjs --check && node scripts/dev-runtime-postflight.mjs --check && node scripts/dev-runtime-hotfix-audit.mjs && node scripts/rpt004-mvp-e2e-harness.mjs --check --no-write && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --check scripts/dev-runtime-hotfix-audit.mjs && node --check scripts/src/dev-runtime-hotfix-audit.mjs && node --test scripts/artifact-runtime-readiness-guard.test.mjs scripts/src/dev-runtime-hotfix-audit.test.mjs", + "check": "node scripts/repo-reports-guard.mjs && node --check scripts/repo-reports-guard.mjs && node --check scripts/src/report-paths.mjs && node --check internal/protocol/index.mjs && node --check internal/build-metadata.mjs && node --check internal/agent/index.mjs && node --check internal/agent/runtime.mjs && node --check internal/audit/index.mjs && node --check internal/db/runtime-store.mjs && node --check internal/db/runtime-store.test.mjs && node --check internal/mvp-gate/summary.mjs && node --check internal/cloud/db-contract.mjs && node --check internal/dev-entrypoint/cloud-web-routes.mjs && node --check internal/dev-entrypoint/cloud-web-proxy.mjs && node --check internal/dev-entrypoint/cloud-web-proxy.test.mjs && node --check internal/dev-entrypoint/http.mjs && node --check internal/dev-entrypoint/http.test.mjs && node --check internal/cloud/code-agent-contract.mjs && node --check internal/cloud/code-agent-session-registry.mjs && node --check internal/cloud/code-agent-session-registry.test.mjs && node --check internal/cloud/code-agent-trace-store.mjs && node --check internal/cloud/codex-stdio-session.mjs && node --check internal/cloud/json-rpc.mjs && node --check internal/cloud/code-agent-chat.mjs && node --check internal/cloud/m3-io-control.mjs && node --check internal/cloud/server.mjs && node --check internal/sim/model.mjs && node --check internal/sim/model.test.mjs && node --check internal/sim/http.mjs && node --check internal/sim/l2-runtime.mjs && node --check internal/patchpanel/model.mjs && node --check internal/patchpanel/runtime.mjs && node --check cmd/hwlab-cloud-api/main.mjs && node --check cmd/hwlab-cloud-api/provision.mjs && node --check cmd/hwlab-cloud-api/migrate.mjs && node --check cmd/hwlab-edge-proxy/main.mjs && node --check cmd/hwlab-agent-mgr/main.mjs && node --check cmd/hwlab-agent-worker/main.mjs && node --check cmd/hwlab-box-simu/main.mjs && node --check cmd/hwlab-gateway/main.mjs && node --check cmd/hwlab-gateway-simu/main.mjs && node --check scripts/cloud-api-runtime-smoke.mjs && node --check scripts/code-agent-chat-smoke.mjs && node --check scripts/dev-edge-health-smoke.mjs && node --check scripts/src/dev-edge-health-smoke-lib.mjs && node --check scripts/validate-contract.mjs && node --check scripts/deploy-contract-plan.test.mjs && node --check scripts/deploy-desired-state-plan.mjs && node --check scripts/src/deploy-desired-state-plan.mjs && node --check scripts/artifact-runtime-readiness-guard.mjs && node --check scripts/src/artifact-runtime-readiness-guard.mjs && node --check scripts/artifact-runtime-readiness-guard.test.mjs && node --check scripts/report-lifecycle.mjs && node --check scripts/report-lifecycle.test.mjs && node --check scripts/validate-dev-gate-report.mjs && node --check scripts/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.mjs && node --check scripts/src/dev-cd-apply.test.mjs && node --check scripts/dev-m3-hardware-loop-smoke.test.mjs && node --check scripts/m3-io-control-e2e.mjs && node --check scripts/src/m3-io-control-e2e.mjs && node --check scripts/m3-io-control-e2e.test.mjs && node --check scripts/dev-cloud-workbench-smoke.test.mjs && node --check scripts/dev-cloud-workbench-layout-smoke.mjs && node --check scripts/rpt004-mvp-e2e-harness.mjs && node --check scripts/src/rpt004-mvp-e2e-harness.mjs && node --check scripts/src/rpt004-mvp-e2e-harness.test.mjs && node --check scripts/validate-dev-m3-cardinality.mjs && node --check scripts/validate-dev-m3-cardinality.test.mjs && node --check scripts/validate-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.mjs && node --check scripts/refresh-artifact-catalog.test.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/dev-runtime-base-image.mjs && node --check scripts/src/dev-artifact-services.mjs && node --check scripts/src/registry-capabilities.mjs && node --check scripts/preflight-dev-base-image.mjs && node --check scripts/src/dev-base-image-preflight.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.mjs && node --check scripts/src/dev-evidence-blocker-aggregator.test.mjs && node --check scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs && node --check scripts/d601-k3s-readonly-observability.mjs && node --check scripts/src/d601-k3s-readonly-observability.mjs && node --check scripts/dev-runtime-provisioning.mjs && node --check scripts/src/dev-runtime-provisioning.mjs && node --check scripts/src/dev-runtime-provisioning.test.mjs && node --check scripts/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.mjs && node --check scripts/src/dev-runtime-migration.test.mjs && node --check scripts/dev-runtime-postflight.mjs && node --check scripts/src/dev-runtime-postflight.mjs && node --check scripts/src/dev-runtime-postflight.test.mjs && node --check scripts/dev-runtime-hotfix-audit.mjs && node --check scripts/src/dev-runtime-hotfix-audit.mjs && node --check scripts/src/dev-runtime-hotfix-audit.test.mjs && node --test scripts/src/dev-runtime-hotfix-audit.test.mjs && node --check scripts/l2-runtime-contract-smoke.mjs && node --check scripts/patch-panel-runtime-smoke.mjs && node --check scripts/export-web-gate-summary.mjs && node --check scripts/l6-cli-web-smoke.mjs && node --check skills/hwlab-agent-runtime/scripts/hwlab-agent-runtime-cli.mjs && node --check skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs && node --check skills/hwlab-agent-runtime/scripts/m3-io-skill-client.test.mjs && node --check tools/hwlab-cli/bin/hwlab-cli.mjs && node --check tools/hwlab-cli/lib/cli.mjs && node --check tools/hwlab-cli/lib/cicd-jobs.mjs && node --check tools/hwlab-cli/lib/cli.test.mjs && node --check web/hwlab-cloud-web/app.mjs && node --check web/hwlab-cloud-web/code-agent-facts.mjs && node --check web/hwlab-cloud-web/code-agent-facts.test.mjs && node --check web/hwlab-cloud-web/code-agent-status.mjs && node --check web/hwlab-cloud-web/code-agent-status.test.mjs && node --check web/hwlab-cloud-web/code-agent-m3-evidence.mjs && node --check web/hwlab-cloud-web/code-agent-m3-evidence.test.mjs && node --check web/hwlab-cloud-web/live-status.mjs && node --check web/hwlab-cloud-web/wiring-status.mjs && node --check web/hwlab-cloud-web/gate-summary.mjs && node --check web/hwlab-cloud-web/scripts/check.mjs && node --check web/hwlab-cloud-web/scripts/live-status-contract.test.mjs && node --check web/hwlab-cloud-web/scripts/build.mjs && node --check web/hwlab-cloud-web/scripts/dist-contract.mjs && node scripts/validate-contract.mjs && node scripts/validate-dev-gate-report.mjs && node scripts/report-lifecycle.test.mjs && node scripts/validate-dev-m3-cardinality.mjs && node scripts/validate-artifact-catalog.mjs && node scripts/deploy-desired-state-plan.mjs --check && node scripts/dev-runtime-provisioning.mjs --check && node scripts/dev-runtime-migration.mjs --check && node cmd/hwlab-cloud-api/provision.mjs --check && node cmd/hwlab-cloud-api/migrate.mjs --check && node scripts/dev-runtime-postflight.mjs --check && node scripts/rpt004-mvp-e2e-harness.mjs --check --no-write && node scripts/dev-evidence-blocker-aggregator.mjs --check && node scripts/l2-runtime-contract-smoke.mjs && node scripts/l6-cli-web-smoke.mjs && node web/hwlab-cloud-web/scripts/check.mjs && node scripts/code-agent-chat-smoke.mjs && node --test web/hwlab-cloud-web/code-agent-facts.test.mjs web/hwlab-cloud-web/code-agent-status.test.mjs web/hwlab-cloud-web/wiring-status.test.mjs web/hwlab-cloud-web/code-agent-m3-evidence.test.mjs scripts/artifact-runtime-readiness-guard.test.mjs scripts/deploy-contract-plan.test.mjs scripts/dev-m3-hardware-loop-smoke.test.mjs scripts/validate-dev-m3-cardinality.test.mjs scripts/dev-cloud-workbench-smoke.test.mjs web/hwlab-cloud-web/scripts/live-status-contract.test.mjs scripts/deploy-desired-state-plan.test.mjs scripts/refresh-artifact-catalog.test.mjs scripts/src/dev-cd-apply.test.mjs scripts/src/dev-deploy-apply.test.mjs scripts/src/dev-runtime-provisioning.test.mjs scripts/src/dev-runtime-migration.test.mjs scripts/src/dev-runtime-postflight.test.mjs scripts/src/dev-m4-agent-loop-smoke-lib.test.mjs scripts/src/dev-evidence-blocker-aggregator.test.mjs skills/hwlab-agent-runtime/scripts/m3-io-skill-client.test.mjs tools/hwlab-cli/lib/cli.test.mjs internal/agent/index.test.mjs internal/audit/index.test.mjs internal/db/schema.test.mjs internal/db/runtime-store.test.mjs internal/cloud/json-rpc.test.mjs internal/cloud/m3-io-control.test.mjs internal/cloud/code-agent-session-registry.test.mjs internal/cloud/server.test.mjs internal/dev-entrypoint/cloud-web-proxy.test.mjs internal/dev-entrypoint/http.test.mjs internal/sim/model.test.mjs internal/patchpanel/model.test.mjs internal/patchpanel/runtime.test.mjs && node scripts/cloud-api-runtime-smoke.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", "dev-base-image:preflight": "node scripts/preflight-dev-base-image.mjs", "dev-runtime-base:build": "node scripts/dev-runtime-base-image.mjs", "cloud-api:smoke": "node scripts/cloud-api-runtime-smoke.mjs", @@ -40,6 +40,7 @@ "dev-runtime:provisioning": "node scripts/dev-runtime-provisioning.mjs", "dev-runtime:migration": "node scripts/dev-runtime-migration.mjs", "dev-runtime:postflight": "node scripts/dev-runtime-postflight.mjs", + "dev-runtime:hotfix-audit": "node scripts/dev-runtime-hotfix-audit.mjs", "rpt004:mvp:e2e": "node scripts/rpt004-mvp-e2e-harness.mjs", "rpt004:mvp:e2e:live": "node scripts/rpt004-mvp-e2e-harness.mjs --live --url http://74.48.78.17:16666/ --api-url http://74.48.78.17:16667/", "m3:io:e2e": "node scripts/m3-io-control-e2e.mjs", diff --git a/scripts/dev-runtime-hotfix-audit.mjs b/scripts/dev-runtime-hotfix-audit.mjs new file mode 100644 index 00000000..21b9fa62 --- /dev/null +++ b/scripts/dev-runtime-hotfix-audit.mjs @@ -0,0 +1,12 @@ +#!/usr/bin/env node +import { + formatDevRuntimeHotfixAuditFailure, + runDevRuntimeHotfixAuditCli +} from "./src/dev-runtime-hotfix-audit.mjs"; + +try { + process.exitCode = await runDevRuntimeHotfixAuditCli(process.argv.slice(2)); +} catch (error) { + process.stdout.write(`${JSON.stringify(formatDevRuntimeHotfixAuditFailure(error), null, 2)}\n`); + process.exitCode = 1; +} diff --git a/scripts/src/dev-runtime-hotfix-audit.mjs b/scripts/src/dev-runtime-hotfix-audit.mjs new file mode 100644 index 00000000..c2a486f1 --- /dev/null +++ b/scripts/src/dev-runtime-hotfix-audit.mjs @@ -0,0 +1,565 @@ +import { execFile } from "node:child_process"; +import { promisify } from "node:util"; + +const execFileAsync = promisify(execFile); + +const defaultKubeconfig = "/etc/rancher/k3s/k3s.yaml"; +const defaultNamespace = "hwlab-dev"; +const defaultDeployment = "hwlab-cloud-api"; +const defaultConfigMap = "hwlab-cloud-api-code-agent-hotfix"; +const defaultMountPath = "/app/internal/cloud/code-agent-chat.mjs"; +const defaultSubPath = "code-agent-chat.mjs"; +const defaultMarker = "runtime-hotfix-pc-gateway-shell"; +const defaultAnnotationKey = "hwlab.pikastech.local/pc-gateway-shell-hotfix"; +const defaultPodLabelSelector = "app.kubernetes.io/name=hwlab-cloud-api"; +const forbiddenKubectlVerbs = new Set([ + "apply", + "patch", + "rollout", + "restart", + "delete", + "create", + "replace", + "scale", + "set", + "edit", + "annotate", + "label" +]); +const allowedKubectlVerbs = new Set(["get", "describe", "exec", "auth", "config", "version"]); +const classificationOrder = Object.freeze([ + "hotfix-configmap-present", + "deployment-mounts-hotfix", + "pod-loads-hotfix-marker", + "source-artifact-expected", + "rollback-required", + "no-hotfix-detected", + "unknown-needs-manual-readonly-check" +]); + +export async function runDevRuntimeHotfixAuditCli(argv = process.argv.slice(2), options = {}) { + const args = parseArgs(argv); + if (args.help) { + process.stdout.write(`${usage()}\n`); + return 0; + } + const report = await buildDevRuntimeHotfixAuditReport(args, options); + process.stdout.write(`${JSON.stringify(report, null, args.pretty ? 2 : 0)}\n`); + return 0; +} + +export function parseArgs(argv = []) { + const args = { + collectReadonly: false, + pretty: false, + timeoutMs: 5000, + kubeconfig: defaultKubeconfig, + namespace: defaultNamespace, + deployment: defaultDeployment, + configMap: defaultConfigMap, + mountPath: defaultMountPath, + subPath: defaultSubPath, + marker: defaultMarker, + annotationKey: defaultAnnotationKey, + podLabelSelector: defaultPodLabelSelector, + help: false + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--collect-readonly") args.collectReadonly = true; + else if (arg === "--plan" || arg === "--dry-run") args.collectReadonly = false; + else if (arg === "--pretty") args.pretty = true; + else if (arg === "--timeout-ms") args.timeoutMs = parsePositiveInt(requireValue(argv[++index], arg), arg); + else if (arg === "--kubeconfig") args.kubeconfig = requireValue(argv[++index], arg); + else if (arg === "--namespace") args.namespace = requireValue(argv[++index], arg); + else if (arg === "--deployment") args.deployment = requireValue(argv[++index], arg); + else if (arg === "--configmap") args.configMap = requireValue(argv[++index], arg); + else if (arg === "--mount-path") args.mountPath = requireValue(argv[++index], arg); + else if (arg === "--sub-path") args.subPath = requireValue(argv[++index], arg); + else if (arg === "--marker") args.marker = requireValue(argv[++index], arg); + else if (arg === "--annotation-key") args.annotationKey = requireValue(argv[++index], arg); + else if (arg === "--pod-label-selector") args.podLabelSelector = requireValue(argv[++index], arg); + else if (arg === "--help" || arg === "-h") args.help = true; + else throw new Error(`unknown argument: ${arg}`); + } + + return args; +} + +export async function buildDevRuntimeHotfixAuditReport(args = {}, options = {}) { + const parsed = { ...parseArgs([]), ...args }; + const report = { + issueRefs: ["pikasTech/HWLAB#462", "pikasTech/HWLAB#465", "pikasTech/HWLAB#7", "pikasTech/HWLAB#239"], + relatedDivisionOfWork: { + capabilityRouting: "pikasTech/HWLAB#460", + sourceRunner: "pikasTech/HWLAB#461", + runtimeHotfixRunbookAndAudit: "pikasTech/HWLAB#462", + serviceIdDiagnostics: "pikasTech/HWLAB#463", + devLiveSmoke: "pikasTech/HWLAB#464" + }, + mode: parsed.collectReadonly ? "collect-readonly" : "plan", + target: { + environment: "DEV", + namespace: parsed.namespace, + deployment: parsed.deployment, + configMap: parsed.configMap, + annotationKey: parsed.annotationKey, + mountPath: parsed.mountPath, + subPath: parsed.subPath, + marker: parsed.marker, + kubeconfig: parsed.kubeconfig, + requiredNode: "d601" + }, + safety: { + prodAllowed: false, + secretValuesRead: false, + secretResourcesRead: false, + k8sWriteAttempted: false, + k8sWriteCommandsExecuted: [], + defaultExecutesKubectl: parsed.collectReadonly, + allowedKubectlVerbs: [...allowedKubectlVerbs].sort(), + forbiddenKubectlVerbs: [...forbiddenKubectlVerbs].sort() + }, + requiredOperatorEvidence: [ + "Deployment", + "pod", + "ConfigMap", + "annotation", + "volume", + "mountPath", + "subPath", + "traceId", + "operationId", + "validationCommand", + "rollbackMethod", + "sourceFollowUp" + ], + commandPlan: buildCommandPlan(parsed), + checks: null, + classifications: [], + conclusion: null + }; + + if (!parsed.collectReadonly) { + report.classifications = orderClassifications([ + "source-artifact-expected", + "unknown-needs-manual-readonly-check" + ]); + report.conclusion = { + status: "unknown", + summary: "Plan mode did not query D601; run --collect-readonly on D601 with the native k3s kubeconfig to classify the current hotfix state." + }; + return report; + } + + const checks = await collectReadonlyChecks(parsed, options); + report.checks = checks; + report.classifications = classifyHotfixState(checks); + report.conclusion = summarizeClassifications(report.classifications); + return report; +} + +async function collectReadonlyChecks(args, options) { + const runCommand = options.runCommand ?? defaultRunCommand; + const nodeProbe = await runKubectl(args, ["get", "nodes", "-o", "json"], runCommand); + const nodeJson = parseJson(nodeProbe); + const nodeNames = (nodeJson?.items ?? []).map((item) => item?.metadata?.name).filter(Boolean); + + const configMapProbe = await runKubectl(args, ["-n", args.namespace, "get", "configmap", args.configMap, "-o", "json"], runCommand); + const configMapJson = parseJson(configMapProbe); + + const deploymentProbe = await runKubectl(args, ["-n", args.namespace, "get", "deployment", args.deployment, "-o", "json"], runCommand); + const deploymentJson = parseJson(deploymentProbe); + const deployment = summarizeDeployment(deploymentJson, args); + + const podsProbe = await runKubectl(args, [ + "-n", + args.namespace, + "get", + "pods", + "-l", + args.podLabelSelector, + "--field-selector=status.phase=Running", + "--sort-by=.metadata.creationTimestamp", + "-o", + "json" + ], runCommand); + const podsJson = parseJson(podsProbe); + const pod = summarizeSelectedPod(podsJson); + + let markerProbe = null; + let syntaxProbe = null; + if (pod?.name) { + markerProbe = await runKubectl(args, [ + "-n", + args.namespace, + "exec", + pod.name, + "--", + "sh", + "-lc", + `grep -n ${shellQuote(args.marker)} ${shellQuote(args.mountPath)} | head` + ], runCommand); + syntaxProbe = await runKubectl(args, [ + "-n", + args.namespace, + "exec", + pod.name, + "--", + "sh", + "-lc", + `node --check ${shellQuote(args.mountPath)}` + ], runCommand); + } + + return { + node: { + ok: nodeProbe.ok, + command: nodeProbe.command, + names: nodeNames, + requiredNodePresent: nodeNames.includes("d601"), + probe: compactProbe(nodeProbe, { includeStdout: false }) + }, + configMap: { + command: configMapProbe.command, + present: configMapProbe.ok && Boolean(configMapJson), + unknown: !configMapProbe.ok && !looksNotFound(configMapProbe), + keys: configMapJson ? Object.keys(configMapJson.data ?? {}).sort() : [], + probe: compactProbe(configMapProbe, { includeStdout: false }) + }, + deployment: { + command: deploymentProbe.command, + present: deploymentProbe.ok && Boolean(deploymentJson), + unknown: !deploymentProbe.ok || !deploymentJson, + ...deployment, + probe: compactProbe(deploymentProbe, { includeStdout: false }) + }, + pod: { + command: podsProbe.command, + selected: pod, + unknown: !podsProbe.ok || !podsJson, + marker: markerProbe ? { + command: markerProbe.command, + found: markerProbe.ok, + matchCount: markerProbe.ok ? markerProbe.stdout.split("\n").filter(Boolean).length : 0, + probe: compactProbe(markerProbe, { includeStdout: false }) + } : null, + syntax: syntaxProbe ? { + command: syntaxProbe.command, + ok: syntaxProbe.ok, + probe: compactProbe(syntaxProbe) + } : null, + probe: compactProbe(podsProbe, { includeStdout: false }) + } + }; +} + +function summarizeDeployment(document, args) { + if (!document) { + return { + generation: null, + annotations: {}, + hotfixVolumes: [], + hotfixMounts: [], + mountsHotfix: false + }; + } + const volumes = document.spec?.template?.spec?.volumes ?? []; + const containers = document.spec?.template?.spec?.containers ?? []; + const hotfixVolumeNames = new Set( + volumes + .filter((volume) => volume?.configMap?.name === args.configMap) + .map((volume) => volume.name) + .filter(Boolean) + ); + const hotfixMounts = containers.flatMap((container, containerIndex) => + (container.volumeMounts ?? []) + .map((mount, mountIndex) => ({ container, containerIndex, mount, mountIndex })) + .filter(({ mount }) => + hotfixVolumeNames.has(mount.name) || + mount.mountPath === args.mountPath || + mount.subPath === args.subPath + ) + .map(({ container, containerIndex, mount, mountIndex }) => ({ + container: container.name ?? `container-${containerIndex}`, + containerIndex, + mountIndex, + name: mount.name, + mountPath: mount.mountPath ?? null, + subPath: mount.subPath ?? null, + readOnly: mount.readOnly === true + })) + ); + const annotations = document.spec?.template?.metadata?.annotations ?? {}; + const hotfixAnnotations = Object.fromEntries( + Object.entries(annotations).filter(([key]) => key === args.annotationKey || key.includes("hotfix")) + ); + + return { + generation: document.metadata?.generation ?? null, + observedImageTags: containers.map((container) => ({ + container: container.name, + image: container.image + })), + annotations: hotfixAnnotations, + hotfixVolumes: volumes + .filter((volume) => hotfixVolumeNames.has(volume.name)) + .map((volume, volumeIndex) => ({ + name: volume.name, + volumeIndex, + configMap: volume.configMap?.name, + keys: (volume.configMap?.items ?? []).map((item) => item.key).filter(Boolean) + })), + hotfixMounts, + mountsHotfix: hotfixMounts.some((mount) => + mount.mountPath === args.mountPath && + mount.subPath === args.subPath + ) + }; +} + +function summarizeSelectedPod(document) { + const pods = document?.items ?? []; + if (pods.length === 0) return null; + const pod = pods[pods.length - 1]; + return { + name: pod.metadata?.name ?? null, + phase: pod.status?.phase ?? null, + nodeName: pod.spec?.nodeName ?? null, + creationTimestamp: pod.metadata?.creationTimestamp ?? null + }; +} + +export function classifyHotfixState(checks) { + const classifications = new Set(["source-artifact-expected"]); + const unknown = !checks || + checks.node?.requiredNodePresent !== true || + checks.configMap?.unknown === true || + checks.deployment?.unknown === true || + checks.pod?.unknown === true; + + if (checks?.configMap?.present === true) classifications.add("hotfix-configmap-present"); + if (checks?.deployment?.mountsHotfix === true) classifications.add("deployment-mounts-hotfix"); + if (checks?.pod?.marker?.found === true) classifications.add("pod-loads-hotfix-marker"); + + const hotfixDetected = classifications.has("hotfix-configmap-present") || + classifications.has("deployment-mounts-hotfix") || + classifications.has("pod-loads-hotfix-marker"); + if (hotfixDetected) classifications.add("rollback-required"); + + if (!hotfixDetected && !unknown) classifications.add("no-hotfix-detected"); + if (unknown) classifications.add("unknown-needs-manual-readonly-check"); + return orderClassifications([...classifications]); +} + +function summarizeClassifications(classifications) { + if (classifications.includes("rollback-required")) { + return { + status: "hotfix-detected", + summary: "DEV runtime still shows hotfix residue or active coverage; remove the override or replace it with a source artifact before treating source/CD as the runtime truth." + }; + } + if (classifications.includes("no-hotfix-detected")) { + return { + status: "clear", + summary: "No ConfigMap or Deployment mount hotfix was detected by the read-only audit; keep official fixes in source/PR/artifact/CD." + }; + } + return { + status: "unknown", + summary: "The audit could not fully classify current DEV hotfix state; run the listed read-only checks on D601 before rollback or source verification." + }; +} + +function orderClassifications(values) { + const set = new Set(values); + return classificationOrder.filter((item) => set.has(item)); +} + +function buildCommandPlan(args) { + return { + readonly: [ + `${kubePrefix(args)} kubectl get nodes -o jsonpath='{.items[*].metadata.name}'`, + `${kubePrefix(args)} kubectl -n ${args.namespace} get configmap ${args.configMap} -o json`, + `${kubePrefix(args)} kubectl -n ${args.namespace} get deployment ${args.deployment} -o json`, + `${kubePrefix(args)} kubectl -n ${args.namespace} get pods -l ${shellQuote(args.podLabelSelector)} --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp -o json`, + `${kubePrefix(args)} kubectl -n ${args.namespace} exec -- sh -lc ${shellQuote(`grep -n ${shellQuote(args.marker)} ${shellQuote(args.mountPath)} | head`)}`, + `${kubePrefix(args)} kubectl -n ${args.namespace} exec -- sh -lc ${shellQuote(`node --check ${shellQuote(args.mountPath)}`)}` + ], + rollbackNotExecutedByThisScript: [ + "Use source/PR/artifact/CD to replace the runtime override whenever possible.", + `If an authorized DEV rollback must remove the override directly, first locate the hotfix volumeMount, volume, and annotation indexes from the read-only deployment JSON for ${args.deployment}.`, + "Then execute the smallest explicit DEV-only rollback outside this audit script, with KUBECONFIG pinned to D601 native k3s, and re-run this audit in --collect-readonly mode." + ], + forbiddenDefaultOperations: [ + "kubectl apply", + "kubectl patch", + "kubectl rollout", + "kubectl rollout restart", + "kubectl delete", + "kubectl create", + "kubectl set image" + ] + }; +} + +async function runKubectl(args, kubectlArgs, runCommand) { + assertReadonlyKubectlArgs(kubectlArgs); + const displayCommand = `${kubePrefix(args)} kubectl ${kubectlArgs.map(shellDisplay).join(" ")}`; + const result = await runCommand({ + command: "env", + args: [`KUBECONFIG=${args.kubeconfig}`, "kubectl", ...kubectlArgs], + timeoutMs: args.timeoutMs, + displayCommand + }); + const probe = normalizeCommandResult(result, displayCommand); + Object.defineProperty(probe, "rawStdout", { + value: String(result.stdout ?? "").trim(), + enumerable: false + }); + return probe; +} + +export function assertReadonlyKubectlArgs(kubectlArgs) { + const verb = kubectlArgs.find((arg) => allowedKubectlVerbs.has(arg) || forbiddenKubectlVerbs.has(arg)); + const forbidden = kubectlArgs.find((arg) => forbiddenKubectlVerbs.has(arg)); + if (forbidden) { + throw new Error(`kubectl write verb is forbidden in hotfix audit: ${forbidden}`); + } + if (!verb || !allowedKubectlVerbs.has(verb)) { + throw new Error(`kubectl command is not in the read-only allowlist: ${kubectlArgs.join(" ")}`); + } +} + +async function defaultRunCommand({ command, args, timeoutMs }) { + try { + const result = await execFileAsync(command, args, { + timeout: timeoutMs, + maxBuffer: 1024 * 1024 + }); + return { + ok: true, + exitCode: 0, + stdout: result.stdout, + stderr: result.stderr + }; + } catch (error) { + return { + ok: false, + exitCode: typeof error.code === "number" ? error.code : 1, + stdout: String(error.stdout ?? ""), + stderr: String(error.stderr ?? ""), + error: error.message + }; + } +} + +function normalizeCommandResult(result, command) { + return { + ok: result.ok === true || result.exitCode === 0, + command, + exitCode: Number.isInteger(result.exitCode) ? result.exitCode : result.ok === true ? 0 : 1, + stdout: redact(String(result.stdout ?? "").trim()), + stderr: redact(String(result.stderr ?? "").trim()), + error: result.error ? redact(String(result.error)) : undefined + }; +} + +function compactProbe(probe, { includeStdout = true } = {}) { + return { + ok: probe.ok, + command: probe.command, + exitCode: probe.exitCode, + stdout: includeStdout && probe.stdout.length <= 200 ? probe.stdout : undefined, + stderr: probe.stderr ? oneLine(probe.stderr) : undefined, + error: probe.error ? oneLine(probe.error) : undefined + }; +} + +function parseJson(probe) { + if (!probe.ok) return null; + const raw = probe.rawStdout ?? probe.stdout; + try { + return raw ? JSON.parse(raw) : null; + } catch { + return null; + } +} + +function looksNotFound(probe) { + return /notfound|not found|notfound/i.test(`${probe.stderr ?? ""} ${probe.error ?? ""}`); +} + +function requireValue(value, flag) { + if (!value) throw new Error(`${flag} requires a value`); + return value; +} + +function parsePositiveInt(value, flag) { + const parsed = Number.parseInt(value, 10); + if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${flag} must be a positive integer`); + return parsed; +} + +function kubePrefix(args) { + return `KUBECONFIG=${args.kubeconfig}`; +} + +function shellQuote(value) { + return `'${String(value).replaceAll("'", "'\\''")}'`; +} + +function shellDisplay(value) { + return /\s|["'\\]/u.test(String(value)) ? shellQuote(value) : String(value); +} + +function oneLine(value) { + return redact(String(value)).replace(/\s+/g, " ").trim().slice(0, 500); +} + +function redact(value) { + return String(value) + .replace(/(bearer\s+)[A-Za-z0-9._~+/=-]+/giu, "$1[REDACTED]") + .replace(/((?:token|password|client-key-data|client-certificate-data|certificate-authority-data)\s*[:=]\s*)\S+/giu, "$1[REDACTED]") + .replace(/(authorization:\s*)\S+/giu, "$1[REDACTED]") + .slice(0, 4000); +} + +function usage() { + return [ + "Usage: node scripts/dev-runtime-hotfix-audit.mjs [--plan|--collect-readonly] [options]", + "", + "Default mode is --plan: it prints the read-only audit plan and performs no kubectl calls.", + "", + "Options:", + " --collect-readonly Run read-only kubectl get/exec checks; never apply/patch/rollout/restart", + " --plan, --dry-run Print plan only (default)", + " --pretty Pretty-print JSON", + " --timeout-ms Per kubectl command timeout", + " --kubeconfig Kubeconfig path, default /etc/rancher/k3s/k3s.yaml", + " --namespace Namespace, default hwlab-dev", + " --deployment Deployment, default hwlab-cloud-api", + " --configmap Hotfix ConfigMap name", + " --mount-path Expected mounted file path", + " --sub-path Expected subPath key", + " --marker Runtime marker to grep in the mounted file", + " --annotation-key Deployment hotfix annotation key", + " --pod-label-selector Pod selector for the Deployment", + " --help Show this help" + ].join("\n"); +} + +export function formatDevRuntimeHotfixAuditFailure(error) { + return { + status: "failed", + script: "dev-runtime-hotfix-audit", + error: { + message: error instanceof Error ? error.message : String(error) + }, + safety: { + k8sWriteAttempted: false, + secretValuesRead: false + } + }; +} diff --git a/scripts/src/dev-runtime-hotfix-audit.test.mjs b/scripts/src/dev-runtime-hotfix-audit.test.mjs new file mode 100644 index 00000000..f8aa87cb --- /dev/null +++ b/scripts/src/dev-runtime-hotfix-audit.test.mjs @@ -0,0 +1,158 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { + assertReadonlyKubectlArgs, + buildDevRuntimeHotfixAuditReport, + parseArgs +} from "./dev-runtime-hotfix-audit.mjs"; + +test("plan mode emits a bounded read-only plan and runs no kubectl commands", async () => { + let commandCount = 0; + const report = await buildDevRuntimeHotfixAuditReport(parseArgs(["--plan"]), { + runCommand: async () => { + commandCount += 1; + throw new Error("plan mode must not execute commands"); + } + }); + + assert.equal(commandCount, 0); + assert.equal(report.mode, "plan"); + assert.equal(report.safety.k8sWriteAttempted, false); + assert.deepEqual(report.classifications, [ + "source-artifact-expected", + "unknown-needs-manual-readonly-check" + ]); + assert.ok(report.commandPlan.readonly.every((command) => command.includes("KUBECONFIG=/etc/rancher/k3s/k3s.yaml"))); + assert.ok(report.commandPlan.forbiddenDefaultOperations.some((command) => command.includes("kubectl patch"))); +}); + +test("collect-readonly classifies active ConfigMap mount and pod marker without write verbs", async () => { + const commands = []; + const report = await buildDevRuntimeHotfixAuditReport(parseArgs(["--collect-readonly"]), { + runCommand: async ({ args, displayCommand }) => { + commands.push(displayCommand); + return fixtureResult(args); + } + }); + + assert.equal(report.mode, "collect-readonly"); + assert.equal(report.safety.k8sWriteAttempted, false); + assert.ok(report.classifications.includes("hotfix-configmap-present")); + assert.ok(report.classifications.includes("deployment-mounts-hotfix")); + assert.ok(report.classifications.includes("pod-loads-hotfix-marker")); + assert.ok(report.classifications.includes("rollback-required")); + assert.equal(report.conclusion.status, "hotfix-detected"); + assert.ok(commands.length > 0); + assertNoWriteVerbs(commands); +}); + +test("collect-readonly reports no-hotfix-detected when ConfigMap and mount are absent", async () => { + const report = await buildDevRuntimeHotfixAuditReport(parseArgs(["--collect-readonly"]), { + runCommand: async ({ args }) => fixtureResult(args, { noHotfix: true }) + }); + + assert.deepEqual(report.classifications, [ + "source-artifact-expected", + "no-hotfix-detected" + ]); + assert.equal(report.conclusion.status, "clear"); +}); + +test("collect-readonly keeps unknown classification when D601 node cannot be proven", async () => { + const report = await buildDevRuntimeHotfixAuditReport(parseArgs(["--collect-readonly"]), { + runCommand: async ({ args }) => { + if (args.includes("nodes")) { + return { ok: false, exitCode: 1, stdout: "", stderr: "unable to connect to the server" }; + } + return fixtureResult(args); + } + }); + + assert.ok(report.classifications.includes("unknown-needs-manual-readonly-check")); + assert.notEqual(report.conclusion.status, "clear"); +}); + +test("kubectl write verbs are rejected by the audit helper", () => { + assert.throws(() => assertReadonlyKubectlArgs(["-n", "hwlab-dev", "patch", "deployment", "hwlab-cloud-api"]), /forbidden/u); + assert.throws(() => assertReadonlyKubectlArgs(["-n", "hwlab-dev", "rollout", "restart", "deployment", "hwlab-cloud-api"]), /forbidden/u); +}); + +function fixtureResult(args, options = {}) { + const text = args.join(" "); + if (text.includes("get nodes")) { + return okJson({ items: [{ metadata: { name: "d601" } }] }); + } + if (text.includes("get configmap")) { + if (options.noHotfix) { + return { ok: false, exitCode: 1, stdout: "", stderr: "Error from server (NotFound): configmaps \"hwlab-cloud-api-code-agent-hotfix\" not found" }; + } + return okJson({ data: { "code-agent-chat.mjs": "// runtime-hotfix-pc-gateway-shell" } }); + } + if (text.includes("get deployment")) { + return okJson(deploymentFixture({ noHotfix: options.noHotfix })); + } + if (text.includes("get pods")) { + return okJson({ + items: [{ + metadata: { name: "hwlab-cloud-api-abc", creationTimestamp: "2026-05-24T00:00:00Z" }, + status: { phase: "Running" }, + spec: { nodeName: "d601" } + }] + }); + } + if (text.includes("exec") && text.includes("grep")) { + return options.noHotfix + ? { ok: false, exitCode: 1, stdout: "", stderr: "" } + : { ok: true, exitCode: 0, stdout: "1:runtime-hotfix-pc-gateway-shell\n", stderr: "" }; + } + if (text.includes("exec") && text.includes("node --check")) { + return { ok: true, exitCode: 0, stdout: "", stderr: "" }; + } + throw new Error(`unexpected fixture command: ${text}`); +} + +function deploymentFixture({ noHotfix = false } = {}) { + return { + metadata: { generation: 9 }, + spec: { + template: { + metadata: { + annotations: noHotfix ? {} : { + "hwlab.pikastech.local/pc-gateway-shell-hotfix": "runtime-hotfix" + } + }, + spec: { + volumes: noHotfix ? [] : [{ + name: "code-agent-hotfix", + configMap: { + name: "hwlab-cloud-api-code-agent-hotfix", + items: [{ key: "code-agent-chat.mjs", path: "code-agent-chat.mjs" }] + } + }], + containers: [{ + name: "hwlab-cloud-api", + image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:dev", + volumeMounts: noHotfix ? [] : [{ + name: "code-agent-hotfix", + mountPath: "/app/internal/cloud/code-agent-chat.mjs", + subPath: "code-agent-chat.mjs", + readOnly: true + }] + }] + } + } + } + }; +} + +function okJson(value) { + return { ok: true, exitCode: 0, stdout: JSON.stringify(value), stderr: "" }; +} + +function assertNoWriteVerbs(commands) { + const forbidden = /\bkubectl\s+(?:apply|patch|rollout|delete|create|replace|scale|set|edit|annotate|label)\b/u; + for (const command of commands) { + assert.doesNotMatch(command, forbidden, command); + } +}