From 3295aa64a02be5141cb83b2b8b9b18c01a2acdb0 Mon Sep 17 00:00:00 2001 From: pikastech Date: Mon, 20 Jul 2026 10:15:38 +0200 Subject: [PATCH] refactor(platform-infra): retire cross-region k3s topology --- .agents/skills/unidesk-cicd/SKILL.md | 47 +- .agents/skills/unidesk-ymalops/SKILL.md | 46 +- config/hwlab-node-control-plane.yaml | 1 - config/platform-infra/host-proxy.yaml | 2 +- config/platform-infra/hyperv-vms.yaml | 2 +- config/platform-infra/k3s-clusters.yaml | 99 +- config/platform-infra/public-edge.yaml | 2 +- docs/reference/platform-infra.md | 161 +- .../specs/PJ2026-01060314-hyperv-ubuntu-vm.md | 25 +- scripts/src/help.ts | 6 +- scripts/src/platform-infra-host-proxy.ts | 7 +- .../src/platform-infra-k3s-cluster-config.ts | 312 +-- scripts/src/platform-infra-k3s-cluster.ts | 1976 ++++------------- 13 files changed, 604 insertions(+), 2082 deletions(-) diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 9db51bf1..8f18765c 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -348,44 +348,29 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - 一旦发现 CI/CD CLI 被误用且可能写入错误状态、产生伪证据或绕过目标运行面，必须立刻先把用法改成更符合直觉的公开入口并更新本 skill/reference，再继续验证或交付；不要只靠口头记忆、隐藏 flag、手动约定或后续小心来避免复发。内部 in-cluster 模式必须只由目标 k8s Job/Pod 调用，操作者从本机只能用公开入口提交目标侧 Job 或读取目标侧摘要。 - PaC migrated consumer 的旧手动 publish/debug 命令不得作为正式 closeout 引导；调查命令默认只能指向只读的 `platform-infra pipelines-as-code status|history`。`closeout` 仅保留只读兼容能力，不得作为主代理、子代理或操作者的 PR 合并后步骤。需要保留的旧 mutation 命令只能属于 YAML 明确的 legacy authority 或独立平台维护，不得标记或引导 manual recovery；单步确认的自动链缺陷必须回到 YAML/controller/源码修复。 - Secret 只通过 YAML sourceRef/targetKey 和受控 CLI 下发；输出只披露 presence/fingerprint。 -- 多主机 k3s 节点变更： - - CI/CD、业务 renderer、PipelineRun 和普通 GitOps workload 不得直接写 WireGuard 配置、k3s systemd unit、agent token 文件或节点 ServiceLB 标签。 - - 节点加入、状态和调度 smoke 只走 `platform-infra k3s-cluster plan|apply|status|smoke`，具体 authority 和控制通道见 `docs/reference/platform-infra.md#多主机-k3s-集群`。 - - 集群 apply 的 control-plane 阶段只允许组合 `hwlab nodes control-plane infra k3s-config` 窄域；禁止调用全量 `infra apply`，避免节点加入顺带覆盖 registry、Tekton、Argo、git-mirror 或运行面对象。 - - k3s drop-in 未变化且节点容量已收敛时必须 `mutation=false`；禁止无条件重启、重装、强写或通过 CI 自动覆盖同一节点配置。 - - WireGuard 配置一致时不得 restart；k3s API/容量暂时不可读时保持 unknown 并停止 mutation，禁止把控制通道抖动解释为配置漂移。 - - worker agent 只在 YAML 派生摘要变化或 service 未运行时安装/恢复，不得在每次集群 apply 中先卸载再重装。 - - worker 调度资格和 taint 只认集群 owning YAML： - - `schedulable=false` 必须由 agent 注册参数与受控 `cordon`/taint 双重收敛； - - CI、PipelineRun、GitOps 和业务 renderer 禁止 `uncordon`、删除该 taint， - 或添加 toleration/`nodeName` 将业务 Pod 强制放回不可调度 worker； - - 清空不可调度 worker 只走 `platform-infra k3s-cluster drain --confirm`， - 保留 DaemonSet/节点系统 Pod，且节点重注册后必须先恢复调度隔离。 - - `Type=notify` agent 的 `activating` 且 MainPID 存在属于运行中；节点 Ready 时不得仅因 systemd 未进入 `active` 而重复重装。 - - worker 必须先由集群入口按 `hostProxyConfigRef` 部署并验证 proxy client，再启动 agent；containerd 通过该 proxy 直接访问上游 registry，禁止用公共 registry mirror 代替出网链路。 +- D601 独立 k3s 集群： + - CI/CD、业务 renderer、PipelineRun 和 GitOps 不得写 k3s systemd unit， + 不得把 D601 注册为 NC01 agent，也不得创建跨地域 CNI 或共享 datastore。 + - D601 只走 + `platform-infra k3s-cluster plan|apply|status|smoke --target d601`； + NC01 只走自己的 control-plane 入口，两者不得互相组合。 + - D601 server 的 YAML 摘要未变化且节点健康时禁止重装、重启或强写。 + - D601 默认是空集群。任何业务进入 D601 必须有独立 workload owning + YAML 和显式 target，禁止继承 NC01 的 PaC、Tekton、Argo、GitOps、 + DaemonSet、namespace 或默认 placement。 + - D601 先通过 host proxy 让 server/containerd 出网，禁止公共 registry + mirror。 - 集群自有 artifact registry 的异地拉取是明确例外： - 内部 publish endpoint 保持唯一写入 authority； - 公网 HTTPS 只允许鉴权 `GET`/`HEAD`，不得成为 push、cache export 或删除入口； - - worker 保持既有镜像引用，由集群 owning YAML 渲染 endpoint 与 SecretRef； - - registry 大流量不得经过 WireGuard/Flannel，也不得使用 CDN 或公共镜像站； + - D601 保持既有镜像引用，由集群 owning YAML 渲染 endpoint 与 SecretRef； + - registry 大流量不得经过 Kubernetes 数据面，也不得使用 CDN 或公共镜像站； - bcrypt hash 必须以 Compose-safe 字面量写入 Secret source，且 public-edge 用脱敏 material fingerprint 驱动凭据变更后的 recreate； 候选校验不得用会保留引号的 `docker run --env-file` 改变 Secret 语义； - 验收统一走 `platform-infra k3s-cluster registry-smoke`，不手工 patch Caddy 或 containerd。 - - worker 安装使用短启动并由 control-plane Node Ready 轮询收口，禁止让 `trans`/SSH 长挂等待 systemd notify。 - - worker 加入集群不改变 PaC consumer 的 source commit authority，也不授权人工 PipelineRun、Argo sync 或业务 rollout。 - - NC01 与 D601 是永久分离的调度域： - - `config/platform-infra/k3s-clusters.yaml` 必须保持 - `topologyMode=separated`，D601 只作为独立单节点 k3s 集群存在； - - CI/CD、PaC、Tekton、Argo、GitOps、VM bootstrap 和业务 renderer - 禁止把 D601 注册为 NC01 agent，禁止恢复跨地域 WireGuard peer， - 禁止把 NC01 workload、DaemonSet 或 namespace 默认复制到 D601； - - 拆分和验收只走 - `platform-infra k3s-cluster separate|status|smoke`； - - 跨地域 artifact 传输使用鉴权、只读的公网 HTTPS registry， - 不得通过 Kubernetes Pod 网络、Service、DNS 或 VXLAN； - - 如未来确需跨地域计算，必须建立独立 workload 级交付规格并显式选择 - D601 集群，不得通过重新组成单一集群获得隐式调度。 + - 跨地域 artifact 传输使用鉴权、只读的公网 HTTPS registry，不得通过 + Kubernetes Pod 网络、Service、DNS 或 VXLAN。 - 长命令用异步 job 或短轮询；不要长时间挂住 trans/ssh。 - Web 哨兵 YAML cadence 变更不新增发布 orchestrator：提交前用 `web-probe sentinel validate --local`；合并后用只读 `web-probe sentinel control-plane status --wait`。线上 desired schedule 必须来自 GitOps 清单，不能用操作者旧 worktree 的本地渲染值做阻塞门禁；收敛差异统一为非阻塞 warning。 diff --git a/.agents/skills/unidesk-ymalops/SKILL.md b/.agents/skills/unidesk-ymalops/SKILL.md index 95d41eb7..8ad23360 100644 --- a/.agents/skills/unidesk-ymalops/SKILL.md +++ b/.agents/skills/unidesk-ymalops/SKILL.md @@ -88,38 +88,20 @@ description: UniDesk YAML-first 运维正规化技能。用户提到 ymal-first/ - 非核心字段可以使用显式领域默认值，但必须同时生成 `blocking=false` warning； - 不得生成不可见默认值，也不得把合并后的大对象写成新的 source of truth。 - 新 VPS 或裸节点 bootstrap 的出网必须优先走 YAML-first 声明的 0 依赖 host proxy client：通过 `trans` 分发零依赖客户端，先让 host route 能出网，再安装 k3s、containerd、Docker、包管理器依赖和 Git/gitrepo 同步；不得把 provider-gateway WebSocket egress 当作大二进制或依赖下载的数据面。 -- 多主机 k3s 集群： - - 跨主机拓扑、WireGuard、agent token、worker host proxy 引用、集群内 artifact registry 映射、ServiceLB 节点和调度 smoke 归独立集群 owning YAML。 - - worker host proxy 归 `config/platform-infra/host-proxy.yaml`；集群 apply 必须先确认 proxy client 连接声明的 `vpn-server` 并通过外网 probe，再启动 agent。 - - worker proxy 前必须先收敛 WireGuard；配置一致时不重启，只有 unit 未运行或握手过期且隧道探针失败时才允许恢复 worker 接口。 - - containerd 通过 host proxy 直接访问公共 registry，禁止为公共依赖新增 registry mirror；集群内 artifact registry 映射不属于公共镜像代理。 - - 集群自有 artifact registry 可由同一 owning YAML 声明公网可信 TLS 的 pull-only endpoint： - - 内部 endpoint 继续拥有 push； - - public-edge 只允许鉴权 `GET`/`HEAD`； - - containerd 凭据与 Caddy hash 必须来自同一 SecretRef； - - 镜像层绕开 WireGuard/Flannel 和 CDN； - - registry 配置变化不得触发 worker agent 卸载重装。 - - 既有 control-plane 的 k3s renderer 通过 `clusterConfigRef` 消费集群网络参数，不得由集群 CLI 再写第二个 systemd drop-in。 - - 无固定公网地址 worker 使用宿主内网 SSH 作为部署控制通道并主动连接 WireGuard endpoint，不使用 FRP 代替三层集群网络。 - - control-plane drop-in 只走 `hwlab nodes control-plane infra k3s-config` 窄域： - - 渲染结果一致且节点容量已收敛时返回 `mutation=false`； - - 只在必要时写入 drop-in、reload 和重启 k3s； - - 确需重启 server 时，先停止服务并备份 k3s 派生 - `server/cred/passwd`；禁止用该文件与 datastore 的 - mtime/freshness 预判阻断变更，备份只输出 - `blocking=false` warning； - - WireGuard 内容一致时不重启，API/容量读取暂时失败时保持 unknown 并停止 mutation； - - worker agent 使用 YAML 派生摘要决定是否重装，禁止每次 apply 先卸载再覆盖； - - 禁止借用全量 infra apply 顺带覆盖 registry、CI/CD 或业务运行面。 - - 日常入口和验收判定统一见 `docs/reference/platform-infra.md#多主机-k3s-集群`。 - - NC01-D601 的 owning YAML 必须声明 `topologyMode=separated`： - - D601 独立 server 的节点名、标签、禁用组件和 kubeconfig mode 必须 - 位于同一 YAML 的 `k3s.standalone`； - - Hyper-V YAML 只能用 `management.mode=standalone` 引用该片段； - - parser、bootstrap、CI/CD 和 GitOps 禁止通过隐藏默认值把 D601 - 恢复为 agent、恢复 WireGuard peer或共享调度域； - - 独立集群间只共享显式声明的只读 artifact endpoint，不共享 Pod、 - Service、DNS、CNI、datastore 或默认 workload。 +- D601 独立 k3s 集群： + - `config/platform-infra/k3s-clusters.yaml#targets.d601` 只声明独立 + server、节点、host proxy 和只读 artifact registry，不得出现 NC01、 + agent token、跨地域 CNI 或共享调度字段。 + - Hyper-V YAML 以 `management.mode=standalone` 引用 D601 k3s 片段； + parser、bootstrap、CI/CD 和 GitOps 禁止用隐藏默认值恢复 agent。 + - NC01 与 D601 不共享 Pod、Service、DNS、CNI、datastore、DaemonSet、 + namespace 或默认 workload。 + - D601 containerd 通过 host proxy 访问公共 registry，禁止公共镜像站； + 集群自有 artifact registry 只通过鉴权、只读公网 HTTPS 拉取。 + - registry 内部 endpoint 继续拥有 push，public-edge 只允许 + `GET`/`HEAD`；containerd 凭据与 Caddy hash 来自同一 SecretRef。 + - 日常入口和验收见 + `docs/reference/platform-infra.md#d601-独立-k3s-集群`。 - 新节点 node-local registry 必须支持从零部署：优先由 YAML 渲染为 k3s workload/PVC；host Docker registry 只能是 legacy/迁移前状态。需要证明从零能力时先停旧 registry，不从旧 registry seed，并在状态中披露 `seededFromOldRegistry`、`zeroSeeded`、PVC、workload、endpoint 与 runtime/tools image preload 结果。 - YAML 配置必须可复用、可继承、可组合；节点/lane 只能作为变量或 YAML value 参与渲染，禁止把具体 node/lane 写入 YAML 文件名、parser/renderer/helper 函数名或长期 schema 名。发现不同节点复制大段配置时，先抽 baseline/configRef/override，再继续扩展节点。 - CLI `plan/status` 应输出 redacted 配置引用图：每个 ref 的文件、path、presence、摘要 hash、缺失字段和下一步 drill-down 命令。不要默认 dump 展开后的完整 YAML 或 Secret。 diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index da7871d0..0d8d933b 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -316,7 +316,6 @@ nodes: serviceName: k3s dropInPath: /etc/systemd/system/k3s.service.d/20-unidesk-node-config.conf nodeStatusName: v2202607378382480008 - clusterConfigRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.controlPlane execStartPre: [] install: enabled: true diff --git a/config/platform-infra/host-proxy.yaml b/config/platform-infra/host-proxy.yaml index d09cef74..8016009b 100644 --- a/config/platform-infra/host-proxy.yaml +++ b/config/platform-infra/host-proxy.yaml @@ -127,7 +127,7 @@ targets: route: D601-VM sourceRef: sources.d601-nc01-vpn-server-shadowsocks files: - k3sSystemdDropIn: /etc/systemd/system/k3s-agent.service.d/10-unidesk-proxy.conf + k3sSystemdDropIn: /etc/systemd/system/k3s.service.d/10-unidesk-proxy.conf env: noProxyRefs: common: noProxy.common diff --git a/config/platform-infra/hyperv-vms.yaml b/config/platform-infra/hyperv-vms.yaml index 2c62a47f..dfc55f6a 100644 --- a/config/platform-infra/hyperv-vms.yaml +++ b/config/platform-infra/hyperv-vms.yaml @@ -74,7 +74,7 @@ targets: k3s: management: mode: standalone - configRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.k3s.standalone + configRef: config/platform-infra/k3s-clusters.yaml#targets.d601.k3s version: v1.36.2+k3s1 installMirror: cn binaryUrl: https://github.com/k3s-io/k3s/releases/download/v1.36.2%2Bk3s1/k3s diff --git a/config/platform-infra/k3s-clusters.yaml b/config/platform-infra/k3s-clusters.yaml index 7a72a9b2..dea7d74c 100644 --- a/config/platform-infra/k3s-clusters.yaml +++ b/config/platform-infra/k3s-clusters.yaml @@ -4,77 +4,32 @@ metadata: owner: unidesk specRef: project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md defaults: - targetId: nc01-d601 + targetId: d601 targets: - nc01-d601: + d601: enabled: true - clusterName: nc01 - topologyMode: separated - controlPlane: - id: NC01 - configRef: config/hwlab-node-control-plane.yaml#nodes.NC01 - lane: v03 - nodeName: v2202607378382480008 - publicAddress: 152.53.229.148 - workers: - D601-VM: - id: D601-VM - bootstrapConfigRef: config/platform-infra/hyperv-vms.yaml#targets.D601-VM - hostProxyConfigRef: config/platform-infra/host-proxy.yaml#targets.D601-VM - nodeName: d601-vm - labels: - unidesk.ai/node-id: D601-VM - unidesk.ai/provider-id: D601-VM - unidesk.ai/runtime-plane: worker - scheduling: - schedulable: false - taints: - - key: unidesk.ai/business-scheduling - value: disabled - effect: NoSchedule - wireguard: - interfaceName: wg0 - packageName: wireguard-tools - mtu: 1380 - listenPort: 51820 - controlPlaneAddress: 10.89.0.1/24 - workerAddress: 10.89.0.2/24 - endpoint: 152.53.229.148:51820 - persistentKeepaliveSeconds: 10 - tcpTuning: - congestionControl: bbr - defaultQdisc: fq - mtuProbing: 1 - sysctlPath: /etc/sysctl.d/90-unidesk-k3s-cross-region.conf - secrets: - sourceRef: /root/.unidesk/.state/secrets/platform-infra/k3s-clusters/nc01-d601.env - controlPlanePrivateKey: NC01_WIREGUARD_PRIVATE_KEY - controlPlanePublicKey: NC01_WIREGUARD_PUBLIC_KEY - workerPrivateKey: D601_VM_WIREGUARD_PRIVATE_KEY - workerPublicKey: D601_VM_WIREGUARD_PUBLIC_KEY - agentToken: K3S_AGENT_TOKEN + clusterName: d601 + node: + id: D601-VM + bootstrapConfigRef: config/platform-infra/hyperv-vms.yaml#targets.D601-VM + hostProxyConfigRef: config/platform-infra/host-proxy.yaml#targets.D601-VM + nodeName: d601-vm + labels: + unidesk.ai/node-id: D601-VM + unidesk.ai/provider-id: D601-VM + unidesk.ai/runtime-plane: standalone k3s: - serverUrl: https://10.89.0.1:6443 - agentTokenFile: /etc/unidesk/k3s-agent-token - flannelInterface: wg0 + writeKubeconfigMode: "0644" maxPods: 500 - standalone: - clusterName: d601 - nodeName: d601-vm - writeKubeconfigMode: "0644" - labels: - unidesk.ai/node-id: D601-VM - unidesk.ai/provider-id: D601-VM - unidesk.ai/runtime-plane: standalone - disableComponents: - - traefik - - servicelb - - metrics-server + disableComponents: + - traefik + - servicelb + - metrics-server registryMirror: source: 127.0.0.1:5000 endpoint: https://registry.hwpod.com auth: - sourceRef: /root/.unidesk/.state/secrets/platform-infra/k3s-clusters/nc01-d601-artifact-registry-pull.env + sourceRef: /root/.unidesk/.state/secrets/platform-infra/k3s-clusters/d601-artifact-registry-pull.env usernameKey: ARTIFACT_REGISTRY_PULL_USERNAME passwordKey: ARTIFACT_REGISTRY_PULL_PASSWORD passwordHashKey: ARTIFACT_REGISTRY_PULL_PASSWORD_HASH @@ -98,21 +53,3 @@ targets: expectedBytes: 195443636 concurrentPulls: 2 timeoutSeconds: 900 - serviceLb: - enabledNodeLabel: svccontroller.k3s.cattle.io/enablelb - enabledNode: v2202607378382480008 - workerDns: - namespace: kube-system - deploymentName: unidesk-worker-coredns - serviceName: unidesk-worker-kube-dns - clusterIp: 10.43.0.53 - image: docker.io/coredns/coredns:1.14.4 - configMapName: coredns - customConfigMapName: coredns-custom - replicas: 1 - timeoutSeconds: 120 - smoke: - namespace: platform-infra-smoke - jobName: d601-runtime-plane-smoke - image: docker.io/library/busybox:1.36.1 - timeoutSeconds: 180 diff --git a/config/platform-infra/public-edge.yaml b/config/platform-infra/public-edge.yaml index e1551dc6..583f5469 100644 --- a/config/platform-infra/public-edge.yaml +++ b/config/platform-infra/public-edge.yaml @@ -70,7 +70,7 @@ targets: sites: - id: artifact-registry configRef: config/platform-infra/k3s-clusters.yaml - path: targets.nc01-d601.k3s.registryMirror + path: targets.d601.k3s.registryMirror - id: pikaoa-development configRef: config/pikaoa.yaml path: developmentRuntime.targets.NC01 diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 039e3923..13c52a14 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -10,125 +10,33 @@ - UniDesk 拥有的 image version、namespace、endpoint、FRP port 与 profile 列表不得隐藏在 helper 常量里；外部工具要求的 TOML、JSON 和 env 仅是边缘渲染格式。 - 新节点出网 bootstrap 遵循 `docs/reference/yaml-first-ops.md` 的零依赖 host proxy 边界，不得依赖尚未安装的 Docker、k3s、containerd 或包管理器网络能力。 -## k3s 集群拓扑 +## D601 独立 k3s 集群 -- 权威配置： - - 集群边界、拓扑模式、历史 WireGuard 迁移参数、独立 server 角色、 - registry endpoint、ServiceLB 节点和 smoke 归 - `config/platform-infra/k3s-clusters.yaml`。 - - worker host proxy 的服务端来源、客户端工件、Secret 引用、代理环境和 systemd drop-in 归 `config/platform-infra/host-proxy.yaml`。 - - Hyper-V VM 资源和宿主内网 SSH 归 `config/platform-infra/hyperv-vms.yaml`。 - - 既有 control-plane 的 k3s systemd drop-in 继续归 `config/hwlab-node-control-plane.yaml`，并通过 `clusterConfigRef` 消费集群网络参数。 -- 受控入口： - - `platform-infra k3s-cluster secret-init --target --confirm` 只在 Secret source 不存在时生成 WireGuard key pair 和 k3s agent token，存在时拒绝覆盖。 - - `platform-infra k3s-cluster plan|apply|separate|drain|status|smoke - --target ` 是部署、拓扑拆分、清空不可调度 worker和验收入口。 - - `topologyMode=separated` 时，`apply --confirm` 与 - `separate --confirm` 都只执行受控拆分： - - 先证明旧 worker 的非 DaemonSet Pod 为零； - - 再清理旧 agent datastore、CNI 和跨地域 WireGuard 配置； - - 在 D601 安装并等待独立单节点 k3s server Ready； - - 最后从 NC01 删除旧 Node 和对端 WireGuard peer。 - - 拆分不得先删除 NC01 Node 或隧道；D601 独立 server 未 Ready 时必须 - 保留旧控制面侧事实并返回失败。 - - 无固定公网地址的 worker 由宿主内网 SSH 控制，主动连接 control-plane WireGuard endpoint；worker provider route 只作为部署后补充验收入口。 - - 集群 apply 先经宿主内网 SSH 分发零依赖 proxy client，确认它连接声明的 `vpn-server` 且外网 probe 成功，再启动或恢复 `k3s-agent`。 - - 集群 apply 在 proxy 前先收敛 worker WireGuard；内容一致时不强写，只有 unit 未运行或握手过期且隧道探针失败时才恢复 worker 接口。 - - worker WireGuard 的业务探针固定使用 control-plane API `:6443`，不以可能被策略丢弃的 ICMP 判断隧道健康。 - - NAT 后 worker 的 WireGuard `PersistentKeepalive` 周期由集群 YAML 声明；跨地域节点优先用较短周期维持映射，但不得据此宣称降低物理 RTT。 - - 跨地域 TCP 调优由集群 YAML 的 `tcpTuning` 统一声明并受控应用到两端： - - 高 RTT、持续丢包链路优先使用内核支持的 BBR、`fq` 与 - `tcp_mtu_probing=1`，同时保留实际吞吐复测； - - sysctl 文件和 live 值均先比较后收敛，不得靠裸 `sysctl`、 - 预拉镜像、导入镜像或反复重启 Pod 掩盖数据面问题； - - status 必须披露两端 live tuning 是否与 YAML 一致。 - - 大型 proxy 工件由 Windows 宿主按 YAML `upstreamUrl` 获取并校验 SHA-256，Provider/trans 只承载控制命令，不作为大二进制数据面。 - - WireGuard 只承载集群三层网络；D601 worker proxy 使用 `vpn-server` 的 - NC01 公网 Shadowsocks TCP 入口，不把应用出网 TCP 再套入 WireGuard， - 也不与集群 WireGuard 竞争 UDP 链路。 - - proxy client 协议实现版本由 host-proxy YAML 声明，并与所连接的 `vpn-server` 协议实现版本保持一致。 - - control-plane 节点配置只组合 `hwlab nodes control-plane infra k3s-config plan|status|apply`： - - 该窄域只比较并安装 owning YAML 渲染的 k3s systemd drop-in； - - 只在 drop-in 或节点 pod 容量需要收敛时重启 k3s； - - 确需重启 k3s server 时，受控入口先停止服务并备份 - `/var/lib/rancher/k3s/server/cred/passwd` 派生文件，再启动服务； - - 禁止用 `passwd` 与 `state.db` 的文件 mtime、datastore freshness - 预判或其他凭据新旧检查阻断 YAML-first 变更；备份动作必须以 - `blocking=false` warning 披露，只有真实 stop/start、Node Ready - 或期望状态收敛失败才可使 apply 失败； - - 同一入口等待节点 Ready； - - 禁止调用 HWLAB 全量 `infra apply`，也不得触碰 Kubernetes manifests、registry、Tekton、Argo、git-mirror 或业务 rollout。 -- 运行边界： - - NC01 与 D601 必须保持两个独立 k3s 集群，禁止把 D601 重新注册为 - NC01 agent，也禁止把 NC01 注册到 D601。 - - D601 独立集群只运行一个 server 节点，初始业务 Pod 数必须为零； - 是否承载新业务必须由新的 workload owning YAML 显式选择，不得继承 - NC01 的 GitOps、PipelineRun、DaemonSet 或默认 namespace。 - - NC01 保留自身 control-plane 与运行面；拆分后可保留无 peer 的本地 - WireGuard 接口以避免重写生产 control-plane 地址，但不得恢复 D601 peer。 - - 跨地域连接只用于受控 SSH/Provider 控制通道和公网 HTTPS - artifact pull，不再承担 Kubernetes API、Pod、Service、DNS 或 VXLAN 数据面。 - - `topologyMode=separated` 是 owning YAML 的架构事实。CI/CD、GitOps、 - VM bootstrap 和普通集群 apply 禁止改回 joined、启动 `k3s-agent`、 - 写入 agent token 或渲染跨地域 Flannel/WireGuard peer。 - - 下列 worker 规则只适用于仍由 owning YAML 明确声明为 joined 的其他 - 集群，不得用于 NC01-D601： - - control-plane 保持唯一 k3s server，同时承担 workload； - - worker 只运行 `k3s-agent`，加入前卸载旧独立 server； - - worker 是否参与业务调度及其 taint 必须由集群 owning YAML 的 - `workers..scheduling` 声明。 - - joined worker 的调度规则： - - `schedulable=false` 时必须同时声明 `NoSchedule` 或 `NoExecute` taint； - - agent 注册参数和 control-plane 的 `cordon`/taint 收敛必须消费同一声明， - 防止节点重装、重注册或 CI 自动覆盖后短暂接收业务 Pod； - - `drain --confirm` 只允许清空声明为不可调度的 worker，并保留 - DaemonSet/节点系统 Pod；普通 apply、CI 和业务 renderer 禁止隐式 - `uncordon` 或移除 owning YAML 声明的 taint； - - drain 与 Node Ready 等待必须由本地 CLI 组合短于 `trans` 上限的 - 目标侧操作和短轮询；节点已不可调度且用户要求彻底清空时允许 - `--disable-eviction` 直接删除非 DaemonSet Pod，但不得扩大到其他节点。 - - 不可调度且无业务 Pod 的 worker 不要求 PodCIDR 代理 probe 成功； - host 本地代理健康与外网 probe 仍须成功，空 DNS Deployment、Service - 和 Endpoints 必须由 drain 一并清理。 - - Flannel 使用 WireGuard 接口承载 VXLAN，不能只用 FRP 转发 Kubernetes API。 - - 参与业务调度的跨地域 worker DNS 使用集群 YAML 声明的独立 - CoreDNS Deployment 和固定 ClusterIP；不可调度 worker 必须移除该 - 非 DaemonSet DNS Pod： - - Deployment 固定调度到 worker，Service 使用 `internalTrafficPolicy: Local`； - - worker kubelet 只在本地 DNS Ready 后通过 `cluster-dns` 参数切换； - - 禁止 patch k3s Addon 拥有的 `coredns` Deployment 或 `kube-dns` Service； - - 受控 apply 先比较声明态哈希，内容一致时跳过 `kubectl apply`，不得触发无意义 rollout。 - - ServiceLB 只在 owning YAML 指定的公网节点启用，防止无公网地址 worker 接管 hostPort。 - - worker 的 containerd 外网访问继承 host proxy systemd 环境，直接访问上游 registry；禁止为公共依赖新增 registry mirror。 - - host proxy 的 Pod 访问监听地址必须由 owning YAML 按目标节点 PodCIDR - 声明为该节点 `cni0` 地址；禁止复用其他节点的 `10.42.x.1`， - 否则 sing-box 会因绑定不存在地址持续重启。 - - `127.0.0.1:5000` 只表示 UniDesk 集群内 artifact registry，worker 的 `registries.yaml` 可把该本地引用映射到集群 endpoint，不得把它扩展为公共镜像代理。 - - 幂等重复执行不得强写： - - WireGuard、systemd drop-in、agent unit、token 文件和节点标签均先比较声明态与运行态； - - 内容一致时保持 `mutation=false`，不得无条件重启、重装或覆盖； - - WireGuard 只在配置变化或 unit 未运行时重启/启动，避免短暂断网触发 control-plane 级联重启； - - worker agent 以 owning YAML、版本、参数、集群内 registry 映射和 token 的脱敏摘要判断是否需要重装； - - `Type=notify` agent 在节点 Ready 时可能保持 `activating`；摘要一致、状态为 `active|activating` 且 MainPID 存在即视为运行中，禁止因此重复重装； - - worker 安装只做短启动，终态由 control-plane 的 Node Ready 轮询判定，不得让宿主 SSH 会话等待 systemd notify； - - k3s API 暂时不可读属于 unknown，不得当作容量漂移触发 restart； - - 集群 CLI 只组合各自 owning domain，不借用全量基础设施入口扩大写入范围。 -- 验收： - - `topologyMode=separated` 的 `status` 必须同时证明： - - D601 只有一个 Ready 节点； - - D601 `k3s` server active 且旧 `k3s-agent` inactive； - - D601 非 `kube-system` Pod 为零； - - NC01 不再存在 D601 Node。 - - joined 拓扑的 `status` 才验证 WireGuard handshake、host proxy、 - server/agent、两个 Ready node、worker InternalIP 和旧 server 退役。 - - 仅当 worker 的 `scheduling.schedulable=true` 时， - `smoke --confirm` 才可把 Job 定向调度到 worker并验证集群 DNS与 - Kubernetes ClusterIP；不可调度 worker 的 smoke 只读验证 taint、 - cordon、Node Ready 和非 DaemonSet Pod 为零，不得创建带 toleration - 的例外 Job。 - - smoke 等待必须使用短 `kubectl get` 轮询，禁止让单次 `trans` 挂住等待 Job 条件。 - - 基础 smoke 要求完整服务 FQDN 的解析结果包含 Pod 注入的 Kubernetes Service IP，并通过 API `ClusterIP:443` TCP connect 验证 Service 数据面；不得接受 `nslookup` 的空应答，也不把容器 TLS 客户端能力混入判定。 - - Secret 输出只显示 sourceRef、presence、key 名和 fingerprint，不显示值。 +- `config/platform-infra/k3s-clusters.yaml#targets.d601` 是 D601 单节点集群的 + 唯一拓扑真相，只包含节点、server、出网 proxy 引用和只读 artifact + registry；不得出现 NC01、agent token、跨地域 CNI 或共享调度参数。 +- NC01 和 D601 是两个完全独立的 Kubernetes 调度域。禁止以 agent、 + WireGuard、Flannel/VXLAN、共享 datastore 或隐藏 bootstrap 参数重新连接。 +- D601 只运行 `k3s` server，不运行 `k3s-agent`。新业务必须由自己的 + owning YAML 显式选择 D601；NC01 的 PaC、Tekton、Argo、GitOps、 + DaemonSet 和 namespace 不得默认复制。 +- 跨地域只保留两类边界： + - 宿主 SSH/Provider 控制通道； + - 鉴权、只读的公网 HTTPS artifact registry。 +- D601 的公共依赖出网由 `config/platform-infra/host-proxy.yaml` 声明， + proxy drop-in 必须归 `k3s.service`；禁止公共 registry mirror。 +- 受控入口为 + `platform-infra k3s-cluster plan|apply|status|smoke|registry-smoke + --target d601`。`apply` 先收敛 host proxy，再比较 server 声明摘要； + 内容一致时不得重装或重启。 +- `status` 和 `smoke` 必须证明： + - 只有一个 Ready 节点； + - server active 且 agent inactive； + - 全部 `kube-system` Deployment Ready； + - 非 `kube-system` Pod 为零。 +- NC01 节点配置独立归 `config/hwlab-node-control-plane.yaml`，不得引用 + D601 集群配置。确需重启 NC01 k3s 时，受控入口先备份并移走派生 + `server/cred/passwd`，该 freshness 事实只能作为非阻塞 warning。 ## Hyper-V Ubuntu Provider VM - `config/platform-infra/hyperv-vms.yaml` 是 Windows Hyper-V Ubuntu Provider VM 的唯一配置真相，完整需求见 `project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md`。 @@ -228,7 +136,7 @@ - NC01 集群内 registry 的内部 endpoint 是唯一写入 authority： - CI、BuildKit 和 publish 继续使用内部 `127.0.0.1:5000` 语义； - - 公网 `registry.hwpod.com` 只用于异地 worker 拉取，不接受 push 或删除。 + - 公网 `registry.hwpod.com` 只用于 D601 独立集群拉取，不接受 push 或删除。 - 公网入口归共享 public-edge 管理： - hostname、DNS、upstream、允许方法和鉴权 SecretRef 来自集群 owning YAML； - Caddy 只允许 `GET`、`HEAD`，其余方法返回拒绝； @@ -239,24 +147,23 @@ 后仅按 key 继承环境，禁止用语义不同的 `docker run --env-file`，也禁止 手工重启容器； - 可信 TLS 由 public-edge 管理，镜像层不得经过 CDN 代理。 -- worker containerd 归集群 CLI 管理： +- D601 containerd 归集群 CLI 管理： - 既有 `127.0.0.1:5000/...` 镜像引用保持不变； - `registries.yaml` 把该引用映射到 YAML 声明的公网 HTTPS endpoint； - 用户名和密码只从同一 SecretRef 渲染； - - registry 配置单独变化时只重启 agent 使配置生效，不卸载或重装 k3s。 + - registry 配置变化时由 server 受控加载，不卸载或重装 k3s。 - 受控入口： ```bash - bun scripts/cli.ts platform-infra k3s-cluster registry-credential-init --target nc01-d601 --confirm - bun scripts/cli.ts platform-infra k3s-cluster apply --target nc01-d601 --confirm - bun scripts/cli.ts platform-infra k3s-cluster registry-smoke --target nc01-d601 --confirm + bun scripts/cli.ts platform-infra k3s-cluster registry-credential-init --target d601 --confirm + bun scripts/cli.ts platform-infra k3s-cluster apply --target d601 --confirm + bun scripts/cli.ts platform-infra k3s-cluster registry-smoke --target d601 --confirm ``` - `registry-smoke` 必须验证： - 未鉴权拒绝、有效凭据、manifest digest、blob range 和写方法拒绝； - - YAML 选定的真实镜像冷拉取、并发拉取和重复拉取； - - 拉取期间 Kubernetes API 连通，前后 control-plane 与 worker 均为 Ready； - - Provider 控制通道能返回结构化终态； + - YAML 选定的真实镜像拉取和摘要； + - 拉取后 D601 Node 仍为 Ready； - 结束后不保留临时 Kubernetes workload。 ## Gitea 与 Pipelines-as-Code 边界 diff --git a/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md b/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md index a701b415..52617f2d 100644 --- a/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md +++ b/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md @@ -32,7 +32,11 @@ ### 3.1 HYPERV-VM-REQ-001 YAML 真相 -虚拟机名称、宿主 route、CPU、内存、磁盘、镜像 URL 与摘要、网络、Ubuntu 用户、k3s 版本和 Provider ID 必须来自 `config/platform-infra/hyperv-vms.yaml`。跨主机集群拓扑、WireGuard、k3s server/agent 角色、SecretRef、registry mirror 和调度 smoke 必须来自独立的集群 owning YAML。实现只能校验、解析引用和渲染，不能维护隐藏运行参数。 +虚拟机名称、宿主 route、CPU、内存、磁盘、镜像 URL 与摘要、网络、 +Ubuntu 用户、k3s 版本和 Provider ID 必须来自 +`config/platform-infra/hyperv-vms.yaml`。D601 独立 server、host proxy、 +registry endpoint 和拉取验收必须来自集群 owning YAML。实现只能校验、 +解析引用和渲染，不能维护隐藏运行参数。 ### 3.2 HYPERV-VM-REQ-002 稳定资源边界 @@ -79,23 +83,20 @@ Provider TCP 数据池恢复必须满足： D601-VM 与 NC01 必须作为两个独立集群运行： -- `config/platform-infra/k3s-clusters.yaml` 必须显式声明 - `topologyMode=separated` 和 D601 `k3s.standalone` 参数。 +- `config/platform-infra/k3s-clusters.yaml#targets.d601` 只声明 D601 + 独立 server，不得出现 NC01、agent token 或跨地域 CNI 参数。 - `config/platform-infra/hyperv-vms.yaml` 必须以 `management.mode=standalone` 引用独立集群配置。 -- D601 只运行单节点 k3s server；旧 `k3s-agent`、agent datastore、CNI - 和跨地域 WireGuard peer 必须由受控拆分入口退役。 -- 拆分前必须证明 D601 非 DaemonSet Pod 为零；D601 独立 server Ready - 后，才能从 NC01 删除旧 Node 和 WireGuard peer。 -- 拆分后 D601 非 `kube-system` Pod 必须为零，NC01 不得存在 D601 Node。 -- CI/CD、GitOps、VM bootstrap 和业务 renderer 禁止重新安装 agent、 - 恢复 agent token、重建跨地域 CNI 或把 NC01 workload 自动投递到 D601。 +- D601 只运行单节点 k3s server，`k3s-agent` 必须 inactive。 +- D601 非 `kube-system` Pod 必须为零，全部系统 Deployment 必须 Ready。 +- CI/CD、GitOps、VM bootstrap 和业务 renderer 禁止安装 agent、创建 + 跨地域 CNI 或把 NC01 workload 自动投递到 D601。 - D601 host proxy 可继续为独立集群提供公共依赖出网，不得依赖公共镜像站 或新增公共 registry mirror。 -- 集群自有 artifact registry 可以通过公网可信 TLS 向异地 worker 提供鉴权只读拉取： +- 集群自有 artifact registry 可以通过公网可信 TLS 向 D601 提供鉴权只读拉取： - 内部 CI/publish 继续使用集群内写入 endpoint，公网入口不得成为第二个 push authority。 - 公网边缘只允许 Registry V2 拉取所需的 `GET` 与 `HEAD`，其他方法必须在反向代理层拒绝。 - endpoint、DNS、上游、允许方法、凭据 `sourceRef`、镜像摘要和拉取验收参数必须来自集群 owning YAML。 - D601 containerd 继续消费既有镜像引用，只把该内部引用映射到声明的公网 HTTPS endpoint。 - - artifact 大文件不得经过 WireGuard、Flannel/VXLAN 或受传输时长和对象大小限制的 CDN。 + - artifact 大文件不得经过 Kubernetes 数据面或受传输时长和对象大小限制的 CDN。 - 宿主内网 SSH 可以作为 D601 的稳定部署控制通道，provider route 只作为部署后验收入口。 diff --git a/scripts/src/help.ts b/scripts/src/help.ts index 56e9eb6f..f0e35f9f 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -894,9 +894,9 @@ function platformInfraHelpSummary(): unknown { "bun scripts/cli.ts platform-infra hyperv-vm plan --target D601-VM", "bun scripts/cli.ts platform-infra hyperv-vm apply --target D601-VM --confirm", "bun scripts/cli.ts platform-infra hyperv-vm status --target D601-VM", - "bun scripts/cli.ts platform-infra k3s-cluster plan --target nc01-d601", - "bun scripts/cli.ts platform-infra k3s-cluster apply --target nc01-d601 --confirm", - "bun scripts/cli.ts platform-infra k3s-cluster smoke --target nc01-d601 --confirm", + "bun scripts/cli.ts platform-infra k3s-cluster plan --target d601", + "bun scripts/cli.ts platform-infra k3s-cluster apply --target d601 --confirm", + "bun scripts/cli.ts platform-infra k3s-cluster smoke --target d601", "bun scripts/cli.ts platform-infra hyperv-vm verify --target D601-VM [--repair-provider-channel --confirm]", "bun scripts/cli.ts platform-infra langbot plan", "bun scripts/cli.ts platform-infra langbot apply --confirm", diff --git a/scripts/src/platform-infra-host-proxy.ts b/scripts/src/platform-infra-host-proxy.ts index bfb301e9..acaaaece 100644 --- a/scripts/src/platform-infra-host-proxy.ts +++ b/scripts/src/platform-infra-host-proxy.ts @@ -201,12 +201,7 @@ export function prepareHostProxyGuestBundle(targetId: string): HostProxyGuestBun remoteArchivePath: target.source.client.archiveInstallPath, archiveSha256: target.source.client.archiveSha256, prepareScript: remotePrepareScript(target.source.client), - applyScript: `${remoteApplyScript(target, material.clientConfigJson)} -if [ "$changed" = true ] && [ -s /etc/systemd/system/k3s-agent.service ]; then - systemctl daemon-reload - systemctl restart --no-block k3s-agent -fi -`, + applyScript: remoteApplyScript(target, material.clientConfigJson), statusScript: remoteStatusScript(target), sourceFingerprint: target.source.fingerprint, materialFingerprint: material.fingerprint, diff --git a/scripts/src/platform-infra-k3s-cluster-config.ts b/scripts/src/platform-infra-k3s-cluster-config.ts index f08f57d7..15473ba0 100644 --- a/scripts/src/platform-infra-k3s-cluster-config.ts +++ b/scripts/src/platform-infra-k3s-cluster-config.ts @@ -6,69 +6,23 @@ export const K3S_CLUSTER_CONFIG_LABEL = "config/platform-infra/k3s-clusters.yaml export interface K3sClusterTarget { id: string; clusterName: string; - topologyMode: "joined" | "separated"; - controlPlane: { - id: string; - configRef: string; - lane: string; - nodeName: string; - publicAddress: string; - route: string; - kubeRoute: string; - }; - worker: { + node: { id: string; bootstrapConfigRef: string; hostProxyConfigRef: string; hostProxyTargetId: string; nodeName: string; labels: Record; - scheduling: { - schedulable: boolean; - taints: Array<{ key: string; value: string; effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute" }>; - }; windowsRoute: string; guestAddress: string; guestUser: string; stateDir: string; k3sVersion: string; }; - wireguard: { - interfaceName: string; - packageName: string; - mtu: number; - listenPort: number; - controlPlaneAddress: string; - workerAddress: string; - endpoint: string; - persistentKeepaliveSeconds: number; - }; - tcpTuning: { - congestionControl: "bbr" | "cubic"; - defaultQdisc: "fq" | "fq_codel"; - mtuProbing: 1 | 2; - sysctlPath: string; - }; - secrets: { - sourceRef: string; - controlPlanePrivateKey: string; - controlPlanePublicKey: string; - workerPrivateKey: string; - workerPublicKey: string; - agentToken: string; - }; k3s: { - serverUrl: string; - agentTokenFile: string; - flannelInterface: string; + writeKubeconfigMode: "0600" | "0640" | "0644"; maxPods: number; - standalone: { - clusterName: string; - nodeName: string; - writeKubeconfigMode: string; - labels: Record; - disableComponents: string[]; - }; + disableComponents: string[]; registryMirror: { source: string; endpoint: string; @@ -86,23 +40,13 @@ export interface K3sClusterTarget { concurrentPulls: number; timeoutSeconds: number; }; - serviceLb: { enabledNodeLabel: string; enabledNode: string }; - workerDns: { - namespace: string; - deploymentName: string; - serviceName: string; - clusterIp: string; - image: string; - configMapName: string; - customConfigMapName: string; - replicas: number; - timeoutSeconds: number; - }; }; - smoke: { namespace: string; jobName: string; image: string; timeoutSeconds: number }; } -export function readK3sClusterConfig(targetId?: string): { defaultTargetId: string; target: K3sClusterTarget } { +export function readK3sClusterConfig(targetId?: string): { + defaultTargetId: string; + target: K3sClusterTarget; +} { const root = record(Bun.YAML.parse(readFileSync(rootPath(K3S_CLUSTER_CONFIG_LABEL), "utf8")), K3S_CLUSTER_CONFIG_LABEL); if (root.kind !== "platform-infra-k3s-clusters") throw new Error(`${K3S_CLUSTER_CONFIG_LABEL}.kind must be platform-infra-k3s-clusters`); const defaults = record(root.defaults, `${K3S_CLUSTER_CONFIG_LABEL}.defaults`); @@ -110,164 +54,63 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri const id = targetId ?? defaultTargetId; const raw = record(record(root.targets, `${K3S_CLUSTER_CONFIG_LABEL}.targets`)[id], `${K3S_CLUSTER_CONFIG_LABEL}.targets.${id}`); if (raw.enabled !== true) throw new Error(`${K3S_CLUSTER_CONFIG_LABEL}.targets.${id}.enabled must be true`); - const control = record(raw.controlPlane, `targets.${id}.controlPlane`); - const workers = record(raw.workers, `targets.${id}.workers`); - const workerEntries = Object.entries(workers); - if (workerEntries.length !== 1) throw new Error(`targets.${id}.workers must contain exactly one worker`); - const [workerId, workerRaw] = workerEntries[0] ?? []; - const worker = record(workerRaw, `targets.${id}.workers.${workerId}`); - const wireguard = record(raw.wireguard, `targets.${id}.wireguard`); - const tcpTuning = record(raw.tcpTuning, `targets.${id}.tcpTuning`); - const secrets = record(raw.secrets, `targets.${id}.secrets`); + const node = record(raw.node, `targets.${id}.node`); const k3s = record(raw.k3s, `targets.${id}.k3s`); - const standalone = record(k3s.standalone, `targets.${id}.k3s.standalone`); const mirror = record(k3s.registryMirror, `targets.${id}.k3s.registryMirror`); const mirrorAuth = record(mirror.auth, `targets.${id}.k3s.registryMirror.auth`); - const registryPullSmoke = record(k3s.registryPullSmoke, `targets.${id}.k3s.registryPullSmoke`); - const serviceLb = record(k3s.serviceLb, `targets.${id}.k3s.serviceLb`); - const workerDns = record(k3s.workerDns, `targets.${id}.k3s.workerDns`); - const smoke = record(raw.smoke, `targets.${id}.smoke`); - const controlSource = resolveRef(text(control, "configRef", `targets.${id}.controlPlane`)); - const bootstrapSource = resolveRef(text(worker, "bootstrapConfigRef", `targets.${id}.workers[0]`)); - const host = record(bootstrapSource.host, "worker bootstrap host"); - const network = record(bootstrapSource.network, "worker bootstrap network"); - const ubuntu = record(bootstrapSource.ubuntu, "worker bootstrap ubuntu"); - const bootstrapK3s = record(bootstrapSource.k3s, "worker bootstrap k3s"); - const labels = stringMap(record(worker.labels, `targets.${id}.workers.${workerId}.labels`), `targets.${id}.workers.${workerId}.labels`); - const scheduling = record(worker.scheduling, `targets.${id}.workers.${workerId}.scheduling`); - const taints = objectArray(scheduling.taints, `targets.${id}.workers.${workerId}.scheduling.taints`).map((item, index) => ({ - key: kubernetesLabelKey(item, "key", `targets.${id}.workers.${workerId}.scheduling.taints[${index}]`), - value: kubernetesLabelValue(item, "value", `targets.${id}.workers.${workerId}.scheduling.taints[${index}]`), - effect: enumText(item, "effect", `targets.${id}.workers.${workerId}.scheduling.taints[${index}]`, ["NoSchedule", "PreferNoSchedule", "NoExecute"]), - })); - const target: K3sClusterTarget = { - id, - clusterName: text(raw, "clusterName", `targets.${id}`), - topologyMode: enumText(raw, "topologyMode", `targets.${id}`, ["joined", "separated"]), - controlPlane: { - id: text(control, "id", `targets.${id}.controlPlane`), - configRef: text(control, "configRef", `targets.${id}.controlPlane`), - lane: text(control, "lane", `targets.${id}.controlPlane`), - nodeName: text(control, "nodeName", `targets.${id}.controlPlane`), - publicAddress: text(control, "publicAddress", `targets.${id}.controlPlane`), - route: text(controlSource, "route", "control plane configRef"), - kubeRoute: text(controlSource, "kubeRoute", "control plane configRef"), - }, - worker: { - id: text(worker, "id", `targets.${id}.workers.${workerId}`), - bootstrapConfigRef: text(worker, "bootstrapConfigRef", `targets.${id}.workers.${workerId}`), - hostProxyConfigRef: text(worker, "hostProxyConfigRef", `targets.${id}.workers.${workerId}`), - hostProxyTargetId: hostProxyTargetId(text(worker, "hostProxyConfigRef", `targets.${id}.workers.${workerId}`)), - nodeName: text(worker, "nodeName", `targets.${id}.workers.${workerId}`), - labels, - scheduling: { - schedulable: boolean(scheduling, "schedulable", `targets.${id}.workers.${workerId}.scheduling`), - taints, + const pullSmoke = record(k3s.registryPullSmoke, `targets.${id}.k3s.registryPullSmoke`); + const bootstrap = resolveRef(text(node, "bootstrapConfigRef", `targets.${id}.node`)); + const host = record(bootstrap.host, "node bootstrap host"); + const network = record(bootstrap.network, "node bootstrap network"); + const ubuntu = record(bootstrap.ubuntu, "node bootstrap ubuntu"); + const bootstrapK3s = record(bootstrap.k3s, "node bootstrap k3s"); + const hostProxyConfigRef = text(node, "hostProxyConfigRef", `targets.${id}.node`); + return { + defaultTargetId, + target: { + id, + clusterName: kubernetesName(raw, "clusterName", `targets.${id}`), + node: { + id: text(node, "id", `targets.${id}.node`), + bootstrapConfigRef: text(node, "bootstrapConfigRef", `targets.${id}.node`), + hostProxyConfigRef, + hostProxyTargetId: hostProxyTargetId(hostProxyConfigRef), + nodeName: kubernetesName(node, "nodeName", `targets.${id}.node`), + labels: stringMap(record(node.labels, `targets.${id}.node.labels`), `targets.${id}.node.labels`), + windowsRoute: text(host, "route", "node bootstrap host"), + guestAddress: text(network, "address", "node bootstrap network"), + guestUser: text(ubuntu, "user", "node bootstrap ubuntu"), + stateDir: text(host, "stateDir", "node bootstrap host"), + k3sVersion: text(bootstrapK3s, "version", "node bootstrap k3s"), }, - windowsRoute: text(host, "route", "worker bootstrap host"), - guestAddress: text(network, "address", "worker bootstrap network"), - guestUser: text(ubuntu, "user", "worker bootstrap ubuntu"), - stateDir: text(host, "stateDir", "worker bootstrap host"), - k3sVersion: text(bootstrapK3s, "version", "worker bootstrap k3s"), - }, - wireguard: { - interfaceName: text(wireguard, "interfaceName", `targets.${id}.wireguard`), - packageName: text(wireguard, "packageName", `targets.${id}.wireguard`), - mtu: integer(wireguard, "mtu", `targets.${id}.wireguard`), - listenPort: integer(wireguard, "listenPort", `targets.${id}.wireguard`), - controlPlaneAddress: cidr(wireguard, "controlPlaneAddress", `targets.${id}.wireguard`), - workerAddress: cidr(wireguard, "workerAddress", `targets.${id}.wireguard`), - endpoint: text(wireguard, "endpoint", `targets.${id}.wireguard`), - persistentKeepaliveSeconds: integer(wireguard, "persistentKeepaliveSeconds", `targets.${id}.wireguard`), - }, - tcpTuning: { - congestionControl: enumText(tcpTuning, "congestionControl", `targets.${id}.tcpTuning`, ["bbr", "cubic"]), - defaultQdisc: enumText(tcpTuning, "defaultQdisc", `targets.${id}.tcpTuning`, ["fq", "fq_codel"]), - mtuProbing: oneOfInteger(tcpTuning, "mtuProbing", `targets.${id}.tcpTuning`, [1, 2]), - sysctlPath: text(tcpTuning, "sysctlPath", `targets.${id}.tcpTuning`), - }, - secrets: { - sourceRef: text(secrets, "sourceRef", `targets.${id}.secrets`), - controlPlanePrivateKey: text(secrets, "controlPlanePrivateKey", `targets.${id}.secrets`), - controlPlanePublicKey: text(secrets, "controlPlanePublicKey", `targets.${id}.secrets`), - workerPrivateKey: text(secrets, "workerPrivateKey", `targets.${id}.secrets`), - workerPublicKey: text(secrets, "workerPublicKey", `targets.${id}.secrets`), - agentToken: text(secrets, "agentToken", `targets.${id}.secrets`), - }, - k3s: { - serverUrl: text(k3s, "serverUrl", `targets.${id}.k3s`), - agentTokenFile: text(k3s, "agentTokenFile", `targets.${id}.k3s`), - flannelInterface: text(k3s, "flannelInterface", `targets.${id}.k3s`), - maxPods: integer(k3s, "maxPods", `targets.${id}.k3s`), - standalone: { - clusterName: kubernetesName(standalone, "clusterName", "k3s.standalone"), - nodeName: kubernetesName(standalone, "nodeName", "k3s.standalone"), - writeKubeconfigMode: enumText(standalone, "writeKubeconfigMode", "k3s.standalone", ["0600", "0640", "0644"]), - labels: stringMap(record(standalone.labels, "k3s.standalone.labels"), "k3s.standalone.labels"), - disableComponents: stringArray(standalone.disableComponents, "k3s.standalone.disableComponents"), - }, - registryMirror: { - source: text(mirror, "source", "registryMirror"), - endpoint: httpsUrl(mirror, "endpoint", "registryMirror"), - auth: { - sourceRef: text(mirrorAuth, "sourceRef", "registryMirror.auth"), - usernameKey: text(mirrorAuth, "usernameKey", "registryMirror.auth"), - passwordKey: text(mirrorAuth, "passwordKey", "registryMirror.auth"), - passwordHashKey: text(mirrorAuth, "passwordHashKey", "registryMirror.auth"), + k3s: { + writeKubeconfigMode: enumText(k3s, "writeKubeconfigMode", `targets.${id}.k3s`, ["0600", "0640", "0644"]), + maxPods: integer(k3s, "maxPods", `targets.${id}.k3s`), + disableComponents: stringArray(k3s.disableComponents, `targets.${id}.k3s.disableComponents`), + registryMirror: { + source: text(mirror, "source", "registryMirror"), + endpoint: httpsUrl(mirror, "endpoint", "registryMirror"), + auth: { + sourceRef: text(mirrorAuth, "sourceRef", "registryMirror.auth"), + usernameKey: text(mirrorAuth, "usernameKey", "registryMirror.auth"), + passwordKey: text(mirrorAuth, "passwordKey", "registryMirror.auth"), + passwordHashKey: text(mirrorAuth, "passwordHashKey", "registryMirror.auth"), + }, + }, + registryPullSmoke: { + image: text(pullSmoke, "image", "registryPullSmoke"), + expectedDigest: sha256Digest(pullSmoke, "expectedDigest", "registryPullSmoke"), + expectedBytes: integer(pullSmoke, "expectedBytes", "registryPullSmoke"), + concurrentPulls: integer(pullSmoke, "concurrentPulls", "registryPullSmoke"), + timeoutSeconds: integer(pullSmoke, "timeoutSeconds", "registryPullSmoke"), }, }, - registryPullSmoke: { - image: text(registryPullSmoke, "image", "registryPullSmoke"), - expectedDigest: sha256Digest(registryPullSmoke, "expectedDigest", "registryPullSmoke"), - expectedBytes: integer(registryPullSmoke, "expectedBytes", "registryPullSmoke"), - concurrentPulls: integer(registryPullSmoke, "concurrentPulls", "registryPullSmoke"), - timeoutSeconds: integer(registryPullSmoke, "timeoutSeconds", "registryPullSmoke"), - }, - serviceLb: { enabledNodeLabel: text(serviceLb, "enabledNodeLabel", "serviceLb"), enabledNode: text(serviceLb, "enabledNode", "serviceLb") }, - workerDns: { - namespace: kubernetesName(workerDns, "namespace", "workerDns"), - deploymentName: kubernetesName(workerDns, "deploymentName", "workerDns"), - serviceName: kubernetesName(workerDns, "serviceName", "workerDns"), - clusterIp: ipv4Address(workerDns, "clusterIp", "workerDns"), - image: text(workerDns, "image", "workerDns"), - configMapName: kubernetesName(workerDns, "configMapName", "workerDns"), - customConfigMapName: kubernetesName(workerDns, "customConfigMapName", "workerDns"), - replicas: integer(workerDns, "replicas", "workerDns"), - timeoutSeconds: integer(workerDns, "timeoutSeconds", "workerDns"), - }, - }, - smoke: { - namespace: text(smoke, "namespace", `targets.${id}.smoke`), - jobName: text(smoke, "jobName", `targets.${id}.smoke`), - image: text(smoke, "image", `targets.${id}.smoke`), - timeoutSeconds: integer(smoke, "timeoutSeconds", `targets.${id}.smoke`), }, }; - if (target.k3s.flannelInterface !== target.wireguard.interfaceName) throw new Error(`targets.${id}.k3s.flannelInterface must match wireguard.interfaceName`); - if (target.k3s.serviceLb.enabledNode !== target.controlPlane.nodeName) throw new Error(`targets.${id}.k3s.serviceLb.enabledNode must match controlPlane.nodeName`); - if (!target.worker.scheduling.schedulable && !target.worker.scheduling.taints.some((taint) => taint.effect === "NoSchedule" || taint.effect === "NoExecute")) { - throw new Error(`targets.${id}.workers.${workerId}.scheduling must declare a NoSchedule or NoExecute taint when schedulable=false`); - } - if (!target.tcpTuning.sysctlPath.startsWith("/etc/sysctl.d/")) throw new Error(`targets.${id}.tcpTuning.sysctlPath must stay under /etc/sysctl.d`); - return { defaultTargetId, target }; } export function clusterServerArgs(configRef: string): string[] { - const match = configRef.match(/^config\/platform-infra\/k3s-clusters\.yaml#targets\.([A-Za-z0-9._-]+)\.controlPlane$/u); - if (match === null) throw new Error(`unsupported k3s clusterConfigRef: ${configRef}`); - const { target } = readK3sClusterConfig(match[1]); - return [ - "--node-ip", address(target.wireguard.controlPlaneAddress), - "--node-external-ip", target.controlPlane.publicAddress, - "--flannel-iface", target.k3s.flannelInterface, - "--tls-san", address(target.wireguard.controlPlaneAddress), - "--tls-san", target.controlPlane.publicAddress, - "--agent-token-file", target.k3s.agentTokenFile, - ]; -} - -export function address(value: string): string { - return value.split("/", 1)[0] ?? value; + throw new Error(`external k3s cluster attachment is retired; remove clusterConfigRef: ${configRef}`); } function resolveRef(value: string): Record { @@ -295,17 +138,6 @@ function integer(value: Record, key: string, label: string): nu return Number(result); } -function boolean(value: Record, key: string, label: string): boolean { - const result = value[key]; - if (typeof result !== "boolean") throw new Error(`${label}.${key} must be a boolean`); - return result; -} - -function objectArray(value: unknown, label: string): Array> { - if (!Array.isArray(value)) throw new Error(`${label} must be an array`); - return value.map((item, index) => record(item, `${label}[${index}]`)); -} - function httpsUrl(value: Record, key: string, label: string): string { const result = text(value, key, label); const parsed = new URL(result); @@ -327,26 +159,6 @@ function enumText(value: Record, key: s return result as T; } -function oneOfInteger(value: Record, key: string, label: string, allowed: readonly T[]): T { - const result = integer(value, key, label); - if (!allowed.includes(result as T)) throw new Error(`${label}.${key} must be one of ${allowed.join(", ")}`); - return result as T; -} - -function cidr(value: Record, key: string, label: string): string { - const result = text(value, key, label); - if (!/^\d{1,3}(?:\.\d{1,3}){3}\/\d{1,2}$/u.test(result)) throw new Error(`${label}.${key} must be an IPv4 CIDR`); - return result; -} - -function ipv4Address(value: Record, key: string, label: string): string { - const result = text(value, key, label); - if (!/^\d{1,3}(?:\.\d{1,3}){3}$/u.test(result) || result.split(".").some((part) => Number(part) > 255)) { - throw new Error(`${label}.${key} must be an IPv4 address`); - } - return result; -} - function kubernetesName(value: Record, key: string, label: string): string { const result = text(value, key, label); if (!/^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/u.test(result) || result.length > 63) { @@ -355,22 +167,6 @@ function kubernetesName(value: Record, key: string, label: stri return result; } -function kubernetesLabelKey(value: Record, key: string, label: string): string { - const result = text(value, key, label); - if (!/^(?:[a-z0-9](?:[-a-z0-9.]*[a-z0-9])?\/)?[A-Za-z0-9](?:[-A-Za-z0-9_.]*[A-Za-z0-9])?$/u.test(result) || result.length > 253) { - throw new Error(`${label}.${key} must be a Kubernetes label key`); - } - return result; -} - -function kubernetesLabelValue(value: Record, key: string, label: string): string { - const result = text(value, key, label); - if (!/^[A-Za-z0-9](?:[-A-Za-z0-9_.]*[A-Za-z0-9])?$/u.test(result) || result.length > 63) { - throw new Error(`${label}.${key} must be a Kubernetes label value`); - } - return result; -} - function stringMap(value: Record, label: string): Record { return Object.fromEntries(Object.entries(value).map(([key, item]) => { if (typeof item !== "string" || item.length === 0) throw new Error(`${label}.${key} must be a non-empty string`); @@ -387,6 +183,6 @@ function stringArray(value: unknown, label: string): string[] { function hostProxyTargetId(ref: string): string { const match = ref.match(/^config\/platform-infra\/host-proxy\.yaml#targets\.([A-Za-z0-9._-]+)$/u); - if (match === null) throw new Error(`worker hostProxyConfigRef must point at config/platform-infra/host-proxy.yaml#targets.`); + if (match === null) throw new Error(`hostProxyConfigRef must point at config/platform-infra/host-proxy.yaml#targets.`); return match[1] ?? ""; } diff --git a/scripts/src/platform-infra-k3s-cluster.ts b/scripts/src/platform-infra-k3s-cluster.ts index eb81cb3d..24ddcb0a 100644 --- a/scripts/src/platform-infra-k3s-cluster.ts +++ b/scripts/src/platform-infra-k3s-cluster.ts @@ -1,19 +1,22 @@ -import { createHash, generateKeyPairSync, randomBytes } from "node:crypto"; -import { appendFileSync, chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { createHash, randomBytes } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; import { dirname } from "node:path"; import type { UniDeskConfig } from "./config"; import { rootPath } from "./config"; import { runCommand } from "./command"; import { parseEnvFile } from "./secrets"; -import { runHwlabNodeControlPlaneInfra } from "./hwlab-node-control-plane"; import { hostProxyGuestStatusScript, prepareHostProxyGuestBundle, type HostProxyGuestBundle, } from "./platform-infra-host-proxy"; -import { address, K3S_CLUSTER_CONFIG_LABEL, readK3sClusterConfig, type K3sClusterTarget } from "./platform-infra-k3s-cluster-config"; +import { + K3S_CLUSTER_CONFIG_LABEL, + readK3sClusterConfig, + type K3sClusterTarget, +} from "./platform-infra-k3s-cluster-config"; -type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "separate" | "drain" | "status" | "smoke" | "registry-smoke"; +type Action = "plan" | "registry-credential-init" | "apply" | "status" | "smoke" | "registry-smoke"; interface Options { action: Action; @@ -21,40 +24,39 @@ interface Options { confirm: boolean; } -interface SecretMaterial { - controlPrivate: string; - controlPublic: string; - workerPrivate: string; - workerPublic: string; - agentToken: string; - registryUsername: string; - registryPassword: string; - registryPasswordHash: string; +interface RegistrySecret { + username: string; + password: string; + passwordHash: string; fingerprint: string; } -export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig, args: string[]): Promise> { +export async function runPlatformInfraK3sClusterCommand( + _config: UniDeskConfig, + args: string[], +): Promise> { const options = parseOptions(args); const { target } = readK3sClusterConfig(options.targetId); if (options.action === "plan") return plan(target); - if (options.action === "secret-init") return secretInit(target, options.confirm); if (options.action === "registry-credential-init") return registryCredentialInit(target, options.confirm); - if (options.action === "separate") return separateCluster(target, options.confirm); - if (options.action === "drain") return drainWorker(target, options.confirm); - if (options.action === "status") return target.topologyMode === "separated" ? separatedStatus(target) : status(target); - if (options.action === "smoke") return target.topologyMode === "separated" ? separatedSmoke(target) : smoke(target, options.confirm); + if (options.action === "status") return status(target); + if (options.action === "smoke") return smoke(target); if (options.action === "registry-smoke") return registrySmoke(target, options.confirm); - if (!options.confirm) return { ...plan(target), mode: "dry-run", mutation: false }; - if (target.topologyMode === "separated") return separateCluster(target, true); + if (!options.confirm) return { ...plan(target), mode: "dry-run" }; return apply(target); } function parseOptions(args: string[]): Options { const action = args[0] ?? "plan"; - if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "separate" && action !== "drain" && action !== "status" && action !== "smoke" && action !== "registry-smoke") { - throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|separate|drain|status|smoke|registry-smoke [--target ] [--confirm]"); + const allowed: Action[] = ["plan", "registry-credential-init", "apply", "status", "smoke", "registry-smoke"]; + if (!allowed.includes(action as Action)) { + throw new Error("platform-infra k3s-cluster usage: plan|registry-credential-init|apply|status|smoke|registry-smoke [--target ] [--confirm]"); } - return { action, targetId: option(args, "--target") ?? undefined, confirm: args.includes("--confirm") }; + return { + action: action as Action, + targetId: option(args, "--target") ?? undefined, + confirm: args.includes("--confirm"), + }; } function plan(target: K3sClusterTarget): Record { @@ -63,130 +65,460 @@ function plan(target: K3sClusterTarget): Record { mutation: false, targetId: target.id, source: K3S_CLUSTER_CONFIG_LABEL, - topologyMode: target.topologyMode, - topology: { - clusterName: target.clusterName, - controlPlane: { id: target.controlPlane.id, nodeName: target.controlPlane.nodeName, route: target.controlPlane.route, kubeRoute: target.controlPlane.kubeRoute }, - worker: { - id: target.worker.id, - nodeName: target.worker.nodeName, - bootstrapConfigRef: target.worker.bootstrapConfigRef, - hostProxyConfigRef: target.worker.hostProxyConfigRef, - controlRoute: target.worker.windowsRoute, - scheduling: target.worker.scheduling, - }, + topology: "standalone-single-node", + clusterName: target.clusterName, + node: { + id: target.node.id, + nodeName: target.node.nodeName, + bootstrapConfigRef: target.node.bootstrapConfigRef, + hostProxyConfigRef: target.node.hostProxyConfigRef, + controlRoute: target.node.windowsRoute, + labels: target.node.labels, }, - wireguard: { ...target.wireguard, endpoint: target.wireguard.endpoint }, - tcpTuning: target.tcpTuning, k3s: { - serverUrl: target.k3s.serverUrl, - agentTokenFile: target.k3s.agentTokenFile, - registryMirror: target.k3s.registryMirror, + version: target.node.k3sVersion, + writeKubeconfigMode: target.k3s.writeKubeconfigMode, + maxPods: target.k3s.maxPods, + disableComponents: target.k3s.disableComponents, + registryMirror: { + source: target.k3s.registryMirror.source, + endpoint: target.k3s.registryMirror.endpoint, + auth: registryAuthSummary(target), + }, registryPullSmoke: target.k3s.registryPullSmoke, - serviceLb: target.k3s.serviceLb, - workerDns: target.k3s.workerDns, }, - secret: secretSummary(target), - registryAuth: registryAuthSummary(target), next: { - secretInit: `bun scripts/cli.ts platform-infra k3s-cluster secret-init --target ${target.id} --confirm`, registryCredentialInit: `bun scripts/cli.ts platform-infra k3s-cluster registry-credential-init --target ${target.id} --confirm`, apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm`, - separate: `bun scripts/cli.ts platform-infra k3s-cluster separate --target ${target.id} --confirm`, - drain: `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm`, status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`, - smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`, + smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id}`, registrySmoke: `bun scripts/cli.ts platform-infra k3s-cluster registry-smoke --target ${target.id} --confirm`, }, }; } -function secretInit(target: K3sClusterTarget, confirm: boolean): Record { - if (!confirm) return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, secret: secretSummary(target) }; - const path = target.secrets.sourceRef; - if (existsSync(path)) throw new Error(`${path} already exists; secret-init never overwrites cluster credentials`); - const control = wireguardKeyPair(); - const worker = wireguardKeyPair(); - const values: Record = { - [target.secrets.controlPlanePrivateKey]: control.privateKey, - [target.secrets.controlPlanePublicKey]: control.publicKey, - [target.secrets.workerPrivateKey]: worker.privateKey, - [target.secrets.workerPublicKey]: worker.publicKey, - [target.secrets.agentToken]: randomBytes(32).toString("base64url"), - }; - mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - writeFileSync(path, Object.entries(values).map(([key, value]) => `${key}=${value}`).join("\n") + "\n", { encoding: "utf8", mode: 0o600 }); - chmodSync(path, 0o600); - return { ok: true, mutation: true, targetId: target.id, secret: secretSummary(target), valuesPrinted: false }; -} - function registryCredentialInit(target: K3sClusterTarget, confirm: boolean): Record { const auth = target.k3s.registryMirror.auth; const path = auth.sourceRef; const keys = [auth.usernameKey, auth.passwordKey, auth.passwordHashKey]; - const values = existsSync(path) ? parseEnvFile(readFileSync(path, "utf8")) : {}; - const present = keys.filter((key) => typeof values[key] === "string" && values[key]!.length > 0); - const clusterSource = readFileSync(target.secrets.sourceRef, "utf8"); - const legacyValues = parseEnvFile(clusterSource); - const legacyPresent = keys.filter((key) => typeof legacyValues[key] === "string" && legacyValues[key]!.length > 0); - if (legacyPresent.length > 0 && legacyPresent.length < keys.length) { - throw new Error(`${target.secrets.sourceRef} contains a partial legacy artifact registry credential; refusing to migrate`); - } - if (present.length === keys.length) { + if (existsSync(path)) { const source = readFileSync(path, "utf8"); - const normalizedSource = normalizeRegistryCredentialSource(source, auth.passwordHashKey, values[auth.passwordHashKey]!); - const normalizationNeeded = normalizedSource !== source; - if (!confirm || (!normalizationNeeded && legacyPresent.length === 0)) { + const values = parseEnvFile(source); + for (const key of keys) { + if (typeof values[key] !== "string" || values[key]!.length === 0) { + throw new Error(`${path} contains a partial artifact registry credential; refusing to overwrite`); + } + } + const normalized = normalizeRegistryCredentialSource(source, auth.passwordHashKey, values[auth.passwordHashKey]!); + if (normalized === source || !confirm) { return { ok: true, mutation: false, - mode: normalizationNeeded - ? "dry-run-normalize-compose-literal" - : legacyPresent.length === 0 ? "already-present" : "dry-run-cleanup-legacy-source", + mode: normalized === source ? "already-present" : "dry-run-normalize-compose-literal", targetId: target.id, registryAuth: registryAuthSummary(target), valuesPrinted: false, }; } - if (normalizationNeeded) { - writeFileSync(path, normalizedSource, { encoding: "utf8", mode: 0o600 }); - chmodSync(path, 0o600); - } - if (legacyPresent.length === keys.length) removeEnvKeys(target.secrets.sourceRef, clusterSource, keys); + writeFileSync(path, normalized, { encoding: "utf8", mode: 0o600 }); + chmodSync(path, 0o600); return { ok: true, mutation: true, - mode: normalizationNeeded ? "normalized-compose-literal" : "cleaned-legacy-source", + mode: "normalized-compose-literal", targetId: target.id, registryAuth: registryAuthSummary(target), valuesPrinted: false, }; } - if (present.length > 0) throw new Error(`${path} contains a partial artifact registry credential; refusing to overwrite`); if (!confirm) { return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, sourceRef: path, keys, valuesPrinted: false }; } - const credential = legacyPresent.length === keys.length - ? { - username: legacyValues[auth.usernameKey]!, - password: legacyValues[auth.passwordKey]!, - passwordHash: legacyValues[auth.passwordHashKey]!, - } - : createRegistryCredential(); + const credential = createRegistryCredential(); mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); - const content = [ + writeFileSync(path, [ `${auth.usernameKey}=${credential.username}`, `${auth.passwordKey}=${credential.password}`, `${auth.passwordHashKey}='${credential.passwordHash}'`, "", - ].join("\n"); - if (existsSync(path)) appendFileSync(path, content, { encoding: "utf8", mode: 0o600 }); - else writeFileSync(path, content, { encoding: "utf8", mode: 0o600 }); + ].join("\n"), { encoding: "utf8", mode: 0o600 }); chmodSync(path, 0o600); - if (legacyPresent.length === keys.length) removeEnvKeys(target.secrets.sourceRef, clusterSource, keys); return { ok: true, mutation: true, targetId: target.id, registryAuth: registryAuthSummary(target), valuesPrinted: false }; } +function apply(target: K3sClusterTarget): Record { + const secret = readRegistrySecret(target); + const proxyBundle = prepareHostProxyGuestBundle(target.node.hostProxyTargetId); + const proxy = runNodeHostProxy(target, proxyBundle, 240_000); + requireSuccess(proxy, "standalone-host-proxy-apply"); + const server = runNode(target, standaloneApplyScript(target, secret, proxyBundle.materialFingerprint), true, 300_000); + requireSuccess(server, "standalone-k3s-apply"); + const converged = waitForStatus(target, 180_000); + requireSuccess(converged, "standalone-k3s-status"); + const data = parseFirstJson(converged.stdout); + if (data.ok !== true) throw new Error(`standalone cluster did not converge: ${bounded(JSON.stringify(data))}`); + return { + ok: true, + mutation: true, + targetId: target.id, + topology: "standalone-single-node", + status: data, + result: { hostProxy: compact(proxy), server: compact(server), status: compact(converged) }, + valuesPrinted: false, + }; +} + +function status(target: K3sClusterTarget): Record { + const result = runNode(target, standaloneStatusScript(target), false, 60_000); + const data = parseFirstJson(result.stdout); + return { + ok: result.exitCode === 0 && data.ok === true, + mutation: false, + targetId: target.id, + topology: "standalone-single-node", + status: data, + result: { exitCode: result.exitCode, timedOut: result.timedOut }, + }; +} + +function smoke(target: K3sClusterTarget): Record { + const current = status(target); + const data = record(current.status); + return { + ...current, + action: "standalone-empty-cluster-smoke", + checks: { + singleNodeReady: data.nodeReady === true && data.nodeCount === 1, + serverOnly: data.serverActive === true && data.oldAgentActive === false, + systemDeploymentsReady: data.systemDeploymentCount === data.systemReadyDeploymentCount, + businessPodCountZero: data.businessPodCount === 0, + }, + }; +} + +function registrySmoke(target: K3sClusterTarget, confirm: boolean): Record { + if (!confirm) { + return { + ok: true, + mutation: false, + mode: "dry-run", + targetId: target.id, + nodeName: target.node.nodeName, + registry: { + source: target.k3s.registryMirror.source, + endpoint: target.k3s.registryMirror.endpoint, + auth: registryAuthSummary(target), + }, + smoke: target.k3s.registryPullSmoke, + }; + } + const secret = readRegistrySecret(target); + const result = runNode( + target, + registrySmokeScript(target, secret), + false, + (target.k3s.registryPullSmoke.timeoutSeconds + 60) * 1000, + ); + const checks = parseFirstJson(result.stdout); + return { + ok: result.exitCode === 0 && checks.ok === true, + mutation: true, + targetId: target.id, + nodeName: target.node.nodeName, + checks, + result: compact(result), + valuesPrinted: false, + }; +} + +function standaloneApplyScript(target: K3sClusterTarget, secret: RegistrySecret, proxyFingerprint: string): string { + const disableArgs = target.k3s.disableComponents.flatMap((component) => ["--disable", component]); + const labels = Object.entries(target.node.labels).flatMap(([key, value]) => ["--node-label", `${key}=${value}`]); + const exec = [ + "server", + ...disableArgs, + "--cluster-init", + "--node-name", target.node.nodeName, + ...labels, + "--write-kubeconfig-mode", target.k3s.writeKubeconfigMode, + "--kubelet-arg", `max-pods=${target.k3s.maxPods}`, + ].map(shellToken).join(" "); + const desiredHash = hash(JSON.stringify({ + version: target.node.k3sVersion, + exec, + registryEndpoint: target.k3s.registryMirror.endpoint, + registryUsername: secret.username, + registryPassword: secret.password, + proxyFingerprint, + })); + return `set -eu +desired_hash=${shQuote(desiredHash)} +desired_hash_file=/etc/unidesk/k3s-standalone-desired.sha256 +install -d -m 0700 /etc/rancher/k3s /etc/unidesk +cat >/etc/rancher/k3s/registries.yaml.candidate <<'EOF' +mirrors: + "${target.k3s.registryMirror.source}": + endpoint: + - "${target.k3s.registryMirror.endpoint}" +configs: + "${new URL(target.k3s.registryMirror.endpoint).host}": + auth: + username: ${JSON.stringify(secret.username)} + password: ${JSON.stringify(secret.password)} +EOF +chmod 0600 /etc/rancher/k3s/registries.yaml.candidate +if ! cmp -s /etc/rancher/k3s/registries.yaml.candidate /etc/rancher/k3s/registries.yaml; then + mv /etc/rancher/k3s/registries.yaml.candidate /etc/rancher/k3s/registries.yaml +else + rm -f /etc/rancher/k3s/registries.yaml.candidate +fi +if [ "$(cat "$desired_hash_file" 2>/dev/null || true)" = "$desired_hash" ] && systemctl is-active --quiet k3s; then + exit 0 +fi +systemctl stop k3s >/dev/null 2>&1 || true +derived_passwd=/var/lib/rancher/k3s/server/cred/passwd +if [ -f "$derived_passwd" ]; then + mv "$derived_passwd" "$derived_passwd.unidesk-pre-restart-$(date -u +%Y%m%dT%H%M%SZ)-$$" +fi +install -m 0755 /tmp/k3s /usr/local/bin/k3s +chmod 0755 /tmp/install-k3s.sh +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_VERSION=${shQuote(target.node.k3sVersion)} INSTALL_K3S_EXEC=${shQuote(exec)} sh /tmp/install-k3s.sh +printf '%s\\n' "$desired_hash" >"$desired_hash_file" +chmod 0600 "$desired_hash_file" +systemctl enable k3s >/dev/null +systemctl start --no-block k3s +`; +} + +function standaloneStatusScript(target: K3sClusterTarget): string { + return `set -eu +active=false; systemctl is-active --quiet k3s && active=true +agent_active=false; systemctl is-active --quiet k3s-agent && agent_active=true +ready="$(k3s kubectl get node ${shQuote(target.node.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" +node_count="$(k3s kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ')" +business_pods="$(k3s kubectl get pods -A --no-headers 2>/dev/null | awk '$1 != "kube-system" {count++} END {print count+0}')" +deployments="$(k3s kubectl get deployments -A --no-headers 2>/dev/null | awk '$1 == "kube-system" {count++} END {print count+0}')" +ready_deployments="$(k3s kubectl get deployments -A --no-headers 2>/dev/null | awk '$1 == "kube-system" && $3 == $4 && $4 != "0" {count++} END {print count+0}')" +python3 - "$active" "$agent_active" "$ready" "$node_count" "$business_pods" "$deployments" "$ready_deployments" <<'PY' +import json, sys +active, agent, ready, nodes, business, deployments, ready_deployments = sys.argv[1:] +ok = ( + active == "true" + and agent == "false" + and ready == "True" + and nodes == "1" + and business == "0" + and deployments == ready_deployments +) +print(json.dumps({ + "ok": ok, + "serverActive": active == "true", + "oldAgentActive": agent == "true", + "nodeReady": ready == "True", + "nodeCount": int(nodes or 0), + "businessPodCount": int(business or 0), + "systemDeploymentCount": int(deployments or 0), + "systemReadyDeploymentCount": int(ready_deployments or 0), +}, separators=(",", ":"))) +PY +`; +} + +function registrySmokeScript(target: K3sClusterTarget, secret: RegistrySecret): string { + const smoke = target.k3s.registryPullSmoke; + const endpoint = new URL(target.k3s.registryMirror.endpoint); + const imagePath = smoke.image.slice(target.k3s.registryMirror.source.length + 1); + const separator = imagePath.lastIndexOf(":"); + if (separator < 1) throw new Error(`registry smoke image must use a tag: ${smoke.image}`); + const repository = imagePath.slice(0, separator); + const tag = imagePath.slice(separator + 1); + return `set -eu +tmp="$(mktemp -d)" +trap 'rm -rf "$tmp"' EXIT +cat >"$tmp/netrc" <<'EOF' +machine ${endpoint.hostname} +login ${secret.username} +password ${secret.password} +EOF +chmod 0600 "$tmp/netrc" +base=${shQuote(target.k3s.registryMirror.endpoint)} +unauth_status="$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 20 "$base/v2/" || true)" +auth_status="$(curl -sS --netrc-file "$tmp/netrc" -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 20 "$base/v2/" || true)" +curl -fsS --netrc-file "$tmp/netrc" -D "$tmp/headers" -o "$tmp/manifest.json" -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' "$base/v2/${repository}/manifests/${tag}" +manifest_digest="$(tr -d '\\r' <"$tmp/headers" | awk 'tolower($1)=="docker-content-digest:" {print $2}' | tail -1)" +layer_digest="$(python3 - "$tmp/manifest.json" <<'PY' +import json, sys +print(json.load(open(sys.argv[1], encoding="utf-8"))["layers"][0]["digest"]) +PY +)" +range_status="$(curl -sS --netrc-file "$tmp/netrc" -H 'Range: bytes=0-1023' -o "$tmp/range.bin" -w '%{http_code}' "$base/v2/${repository}/blobs/$layer_digest" || true)" +range_bytes="$(wc -c <"$tmp/range.bin" | tr -d ' ')" +write_status="$(curl -sS --netrc-file "$tmp/netrc" -X POST -o /dev/null -w '%{http_code}' "$base/v2/unidesk-registry-smoke/blobs/uploads/" || true)" +k3s crictl pull ${shQuote(smoke.image)} >"$tmp/pull.log" 2>&1 +k3s crictl inspecti ${shQuote(smoke.image)} >"$tmp/inspect.json" +grep -F ${shQuote(smoke.expectedDigest)} "$tmp/inspect.json" >/dev/null +node_ready="$(k3s kubectl get node ${shQuote(target.node.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')" +python3 - "$unauth_status" "$auth_status" "$manifest_digest" "$range_status" "$range_bytes" "$write_status" "$node_ready" <<'PY' +import json, sys +unauth, auth, digest, range_status, range_bytes, write_status, ready = sys.argv[1:] +checks = { + "unauthenticatedRejected": unauth == "401", + "authenticatedV2": auth == "200", + "manifestDigest": digest == ${JSON.stringify(smoke.expectedDigest)}, + "blobRange": range_status == "206" and int(range_bytes or 0) == 1024, + "writesRejected": write_status in {"403", "405"}, + "imagePulled": True, + "nodeReady": ready == "True", +} +print(json.dumps({"ok": all(checks.values()), "checks": checks, "valuesPrinted": False}, separators=(",", ":"))) +PY +`; +} + +function waitForStatus(target: K3sClusterTarget, timeoutMs: number) { + const deadline = Date.now() + timeoutMs; + let current = runNode(target, standaloneStatusScript(target), false, 55_000); + while ((current.exitCode !== 0 || parseFirstJson(current.stdout).ok !== true) && Date.now() < deadline) { + runCommand(["sleep", "3"], rootPath(), { timeoutMs: 5_000 }); + current = runNode(target, standaloneStatusScript(target), false, 55_000); + } + return current; +} + +function runNode(target: K3sClusterTarget, script: string, copyArtifacts: boolean, timeoutMs: number) { + const encoded = Buffer.from(script, "utf8").toString("base64"); + const keyPath = `${target.node.stateDir}\\id_ed25519`; + const guest = `${target.node.guestUser}@${target.node.guestAddress}`; + if (!copyArtifacts) { + const lines = [ + "$ErrorActionPreference='Stop'", + `$key=${psQuote(keyPath)}`, + `$guest=${psQuote(guest)}`, + `$payload=${psQuote(encoded)}`, + "$start=[Diagnostics.ProcessStartInfo]::new()", + "$start.FileName='ssh.exe'", + '$start.Arguments="-i `"$key`" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 $guest `"sudo sh -s`""', + "$start.UseShellExecute=$false", + "$start.RedirectStandardInput=$true", + "$start.RedirectStandardOutput=$true", + "$start.RedirectStandardError=$true", + "$process=[Diagnostics.Process]::new()", + "$process.StartInfo=$start", + "if(!$process.Start()){throw 'node-inline-ssh-start-failed'}", + "$stdout=$process.StandardOutput.ReadToEndAsync()", + "$stderr=$process.StandardError.ReadToEndAsync()", + "$bytes=[Convert]::FromBase64String($payload)", + "$process.StandardInput.BaseStream.Write($bytes,0,$bytes.Length)", + "$process.StandardInput.Close()", + "$process.WaitForExit()", + "[Console]::Out.Write($stdout.Result)", + "[Console]::Error.Write($stderr.Result)", + "exit $process.ExitCode", + ]; + return runCommand(["trans", target.node.windowsRoute, "ps"], rootPath(), { input: `${lines.join("\n")}\n`, timeoutMs }); + } + const remoteScript = `/tmp/unidesk-k3s-cluster-${target.id}.sh`; + const lines = [ + "$ErrorActionPreference='Stop'", + `$key=${psQuote(keyPath)}`, + `$guest=${psQuote(guest)}`, + `$temp=Join-Path $env:TEMP ${psQuote(`unidesk-k3s-cluster-${target.id}.sh`)}`, + `[IO.File]::WriteAllBytes($temp,[Convert]::FromBase64String(${psQuote(encoded)}))`, + `$binary=${psQuote(`${target.node.stateDir}\\k3s`)}`, + `$installer=${psQuote(`${target.node.stateDir}\\install-k3s.sh`)}`, + "& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $binary \"$($guest):/tmp/k3s\"", + "if($LASTEXITCODE -ne 0){throw 'node-k3s-binary-scp-failed'}", + "& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $installer \"$($guest):/tmp/install-k3s.sh\"", + "if($LASTEXITCODE -ne 0){throw 'node-k3s-installer-scp-failed'}", + `& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $temp "$($guest):${remoteScript}"`, + "if($LASTEXITCODE -ne 0){throw 'node-script-scp-failed'}", + `& ssh.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 $guest ${psQuote(`sudo sh ${remoteScript}`)}`, + "$rc=$LASTEXITCODE", + "Remove-Item -Force -LiteralPath $temp -ErrorAction SilentlyContinue", + `& ssh.exe -i $key -o BatchMode=yes $guest ${psQuote(`rm -f ${remoteScript}`)} | Out-Null`, + "exit $rc", + ]; + return runCommand(["trans", target.node.windowsRoute, "ps"], rootPath(), { input: `${lines.join("\n")}\n`, timeoutMs }); +} + +function runNodeHostProxy(target: K3sClusterTarget, bundle: HostProxyGuestBundle, timeoutMs: number) { + const windowsArchive = `${target.node.stateDir}\\host-proxy-${bundle.targetId}.tar.gz`; + const guestArchive = `/tmp/unidesk-host-proxy-${bundle.targetId}.tar.gz`; + const remoteScript = `/tmp/unidesk-host-proxy-${bundle.targetId}.sh`; + const script = `${bundle.prepareScript} +remote_archive_sha="$(sha256sum ${shQuote(bundle.remoteArchivePath)} 2>/dev/null | awk '{print $1}' || true)" +if [ "$remote_archive_sha" != ${shQuote(bundle.archiveSha256)} ]; then + install -m 0644 ${shQuote(guestArchive)} ${shQuote(bundle.remoteArchivePath)} +fi +${bundle.applyScript} +${hostProxyGuestStatusScript(bundle.targetId)} +`; + const encoded = Buffer.from(script, "utf8").toString("base64"); + const lines = [ + "$ErrorActionPreference='Stop'", + `$key=${psQuote(`${target.node.stateDir}\\id_ed25519`)}`, + `$guest=${psQuote(`${target.node.guestUser}@${target.node.guestAddress}`)}`, + `$archive=${psQuote(windowsArchive)}`, + `$temp=Join-Path $env:TEMP ${psQuote(`unidesk-host-proxy-${bundle.targetId}.sh`)}`, + `[IO.File]::WriteAllBytes($temp,[Convert]::FromBase64String(${psQuote(encoded)}))`, + `if(!(Test-Path -LiteralPath $archive)){`, + ` & curl.exe -fL --retry 2 --connect-timeout 15 --max-time 180 -o $archive ${psQuote(bundle.upstreamUrl)}`, + " if($LASTEXITCODE -ne 0){throw 'host-proxy-artifact-download-failed'}", + "}", + `& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $archive "$($guest):${guestArchive}"`, + "if($LASTEXITCODE -ne 0){throw 'host-proxy-artifact-scp-failed'}", + `& scp.exe -i $key -o BatchMode=yes $temp "$($guest):${remoteScript}"`, + "if($LASTEXITCODE -ne 0){throw 'host-proxy-script-scp-failed'}", + `& ssh.exe -i $key -o BatchMode=yes $guest ${psQuote(`sudo sh ${remoteScript}`)}`, + "$rc=$LASTEXITCODE", + "Remove-Item -Force -LiteralPath $temp -ErrorAction SilentlyContinue", + "exit $rc", + ]; + return runCommand(["trans", target.node.windowsRoute, "ps"], rootPath(), { input: `${lines.join("\n")}\n`, timeoutMs }); +} + +function readRegistrySecret(target: K3sClusterTarget): RegistrySecret { + const auth = target.k3s.registryMirror.auth; + if (!existsSync(auth.sourceRef)) throw new Error(`${auth.sourceRef} is missing; run registry-credential-init --confirm`); + const values = parseEnvFile(readFileSync(auth.sourceRef, "utf8")); + const required = (key: string, minimumLength: number) => { + const value = values[key]; + if (typeof value !== "string" || value.length < minimumLength) throw new Error(`${auth.sourceRef}.${key} is missing or invalid`); + return value; + }; + const material = { + username: required(auth.usernameKey, 1), + password: required(auth.passwordKey, 32), + passwordHash: required(auth.passwordHashKey, 32), + }; + return { ...material, fingerprint: hash(Object.entries(material).sort().map(([key, value]) => `${key}=${value}`).join("\n")) }; +} + +function registryAuthSummary(target: K3sClusterTarget): Record { + const auth = target.k3s.registryMirror.auth; + if (!existsSync(auth.sourceRef)) { + return { sourceRef: auth.sourceRef, presence: false, fingerprint: null, valuesPrinted: false }; + } + try { + const secret = readRegistrySecret(target); + return { + sourceRef: auth.sourceRef, + presence: true, + fingerprint: secret.fingerprint, + keys: [auth.usernameKey, auth.passwordKey, auth.passwordHashKey], + valuesPrinted: false, + }; + } catch (error) { + return { + sourceRef: auth.sourceRef, + presence: true, + fingerprint: null, + error: error instanceof Error ? error.message : String(error), + valuesPrinted: false, + }; + } +} + function normalizeRegistryCredentialSource(source: string, passwordHashKey: string, passwordHash: string): string { const lines = source.split(/\r?\n/u).map((line) => { const match = /^([A-Za-z_][A-Za-z0-9_]*)=/u.exec(line); @@ -196,17 +528,6 @@ function normalizeRegistryCredentialSource(source: string, passwordHashKey: stri return `${lines.join("\n")}\n`; } -function removeEnvKeys(path: string, source: string, keys: string[]): void { - const selected = new Set(keys); - const retained = source.split(/\r?\n/u).filter((line) => { - const match = /^([A-Za-z_][A-Za-z0-9_]*)=/u.exec(line); - return match === null || !selected.has(match[1]!); - }); - while (retained.at(-1) === "") retained.pop(); - writeFileSync(path, `${retained.join("\n")}\n`, { encoding: "utf8", mode: 0o600 }); - chmodSync(path, 0o600); -} - function createRegistryCredential(): { username: string; password: string; passwordHash: string } { const username = "unidesk-pull"; const password = randomBytes(32).toString("base64url"); @@ -221,1430 +542,23 @@ function createRegistryCredential(): { username: string; password: string; passw return { username, password, passwordHash: result.stdout.trim() }; } -function apply(target: K3sClusterTarget): Record { - const secret = readSecrets(target); - const workerNetwork = runWorker(target, workerWireguardApplyScript(target, secret), false, 120_000); - requireSuccess(workerNetwork, "worker-wireguard-apply"); - - const hostProxy = prepareHostProxyGuestBundle(target.worker.hostProxyTargetId); - const workerHostProxy = runWorkerHostProxy(target, hostProxy, 240_000); - requireSuccess(workerHostProxy, "worker-host-proxy-apply"); - - const controlWireguard = runControlPlane(target, controlPlaneApplyScript(target, secret), 180_000); - requireSuccess(controlWireguard, "control-plane-wireguard-apply"); - - const controlPlane = runHwlabNodeControlPlaneInfra([ - "k3s-config", "apply", "--node", target.controlPlane.id, "--lane", target.controlPlane.lane, "--confirm", - ]) as Record; - if (controlPlane.ok !== true) throw new Error(`control-plane-k3s-apply failed: ${bounded(JSON.stringify(compactDomain(controlPlane)))}`); - - const workerSchedulingBeforeAgent = reconcileWorkerScheduling(target); - const workerDns = reconcileWorkerDns(target); - - const worker = runWorker(target, workerApplyScript(target, secret), true, 300_000); - requireSuccess(worker, "worker-agent-apply"); - const workerSchedulingAfterAgent = reconcileWorkerScheduling(target); - - const labels = [ - "label", "node", target.k3s.serviceLb.enabledNode, - `${target.k3s.serviceLb.enabledNodeLabel}=true`, "--overwrite", - ]; - const labelResult = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", ...labels], rootPath(), { timeoutMs: 60_000 }); - requireSuccess(labelResult, "servicelb-node-label"); - - const ready = waitForNodes(target, 180_000); - requireSuccess(ready, "cluster-node-readiness"); - return { - ok: true, - mutation: true, - targetId: target.id, - stages: { - workerWireguard: compact(workerNetwork), - workerHostProxy: { - ...compact(workerHostProxy), - configRef: hostProxy.configRef, - sourceFingerprint: hostProxy.sourceFingerprint, - materialFingerprint: hostProxy.materialFingerprint, - valuesPrinted: false, - }, - controlPlaneWireguard: compact(controlWireguard), - controlPlaneK3s: compactDomain(controlPlane), - workerDns, - workerAgent: compact(worker), - workerSchedulingBeforeAgent, - workerSchedulingAfterAgent, - serviceLbPlacement: compact(labelResult), - nodeReadiness: compact(ready), - }, - secret: { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: secret.fingerprint, valuesPrinted: false }, - registryAuth: registryAuthSummary(target), - next: { - drain: target.worker.scheduling.schedulable ? null : `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm`, - status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`, - smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`, - }, - }; -} - -function separateCluster(target: K3sClusterTarget, confirm: boolean): Record { - if (target.topologyMode !== "separated") throw new Error("separate requires topologyMode=separated in the owning YAML"); - const workloads = workerWorkloadStatus(target); - if (workloads.ok !== true) throw new Error(`worker must be empty before separation: ${bounded(JSON.stringify(workloads))}`); - if (!confirm) { - return { - ok: true, - mutation: false, - mode: "dry-run", - targetId: target.id, - topologyMode: target.topologyMode, - workerWorkloads: workloads, - standalone: target.k3s.standalone, - }; - } - const secret = readSecrets(target); - const standalone = runWorker(target, standaloneApplyScript(target, secret), true, 300_000); - requireSuccess(standalone, "worker-standalone-k3s-apply"); - const localStatus = waitForStandalone(target, 180_000); - requireSuccess(localStatus, "worker-standalone-k3s-status"); - const localData = parseFirstJson(localStatus.stdout); - if (localData.ok !== true) throw new Error(`standalone cluster did not converge: ${bounded(JSON.stringify(localData))}`); - - const deleteOldNode = runCommand([ - "trans", target.controlPlane.kubeRoute, "kubectl", "delete", "node", target.worker.nodeName, "--ignore-not-found=true", - ], rootPath(), { timeoutMs: 60_000 }); - requireSuccess(deleteOldNode, "old-cluster-worker-delete"); - const retirePeer = runControlPlane(target, controlPlaneRetireWorkerPeerScript(target, secret), 60_000); - requireSuccess(retirePeer, "control-plane-wireguard-peer-retire"); - const result = separatedStatus(target); - if (result.ok !== true) throw new Error(`cluster separation final status failed: ${bounded(JSON.stringify(result))}`); - return { - ok: true, - mutation: true, - targetId: target.id, - topologyMode: target.topologyMode, - standalone: localData, - oldClusterNodeDeleted: true, - wireguardPeerRetired: true, - stages: { - standalone: compact(standalone), - oldClusterNode: compact(deleteOldNode), - controlPlaneWireguard: compact(retirePeer), - }, - next: { - status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`, - smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`, - }, - }; -} - -function separatedStatus(target: K3sClusterTarget): Record { - const standalone = runWorker(target, standaloneStatusScript(target), false, 60_000); - const standaloneData = parseFirstJson(standalone.stdout); - const oldNode = runCommand([ - "trans", target.controlPlane.kubeRoute, "kubectl", "get", "node", target.worker.nodeName, "-o", "name", - ], rootPath(), { timeoutMs: 30_000 }); - const oldNodeAbsent = oldNode.exitCode !== 0; - return { - ok: standalone.exitCode === 0 && standaloneData.ok === true && oldNodeAbsent, - mutation: false, - targetId: target.id, - topologyMode: target.topologyMode, - standalone: standaloneData, - oldCluster: { - controlPlane: target.controlPlane.id, - workerNodeAbsent: oldNodeAbsent, - }, - result: { - standalone: { exitCode: standalone.exitCode }, - oldClusterNode: { exitCode: oldNode.exitCode }, - }, - }; -} - -function waitForStandalone(target: K3sClusterTarget, timeoutMs: number): ReturnType { - const deadline = Date.now() + timeoutMs; - let current = runWorker(target, standaloneStatusScript(target), false, 55_000); - while ((current.exitCode !== 0 || parseFirstJson(current.stdout).ok !== true) && Date.now() < deadline) { - runCommand(["sleep", "5"], rootPath(), { timeoutMs: 10_000 }); - current = runWorker(target, standaloneStatusScript(target), false, 55_000); - } - if (current.exitCode === 0 && parseFirstJson(current.stdout).ok === true) return current; - return { ...current, exitCode: 1, stderr: current.stderr || `standalone cluster did not become ready within ${timeoutMs}ms` }; -} - -function separatedSmoke(target: K3sClusterTarget): Record { - const status = separatedStatus(target); - return { - ...status, - action: "separated-cluster-smoke", - checks: { - standaloneSingleNodeReady: (status.standalone as Record)?.ok === true, - oldClusterNodeAbsent: (status.oldCluster as Record)?.workerNodeAbsent === true, - businessPodCountZero: (status.standalone as Record)?.businessPodCount === 0, - }, - }; -} - -function status(target: K3sClusterTarget): Record { - const control = runControlPlane(target, statusScript(target), 60_000); - const workerProxy = runWorker(target, hostProxyGuestStatusScript(target.worker.hostProxyTargetId), false, 60_000); - const worker = runWorker(target, workerStatusScript(target), false, 60_000); - const workerDns = workerDnsStatus(target); - const controlData = parseFirstJson(control.stdout); - const workerProxyData = parseFirstJson(workerProxy.stdout); - const workerData = parseFirstJson(worker.stdout); - const workerScheduling = workerSchedulingStatus(target); - const workerWorkloads = workerWorkloadStatus(target); - const proxyHealth = workerProxyData.health as Record | undefined; - const proxyExternalProbe = workerProxyData.externalProbe as Record | undefined; - const workerProxyReady = workerProxyData.ok === true - || (!target.worker.scheduling.schedulable && proxyHealth?.exitCode === 0 && proxyExternalProbe?.exitCode === 0); - const ok = control.exitCode === 0 - && (workerProxy.exitCode === 0 || (!target.worker.scheduling.schedulable && workerProxyReady)) - && worker.exitCode === 0 - && controlData.ok === true - && workerProxyReady - && workerData.ok === true - && workerDns.data.ok === true - && workerScheduling.ok === true - && workerWorkloads.ok === true; - return { - ok, - mutation: false, - targetId: target.id, - controlPlane: controlData, - workerHostProxy: { ...workerProxyData, ok: workerProxyReady, podAccessRequired: target.worker.scheduling.schedulable }, - worker: workerData, - workerDns: workerDns.data, - workerScheduling, - workerWorkloads, - secret: secretSummary(target), - registryAuth: registryAuthSummary(target), - result: { - controlPlane: { exitCode: control.exitCode }, - workerHostProxy: { exitCode: workerProxy.exitCode }, - worker: { exitCode: worker.exitCode }, - workerDns: { exitCode: workerDns.result.exitCode }, - }, - next: ok - ? { smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm` } - : !target.worker.scheduling.schedulable && workerScheduling.ok === true && workerWorkloads.ok !== true - ? { drain: `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm` } - : { apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm` }, - }; -} - -function smoke(target: K3sClusterTarget, confirm: boolean): Record { - if (!target.worker.scheduling.schedulable) { - const scheduling = workerSchedulingStatus(target); - const workloads = workerWorkloadStatus(target); - const readiness = parseFirstJson(runControlPlane(target, nodeReadinessScript(target), 30_000).stdout); - return { - ok: scheduling.ok === true && workloads.ok === true && readiness.ok === true, - mutation: false, - mode: confirm ? "confirmed-read-only" : "dry-run", - targetId: target.id, - workerNode: target.worker.nodeName, - checks: { scheduling, workloads, readiness }, - }; - } - const manifest = smokeManifest(target); - if (!confirm) return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, workerNode: target.worker.nodeName, manifestSha256: hash(manifest) }; - const namespace = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "create", "namespace", target.smoke.namespace, "--dry-run=client", "-o", "yaml"], rootPath(), { timeoutMs: 30_000 }); - requireSuccess(namespace, "smoke-namespace-render"); - const applyNamespace = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "apply", "-f", "-"], rootPath(), { input: namespace.stdout, timeoutMs: 30_000 }); - requireSuccess(applyNamespace, "smoke-namespace-apply"); - runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "delete", "job", target.smoke.jobName, "--ignore-not-found=true", "--wait=true"], rootPath(), { timeoutMs: 60_000 }); - const applyJob = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "apply", "-f", "-"], rootPath(), { input: manifest, timeoutMs: 30_000 }); - requireSuccess(applyJob, "smoke-job-apply"); - const deadline = Date.now() + target.smoke.timeoutSeconds * 1000; - const conditionArgs = ["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "get", `job/${target.smoke.jobName}`, "-o", "jsonpath={range .status.conditions[*]}{.type}={.status};{end}"]; - let wait = runCommand( - conditionArgs, - rootPath(), - { timeoutMs: 30_000 }, - ); - while (wait.exitCode === 0 && !wait.stdout.includes("Complete=True") && !wait.stdout.includes("Failed=True") && Date.now() < deadline) { - runCommand(["sleep", "5"], rootPath(), { timeoutMs: 10_000 }); - wait = runCommand( - conditionArgs, - rootPath(), - { timeoutMs: 30_000 }, - ); - } - if (wait.exitCode === 0 && !wait.stdout.includes("Complete=True")) { - const reason = wait.stdout.includes("Failed=True") ? "smoke job failed" : `smoke job did not complete within ${target.smoke.timeoutSeconds}s`; - wait = { ...wait, exitCode: 1, stderr: reason }; - } - const logs = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "logs", `job/${target.smoke.jobName}`], rootPath(), { timeoutMs: 30_000 }); - const placement = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "get", "pod", "-l", `job-name=${target.smoke.jobName}`, "-o", "jsonpath={.items[0].spec.nodeName}"], rootPath(), { timeoutMs: 30_000 }); - const ok = wait.exitCode === 0 && logs.exitCode === 0 && logs.stdout.includes("D601_RUNTIME_SMOKE_OK") && placement.stdout.trim() === target.worker.nodeName; - if (ok) runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "delete", "job", target.smoke.jobName, "--wait=true"], rootPath(), { timeoutMs: 60_000 }); - return { - ok, - mutation: true, - targetId: target.id, - workerNode: target.worker.nodeName, - scheduledNode: placement.stdout.trim(), - checks: { jobCompleted: wait.exitCode === 0, dnsAndClusterIp: logs.stdout.includes("D601_RUNTIME_SMOKE_OK"), workerPlacement: placement.stdout.trim() === target.worker.nodeName }, - logs: bounded(logs.stdout, 4_000), - result: { apply: compact(applyJob), wait: compact(wait), placement: compact(placement), cleanedUp: ok }, - }; -} - -function registrySmoke(target: K3sClusterTarget, confirm: boolean): Record { - const smoke = target.k3s.registryPullSmoke; - if (!confirm) { - return { - ok: true, - mutation: false, - mode: "dry-run", - targetId: target.id, - workerNode: target.worker.nodeName, - registry: { ...target.k3s.registryMirror, auth: { ...target.k3s.registryMirror.auth, valuesPrinted: false } }, - smoke, - valuesPrinted: false, - }; - } - const secret = readSecrets(target); - const before = runControlPlane(target, nodeReadinessScript(target), 30_000); - requireSuccess(before, "registry-smoke-node-readiness-before"); - const worker = runWorker(target, registrySmokeScript(target, secret), true, (smoke.timeoutSeconds + 120) * 1000); - const after = runControlPlane(target, nodeReadinessScript(target), 30_000); - const checks = parseFirstJson(worker.stdout); - const beforeChecks = parseFirstJson(before.stdout); - const afterChecks = parseFirstJson(after.stdout); - const ok = worker.exitCode === 0 - && beforeChecks.ok === true - && after.exitCode === 0 - && afterChecks.ok === true - && checks.ok === true; - return { - ok, - mutation: true, - targetId: target.id, - workerNode: target.worker.nodeName, - registry: { - source: target.k3s.registryMirror.source, - endpoint: target.k3s.registryMirror.endpoint, - authSourceRef: target.k3s.registryMirror.auth.sourceRef, - valuesPrinted: false, - }, - smoke, - checks, - clusterBefore: beforeChecks, - clusterAfter: afterChecks, - providerControlChannelReturned: worker.exitCode === 0, - result: { worker: compact(worker), before: compact(before), after: compact(after) }, - valuesPrinted: false, - }; -} - -function nodeReadinessScript(target: K3sClusterTarget): string { - return `set -eu -cp_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.controlPlane.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') -worker_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}') -python3 - "$cp_ready" "$worker_ready" <<'PY' -import json, sys -cp, worker = sys.argv[1:] -print(json.dumps({"ok": cp == "True" and worker == "True", "controlPlaneReady": cp == "True", "workerReady": worker == "True"}, separators=(",", ":"))) -PY -`; -} - -function registrySmokeScript(target: K3sClusterTarget, secret: SecretMaterial): string { - const smoke = target.k3s.registryPullSmoke; - const endpoint = new URL(target.k3s.registryMirror.endpoint); - const imagePath = smoke.image.slice(target.k3s.registryMirror.source.length + 1); - const separator = imagePath.lastIndexOf(":"); - if (separator < 1) throw new Error(`registry smoke image must use a tag: ${smoke.image}`); - const repository = imagePath.slice(0, separator); - const tag = imagePath.slice(separator + 1); - return `set -eu -tmp="$(mktemp -d)" -monitor_pid="" -cleanup() { - touch "$tmp/stop" - if [ -n "$monitor_pid" ]; then wait "$monitor_pid" 2>/dev/null || true; fi - rm -rf "$tmp" -} -trap cleanup EXIT -cat >"$tmp/netrc" <<'EOF' -machine ${endpoint.hostname} -login ${secret.registryUsername} -password ${secret.registryPassword} -EOF -chmod 0600 "$tmp/netrc" -base=${shQuote(target.k3s.registryMirror.endpoint)} -unauth_status="$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 20 "$base/v2/" || true)" -auth_status="$(curl -sS --netrc-file "$tmp/netrc" -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 20 "$base/v2/" || true)" -manifest_headers="$tmp/manifest.headers" -curl -fsS --netrc-file "$tmp/netrc" -D "$manifest_headers" -o "$tmp/manifest.json" -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' "$base/v2/${repository}/manifests/${tag}" -manifest_digest="$(tr -d '\\r' <"$manifest_headers" | awk 'tolower($1)=="docker-content-digest:" {print $2}' | tail -1)" -layer_digest="$(python3 - "$tmp/manifest.json" <<'PY' -import json, sys -manifest = json.load(open(sys.argv[1], encoding="utf-8")) -print(manifest["layers"][0]["digest"]) -PY -)" -range_status="$(curl -sS --netrc-file "$tmp/netrc" -H 'Range: bytes=0-1023' -o "$tmp/range.bin" -w '%{http_code}' "$base/v2/${repository}/blobs/$layer_digest" || true)" -range_bytes="$(wc -c <"$tmp/range.bin" | tr -d ' ')" -write_status="$(curl -sS --netrc-file "$tmp/netrc" -X POST -o /dev/null -w '%{http_code}' "$base/v2/unidesk-registry-smoke/blobs/uploads/" || true)" -( - : >"$tmp/api.failures" - while [ ! -f "$tmp/stop" ]; do - nc -z -w 3 ${address(target.wireguard.controlPlaneAddress)} 6443 || printf x >>"$tmp/api.failures" - sleep 2 - done -) & -monitor_pid=$! -k3s crictl rmi ${shQuote(smoke.image)} >/dev/null 2>&1 || true -cold_start="$(date +%s%3N)" -k3s crictl pull ${shQuote(smoke.image)} >"$tmp/cold.log" 2>&1 -cold_end="$(date +%s%3N)" -k3s crictl inspecti ${shQuote(smoke.image)} >"$tmp/inspect.json" -grep -F ${shQuote(smoke.expectedDigest)} "$tmp/inspect.json" >/dev/null -k3s crictl rmi ${shQuote(smoke.image)} >/dev/null 2>&1 || true -concurrent_start="$(date +%s%3N)" -pids="" -for index in $(seq 1 ${smoke.concurrentPulls}); do - k3s crictl pull ${shQuote(smoke.image)} >"$tmp/concurrent-$index.log" 2>&1 & - pids="$pids $!" -done -concurrent_ok=true -for pid in $pids; do wait "$pid" || concurrent_ok=false; done -concurrent_end="$(date +%s%3N)" -hot_start="$(date +%s%3N)" -k3s crictl pull ${shQuote(smoke.image)} >"$tmp/hot.log" 2>&1 -hot_end="$(date +%s%3N)" -touch "$tmp/stop" -wait "$monitor_pid" 2>/dev/null || true -monitor_pid="" -api_failures="$(wc -c <"$tmp/api.failures" | tr -d ' ')" -python3 - "$unauth_status" "$auth_status" "$manifest_digest" "$range_status" "$range_bytes" "$write_status" "$cold_start" "$cold_end" "$concurrent_start" "$concurrent_end" "$hot_start" "$hot_end" "$concurrent_ok" "$api_failures" <<'PY' -import json, sys -(unauth, auth, digest, range_status, range_bytes, write_status, cold_start, cold_end, - concurrent_start, concurrent_end, hot_start, hot_end, concurrent_ok, api_failures) = sys.argv[1:] -cold_ms = int(cold_end) - int(cold_start) -concurrent_ms = int(concurrent_end) - int(concurrent_start) -hot_ms = int(hot_end) - int(hot_start) -expected_bytes = ${smoke.expectedBytes} -throughput = None if cold_ms <= 0 else round(expected_bytes / (cold_ms / 1000), 2) -checks = { - "unauthenticatedRejected": unauth == "401", - "authenticatedV2": auth == "200", - "manifestDigest": digest == ${JSON.stringify(smoke.expectedDigest)}, - "blobRange": range_status == "206" and int(range_bytes or 0) == 1024, - "writesRejected": write_status in {"403", "405"}, - "coldPull": cold_ms > 0, - "concurrentPulls": concurrent_ok == "true", - "repeatPull": hot_ms >= 0, - "apiStable": int(api_failures) == 0, -} -print(json.dumps({ - "ok": all(checks.values()), - "checks": checks, - "unauthenticatedStatus": unauth, - "authenticatedStatus": auth, - "manifestDigest": digest, - "rangeStatus": range_status, - "rangeBytes": int(range_bytes or 0), - "writeStatus": write_status, - "coldPullDurationMs": cold_ms, - "coldPullBytesPerSecond": throughput, - "concurrentPullDurationMs": concurrent_ms, - "hotPullDurationMs": hot_ms, - "apiProbeFailures": int(api_failures), - "valuesPrinted": False, -}, separators=(",", ":"))) -PY -`; -} - -function reconcileWorkerDns(target: K3sClusterTarget): Record { - const before = workerDnsStatus(target); - if (!target.worker.scheduling.schedulable) { - if (before.data.presence !== true) { - return { ok: true, mutation: false, enabled: false, reason: "worker-unschedulable", presence: false }; - } - const deploymentDeletion = runCommand([ - "trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.k3s.workerDns.namespace, - "delete", "deployment", target.k3s.workerDns.deploymentName, "--ignore-not-found=true", "--wait=true", - ], rootPath(), { timeoutMs: 120_000 }); - requireSuccess(deploymentDeletion, "worker-dns-deployment-delete"); - const serviceDeletion = runCommand([ - "trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.k3s.workerDns.namespace, - "delete", "service", target.k3s.workerDns.serviceName, "--ignore-not-found=true", "--wait=true", - ], rootPath(), { timeoutMs: 120_000 }); - requireSuccess(serviceDeletion, "worker-dns-service-delete"); - let after = workerDnsStatus(target); - const deadline = Date.now() + 30_000; - while (after.data.ok !== true && Date.now() < deadline) { - runCommand(["sleep", "2"], rootPath(), { timeoutMs: 5_000 }); - after = workerDnsStatus(target); - } - if (after.data.ok !== true) throw new Error(`worker-dns-removal failed: ${bounded(JSON.stringify(after.data))}`); - return { - ok: true, - mutation: true, - enabled: false, - reason: "worker-unschedulable", - presence: false, - deletion: { deployment: compact(deploymentDeletion), service: compact(serviceDeletion) }, - }; - } - const mutation = before.data.configurationCurrent !== true; - let applyResult: ReturnType | null = null; - if (mutation) { - applyResult = runCommand( - ["trans", target.controlPlane.kubeRoute, "kubectl", "apply", "-f", "-"], - rootPath(), - { input: workerDnsManifest(target), timeoutMs: 60_000 }, - ); - requireSuccess(applyResult, "worker-dns-apply"); - } - - const deadline = Date.now() + target.k3s.workerDns.timeoutSeconds * 1000; - let current = workerDnsStatus(target); - while (current.data.ok !== true && Date.now() < deadline) { - runCommand(["sleep", "3"], rootPath(), { timeoutMs: 5_000 }); - current = workerDnsStatus(target); - } - if (current.data.ok !== true) throw new Error(`worker-dns-readiness failed: ${bounded(JSON.stringify(current.data))}`); - return { - mutation, - desiredHash: workerDnsDesiredHash(target), - ...current.data, - apply: applyResult === null ? { skipped: true, reason: "configuration-current" } : compact(applyResult), - }; -} - -function workerSchedulingStatus(target: K3sClusterTarget): Record { - const result = runCommand([ - "trans", target.controlPlane.kubeRoute, "kubectl", "get", "node", target.worker.nodeName, - "-o", String.raw`jsonpath={.spec.unschedulable}|{range .spec.taints[*]}{.key}={.value}:{.effect},{end}`, - ], rootPath(), { timeoutMs: 60_000 }); - if (result.exitCode !== 0) return { ok: false, nodePresent: false, result: compact(result) }; - const [unschedulableRaw = "", taintsRaw = ""] = result.stdout.trim().split("|", 2); - const actualTaints = taintsRaw.split(",").filter(Boolean); - const desiredTaintsPresent = target.worker.scheduling.taints.every((desired) => - actualTaints.includes(`${desired.key}=${desired.value}:${desired.effect}`)); - const unschedulable = unschedulableRaw === "true"; - return { - ok: desiredTaintsPresent && unschedulable === !target.worker.scheduling.schedulable, - nodePresent: true, - schedulable: !unschedulable, - desiredSchedulable: target.worker.scheduling.schedulable, - desiredTaintsPresent, - desiredTaints: target.worker.scheduling.taints, - }; -} - -function reconcileWorkerScheduling(target: K3sClusterTarget): Record { - const before = workerSchedulingStatus(target); - if (before.nodePresent !== true) throw new Error(`worker node is absent: ${target.worker.nodeName}`); - const mutations: string[] = []; - if (before.schedulable !== target.worker.scheduling.schedulable) { - const action = target.worker.scheduling.schedulable ? "uncordon" : "cordon"; - const result = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", action, target.worker.nodeName], rootPath(), { timeoutMs: 60_000 }); - requireSuccess(result, `worker-${action}`); - mutations.push(action); - } - if (before.desiredTaintsPresent !== true) { - for (const taint of target.worker.scheduling.taints) { - const desired = `${taint.key}=${taint.value}:${taint.effect}`; - const result = runCommand([ - "trans", target.controlPlane.kubeRoute, "kubectl", "taint", "node", target.worker.nodeName, desired, "--overwrite", - ], rootPath(), { timeoutMs: 60_000 }); - requireSuccess(result, `worker-taint-${taint.key}`); - mutations.push(`taint:${taint.key}`); - } - } - const after = workerSchedulingStatus(target); - if (after.ok !== true) throw new Error(`worker scheduling did not converge: ${bounded(JSON.stringify(after))}`); - return { ok: true, mutation: mutations.length > 0, mutations, before, after }; -} - -function workerWorkloadStatus(target: K3sClusterTarget): Record { - const result = runCommand([ - "trans", target.controlPlane.kubeRoute, "kubectl", "get", "pods", "-A", - "--field-selector", `spec.nodeName=${target.worker.nodeName}`, - "-o", "custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,OWNER:.metadata.ownerReferences[0].kind,PHASE:.status.phase", - "--no-headers", - ], rootPath(), { timeoutMs: 60_000 }); - if (result.exitCode !== 0) return { ok: false, result: compact(result) }; - const items = result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => { - const [namespace = "", name = "", ownerKind = "", phase = ""] = line.split(/\s+/u, 4); - return { namespace, name, ownerKind, phase }; - }); - const retainedSystemPods = items.filter((pod) => pod.ownerKind === "DaemonSet" || pod.ownerKind === "Node"); - const nonDaemonSetPods = items.filter((pod) => !retainedSystemPods.includes(pod)); - const requiredEmpty = !target.worker.scheduling.schedulable; - return { - ok: !requiredEmpty || nonDaemonSetPods.length === 0, - requiredEmpty, - nonDaemonSetPodCount: nonDaemonSetPods.length, - retainedSystemPodCount: retainedSystemPods.length, - nonDaemonSetPods: nonDaemonSetPods.slice(0, 40), - valuesPrinted: false, - }; -} - -function drainWorker(target: K3sClusterTarget, confirm: boolean): Record { - if (target.worker.scheduling.schedulable) throw new Error("worker drain requires scheduling.schedulable=false in the owning YAML"); - if (!confirm) { - return { - ok: true, - mutation: false, - mode: "dry-run", - targetId: target.id, - workerNode: target.worker.nodeName, - scheduling: workerSchedulingStatus(target), - workloads: workerWorkloadStatus(target), - }; - } - const scheduling = reconcileWorkerScheduling(target); - const dns = reconcileWorkerDns(target); - const before = workerWorkloadStatus(target); - const drainAttempts: Array> = []; - let current = before; - const drainDeadline = Date.now() + 10 * 60_000; - while (current.nonDaemonSetPodCount !== 0 && Date.now() < drainDeadline) { - const drain = runCommand([ - "trans", target.controlPlane.kubeRoute, "kubectl", "drain", target.worker.nodeName, - "--ignore-daemonsets", "--delete-emptydir-data", "--force", "--disable-eviction", - "--grace-period=10", "--skip-wait-for-delete-timeout=10", "--timeout=45s", - ], rootPath(), { timeoutMs: 55_000 }); - const next = workerWorkloadStatus(target); - drainAttempts.push({ - exitCode: drain.exitCode, - beforeCount: current.nonDaemonSetPodCount, - afterCount: next.nonDaemonSetPodCount, - stderr: bounded(drain.stderr, 1_000), - }); - if (next.nonDaemonSetPodCount === current.nonDaemonSetPodCount && drain.exitCode !== 0) { - throw new Error(`worker drain made no progress: ${bounded(JSON.stringify(drainAttempts.at(-1)))}`); - } - current = next; - } - const after = current; - if (after.ok !== true) throw new Error(`worker drain did not converge: ${bounded(JSON.stringify(after))}`); - return { - ok: true, - mutation: before.nonDaemonSetPodCount !== 0 || scheduling.mutation === true || dns.mutation === true, - targetId: target.id, - workerNode: target.worker.nodeName, - scheduling, - workerDns: dns, - before, - after, - drain: drainAttempts.length === 0 ? { skipped: true, reason: "no-non-daemonset-pods" } : { attempts: drainAttempts }, - }; -} - -function workerDnsStatus(target: K3sClusterTarget): { data: Record; result: ReturnType } { - const dns = target.k3s.workerDns; - const result = runCommand([ - "trans", target.controlPlane.kubeRoute, "kubectl", "-n", dns.namespace, "get", "deployment,service,endpoints", - "-l", `app.kubernetes.io/name=${dns.deploymentName}`, - "-o", String.raw`jsonpath={range .items[*]}{.kind}|{.metadata.annotations.unidesk\.ai/config-sha256}|{.status.readyReplicas}|{.spec.clusterIP}|{.spec.internalTrafficPolicy}|{range .subsets[*].addresses[*]}{.nodeName},{end}{"\n"}{end}`, - ], rootPath(), { timeoutMs: 30_000 }); - if (result.exitCode !== 0) { - return { - data: { ok: false, presence: false, configurationCurrent: false, error: bounded(result.stderr || result.stdout) }, - result, - }; - } - const rows = new Map( - result.stdout.split(/\r?\n/u) - .map((line) => line.trim()) - .filter((line) => /^(Deployment|Service|Endpoints)\|/u.test(line)) - .map((line) => { - const fields = line.split("|"); - return [fields[0], fields] as const; - }), - ); - const deployment = rows.get("Deployment"); - const service = rows.get("Service"); - const endpoints = rows.get("Endpoints"); - if (deployment === undefined || service === undefined || endpoints === undefined) { - if (!target.worker.scheduling.schedulable && deployment === undefined && service === undefined) { - return { data: { ok: true, presence: false, configurationCurrent: true, enabled: false, reason: "worker-unschedulable" }, result }; - } - return { - data: { - ok: false, - presence: deployment !== undefined || service !== undefined || endpoints !== undefined, - configurationCurrent: false, - error: "worker-dns-status-incomplete", - }, - result, - }; - } - if (!target.worker.scheduling.schedulable) { - return { - data: { - ok: false, - presence: true, - configurationCurrent: false, - enabled: false, - reason: "worker-unschedulable-cleanup-required", - }, - result, - }; - } - const desiredHash = workerDnsDesiredHash(target); - const deploymentHash = deployment[1] || null; - const serviceHash = service[1] || null; - const readyReplicas = Number(deployment[2] || 0); - const endpointNodes = (endpoints[5] ?? "").split(",").filter(Boolean); - const configurationCurrent = deploymentHash === desiredHash && serviceHash === desiredHash; - const deploymentReady = readyReplicas === dns.replicas; - const endpointLocal = endpointNodes.includes(target.worker.nodeName); - const serviceIp = service[3] || null; - const localTrafficPolicy = service[4] || null; - return { - data: { - ok: configurationCurrent && deploymentReady && endpointLocal && serviceIp === dns.clusterIp && localTrafficPolicy === "Local", - presence: true, - configurationCurrent, - deploymentReady, - readyReplicas, - desiredReplicas: dns.replicas, - serviceIp, - localTrafficPolicy, - endpointNodes, - workerNode: target.worker.nodeName, - desiredHash, - }, - result, - }; -} - -function workerDnsDesiredHash(target: K3sClusterTarget): string { - return hash(JSON.stringify({ workerNode: target.worker.nodeName, ...target.k3s.workerDns })); -} - -function workerDnsManifest(target: K3sClusterTarget): string { - const dns = target.k3s.workerDns; - const desiredHash = workerDnsDesiredHash(target); - const labels = `app.kubernetes.io/name: ${dns.deploymentName}\n app.kubernetes.io/managed-by: unidesk`; - return `apiVersion: apps/v1 -kind: Deployment -metadata: - name: ${dns.deploymentName} - namespace: ${dns.namespace} - annotations: - unidesk.ai/config-sha256: ${desiredHash} - labels: - ${labels} -spec: - replicas: ${dns.replicas} - revisionHistoryLimit: 1 - strategy: - type: RollingUpdate - rollingUpdate: - maxUnavailable: 0 - maxSurge: 1 - selector: - matchLabels: - app.kubernetes.io/name: ${dns.deploymentName} - template: - metadata: - annotations: - unidesk.ai/config-sha256: ${desiredHash} - labels: - app.kubernetes.io/name: ${dns.deploymentName} - app.kubernetes.io/managed-by: unidesk - spec: - serviceAccountName: coredns - priorityClassName: system-cluster-critical - dnsPolicy: Default - nodeSelector: - kubernetes.io/hostname: ${target.worker.nodeName} - tolerations: - - key: CriticalAddonsOnly - operator: Exists - containers: - - name: coredns - image: ${dns.image} - imagePullPolicy: IfNotPresent - args: ["-conf", "/etc/coredns/Corefile"] - ports: - - { name: dns, containerPort: 53, protocol: UDP } - - { name: dns-tcp, containerPort: 53, protocol: TCP } - - { name: metrics, containerPort: 9153, protocol: TCP } - readinessProbe: - httpGet: { path: /ready, port: 8181 } - periodSeconds: 2 - timeoutSeconds: 1 - failureThreshold: 3 - livenessProbe: - httpGet: { path: /health, port: 8080 } - initialDelaySeconds: 60 - periodSeconds: 10 - timeoutSeconds: 1 - failureThreshold: 3 - resources: - requests: { cpu: 100m, memory: 70Mi } - limits: { memory: 170Mi } - securityContext: - allowPrivilegeEscalation: false - readOnlyRootFilesystem: true - capabilities: - add: [NET_BIND_SERVICE] - drop: [ALL] - volumeMounts: - - { name: config-volume, mountPath: /etc/coredns, readOnly: true } - - { name: custom-config-volume, mountPath: /etc/coredns/custom, readOnly: true } - volumes: - - name: config-volume - configMap: - name: ${dns.configMapName} - items: - - { key: Corefile, path: Corefile } - - { key: NodeHosts, path: NodeHosts } - - name: custom-config-volume - configMap: - name: ${dns.customConfigMapName} - optional: true ---- -apiVersion: v1 -kind: Service -metadata: - name: ${dns.serviceName} - namespace: ${dns.namespace} - annotations: - unidesk.ai/config-sha256: ${desiredHash} - labels: - app.kubernetes.io/name: ${dns.deploymentName} - app.kubernetes.io/managed-by: unidesk -spec: - clusterIP: ${dns.clusterIp} - internalTrafficPolicy: Local - selector: - app.kubernetes.io/name: ${dns.deploymentName} - ports: - - { name: dns, port: 53, targetPort: 53, protocol: UDP } - - { name: dns-tcp, port: 53, targetPort: 53, protocol: TCP } - - { name: metrics, port: 9153, targetPort: 9153, protocol: TCP } -`; -} - -function workerWireguardApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string { - const wg = target.wireguard; - return `set -eu -${tcpTuningApplyScript(target)} -if ! command -v wg >/dev/null 2>&1; then - DEBIAN_FRONTEND=noninteractive apt-get update -qq - DEBIAN_FRONTEND=noninteractive apt-get install -y -qq ${shellToken(wg.packageName)} -fi -install -d -m 700 /etc/wireguard -candidate="$(mktemp)" -trap 'rm -f "$candidate"' EXIT -cat >"$candidate" <<'EOF' -[Interface] -Address = ${wg.workerAddress} -PrivateKey = ${secret.workerPrivate} -MTU = ${wg.mtu} - -[Peer] -PublicKey = ${secret.controlPublic} -AllowedIPs = ${address(wg.controlPlaneAddress)}/32 -Endpoint = ${wg.endpoint} -PersistentKeepalive = ${wg.persistentKeepaliveSeconds} -EOF -changed=false -if ! cmp -s "$candidate" /etc/wireguard/${wg.interfaceName}.conf 2>/dev/null; then - install -m 0600 "$candidate" /etc/wireguard/${wg.interfaceName}.conf - changed=true -fi -systemctl enable wg-quick@${wg.interfaceName} >/dev/null 2>&1 || true -if [ "$changed" = true ]; then - systemctl restart wg-quick@${wg.interfaceName} -elif ! systemctl is-active --quiet wg-quick@${wg.interfaceName}; then - systemctl start wg-quick@${wg.interfaceName} -elif ! timeout 5 nc -z ${address(wg.controlPlaneAddress)} 6443 >/dev/null 2>&1; then - handshake="$(wg show ${wg.interfaceName} latest-handshakes 2>/dev/null | awk '{print $2}' | head -n1)" - now="$(date +%s)" - if [ -z "$handshake" ] || [ $((now - handshake)) -ge 180 ]; then - systemctl restart wg-quick@${wg.interfaceName} - fi -fi -timeout 15 nc -z ${address(wg.controlPlaneAddress)} 6443 -`; -} - -function controlPlaneApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string { - const wg = target.wireguard; - return `set -eu -export DEBIAN_FRONTEND=noninteractive -${tcpTuningApplyScript(target)} -if ! command -v wg >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq ${shQuote(wg.packageName)}; fi -umask 077 -install -d -m 700 /etc/wireguard /etc/unidesk -wg_candidate=$(mktemp /tmp/unidesk-${wg.interfaceName}.XXXXXX) -token_candidate=$(mktemp /tmp/unidesk-k3s-agent-token.XXXXXX) -trap 'rm -f "$wg_candidate" "$token_candidate"' EXIT -cat >"$wg_candidate" <<'EOF' -[Interface] -Address = ${wg.controlPlaneAddress} -ListenPort = ${wg.listenPort} -PrivateKey = ${secret.controlPrivate} -MTU = ${wg.mtu} - -[Peer] -PublicKey = ${secret.workerPublic} -AllowedIPs = ${address(wg.workerAddress)}/32 -EOF -printf '%s\n' ${shQuote(secret.agentToken)} >"$token_candidate" -wg_changed=false -if ! cmp -s "$wg_candidate" /etc/wireguard/${wg.interfaceName}.conf 2>/dev/null; then - install -m 0600 "$wg_candidate" /etc/wireguard/${wg.interfaceName}.conf - wg_changed=true -fi -if ! cmp -s "$token_candidate" ${shQuote(target.k3s.agentTokenFile)} 2>/dev/null; then - install -m 0600 "$token_candidate" ${shQuote(target.k3s.agentTokenFile)} -fi -systemctl enable wg-quick@${wg.interfaceName} >/dev/null 2>&1 || true -if [ "$wg_changed" = true ]; then - systemctl restart wg-quick@${wg.interfaceName} -elif ! systemctl is-active --quiet wg-quick@${wg.interfaceName}; then - systemctl start wg-quick@${wg.interfaceName} -fi -wg show ${wg.interfaceName} >/dev/null -`; -} - -function controlPlaneRetireWorkerPeerScript(target: K3sClusterTarget, secret: SecretMaterial): string { - const wg = target.wireguard; - return `set -eu -umask 077 -candidate="$(mktemp)" -trap 'rm -f "$candidate"' EXIT -cat >"$candidate" <<'EOF' -[Interface] -Address = ${wg.controlPlaneAddress} -ListenPort = ${wg.listenPort} -PrivateKey = ${secret.controlPrivate} -MTU = ${wg.mtu} -EOF -if ! cmp -s "$candidate" /etc/wireguard/${wg.interfaceName}.conf; then - install -m 0600 "$candidate" /etc/wireguard/${wg.interfaceName}.conf - systemctl restart wg-quick@${wg.interfaceName} -fi -wg show ${wg.interfaceName} peers | grep -q . && { echo "retired worker peer still present" >&2; exit 1; } -`; -} - -function standaloneApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string { - const standalone = target.k3s.standalone; - const disableArgs = standalone.disableComponents.flatMap((component) => ["--disable", component]); - const labels = Object.entries(standalone.labels).flatMap(([key, value]) => ["--node-label", `${key}=${value}`]); - const exec = [ - "server", - ...disableArgs, - "--cluster-init", - "--node-name", standalone.nodeName, - ...labels, - "--write-kubeconfig-mode", standalone.writeKubeconfigMode, - "--kubelet-arg", `max-pods=${target.k3s.maxPods}`, - ].map(shellToken).join(" "); - const desiredHash = hash(JSON.stringify({ - version: target.worker.k3sVersion, - exec, - registryEndpoint: target.k3s.registryMirror.endpoint, - registryUsername: secret.registryUsername, - registryPassword: secret.registryPassword, - })); - return `set -eu -desired_hash=${shQuote(desiredHash)} -desired_hash_file=/etc/unidesk/k3s-standalone-desired.sha256 -if [ "$(cat "$desired_hash_file" 2>/dev/null || true)" = "$desired_hash" ] && systemctl is-active --quiet k3s; then - exit 0 -fi -systemctl disable --now wg-quick@${target.wireguard.interfaceName} >/dev/null 2>&1 || true -if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh; fi -if [ -x /usr/local/bin/k3s-uninstall.sh ]; then /usr/local/bin/k3s-uninstall.sh; fi -rm -rf /var/lib/rancher/k3s /etc/rancher/k3s /var/lib/kubelet /var/lib/cni /etc/cni/net.d /run/k3s /run/flannel -ip link delete cni0 >/dev/null 2>&1 || true -ip link delete flannel.1 >/dev/null 2>&1 || true -rm -f /etc/wireguard/${target.wireguard.interfaceName}.conf -install -d -m 0700 /etc/rancher/k3s /etc/unidesk -cat >/etc/rancher/k3s/registries.yaml <<'EOF' -mirrors: - "${target.k3s.registryMirror.source}": - endpoint: - - "${target.k3s.registryMirror.endpoint}" -configs: - "${new URL(target.k3s.registryMirror.endpoint).host}": - auth: - username: ${JSON.stringify(secret.registryUsername)} - password: ${JSON.stringify(secret.registryPassword)} -EOF -chmod 0600 /etc/rancher/k3s/registries.yaml -install -m 0755 /tmp/k3s /usr/local/bin/k3s -chmod 0755 /tmp/install-k3s.sh -INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_VERSION=${shQuote(target.worker.k3sVersion)} INSTALL_K3S_EXEC=${shQuote(exec)} sh /tmp/install-k3s.sh -printf '%s\n' "$desired_hash" >"$desired_hash_file" -chmod 0600 "$desired_hash_file" -systemctl enable k3s >/dev/null -systemctl restart --no-block k3s -`; -} - -function standaloneStatusScript(target: K3sClusterTarget): string { - const nodeName = target.k3s.standalone.nodeName; - return `set -eu -active=false; systemctl is-active --quiet k3s && active=true -agent_active=false; systemctl is-active --quiet k3s-agent && agent_active=true -ready="$(k3s kubectl get node ${shQuote(nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" -node_count="$(k3s kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ')" -business_pods="$(k3s kubectl get pods -A --no-headers 2>/dev/null | awk '$1 != "kube-system" {count++} END {print count+0}')" -python3 - "$active" "$agent_active" "$ready" "$node_count" "$business_pods" <<'PY' -import json, sys -active, agent, ready, nodes, business = sys.argv[1:] -ok = active == "true" and agent == "false" and ready == "True" and nodes == "1" and business == "0" -print(json.dumps({"ok":ok,"serverActive":active=="true","oldAgentActive":agent=="true","nodeReady":ready=="True","nodeCount":int(nodes or 0),"businessPodCount":int(business or 0)}, separators=(",",":"))) -PY -`; -} - -function tcpTuningApplyScript(target: K3sClusterTarget): string { - const tuning = target.tcpTuning; - const content = [ - `net.core.default_qdisc=${tuning.defaultQdisc}`, - `net.ipv4.tcp_congestion_control=${tuning.congestionControl}`, - `net.ipv4.tcp_mtu_probing=${tuning.mtuProbing}`, - "", - ].join("\n"); - const encoded = Buffer.from(content, "utf8").toString("base64"); - return `modprobe tcp_${tuning.congestionControl} 2>/dev/null || true -available="$(sysctl -n net.ipv4.tcp_available_congestion_control)" -case " $available " in - *" ${tuning.congestionControl} "*) ;; - *) echo "configured TCP congestion control is unavailable: ${tuning.congestionControl}" >&2; exit 1 ;; -esac -sysctl_candidate="$(mktemp)" -printf %s ${shQuote(encoded)} | base64 -d >"$sysctl_candidate" -if ! cmp -s "$sysctl_candidate" ${shQuote(tuning.sysctlPath)} 2>/dev/null; then - install -D -m 0644 "$sysctl_candidate" ${shQuote(tuning.sysctlPath)} -fi -rm -f "$sysctl_candidate" -[ "$(sysctl -n net.core.default_qdisc)" = ${shQuote(tuning.defaultQdisc)} ] || sysctl -q -w net.core.default_qdisc=${shellToken(tuning.defaultQdisc)} -[ "$(sysctl -n net.ipv4.tcp_congestion_control)" = ${shQuote(tuning.congestionControl)} ] || sysctl -q -w net.ipv4.tcp_congestion_control=${shellToken(tuning.congestionControl)} -[ "$(sysctl -n net.ipv4.tcp_mtu_probing)" = ${shQuote(String(tuning.mtuProbing))} ] || sysctl -q -w net.ipv4.tcp_mtu_probing=${tuning.mtuProbing} -`; -} - -function workerApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string { - const wg = target.wireguard; - const labels = Object.entries(target.worker.labels).flatMap(([key, value]) => ["--node-label", `${key}=${value}`]); - const taints = target.worker.scheduling.taints.flatMap((taint) => ["--node-taint", `${taint.key}=${taint.value}:${taint.effect}`]); - const dnsArgs = target.worker.scheduling.schedulable ? ["--kubelet-arg", `cluster-dns=${target.k3s.workerDns.clusterIp}`] : []; - const exec = [ - "agent", "--server", target.k3s.serverUrl, "--token-file", target.k3s.agentTokenFile, - "--node-name", target.worker.nodeName, "--node-ip", address(wg.workerAddress), - "--flannel-iface", target.k3s.flannelInterface, ...labels, ...taints, - "--kubelet-arg", `max-pods=${target.k3s.maxPods}`, - ...dnsArgs, - ].map(shellToken).join(" "); - const desiredHash = createHash("sha256").update(JSON.stringify({ - version: target.worker.k3sVersion, - exec, - agentToken: secret.agentToken, - })).digest("hex"); - return `set -eu -export DEBIAN_FRONTEND=noninteractive -if ! command -v wg >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq ${shQuote(wg.packageName)}; fi -umask 077 -install -d -m 700 /etc/wireguard /etc/unidesk /etc/rancher/k3s -wg_candidate=$(mktemp /tmp/unidesk-${wg.interfaceName}.XXXXXX) -token_candidate=$(mktemp /tmp/unidesk-k3s-agent-token.XXXXXX) -registry_candidate=$(mktemp /tmp/unidesk-k3s-registries.XXXXXX) -trap 'rm -f "$wg_candidate" "$token_candidate" "$registry_candidate"' EXIT -cat >"$wg_candidate" <<'EOF' -[Interface] -Address = ${wg.workerAddress} -PrivateKey = ${secret.workerPrivate} -MTU = ${wg.mtu} - -[Peer] -PublicKey = ${secret.controlPublic} -Endpoint = ${wg.endpoint} -AllowedIPs = ${address(wg.controlPlaneAddress)}/32 -PersistentKeepalive = ${wg.persistentKeepaliveSeconds} -EOF -printf '%s\n' ${shQuote(secret.agentToken)} >"$token_candidate" -cat >"$registry_candidate" <<'EOF' -mirrors: - "${target.k3s.registryMirror.source}": - endpoint: - - "${target.k3s.registryMirror.endpoint}" -configs: - "${new URL(target.k3s.registryMirror.endpoint).host}": - auth: - username: ${JSON.stringify(secret.registryUsername)} - password: ${JSON.stringify(secret.registryPassword)} -EOF -wg_changed=false -registry_changed=false -if ! cmp -s "$wg_candidate" /etc/wireguard/${wg.interfaceName}.conf 2>/dev/null; then - install -m 0600 "$wg_candidate" /etc/wireguard/${wg.interfaceName}.conf - wg_changed=true -fi -if ! cmp -s "$token_candidate" ${shQuote(target.k3s.agentTokenFile)} 2>/dev/null; then - install -m 0600 "$token_candidate" ${shQuote(target.k3s.agentTokenFile)} -fi -if ! cmp -s "$registry_candidate" /etc/rancher/k3s/registries.yaml 2>/dev/null; then - install -m 0600 "$registry_candidate" /etc/rancher/k3s/registries.yaml - registry_changed=true -fi -systemctl enable wg-quick@${wg.interfaceName} >/dev/null 2>&1 || true -if [ "$wg_changed" = true ]; then - systemctl restart wg-quick@${wg.interfaceName} -elif ! systemctl is-active --quiet wg-quick@${wg.interfaceName}; then - systemctl start wg-quick@${wg.interfaceName} -fi -timeout 15 nc -z ${address(wg.controlPlaneAddress)} 6443 -desired_hash=${shQuote(desiredHash)} -desired_hash_file=/etc/unidesk/k3s-agent-desired.sha256 -current_hash=$(cat "$desired_hash_file" 2>/dev/null || true) -agent_state="$(systemctl show k3s-agent -p ActiveState --value 2>/dev/null || true)" -agent_pid="$(systemctl show k3s-agent -p MainPID --value 2>/dev/null || true)" -if [ "$current_hash" = "$desired_hash" ] && { [ "$agent_state" = active ] || [ "$agent_state" = activating ]; } && [ "\${agent_pid:-0}" -gt 0 ]; then - if [ "$registry_changed" = true ]; then systemctl restart --no-block k3s-agent; fi - exit 0 -fi -if [ -x /usr/local/bin/k3s-uninstall.sh ]; then /usr/local/bin/k3s-uninstall.sh; fi -if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh; fi -install -d -m 700 /etc/rancher/k3s -install -m 0600 "$registry_candidate" /etc/rancher/k3s/registries.yaml -install -m 0755 /tmp/k3s /usr/local/bin/k3s -chmod 0755 /tmp/install-k3s.sh -INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_VERSION=${shQuote(target.worker.k3sVersion)} INSTALL_K3S_EXEC=${shQuote(exec)} sh /tmp/install-k3s.sh -printf '%s\n' "$desired_hash" >"$desired_hash_file" -chmod 0600 "$desired_hash_file" -systemctl enable k3s-agent >/dev/null -systemctl restart --no-block k3s-agent -`; -} - -function waitForNodes(target: K3sClusterTarget, timeoutMs: number): ReturnType { - const deadline = Date.now() + timeoutMs; - let current = runControlPlane(target, nodeReadinessScript(target), 30_000); - while ((current.exitCode !== 0 || parseFirstJson(current.stdout).ok !== true) && Date.now() < deadline) { - runCommand(["sleep", "5"], rootPath(), { timeoutMs: 10_000 }); - current = runControlPlane(target, nodeReadinessScript(target), 30_000); - } - if (current.exitCode === 0 && parseFirstJson(current.stdout).ok === true) return current; - return { ...current, exitCode: 1, stderr: current.stderr || `nodes did not become Ready within ${timeoutMs}ms` }; -} - -function statusScript(target: K3sClusterTarget): string { - const cp = address(target.wireguard.controlPlaneAddress); - return `set -eu -wg_active=false; systemctl is-active --quiet wg-quick@${target.wireguard.interfaceName} && wg_active=true -k3s_active=false; systemctl is-active --quiet k3s && k3s_active=true -cp_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.controlPlane.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) -worker_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) -worker_ip=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null || true) -handshake=$(wg show ${target.wireguard.interfaceName} latest-handshakes 2>/dev/null | awk '{print $2}' | head -n1) -tuning="$(sysctl -n net.ipv4.tcp_congestion_control)/$(sysctl -n net.core.default_qdisc)/$(sysctl -n net.ipv4.tcp_mtu_probing)" -python3 - "$wg_active" "$k3s_active" "$cp_ready" "$worker_ready" "$worker_ip" "$handshake" "$tuning" <<'PY' -import json, sys, time -wg, k3s, cp, worker, worker_ip, handshake, tuning = sys.argv[1:] -stamp = int(handshake or 0) -age = None if stamp == 0 else max(0, int(time.time()) - stamp) -tuning_ready = tuning == '${target.tcpTuning.congestionControl}/${target.tcpTuning.defaultQdisc}/${target.tcpTuning.mtuProbing}' -ok = wg == 'true' and k3s == 'true' and cp == 'True' and worker == 'True' and worker_ip == '${address(target.wireguard.workerAddress)}' and age is not None and age < 180 and tuning_ready -print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'k3sActive': k3s == 'true', 'controlPlaneReady': cp == 'True', 'workerReady': worker == 'True', 'workerInternalIp': worker_ip, 'handshakeAgeSeconds': age, 'controlPlaneAddress': '${cp}', 'tcpTuningReady': tuning_ready, 'tcpTuning': tuning}, separators=(',', ':'))) -PY -`; -} - -function workerStatusScript(target: K3sClusterTarget): string { - return `set -eu -wg_active=false; systemctl is-active --quiet wg-quick@${target.wireguard.interfaceName} && wg_active=true -agent_state="$(systemctl show k3s-agent -p ActiveState --value 2>/dev/null || true)" -agent_pid="$(systemctl show k3s-agent -p MainPID --value 2>/dev/null || true)" -old_server=false; systemctl is-active --quiet k3s && old_server=true -ip_present=false; ip -4 address show dev ${target.wireguard.interfaceName} | grep -F ${shQuote(address(target.wireguard.workerAddress))} >/dev/null 2>&1 && ip_present=true -tuning="$(sysctl -n net.ipv4.tcp_congestion_control)/$(sysctl -n net.core.default_qdisc)/$(sysctl -n net.ipv4.tcp_mtu_probing)" -python3 - "$wg_active" "$agent_state" "$agent_pid" "$old_server" "$ip_present" "$tuning" <<'PY' -import json, sys -wg, agent_state, agent_pid, old_server, ip_present, tuning = sys.argv[1:] -agent_running = agent_state in ('active', 'activating') and int(agent_pid or 0) > 0 -tuning_ready = tuning == '${target.tcpTuning.congestionControl}/${target.tcpTuning.defaultQdisc}/${target.tcpTuning.mtuProbing}' -ok = wg == 'true' and agent_running and old_server == 'false' and ip_present == 'true' and tuning_ready -print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'agentActive': agent_state == 'active', 'agentRunning': agent_running, 'agentState': agent_state, 'agentMainPidPresent': int(agent_pid or 0) > 0, 'standaloneServerActive': old_server == 'true', 'wireguardAddressPresent': ip_present == 'true', 'tcpTuningReady': tuning_ready, 'tcpTuning': tuning}, separators=(',', ':'))) -PY -`; -} - -function smokeManifest(target: K3sClusterTarget): string { - return `apiVersion: batch/v1 -kind: Job -metadata: - name: ${target.smoke.jobName} - namespace: ${target.smoke.namespace} -spec: - backoffLimit: 0 - template: - metadata: - labels: - app.kubernetes.io/name: ${target.smoke.jobName} - spec: - restartPolicy: Never - nodeSelector: - kubernetes.io/hostname: ${target.worker.nodeName} - containers: - - name: smoke - image: ${target.smoke.image} - imagePullPolicy: IfNotPresent - command: - - sh - - -ec - - | - grep -F ${shQuote(`nameserver ${target.k3s.workerDns.clusterIp}`)} /etc/resolv.conf - dns_output="$(nslookup kubernetes.default.svc.cluster.local 2>&1)" - printf '%s\\n' "$dns_output" - printf '%s\\n' "$dns_output" | grep -F "$KUBERNETES_SERVICE_HOST" - nc -z -w 15 "\${KUBERNETES_SERVICE_HOST}" "\${KUBERNETES_SERVICE_PORT_HTTPS}" - echo D601_RUNTIME_SMOKE_OK -`; -} - -function runControlPlane(target: K3sClusterTarget, script: string, timeoutMs: number) { - return runCommand(["trans", target.controlPlane.route, "sh"], rootPath(), { input: script, timeoutMs }); -} - -function runWorker(target: K3sClusterTarget, script: string, copyArtifacts: boolean, timeoutMs: number) { - const encoded = Buffer.from(script, "utf8").toString("base64"); - const keyPath = `${target.worker.stateDir}\\id_ed25519`; - if (!copyArtifacts) { - const lines = [ - "$ErrorActionPreference='Stop'", - `$key=${psQuote(keyPath)}`, - `$guest=${psQuote(`${target.worker.guestUser}@${target.worker.guestAddress}`)}`, - `$payload=${psQuote(encoded)}`, - "$start=[Diagnostics.ProcessStartInfo]::new()", - "$start.FileName='ssh.exe'", - '$start.Arguments="-i `"$key`" -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 $guest `"sudo sh -s`""', - "$start.UseShellExecute=$false", - "$start.RedirectStandardInput=$true", - "$start.RedirectStandardOutput=$true", - "$start.RedirectStandardError=$true", - "$process=[Diagnostics.Process]::new()", - "$process.StartInfo=$start", - "if(!$process.Start()){throw 'worker-inline-ssh-start-failed'}", - "$stdout=$process.StandardOutput.ReadToEndAsync()", - "$stderr=$process.StandardError.ReadToEndAsync()", - "$bytes=[Convert]::FromBase64String($payload)", - "$process.StandardInput.BaseStream.Write($bytes,0,$bytes.Length)", - "$process.StandardInput.Close()", - "$process.WaitForExit()", - "[Console]::Out.Write($stdout.Result)", - "[Console]::Error.Write($stderr.Result)", - "exit $process.ExitCode", - ]; - return runCommand(["trans", target.worker.windowsRoute, "ps"], rootPath(), { input: lines.join("\n") + "\n", timeoutMs }); - } - const remoteScript = `/tmp/unidesk-k3s-cluster-${target.id}.sh`; - const lines = [ - "$ErrorActionPreference='Stop'", - `$key=${psQuote(keyPath)}`, - `$guest=${psQuote(`${target.worker.guestUser}@${target.worker.guestAddress}`)}`, - `$temp=Join-Path $env:TEMP ${psQuote(`unidesk-k3s-cluster-${target.id}.sh`)}`, - `[IO.File]::WriteAllBytes($temp,[Convert]::FromBase64String(${psQuote(encoded)}))`, - ]; - lines.push( - `$binary=${psQuote(`${target.worker.stateDir}\\k3s`)}`, - `$installer=${psQuote(`${target.worker.stateDir}\\install-k3s.sh`)}`, - "& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $binary \"$($guest):/tmp/k3s\"", - "if($LASTEXITCODE -ne 0){throw 'worker-k3s-binary-scp-failed'}", - "& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $installer \"$($guest):/tmp/install-k3s.sh\"", - "if($LASTEXITCODE -ne 0){throw 'worker-k3s-installer-scp-failed'}", - ); - lines.push( - `& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $temp "$($guest):${remoteScript}"`, - "if($LASTEXITCODE -ne 0){throw 'worker-script-scp-failed'}", - `& ssh.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 $guest ${psQuote(`sudo sh ${remoteScript}`)}`, - "$rc=$LASTEXITCODE", - "Remove-Item -Force -LiteralPath $temp -ErrorAction SilentlyContinue", - `& ssh.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $guest ${psQuote(`rm -f ${remoteScript}`)} | Out-Null`, - "exit $rc", - ); - return runCommand(["trans", target.worker.windowsRoute, "ps"], rootPath(), { input: lines.join("\n") + "\n", timeoutMs }); -} - -function runWorkerHostProxy(target: K3sClusterTarget, bundle: HostProxyGuestBundle, timeoutMs: number) { - const windowsArchive = `${target.worker.stateDir}\\host-proxy-${bundle.targetId}.tar.gz`; - const archiveStatus = runCommand( - ["trans", target.worker.windowsRoute, "ps"], - rootPath(), - { - input: `$path=${psQuote(windowsArchive)} -if(!(Test-Path -LiteralPath $path -PathType Leaf)){exit 3} -$sha=(Get-FileHash -Algorithm SHA256 -LiteralPath $path).Hash.ToLowerInvariant() -if($sha -ne ${psQuote(bundle.archiveSha256)}){exit 4} -`, - timeoutMs: 60_000, - }, - ); - if (archiveStatus.exitCode !== 0) { - const fetch = runCommand( - ["trans", target.worker.windowsRoute, "ps"], - rootPath(), - { - input: `$ErrorActionPreference='Stop' -$path=${psQuote(windowsArchive)} -$temp="$path.tmp" -New-Item -ItemType Directory -Force -Path (Split-Path -Parent $path) | Out-Null -Remove-Item -Force -LiteralPath $temp -ErrorAction SilentlyContinue -& curl.exe -fL --retry 2 --connect-timeout 15 --max-time 180 -o $temp ${psQuote(bundle.upstreamUrl)} -if($LASTEXITCODE -ne 0){throw 'worker-host-proxy-artifact-download-failed'} -$sha=(Get-FileHash -Algorithm SHA256 -LiteralPath $temp).Hash.ToLowerInvariant() -if($sha -ne ${psQuote(bundle.archiveSha256)}){throw 'worker-host-proxy-artifact-sha256-mismatch'} -Move-Item -Force -LiteralPath $temp -Destination $path -`, - timeoutMs, - }, - ); - if (fetch.exitCode !== 0) return fetch; - } - - const guestArchive = `/tmp/unidesk-host-proxy-${bundle.targetId}.tar.gz`; - const remoteScript = `/tmp/unidesk-host-proxy-${bundle.targetId}.sh`; - const script = `${bundle.prepareScript} -remote_archive_sha="$(sha256sum ${shQuote(bundle.remoteArchivePath)} 2>/dev/null | awk '{print $1}' || true)" -if [ "$remote_archive_sha" != ${shQuote(bundle.archiveSha256)} ]; then - install -m 0644 ${shQuote(guestArchive)} ${shQuote(bundle.remoteArchivePath)} -fi -${bundle.applyScript} -`; - const encoded = Buffer.from(script, "utf8").toString("base64"); - const keyPath = `${target.worker.stateDir}\\id_ed25519`; - const lines = [ - "$ErrorActionPreference='Stop'", - `$key=${psQuote(keyPath)}`, - `$guest=${psQuote(`${target.worker.guestUser}@${target.worker.guestAddress}`)}`, - `$archive=${psQuote(windowsArchive)}`, - `$temp=Join-Path $env:TEMP ${psQuote(`unidesk-host-proxy-${bundle.targetId}.sh`)}`, - `[IO.File]::WriteAllBytes($temp,[Convert]::FromBase64String(${psQuote(encoded)}))`, - `& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $archive "$($guest):${guestArchive}"`, - "if($LASTEXITCODE -ne 0){throw 'worker-host-proxy-archive-scp-failed'}", - `& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $temp "$($guest):${remoteScript}"`, - "if($LASTEXITCODE -ne 0){throw 'worker-host-proxy-script-scp-failed'}", - `& ssh.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 $guest ${psQuote(`sudo sh ${remoteScript}`)}`, - "$rc=$LASTEXITCODE", - "Remove-Item -Force -LiteralPath $temp -ErrorAction SilentlyContinue", - `& ssh.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $guest ${psQuote(`rm -f ${remoteScript} ${guestArchive}`)} | Out-Null`, - "exit $rc", - ]; - return runCommand(["trans", target.worker.windowsRoute, "ps"], rootPath(), { - input: lines.join("\n") + "\n", - timeoutMs, - }); -} - -function readSecrets(target: K3sClusterTarget): SecretMaterial { - if (!existsSync(target.secrets.sourceRef)) throw new Error(`${target.secrets.sourceRef} is missing; run platform-infra k3s-cluster secret-init --confirm`); - const values = parseEnvFile(readFileSync(target.secrets.sourceRef, "utf8")); - const auth = target.k3s.registryMirror.auth; - if (!existsSync(auth.sourceRef)) throw new Error(`${auth.sourceRef} is missing; run platform-infra k3s-cluster registry-credential-init --confirm`); - const authValues = parseEnvFile(readFileSync(auth.sourceRef, "utf8")); - const required = (key: string, minimumLength = 32) => { - const value = values[key]; - if (typeof value !== "string" || value.length < minimumLength) throw new Error(`${target.secrets.sourceRef}.${key} is missing or invalid`); - return value; - }; - const requiredAuth = (key: string, minimumLength = 32) => { - const value = authValues[key]; - if (typeof value !== "string" || value.length < minimumLength) throw new Error(`${auth.sourceRef}.${key} is missing or invalid`); - return value; - }; - const material = { - controlPrivate: required(target.secrets.controlPlanePrivateKey), - controlPublic: required(target.secrets.controlPlanePublicKey), - workerPrivate: required(target.secrets.workerPrivateKey), - workerPublic: required(target.secrets.workerPublicKey), - agentToken: required(target.secrets.agentToken), - registryUsername: requiredAuth(auth.usernameKey, 1), - registryPassword: requiredAuth(auth.passwordKey), - registryPasswordHash: requiredAuth(auth.passwordHashKey), - }; - return { ...material, fingerprint: hash(Object.entries(material).sort().map(([key, value]) => `${key}=${value}`).join("\n")) }; -} - -function registryAuthSummary(target: K3sClusterTarget): Record { - const auth = target.k3s.registryMirror.auth; - if (!existsSync(auth.sourceRef)) { - return { sourceRef: auth.sourceRef, presence: false, fingerprint: null, keys: [auth.usernameKey, auth.passwordKey, auth.passwordHashKey], valuesPrinted: false }; - } - try { - const values = parseEnvFile(readFileSync(auth.sourceRef, "utf8")); - const selected = [auth.usernameKey, auth.passwordKey, auth.passwordHashKey].map((key) => { - const value = values[key]; - if (typeof value !== "string" || value.length === 0) throw new Error(`${auth.sourceRef}.${key} is missing or invalid`); - return `${key}=${value}`; - }); - return { - sourceRef: auth.sourceRef, - presence: true, - fingerprint: hash(selected.join("\n")), - keys: [auth.usernameKey, auth.passwordKey, auth.passwordHashKey], - valuesPrinted: false, - }; - } catch (error) { - return { sourceRef: auth.sourceRef, presence: true, fingerprint: null, error: error instanceof Error ? error.message : String(error), valuesPrinted: false }; - } -} - -function secretSummary(target: K3sClusterTarget): Record { - if (!existsSync(target.secrets.sourceRef)) return { sourceRef: target.secrets.sourceRef, presence: false, fingerprint: null, valuesPrinted: false }; - try { - const values = parseEnvFile(readFileSync(target.secrets.sourceRef, "utf8")); - const keys = Object.values(target.secrets).filter((value) => value !== target.secrets.sourceRef); - const selected = keys.map((key) => { - const value = values[key]; - if (typeof value !== "string" || value.length === 0) throw new Error(`${target.secrets.sourceRef}.${key} is missing or invalid`); - return `${key}=${value}`; - }); - return { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: hash(selected.join("\n")), keys, valuesPrinted: false }; - } catch (error) { - return { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: null, error: error instanceof Error ? error.message : String(error), valuesPrinted: false }; - } -} - -function wireguardKeyPair(): { privateKey: string; publicKey: string } { - const pair = generateKeyPairSync("x25519"); - const privateJwk = pair.privateKey.export({ format: "jwk" }) as JsonWebKey; - const publicJwk = pair.publicKey.export({ format: "jwk" }) as JsonWebKey; - if (!privateJwk.d || !publicJwk.x) throw new Error("x25519 key generation did not produce raw JWK keys"); - return { privateKey: base64UrlToBase64(privateJwk.d), publicKey: base64UrlToBase64(publicJwk.x) }; -} - -function base64UrlToBase64(value: string): string { - return Buffer.from(value, "base64url").toString("base64"); -} - function requireSuccess(result: ReturnType, stage: string): void { if (result.exitCode !== 0) throw new Error(`${stage} failed: ${bounded(result.stderr || result.stdout)}`); } function compact(result: ReturnType): Record { - return { exitCode: result.exitCode, timedOut: result.timedOut, stdout: bounded(result.stdout, 2_000), stderr: bounded(result.stderr, 2_000) }; -} - -function compactDomain(result: Record): Record { - return Object.fromEntries(["ok", "command", "mode", "mutation", "node", "lane"].map((key) => [key, result[key] ?? null])); + return { + exitCode: result.exitCode, + timedOut: result.timedOut, + stdout: bounded(result.stdout, 2_000), + stderr: bounded(result.stderr, 2_000), + }; } function parseFirstJson(value: string): Record { const complete = value.trim(); if (complete.startsWith("{")) { - try { return JSON.parse(complete) as Record; } catch { /* try line-delimited output */ } + try { return JSON.parse(complete) as Record; } catch { /* continue */ } } for (const line of value.split(/\r?\n/u)) { const text = line.trim(); @@ -1654,6 +568,12 @@ function parseFirstJson(value: string): Record { return { ok: false, error: "remote-json-missing", output: bounded(value) }; } +function record(value: unknown): Record { + return typeof value === "object" && value !== null && !Array.isArray(value) + ? value as Record + : {}; +} + function option(args: string[], name: string): string | null { const index = args.indexOf(name); if (index < 0) return null;