From d3972699b05056931bd4cafff1fb0f176ee85027 Mon Sep 17 00:00:00 2001 From: pikastech Date: Mon, 20 Jul 2026 09:50:06 +0200 Subject: [PATCH] fix(platform-infra): split D601 from NC01 cluster --- .agents/skills/unidesk-cicd/SKILL.md | 12 + .agents/skills/unidesk-ymalops/SKILL.md | 8 + config/platform-infra/hyperv-vms.yaml | 4 +- config/platform-infra/k3s-clusters.yaml | 13 ++ docs/reference/platform-infra.md | 48 +++- .../specs/PJ2026-01060314-hyperv-ubuntu-vm.md | 38 ++-- .../platform-infra/hyperv-vm-bootstrap.ps1 | 4 +- scripts/src/platform-infra-hyperv-vm.ts | 4 +- .../src/platform-infra-k3s-cluster-config.ts | 24 ++ scripts/src/platform-infra-k3s-cluster.ts | 207 +++++++++++++++++- 10 files changed, 328 insertions(+), 34 deletions(-) diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index cb3cbbd5..9db51bf1 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -374,6 +374,18 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - 验收统一走 `platform-infra k3s-cluster registry-smoke`，不手工 patch Caddy 或 containerd。 - worker 安装使用短启动并由 control-plane Node Ready 轮询收口，禁止让 `trans`/SSH 长挂等待 systemd notify。 - worker 加入集群不改变 PaC consumer 的 source commit authority，也不授权人工 PipelineRun、Argo sync 或业务 rollout。 + - NC01 与 D601 是永久分离的调度域： + - `config/platform-infra/k3s-clusters.yaml` 必须保持 + `topologyMode=separated`，D601 只作为独立单节点 k3s 集群存在； + - CI/CD、PaC、Tekton、Argo、GitOps、VM bootstrap 和业务 renderer + 禁止把 D601 注册为 NC01 agent，禁止恢复跨地域 WireGuard peer， + 禁止把 NC01 workload、DaemonSet 或 namespace 默认复制到 D601； + - 拆分和验收只走 + `platform-infra k3s-cluster separate|status|smoke`； + - 跨地域 artifact 传输使用鉴权、只读的公网 HTTPS registry， + 不得通过 Kubernetes Pod 网络、Service、DNS 或 VXLAN； + - 如未来确需跨地域计算，必须建立独立 workload 级交付规格并显式选择 + D601 集群，不得通过重新组成单一集群获得隐式调度。 - 长命令用异步 job 或短轮询；不要长时间挂住 trans/ssh。 - Web 哨兵 YAML cadence 变更不新增发布 orchestrator：提交前用 `web-probe sentinel validate --local`；合并后用只读 `web-probe sentinel control-plane status --wait`。线上 desired schedule 必须来自 GitOps 清单，不能用操作者旧 worktree 的本地渲染值做阻塞门禁；收敛差异统一为非阻塞 warning。 diff --git a/.agents/skills/unidesk-ymalops/SKILL.md b/.agents/skills/unidesk-ymalops/SKILL.md index 654d14b6..95d41eb7 100644 --- a/.agents/skills/unidesk-ymalops/SKILL.md +++ b/.agents/skills/unidesk-ymalops/SKILL.md @@ -112,6 +112,14 @@ description: UniDesk YAML-first 运维正规化技能。用户提到 ymal-first/ - worker agent 使用 YAML 派生摘要决定是否重装，禁止每次 apply 先卸载再覆盖； - 禁止借用全量 infra apply 顺带覆盖 registry、CI/CD 或业务运行面。 - 日常入口和验收判定统一见 `docs/reference/platform-infra.md#多主机-k3s-集群`。 + - NC01-D601 的 owning YAML 必须声明 `topologyMode=separated`： + - D601 独立 server 的节点名、标签、禁用组件和 kubeconfig mode 必须 + 位于同一 YAML 的 `k3s.standalone`； + - Hyper-V YAML 只能用 `management.mode=standalone` 引用该片段； + - parser、bootstrap、CI/CD 和 GitOps 禁止通过隐藏默认值把 D601 + 恢复为 agent、恢复 WireGuard peer或共享调度域； + - 独立集群间只共享显式声明的只读 artifact endpoint，不共享 Pod、 + Service、DNS、CNI、datastore 或默认 workload。 - 新节点 node-local registry 必须支持从零部署：优先由 YAML 渲染为 k3s workload/PVC；host Docker registry 只能是 legacy/迁移前状态。需要证明从零能力时先停旧 registry，不从旧 registry seed，并在状态中披露 `seededFromOldRegistry`、`zeroSeeded`、PVC、workload、endpoint 与 runtime/tools image preload 结果。 - YAML 配置必须可复用、可继承、可组合；节点/lane 只能作为变量或 YAML value 参与渲染，禁止把具体 node/lane 写入 YAML 文件名、parser/renderer/helper 函数名或长期 schema 名。发现不同节点复制大段配置时，先抽 baseline/configRef/override，再继续扩展节点。 - CLI `plan/status` 应输出 redacted 配置引用图：每个 ref 的文件、path、presence、摘要 hash、缺失字段和下一步 drill-down 命令。不要默认 dump 展开后的完整 YAML 或 Secret。 diff --git a/config/platform-infra/hyperv-vms.yaml b/config/platform-infra/hyperv-vms.yaml index 2fe3ed91..2c62a47f 100644 --- a/config/platform-infra/hyperv-vms.yaml +++ b/config/platform-infra/hyperv-vms.yaml @@ -73,8 +73,8 @@ targets: - https://docker.m.daocloud.io k3s: management: - mode: external-cluster - configRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.workers.D601-VM + mode: standalone + configRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.k3s.standalone version: v1.36.2+k3s1 installMirror: cn binaryUrl: https://github.com/k3s-io/k3s/releases/download/v1.36.2%2Bk3s1/k3s diff --git a/config/platform-infra/k3s-clusters.yaml b/config/platform-infra/k3s-clusters.yaml index d89ed446..7a72a9b2 100644 --- a/config/platform-infra/k3s-clusters.yaml +++ b/config/platform-infra/k3s-clusters.yaml @@ -9,6 +9,7 @@ targets: nc01-d601: enabled: true clusterName: nc01 + topologyMode: separated controlPlane: id: NC01 configRef: config/hwlab-node-control-plane.yaml#nodes.NC01 @@ -57,6 +58,18 @@ targets: agentTokenFile: /etc/unidesk/k3s-agent-token flannelInterface: wg0 maxPods: 500 + standalone: + clusterName: d601 + nodeName: d601-vm + writeKubeconfigMode: "0644" + labels: + unidesk.ai/node-id: D601-VM + unidesk.ai/provider-id: D601-VM + unidesk.ai/runtime-plane: standalone + disableComponents: + - traefik + - servicelb + - metrics-server registryMirror: source: 127.0.0.1:5000 endpoint: https://registry.hwpod.com diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 4bfdcabc..039e3923 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -10,16 +10,27 @@ - UniDesk 拥有的 image version、namespace、endpoint、FRP port 与 profile 列表不得隐藏在 helper 常量里；外部工具要求的 TOML、JSON 和 env 仅是边缘渲染格式。 - 新节点出网 bootstrap 遵循 `docs/reference/yaml-first-ops.md` 的零依赖 host proxy 边界，不得依赖尚未安装的 Docker、k3s、containerd 或包管理器网络能力。 -## 多主机 k3s 集群 +## k3s 集群拓扑 - 权威配置： - - 跨主机拓扑、WireGuard、agent token `sourceRef`、worker host proxy 引用、集群内 artifact registry 映射、ServiceLB 节点和 smoke 归 `config/platform-infra/k3s-clusters.yaml`。 + - 集群边界、拓扑模式、历史 WireGuard 迁移参数、独立 server 角色、 + registry endpoint、ServiceLB 节点和 smoke 归 + `config/platform-infra/k3s-clusters.yaml`。 - worker host proxy 的服务端来源、客户端工件、Secret 引用、代理环境和 systemd drop-in 归 `config/platform-infra/host-proxy.yaml`。 - Hyper-V VM 资源和宿主内网 SSH 归 `config/platform-infra/hyperv-vms.yaml`。 - 既有 control-plane 的 k3s systemd drop-in 继续归 `config/hwlab-node-control-plane.yaml`，并通过 `clusterConfigRef` 消费集群网络参数。 - 受控入口： - `platform-infra k3s-cluster secret-init --target --confirm` 只在 Secret source 不存在时生成 WireGuard key pair 和 k3s agent token，存在时拒绝覆盖。 - - `platform-infra k3s-cluster plan|apply|drain|status|smoke --target ` 是部署、清空不可调度 worker 和验收入口。 + - `platform-infra k3s-cluster plan|apply|separate|drain|status|smoke + --target ` 是部署、拓扑拆分、清空不可调度 worker和验收入口。 + - `topologyMode=separated` 时，`apply --confirm` 与 + `separate --confirm` 都只执行受控拆分： + - 先证明旧 worker 的非 DaemonSet Pod 为零； + - 再清理旧 agent datastore、CNI 和跨地域 WireGuard 配置； + - 在 D601 安装并等待独立单节点 k3s server Ready； + - 最后从 NC01 删除旧 Node 和对端 WireGuard peer。 + - 拆分不得先删除 NC01 Node 或隧道；D601 独立 server 未 Ready 时必须 + 保留旧控制面侧事实并返回失败。 - 无固定公网地址的 worker 由宿主内网 SSH 控制，主动连接 control-plane WireGuard endpoint；worker provider route 只作为部署后补充验收入口。 - 集群 apply 先经宿主内网 SSH 分发零依赖 proxy client，确认它连接声明的 `vpn-server` 且外网 probe 成功，再启动或恢复 `k3s-agent`。 - 集群 apply 在 proxy 前先收敛 worker WireGuard；内容一致时不强写，只有 unit 未运行或握手过期且隧道探针失败时才恢复 worker 接口。 @@ -48,10 +59,25 @@ - 同一入口等待节点 Ready； - 禁止调用 HWLAB 全量 `infra apply`，也不得触碰 Kubernetes manifests、registry、Tekton、Argo、git-mirror 或业务 rollout。 - 运行边界： - - control-plane 保持唯一 k3s server，同时承担 workload。 - - worker 只运行 `k3s-agent`；加入前必须卸载旧独立 server，禁止保留第二 datastore/control-plane。 - - worker 是否参与业务调度及其 taint 必须由集群 owning YAML 的 - `workers..scheduling` 声明： + - NC01 与 D601 必须保持两个独立 k3s 集群，禁止把 D601 重新注册为 + NC01 agent，也禁止把 NC01 注册到 D601。 + - D601 独立集群只运行一个 server 节点，初始业务 Pod 数必须为零； + 是否承载新业务必须由新的 workload owning YAML 显式选择，不得继承 + NC01 的 GitOps、PipelineRun、DaemonSet 或默认 namespace。 + - NC01 保留自身 control-plane 与运行面；拆分后可保留无 peer 的本地 + WireGuard 接口以避免重写生产 control-plane 地址，但不得恢复 D601 peer。 + - 跨地域连接只用于受控 SSH/Provider 控制通道和公网 HTTPS + artifact pull，不再承担 Kubernetes API、Pod、Service、DNS 或 VXLAN 数据面。 + - `topologyMode=separated` 是 owning YAML 的架构事实。CI/CD、GitOps、 + VM bootstrap 和普通集群 apply 禁止改回 joined、启动 `k3s-agent`、 + 写入 agent token 或渲染跨地域 Flannel/WireGuard peer。 + - 下列 worker 规则只适用于仍由 owning YAML 明确声明为 joined 的其他 + 集群，不得用于 NC01-D601： + - control-plane 保持唯一 k3s server，同时承担 workload； + - worker 只运行 `k3s-agent`，加入前卸载旧独立 server； + - worker 是否参与业务调度及其 taint 必须由集群 owning YAML 的 + `workers..scheduling` 声明。 + - joined worker 的调度规则： - `schedulable=false` 时必须同时声明 `NoSchedule` 或 `NoExecute` taint； - agent 注册参数和 control-plane 的 `cordon`/taint 收敛必须消费同一声明， 防止节点重装、重注册或 CI 自动覆盖后短暂接收业务 Pod； @@ -88,7 +114,13 @@ - k3s API 暂时不可读属于 unknown，不得当作容量漂移触发 restart； - 集群 CLI 只组合各自 owning domain，不借用全量基础设施入口扩大写入范围。 - 验收： - - `status` 必须同时证明 WireGuard handshake、worker host proxy、server/agent systemd、两个 Ready node、worker InternalIP 和旧 server 退役。 + - `topologyMode=separated` 的 `status` 必须同时证明： + - D601 只有一个 Ready 节点； + - D601 `k3s` server active 且旧 `k3s-agent` inactive； + - D601 非 `kube-system` Pod 为零； + - NC01 不再存在 D601 Node。 + - joined 拓扑的 `status` 才验证 WireGuard handshake、host proxy、 + server/agent、两个 Ready node、worker InternalIP 和旧 server 退役。 - 仅当 worker 的 `scheduling.schedulable=true` 时， `smoke --confirm` 才可把 Job 定向调度到 worker并验证集群 DNS与 Kubernetes ClusterIP；不可调度 worker 的 smoke 只读验证 taint、 diff --git a/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md b/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md index 2163fc34..a701b415 100644 --- a/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md +++ b/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md @@ -18,7 +18,7 @@ - Hyper-V 虚拟机、VHDX、处理器、内存和自动启动策略。 - 独立内部交换机、NAT、固定地址和出网连通性。 - Ubuntu cloud-init、SSH、Docker 和受控 k3s 运行角色初始化。 -- 无固定公网地址 worker 通过原生 WireGuard 主动接入既有 k3s 控制面。 +- D601-VM 运行 YAML 声明的独立单节点 k3s 空集群。 - provider-gateway 的节点身份、常驻恢复和 trans 验收。 - 后台安装的阶段、日志、错误和终态可见性。 @@ -57,8 +57,9 @@ Hyper-V 虚拟机必须配置为宿主启动后自动启动、宿主关机时保 - VM 为运行状态且资源与 YAML 一致。 - Ubuntu 固定地址可达并能正常出网。 - Docker daemon 和 k3s 节点健康。 -- worker 以 agent 身份加入声明的控制面，且独立 server 不再运行。 -- 定向调度到 worker 的 smoke 能完成集群 DNS 与 ClusterIP 访问。 +- D601 只运行独立 k3s server，旧 agent 已停止。 +- 独立集群只有一个 Ready 节点，且非 `kube-system` Pod 为零。 +- NC01 集群内不存在 D601 Node。 - provider-gateway 以 YAML 声明的 Provider ID 在线。 - `trans hostname` 和 k3s route 均可用。 @@ -74,24 +75,27 @@ Provider TCP 数据池恢复必须满足： - 修复范围只包含 Provider 服务，不得连带重启 VM、Docker 或 k3s。 - 同一入口必须在修复后重新执行 host 和 k3s route 验收。 -### 3.8 HYPERV-VM-REQ-008 多主机 k3s 运行面 +### 3.8 HYPERV-VM-REQ-008 独立 k3s 运行面 -既有控制面与 Hyper-V worker 组成集群时必须满足： +D601-VM 与 NC01 必须作为两个独立集群运行： -- 控制面保持唯一 server，并继续承担 workload。 -- Hyper-V 节点只能安装为 agent，不得保留独立 server。 -- 无固定公网地址的 worker 通过主动拨号的原生 WireGuard 三层隧道接入控制面。 -- WireGuard 私钥和 k3s agent token 只能来自 YAML `sourceRef`，不得从运行面反解或打印。 -- k3s 节点地址、Flannel 接口、API TLS SAN、集群内 artifact registry 映射和 ServiceLB 节点选择必须由 owning YAML 渲染。 -- worker 的业务调度资格、cordon 和 taint 必须由 owning YAML 声明并由 - agent 注册参数与受控集群入口共同收敛。声明为不可调度的 worker 不得承载 - 非 DaemonSet 业务 Pod；CI/CD、GitOps 和 smoke 不得通过 toleration、 - `nodeName` 或删除 taint 绕过该隔离。 -- worker 必须先受控部署 host proxy client 并连接 YAML 引用的 `vpn-server`，再启动 agent；containerd 直接通过该 proxy 访问公共 registry，不得依赖公共镜像站或新增 registry mirror。 +- `config/platform-infra/k3s-clusters.yaml` 必须显式声明 + `topologyMode=separated` 和 D601 `k3s.standalone` 参数。 +- `config/platform-infra/hyperv-vms.yaml` 必须以 + `management.mode=standalone` 引用独立集群配置。 +- D601 只运行单节点 k3s server；旧 `k3s-agent`、agent datastore、CNI + 和跨地域 WireGuard peer 必须由受控拆分入口退役。 +- 拆分前必须证明 D601 非 DaemonSet Pod 为零；D601 独立 server Ready + 后，才能从 NC01 删除旧 Node 和 WireGuard peer。 +- 拆分后 D601 非 `kube-system` Pod 必须为零，NC01 不得存在 D601 Node。 +- CI/CD、GitOps、VM bootstrap 和业务 renderer 禁止重新安装 agent、 + 恢复 agent token、重建跨地域 CNI 或把 NC01 workload 自动投递到 D601。 +- D601 host proxy 可继续为独立集群提供公共依赖出网，不得依赖公共镜像站 + 或新增公共 registry mirror。 - 集群自有 artifact registry 可以通过公网可信 TLS 向异地 worker 提供鉴权只读拉取： - 内部 CI/publish 继续使用集群内写入 endpoint，公网入口不得成为第二个 push authority。 - 公网边缘只允许 Registry V2 拉取所需的 `GET` 与 `HEAD`，其他方法必须在反向代理层拒绝。 - endpoint、DNS、上游、允许方法、凭据 `sourceRef`、镜像摘要和拉取验收参数必须来自集群 owning YAML。 - - worker containerd 继续消费既有镜像引用，只把该内部引用映射到声明的公网 HTTPS endpoint。 + - D601 containerd 继续消费既有镜像引用，只把该内部引用映射到声明的公网 HTTPS endpoint。 - artifact 大文件不得经过 WireGuard、Flannel/VXLAN 或受传输时长和对象大小限制的 CDN。 -- 宿主内网 SSH 可以作为 worker 的稳定部署控制通道，provider route 只作为部署后验收入口。 +- 宿主内网 SSH 可以作为 D601 的稳定部署控制通道，provider route 只作为部署后验收入口。 diff --git a/scripts/assets/platform-infra/hyperv-vm-bootstrap.ps1 b/scripts/assets/platform-infra/hyperv-vm-bootstrap.ps1 index 0c2e99da..23a265d2 100644 --- a/scripts/assets/platform-infra/hyperv-vm-bootstrap.ps1 +++ b/scripts/assets/platform-infra/hyperv-vm-bootstrap.ps1 @@ -492,12 +492,14 @@ try { Wait-For-Ssh -KeyPath $key.private if ([string]$Config.k3s.management.mode -eq 'external-cluster') { Write-State -Stage 'staging-k3s-agent' -Message 'Pinned k3s artifacts are staged; the YAML-controlled cluster manager owns the agent role' + } elseif ([string]$Config.k3s.management.mode -eq 'standalone') { + Write-State -Stage 'staging-k3s-standalone' -Message 'Pinned k3s artifacts are staged; the YAML-controlled cluster manager owns the standalone role' } else { throw "unsupported-k3s-management-mode: $($Config.k3s.management.mode)" } Install-Provider -KeyPath $key.private - Write-State -Stage 'completed' -State 'completed' -Message 'Ubuntu VM, Docker, pinned k3s artifacts and provider-gateway are ready for the external cluster manager' -Extra @{ + Write-State -Stage 'completed' -State 'completed' -Message 'Ubuntu VM, Docker, YAML-managed k3s and provider-gateway are ready' -Extra @{ address = [string]$Config.network.address providerId = [string]$Config.provider.id processorCount = [int]$Config.vm.processorCount diff --git a/scripts/src/platform-infra-hyperv-vm.ts b/scripts/src/platform-infra-hyperv-vm.ts index afd44a20..893e3348 100644 --- a/scripts/src/platform-infra-hyperv-vm.ts +++ b/scripts/src/platform-infra-hyperv-vm.ts @@ -100,7 +100,9 @@ function validateTarget(id: string, target: HyperVTarget): void { if (!/^[a-f0-9]{64}$/u.test(target.image.sha256)) throw new Error(`${configLabel}.targets.${id}.image.sha256 must be lowercase SHA-256`); if (target.image.sources.length < 2) throw new Error(`${configLabel}.targets.${id}.image.sources must include mirror and canonical fallback`); const k3sManagement = record(target.k3s.management, `${configLabel}.targets.${id}.k3s.management`); - if (k3sManagement.mode !== "external-cluster") throw new Error(`${configLabel}.targets.${id}.k3s.management.mode must be external-cluster`); + if (k3sManagement.mode !== "external-cluster" && k3sManagement.mode !== "standalone") { + throw new Error(`${configLabel}.targets.${id}.k3s.management.mode must be external-cluster or standalone`); + } if (typeof k3sManagement.configRef !== "string" || !k3sManagement.configRef.startsWith("config/platform-infra/k3s-clusters.yaml#")) { throw new Error(`${configLabel}.targets.${id}.k3s.management.configRef must reference the k3s cluster owning YAML`); } diff --git a/scripts/src/platform-infra-k3s-cluster-config.ts b/scripts/src/platform-infra-k3s-cluster-config.ts index 39fa807d..f08f57d7 100644 --- a/scripts/src/platform-infra-k3s-cluster-config.ts +++ b/scripts/src/platform-infra-k3s-cluster-config.ts @@ -6,6 +6,7 @@ export const K3S_CLUSTER_CONFIG_LABEL = "config/platform-infra/k3s-clusters.yaml export interface K3sClusterTarget { id: string; clusterName: string; + topologyMode: "joined" | "separated"; controlPlane: { id: string; configRef: string; @@ -61,6 +62,13 @@ export interface K3sClusterTarget { agentTokenFile: string; flannelInterface: string; maxPods: number; + standalone: { + clusterName: string; + nodeName: string; + writeKubeconfigMode: string; + labels: Record; + disableComponents: string[]; + }; registryMirror: { source: string; endpoint: string; @@ -112,6 +120,7 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri const tcpTuning = record(raw.tcpTuning, `targets.${id}.tcpTuning`); const secrets = record(raw.secrets, `targets.${id}.secrets`); const k3s = record(raw.k3s, `targets.${id}.k3s`); + const standalone = record(k3s.standalone, `targets.${id}.k3s.standalone`); const mirror = record(k3s.registryMirror, `targets.${id}.k3s.registryMirror`); const mirrorAuth = record(mirror.auth, `targets.${id}.k3s.registryMirror.auth`); const registryPullSmoke = record(k3s.registryPullSmoke, `targets.${id}.k3s.registryPullSmoke`); @@ -134,6 +143,7 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri const target: K3sClusterTarget = { id, clusterName: text(raw, "clusterName", `targets.${id}`), + topologyMode: enumText(raw, "topologyMode", `targets.${id}`, ["joined", "separated"]), controlPlane: { id: text(control, "id", `targets.${id}.controlPlane`), configRef: text(control, "configRef", `targets.${id}.controlPlane`), @@ -189,6 +199,13 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri agentTokenFile: text(k3s, "agentTokenFile", `targets.${id}.k3s`), flannelInterface: text(k3s, "flannelInterface", `targets.${id}.k3s`), maxPods: integer(k3s, "maxPods", `targets.${id}.k3s`), + standalone: { + clusterName: kubernetesName(standalone, "clusterName", "k3s.standalone"), + nodeName: kubernetesName(standalone, "nodeName", "k3s.standalone"), + writeKubeconfigMode: enumText(standalone, "writeKubeconfigMode", "k3s.standalone", ["0600", "0640", "0644"]), + labels: stringMap(record(standalone.labels, "k3s.standalone.labels"), "k3s.standalone.labels"), + disableComponents: stringArray(standalone.disableComponents, "k3s.standalone.disableComponents"), + }, registryMirror: { source: text(mirror, "source", "registryMirror"), endpoint: httpsUrl(mirror, "endpoint", "registryMirror"), @@ -361,6 +378,13 @@ function stringMap(value: Record, label: string): Record typeof item !== "string" || item.length === 0)) { + throw new Error(`${label} must be a non-empty string array`); + } + return value as string[]; +} + function hostProxyTargetId(ref: string): string { const match = ref.match(/^config\/platform-infra\/host-proxy\.yaml#targets\.([A-Za-z0-9._-]+)$/u); if (match === null) throw new Error(`worker hostProxyConfigRef must point at config/platform-infra/host-proxy.yaml#targets.`); diff --git a/scripts/src/platform-infra-k3s-cluster.ts b/scripts/src/platform-infra-k3s-cluster.ts index ed9be5c0..eb81cb3d 100644 --- a/scripts/src/platform-infra-k3s-cluster.ts +++ b/scripts/src/platform-infra-k3s-cluster.ts @@ -13,7 +13,7 @@ import { } from "./platform-infra-host-proxy"; import { address, K3S_CLUSTER_CONFIG_LABEL, readK3sClusterConfig, type K3sClusterTarget } from "./platform-infra-k3s-cluster-config"; -type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "drain" | "status" | "smoke" | "registry-smoke"; +type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "separate" | "drain" | "status" | "smoke" | "registry-smoke"; interface Options { action: Action; @@ -39,18 +39,20 @@ export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig, if (options.action === "plan") return plan(target); if (options.action === "secret-init") return secretInit(target, options.confirm); if (options.action === "registry-credential-init") return registryCredentialInit(target, options.confirm); + if (options.action === "separate") return separateCluster(target, options.confirm); if (options.action === "drain") return drainWorker(target, options.confirm); - if (options.action === "status") return status(target); - if (options.action === "smoke") return smoke(target, options.confirm); + if (options.action === "status") return target.topologyMode === "separated" ? separatedStatus(target) : status(target); + if (options.action === "smoke") return target.topologyMode === "separated" ? separatedSmoke(target) : smoke(target, options.confirm); if (options.action === "registry-smoke") return registrySmoke(target, options.confirm); if (!options.confirm) return { ...plan(target), mode: "dry-run", mutation: false }; + if (target.topologyMode === "separated") return separateCluster(target, true); return apply(target); } function parseOptions(args: string[]): Options { const action = args[0] ?? "plan"; - if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "drain" && action !== "status" && action !== "smoke" && action !== "registry-smoke") { - throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|drain|status|smoke|registry-smoke [--target ] [--confirm]"); + if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "separate" && action !== "drain" && action !== "status" && action !== "smoke" && action !== "registry-smoke") { + throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|separate|drain|status|smoke|registry-smoke [--target ] [--confirm]"); } return { action, targetId: option(args, "--target") ?? undefined, confirm: args.includes("--confirm") }; } @@ -61,6 +63,7 @@ function plan(target: K3sClusterTarget): Record { mutation: false, targetId: target.id, source: K3S_CLUSTER_CONFIG_LABEL, + topologyMode: target.topologyMode, topology: { clusterName: target.clusterName, controlPlane: { id: target.controlPlane.id, nodeName: target.controlPlane.nodeName, route: target.controlPlane.route, kubeRoute: target.controlPlane.kubeRoute }, @@ -89,6 +92,7 @@ function plan(target: K3sClusterTarget): Record { secretInit: `bun scripts/cli.ts platform-infra k3s-cluster secret-init --target ${target.id} --confirm`, registryCredentialInit: `bun scripts/cli.ts platform-infra k3s-cluster registry-credential-init --target ${target.id} --confirm`, apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm`, + separate: `bun scripts/cli.ts platform-infra k3s-cluster separate --target ${target.id} --confirm`, drain: `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm`, status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`, smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`, @@ -282,6 +286,105 @@ function apply(target: K3sClusterTarget): Record { }; } +function separateCluster(target: K3sClusterTarget, confirm: boolean): Record { + if (target.topologyMode !== "separated") throw new Error("separate requires topologyMode=separated in the owning YAML"); + const workloads = workerWorkloadStatus(target); + if (workloads.ok !== true) throw new Error(`worker must be empty before separation: ${bounded(JSON.stringify(workloads))}`); + if (!confirm) { + return { + ok: true, + mutation: false, + mode: "dry-run", + targetId: target.id, + topologyMode: target.topologyMode, + workerWorkloads: workloads, + standalone: target.k3s.standalone, + }; + } + const secret = readSecrets(target); + const standalone = runWorker(target, standaloneApplyScript(target, secret), true, 300_000); + requireSuccess(standalone, "worker-standalone-k3s-apply"); + const localStatus = waitForStandalone(target, 180_000); + requireSuccess(localStatus, "worker-standalone-k3s-status"); + const localData = parseFirstJson(localStatus.stdout); + if (localData.ok !== true) throw new Error(`standalone cluster did not converge: ${bounded(JSON.stringify(localData))}`); + + const deleteOldNode = runCommand([ + "trans", target.controlPlane.kubeRoute, "kubectl", "delete", "node", target.worker.nodeName, "--ignore-not-found=true", + ], rootPath(), { timeoutMs: 60_000 }); + requireSuccess(deleteOldNode, "old-cluster-worker-delete"); + const retirePeer = runControlPlane(target, controlPlaneRetireWorkerPeerScript(target, secret), 60_000); + requireSuccess(retirePeer, "control-plane-wireguard-peer-retire"); + const result = separatedStatus(target); + if (result.ok !== true) throw new Error(`cluster separation final status failed: ${bounded(JSON.stringify(result))}`); + return { + ok: true, + mutation: true, + targetId: target.id, + topologyMode: target.topologyMode, + standalone: localData, + oldClusterNodeDeleted: true, + wireguardPeerRetired: true, + stages: { + standalone: compact(standalone), + oldClusterNode: compact(deleteOldNode), + controlPlaneWireguard: compact(retirePeer), + }, + next: { + status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`, + smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`, + }, + }; +} + +function separatedStatus(target: K3sClusterTarget): Record { + const standalone = runWorker(target, standaloneStatusScript(target), false, 60_000); + const standaloneData = parseFirstJson(standalone.stdout); + const oldNode = runCommand([ + "trans", target.controlPlane.kubeRoute, "kubectl", "get", "node", target.worker.nodeName, "-o", "name", + ], rootPath(), { timeoutMs: 30_000 }); + const oldNodeAbsent = oldNode.exitCode !== 0; + return { + ok: standalone.exitCode === 0 && standaloneData.ok === true && oldNodeAbsent, + mutation: false, + targetId: target.id, + topologyMode: target.topologyMode, + standalone: standaloneData, + oldCluster: { + controlPlane: target.controlPlane.id, + workerNodeAbsent: oldNodeAbsent, + }, + result: { + standalone: { exitCode: standalone.exitCode }, + oldClusterNode: { exitCode: oldNode.exitCode }, + }, + }; +} + +function waitForStandalone(target: K3sClusterTarget, timeoutMs: number): ReturnType { + const deadline = Date.now() + timeoutMs; + let current = runWorker(target, standaloneStatusScript(target), false, 55_000); + while ((current.exitCode !== 0 || parseFirstJson(current.stdout).ok !== true) && Date.now() < deadline) { + runCommand(["sleep", "5"], rootPath(), { timeoutMs: 10_000 }); + current = runWorker(target, standaloneStatusScript(target), false, 55_000); + } + if (current.exitCode === 0 && parseFirstJson(current.stdout).ok === true) return current; + return { ...current, exitCode: 1, stderr: current.stderr || `standalone cluster did not become ready within ${timeoutMs}ms` }; +} + +function separatedSmoke(target: K3sClusterTarget): Record { + const status = separatedStatus(target); + return { + ...status, + action: "separated-cluster-smoke", + checks: { + standaloneSingleNodeReady: (status.standalone as Record)?.ok === true, + oldClusterNodeAbsent: (status.oldCluster as Record)?.workerNodeAbsent === true, + businessPodCountZero: (status.standalone as Record)?.businessPodCount === 0, + }, + }; +} + function status(target: K3sClusterTarget): Record { const control = runControlPlane(target, statusScript(target), 60_000); const workerProxy = runWorker(target, hostProxyGuestStatusScript(target.worker.hostProxyTargetId), false, 60_000); @@ -1013,6 +1116,100 @@ wg show ${wg.interfaceName} >/dev/null `; } +function controlPlaneRetireWorkerPeerScript(target: K3sClusterTarget, secret: SecretMaterial): string { + const wg = target.wireguard; + return `set -eu +umask 077 +candidate="$(mktemp)" +trap 'rm -f "$candidate"' EXIT +cat >"$candidate" <<'EOF' +[Interface] +Address = ${wg.controlPlaneAddress} +ListenPort = ${wg.listenPort} +PrivateKey = ${secret.controlPrivate} +MTU = ${wg.mtu} +EOF +if ! cmp -s "$candidate" /etc/wireguard/${wg.interfaceName}.conf; then + install -m 0600 "$candidate" /etc/wireguard/${wg.interfaceName}.conf + systemctl restart wg-quick@${wg.interfaceName} +fi +wg show ${wg.interfaceName} peers | grep -q . && { echo "retired worker peer still present" >&2; exit 1; } +`; +} + +function standaloneApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string { + const standalone = target.k3s.standalone; + const disableArgs = standalone.disableComponents.flatMap((component) => ["--disable", component]); + const labels = Object.entries(standalone.labels).flatMap(([key, value]) => ["--node-label", `${key}=${value}`]); + const exec = [ + "server", + ...disableArgs, + "--cluster-init", + "--node-name", standalone.nodeName, + ...labels, + "--write-kubeconfig-mode", standalone.writeKubeconfigMode, + "--kubelet-arg", `max-pods=${target.k3s.maxPods}`, + ].map(shellToken).join(" "); + const desiredHash = hash(JSON.stringify({ + version: target.worker.k3sVersion, + exec, + registryEndpoint: target.k3s.registryMirror.endpoint, + registryUsername: secret.registryUsername, + registryPassword: secret.registryPassword, + })); + return `set -eu +desired_hash=${shQuote(desiredHash)} +desired_hash_file=/etc/unidesk/k3s-standalone-desired.sha256 +if [ "$(cat "$desired_hash_file" 2>/dev/null || true)" = "$desired_hash" ] && systemctl is-active --quiet k3s; then + exit 0 +fi +systemctl disable --now wg-quick@${target.wireguard.interfaceName} >/dev/null 2>&1 || true +if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh; fi +if [ -x /usr/local/bin/k3s-uninstall.sh ]; then /usr/local/bin/k3s-uninstall.sh; fi +rm -rf /var/lib/rancher/k3s /etc/rancher/k3s /var/lib/kubelet /var/lib/cni /etc/cni/net.d /run/k3s /run/flannel +ip link delete cni0 >/dev/null 2>&1 || true +ip link delete flannel.1 >/dev/null 2>&1 || true +rm -f /etc/wireguard/${target.wireguard.interfaceName}.conf +install -d -m 0700 /etc/rancher/k3s /etc/unidesk +cat >/etc/rancher/k3s/registries.yaml <<'EOF' +mirrors: + "${target.k3s.registryMirror.source}": + endpoint: + - "${target.k3s.registryMirror.endpoint}" +configs: + "${new URL(target.k3s.registryMirror.endpoint).host}": + auth: + username: ${JSON.stringify(secret.registryUsername)} + password: ${JSON.stringify(secret.registryPassword)} +EOF +chmod 0600 /etc/rancher/k3s/registries.yaml +install -m 0755 /tmp/k3s /usr/local/bin/k3s +chmod 0755 /tmp/install-k3s.sh +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_VERSION=${shQuote(target.worker.k3sVersion)} INSTALL_K3S_EXEC=${shQuote(exec)} sh /tmp/install-k3s.sh +printf '%s\n' "$desired_hash" >"$desired_hash_file" +chmod 0600 "$desired_hash_file" +systemctl enable k3s >/dev/null +systemctl restart --no-block k3s +`; +} + +function standaloneStatusScript(target: K3sClusterTarget): string { + const nodeName = target.k3s.standalone.nodeName; + return `set -eu +active=false; systemctl is-active --quiet k3s && active=true +agent_active=false; systemctl is-active --quiet k3s-agent && agent_active=true +ready="$(k3s kubectl get node ${shQuote(nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" +node_count="$(k3s kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ')" +business_pods="$(k3s kubectl get pods -A --no-headers 2>/dev/null | awk '$1 != "kube-system" {count++} END {print count+0}')" +python3 - "$active" "$agent_active" "$ready" "$node_count" "$business_pods" <<'PY' +import json, sys +active, agent, ready, nodes, business = sys.argv[1:] +ok = active == "true" and agent == "false" and ready == "True" and nodes == "1" and business == "0" +print(json.dumps({"ok":ok,"serverActive":active=="true","oldAgentActive":agent=="true","nodeReady":ready=="True","nodeCount":int(nodes or 0),"businessPodCount":int(business or 0)}, separators=(",",":"))) +PY +`; +} + function tcpTuningApplyScript(target: K3sClusterTarget): string { const tuning = target.tcpTuning; const content = [