fix(platform-infra): split D601 from NC01 cluster

This commit is contained in:
pikastech
2026-07-20 09:50:06 +02:00
parent 4912abe5de
commit d3972699b0
10 changed files with 328 additions and 34 deletions
+12
View File
@@ -374,6 +374,18 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help
- 验收统一走 `platform-infra k3s-cluster registry-smoke`,不手工 patch Caddy 或 containerd。
- worker 安装使用短启动并由 control-plane Node Ready 轮询收口,禁止让 `trans`/SSH 长挂等待 systemd notify。
- worker 加入集群不改变 PaC consumer 的 source commit authority,也不授权人工 PipelineRun、Argo sync 或业务 rollout。
- NC01 与 D601 是永久分离的调度域:
- `config/platform-infra/k3s-clusters.yaml` 必须保持
`topologyMode=separated`,D601 只作为独立单节点 k3s 集群存在;
- CI/CD、PaC、Tekton、Argo、GitOps、VM bootstrap 和业务 renderer
禁止把 D601 注册为 NC01 agent,禁止恢复跨地域 WireGuard peer
禁止把 NC01 workload、DaemonSet 或 namespace 默认复制到 D601
- 拆分和验收只走
`platform-infra k3s-cluster separate|status|smoke`
- 跨地域 artifact 传输使用鉴权、只读的公网 HTTPS registry
不得通过 Kubernetes Pod 网络、Service、DNS 或 VXLAN
- 如未来确需跨地域计算,必须建立独立 workload 级交付规格并显式选择
D601 集群,不得通过重新组成单一集群获得隐式调度。
- 长命令用异步 job 或短轮询;不要长时间挂住 trans/ssh。
- Web 哨兵 YAML cadence 变更不新增发布 orchestrator:提交前用 `web-probe sentinel validate --local`;合并后用只读 `web-probe sentinel control-plane status --wait`。线上 desired schedule 必须来自 GitOps 清单,不能用操作者旧 worktree 的本地渲染值做阻塞门禁;收敛差异统一为非阻塞 warning。
+8
View File
@@ -112,6 +112,14 @@ description: UniDesk YAML-first 运维正规化技能。用户提到 ymal-first/
- worker agent 使用 YAML 派生摘要决定是否重装,禁止每次 apply 先卸载再覆盖;
- 禁止借用全量 infra apply 顺带覆盖 registry、CI/CD 或业务运行面。
- 日常入口和验收判定统一见 `docs/reference/platform-infra.md#多主机-k3s-集群`
- NC01-D601 的 owning YAML 必须声明 `topologyMode=separated`
- D601 独立 server 的节点名、标签、禁用组件和 kubeconfig mode 必须
位于同一 YAML 的 `k3s.standalone`
- Hyper-V YAML 只能用 `management.mode=standalone` 引用该片段;
- parser、bootstrap、CI/CD 和 GitOps 禁止通过隐藏默认值把 D601
恢复为 agent、恢复 WireGuard peer或共享调度域;
- 独立集群间只共享显式声明的只读 artifact endpoint,不共享 Pod、
Service、DNS、CNI、datastore 或默认 workload。
- 新节点 node-local registry 必须支持从零部署:优先由 YAML 渲染为 k3s workload/PVChost Docker registry 只能是 legacy/迁移前状态。需要证明从零能力时先停旧 registry,不从旧 registry seed,并在状态中披露 `seededFromOldRegistry``zeroSeeded`、PVC、workload、endpoint 与 runtime/tools image preload 结果。
- YAML 配置必须可复用、可继承、可组合;节点/lane 只能作为变量或 YAML value 参与渲染,禁止把具体 node/lane 写入 YAML 文件名、parser/renderer/helper 函数名或长期 schema 名。发现不同节点复制大段配置时,先抽 baseline/configRef/override,再继续扩展节点。
- CLI `plan/status` 应输出 redacted 配置引用图:每个 ref 的文件、path、presence、摘要 hash、缺失字段和下一步 drill-down 命令。不要默认 dump 展开后的完整 YAML 或 Secret。
+2 -2
View File
@@ -73,8 +73,8 @@ targets:
- https://docker.m.daocloud.io
k3s:
management:
mode: external-cluster
configRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.workers.D601-VM
mode: standalone
configRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.k3s.standalone
version: v1.36.2+k3s1
installMirror: cn
binaryUrl: https://github.com/k3s-io/k3s/releases/download/v1.36.2%2Bk3s1/k3s
+13
View File
@@ -9,6 +9,7 @@ targets:
nc01-d601:
enabled: true
clusterName: nc01
topologyMode: separated
controlPlane:
id: NC01
configRef: config/hwlab-node-control-plane.yaml#nodes.NC01
@@ -57,6 +58,18 @@ targets:
agentTokenFile: /etc/unidesk/k3s-agent-token
flannelInterface: wg0
maxPods: 500
standalone:
clusterName: d601
nodeName: d601-vm
writeKubeconfigMode: "0644"
labels:
unidesk.ai/node-id: D601-VM
unidesk.ai/provider-id: D601-VM
unidesk.ai/runtime-plane: standalone
disableComponents:
- traefik
- servicelb
- metrics-server
registryMirror:
source: 127.0.0.1:5000
endpoint: https://registry.hwpod.com
+40 -8
View File
@@ -10,16 +10,27 @@
- UniDesk 拥有的 image version、namespace、endpoint、FRP port 与 profile 列表不得隐藏在 helper 常量里;外部工具要求的 TOML、JSON 和 env 仅是边缘渲染格式。
- 新节点出网 bootstrap 遵循 `docs/reference/yaml-first-ops.md` 的零依赖 host proxy 边界,不得依赖尚未安装的 Docker、k3s、containerd 或包管理器网络能力。
## 多主机 k3s 集群
## k3s 集群拓扑
- 权威配置:
- 跨主机拓扑、WireGuard、agent token `sourceRef`、worker host proxy 引用、集群内 artifact registry 映射、ServiceLB 节点和 smoke 归 `config/platform-infra/k3s-clusters.yaml`
- 集群边界、拓扑模式、历史 WireGuard 迁移参数、独立 server 角色、
registry endpoint、ServiceLB 节点和 smoke 归
`config/platform-infra/k3s-clusters.yaml`
- worker host proxy 的服务端来源、客户端工件、Secret 引用、代理环境和 systemd drop-in 归 `config/platform-infra/host-proxy.yaml`
- Hyper-V VM 资源和宿主内网 SSH 归 `config/platform-infra/hyperv-vms.yaml`
- 既有 control-plane 的 k3s systemd drop-in 继续归 `config/hwlab-node-control-plane.yaml`,并通过 `clusterConfigRef` 消费集群网络参数。
- 受控入口:
- `platform-infra k3s-cluster secret-init --target <id> --confirm` 只在 Secret source 不存在时生成 WireGuard key pair 和 k3s agent token,存在时拒绝覆盖。
- `platform-infra k3s-cluster plan|apply|drain|status|smoke --target <id>` 是部署、清空不可调度 worker 和验收入口。
- `platform-infra k3s-cluster plan|apply|separate|drain|status|smoke
--target <id>` 是部署、拓扑拆分、清空不可调度 worker和验收入口。
- `topologyMode=separated` 时,`apply --confirm`
`separate --confirm` 都只执行受控拆分:
- 先证明旧 worker 的非 DaemonSet Pod 为零;
- 再清理旧 agent datastore、CNI 和跨地域 WireGuard 配置;
- 在 D601 安装并等待独立单节点 k3s server Ready
- 最后从 NC01 删除旧 Node 和对端 WireGuard peer。
- 拆分不得先删除 NC01 Node 或隧道;D601 独立 server 未 Ready 时必须
保留旧控制面侧事实并返回失败。
- 无固定公网地址的 worker 由宿主内网 SSH 控制,主动连接 control-plane WireGuard endpointworker provider route 只作为部署后补充验收入口。
- 集群 apply 先经宿主内网 SSH 分发零依赖 proxy client,确认它连接声明的 `vpn-server` 且外网 probe 成功,再启动或恢复 `k3s-agent`
- 集群 apply 在 proxy 前先收敛 worker WireGuard;内容一致时不强写,只有 unit 未运行或握手过期且隧道探针失败时才恢复 worker 接口。
@@ -48,10 +59,25 @@
- 同一入口等待节点 Ready;
- 禁止调用 HWLAB 全量 `infra apply`,也不得触碰 Kubernetes manifests、registry、Tekton、Argo、git-mirror 或业务 rollout。
- 运行边界:
- control-plane 保持唯一 k3s server,同时承担 workload。
- worker 只运行 `k3s-agent`;加入前必须卸载旧独立 server,禁止保留第二 datastore/control-plane
- worker 是否参与业务调度及其 taint 必须由集群 owning YAML 的
`workers.<id>.scheduling` 声明:
- NC01 与 D601 必须保持两个独立 k3s 集群,禁止把 D601 重新注册为
NC01 agent,也禁止把 NC01 注册到 D601
- D601 独立集群只运行一个 server 节点,初始业务 Pod 数必须为零;
是否承载新业务必须由新的 workload owning YAML 显式选择,不得继承
NC01 的 GitOps、PipelineRun、DaemonSet 或默认 namespace。
- NC01 保留自身 control-plane 与运行面;拆分后可保留无 peer 的本地
WireGuard 接口以避免重写生产 control-plane 地址,但不得恢复 D601 peer。
- 跨地域连接只用于受控 SSH/Provider 控制通道和公网 HTTPS
artifact pull,不再承担 Kubernetes API、Pod、Service、DNS 或 VXLAN 数据面。
- `topologyMode=separated` 是 owning YAML 的架构事实。CI/CD、GitOps、
VM bootstrap 和普通集群 apply 禁止改回 joined、启动 `k3s-agent`
写入 agent token 或渲染跨地域 Flannel/WireGuard peer。
- 下列 worker 规则只适用于仍由 owning YAML 明确声明为 joined 的其他
集群,不得用于 NC01-D601
- control-plane 保持唯一 k3s server,同时承担 workload
- worker 只运行 `k3s-agent`,加入前卸载旧独立 server
- worker 是否参与业务调度及其 taint 必须由集群 owning YAML 的
`workers.<id>.scheduling` 声明。
- joined worker 的调度规则:
- `schedulable=false` 时必须同时声明 `NoSchedule``NoExecute` taint
- agent 注册参数和 control-plane 的 `cordon`/taint 收敛必须消费同一声明,
防止节点重装、重注册或 CI 自动覆盖后短暂接收业务 Pod;
@@ -88,7 +114,13 @@
- k3s API 暂时不可读属于 unknown,不得当作容量漂移触发 restart;
- 集群 CLI 只组合各自 owning domain,不借用全量基础设施入口扩大写入范围。
- 验收:
- `status` 必须同时证明 WireGuard handshake、worker host proxy、server/agent systemd、两个 Ready node、worker InternalIP 和旧 server 退役。
- `topologyMode=separated``status` 必须同时证明:
- D601 只有一个 Ready 节点;
- D601 `k3s` server active 且旧 `k3s-agent` inactive
- D601 非 `kube-system` Pod 为零;
- NC01 不再存在 D601 Node。
- joined 拓扑的 `status` 才验证 WireGuard handshake、host proxy、
server/agent、两个 Ready node、worker InternalIP 和旧 server 退役。
- 仅当 worker 的 `scheduling.schedulable=true` 时,
`smoke --confirm` 才可把 Job 定向调度到 worker并验证集群 DNS与
Kubernetes ClusterIP;不可调度 worker 的 smoke 只读验证 taint、
@@ -18,7 +18,7 @@
- Hyper-V 虚拟机、VHDX、处理器、内存和自动启动策略。
- 独立内部交换机、NAT、固定地址和出网连通性。
- Ubuntu cloud-init、SSH、Docker 和受控 k3s 运行角色初始化。
- 无固定公网地址 worker 通过原生 WireGuard 主动接入既有 k3s 控制面
- D601-VM 运行 YAML 声明的独立单节点 k3s 空集群
- provider-gateway 的节点身份、常驻恢复和 trans 验收。
- 后台安装的阶段、日志、错误和终态可见性。
@@ -57,8 +57,9 @@ Hyper-V 虚拟机必须配置为宿主启动后自动启动、宿主关机时保
- VM 为运行状态且资源与 YAML 一致。
- Ubuntu 固定地址可达并能正常出网。
- Docker daemon 和 k3s 节点健康。
- worker 以 agent 身份加入声明的控制面,且独立 server 不再运行
- 定向调度到 worker 的 smoke 能完成集群 DNS 与 ClusterIP 访问
- D601 只运行独立 k3s server,旧 agent 已停止
- 独立集群只有一个 Ready 节点,且非 `kube-system` Pod 为零
- NC01 集群内不存在 D601 Node。
- provider-gateway 以 YAML 声明的 Provider ID 在线。
- `trans <provider-id> hostname` 和 k3s route 均可用。
@@ -74,24 +75,27 @@ Provider TCP 数据池恢复必须满足:
- 修复范围只包含 Provider 服务,不得连带重启 VM、Docker 或 k3s。
- 同一入口必须在修复后重新执行 host 和 k3s route 验收。
### 3.8 HYPERV-VM-REQ-008 多主机 k3s 运行面
### 3.8 HYPERV-VM-REQ-008 独立 k3s 运行面
既有控制面与 Hyper-V worker 组成集群时必须满足
D601-VM 与 NC01 必须作为两个独立集群运行
- 控制面保持唯一 server,并继续承担 workload。
- Hyper-V 节点只能安装为 agent,不得保留独立 server
- 无固定公网地址的 worker 通过主动拨号的原生 WireGuard 三层隧道接入控制面。
- WireGuard 私钥和 k3s agent token 只能来自 YAML `sourceRef`,不得从运行面反解或打印
- k3s 节点地址、Flannel 接口、API TLS SAN、集群内 artifact registry 映射和 ServiceLB 节点选择必须由 owning YAML 渲染。
- worker 的业务调度资格、cordon 和 taint 必须由 owning YAML 声明并由
agent 注册参数与受控集群入口共同收敛。声明为不可调度的 worker 不得承载
非 DaemonSet 业务 PodCI/CD、GitOps 和 smoke 不得通过 toleration、
`nodeName` 或删除 taint 绕过该隔离
- worker 必须先受控部署 host proxy client 并连接 YAML 引用的 `vpn-server`,再启动 agentcontainerd 直接通过该 proxy 访问公共 registry,不得依赖公共镜像站或新增 registry mirror。
- `config/platform-infra/k3s-clusters.yaml` 必须显式声明
`topologyMode=separated` 和 D601 `k3s.standalone` 参数
- `config/platform-infra/hyperv-vms.yaml` 必须以
`management.mode=standalone` 引用独立集群配置
- D601 只运行单节点 k3s server;旧 `k3s-agent`、agent datastore、CNI
和跨地域 WireGuard peer 必须由受控拆分入口退役。
- 拆分前必须证明 D601 非 DaemonSet Pod 为零;D601 独立 server Ready
后,才能从 NC01 删除旧 Node 和 WireGuard peer。
- 拆分后 D601 非 `kube-system` Pod 必须为零,NC01 不得存在 D601 Node
- CI/CD、GitOps、VM bootstrap 和业务 renderer 禁止重新安装 agent、
恢复 agent token、重建跨地域 CNI 或把 NC01 workload 自动投递到 D601。
- D601 host proxy 可继续为独立集群提供公共依赖出网,不得依赖公共镜像站
或新增公共 registry mirror。
- 集群自有 artifact registry 可以通过公网可信 TLS 向异地 worker 提供鉴权只读拉取:
- 内部 CI/publish 继续使用集群内写入 endpoint,公网入口不得成为第二个 push authority。
- 公网边缘只允许 Registry V2 拉取所需的 `GET``HEAD`,其他方法必须在反向代理层拒绝。
- endpoint、DNS、上游、允许方法、凭据 `sourceRef`、镜像摘要和拉取验收参数必须来自集群 owning YAML。
- worker containerd 继续消费既有镜像引用,只把该内部引用映射到声明的公网 HTTPS endpoint。
- D601 containerd 继续消费既有镜像引用,只把该内部引用映射到声明的公网 HTTPS endpoint。
- artifact 大文件不得经过 WireGuard、Flannel/VXLAN 或受传输时长和对象大小限制的 CDN。
- 宿主内网 SSH 可以作为 worker 的稳定部署控制通道,provider route 只作为部署后验收入口。
- 宿主内网 SSH 可以作为 D601 的稳定部署控制通道,provider route 只作为部署后验收入口。
@@ -492,12 +492,14 @@ try {
Wait-For-Ssh -KeyPath $key.private
if ([string]$Config.k3s.management.mode -eq 'external-cluster') {
Write-State -Stage 'staging-k3s-agent' -Message 'Pinned k3s artifacts are staged; the YAML-controlled cluster manager owns the agent role'
} elseif ([string]$Config.k3s.management.mode -eq 'standalone') {
Write-State -Stage 'staging-k3s-standalone' -Message 'Pinned k3s artifacts are staged; the YAML-controlled cluster manager owns the standalone role'
} else {
throw "unsupported-k3s-management-mode: $($Config.k3s.management.mode)"
}
Install-Provider -KeyPath $key.private
Write-State -Stage 'completed' -State 'completed' -Message 'Ubuntu VM, Docker, pinned k3s artifacts and provider-gateway are ready for the external cluster manager' -Extra @{
Write-State -Stage 'completed' -State 'completed' -Message 'Ubuntu VM, Docker, YAML-managed k3s and provider-gateway are ready' -Extra @{
address = [string]$Config.network.address
providerId = [string]$Config.provider.id
processorCount = [int]$Config.vm.processorCount
+3 -1
View File
@@ -100,7 +100,9 @@ function validateTarget(id: string, target: HyperVTarget): void {
if (!/^[a-f0-9]{64}$/u.test(target.image.sha256)) throw new Error(`${configLabel}.targets.${id}.image.sha256 must be lowercase SHA-256`);
if (target.image.sources.length < 2) throw new Error(`${configLabel}.targets.${id}.image.sources must include mirror and canonical fallback`);
const k3sManagement = record(target.k3s.management, `${configLabel}.targets.${id}.k3s.management`);
if (k3sManagement.mode !== "external-cluster") throw new Error(`${configLabel}.targets.${id}.k3s.management.mode must be external-cluster`);
if (k3sManagement.mode !== "external-cluster" && k3sManagement.mode !== "standalone") {
throw new Error(`${configLabel}.targets.${id}.k3s.management.mode must be external-cluster or standalone`);
}
if (typeof k3sManagement.configRef !== "string" || !k3sManagement.configRef.startsWith("config/platform-infra/k3s-clusters.yaml#")) {
throw new Error(`${configLabel}.targets.${id}.k3s.management.configRef must reference the k3s cluster owning YAML`);
}
@@ -6,6 +6,7 @@ export const K3S_CLUSTER_CONFIG_LABEL = "config/platform-infra/k3s-clusters.yaml
export interface K3sClusterTarget {
id: string;
clusterName: string;
topologyMode: "joined" | "separated";
controlPlane: {
id: string;
configRef: string;
@@ -61,6 +62,13 @@ export interface K3sClusterTarget {
agentTokenFile: string;
flannelInterface: string;
maxPods: number;
standalone: {
clusterName: string;
nodeName: string;
writeKubeconfigMode: string;
labels: Record<string, string>;
disableComponents: string[];
};
registryMirror: {
source: string;
endpoint: string;
@@ -112,6 +120,7 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri
const tcpTuning = record(raw.tcpTuning, `targets.${id}.tcpTuning`);
const secrets = record(raw.secrets, `targets.${id}.secrets`);
const k3s = record(raw.k3s, `targets.${id}.k3s`);
const standalone = record(k3s.standalone, `targets.${id}.k3s.standalone`);
const mirror = record(k3s.registryMirror, `targets.${id}.k3s.registryMirror`);
const mirrorAuth = record(mirror.auth, `targets.${id}.k3s.registryMirror.auth`);
const registryPullSmoke = record(k3s.registryPullSmoke, `targets.${id}.k3s.registryPullSmoke`);
@@ -134,6 +143,7 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri
const target: K3sClusterTarget = {
id,
clusterName: text(raw, "clusterName", `targets.${id}`),
topologyMode: enumText(raw, "topologyMode", `targets.${id}`, ["joined", "separated"]),
controlPlane: {
id: text(control, "id", `targets.${id}.controlPlane`),
configRef: text(control, "configRef", `targets.${id}.controlPlane`),
@@ -189,6 +199,13 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri
agentTokenFile: text(k3s, "agentTokenFile", `targets.${id}.k3s`),
flannelInterface: text(k3s, "flannelInterface", `targets.${id}.k3s`),
maxPods: integer(k3s, "maxPods", `targets.${id}.k3s`),
standalone: {
clusterName: kubernetesName(standalone, "clusterName", "k3s.standalone"),
nodeName: kubernetesName(standalone, "nodeName", "k3s.standalone"),
writeKubeconfigMode: enumText(standalone, "writeKubeconfigMode", "k3s.standalone", ["0600", "0640", "0644"]),
labels: stringMap(record(standalone.labels, "k3s.standalone.labels"), "k3s.standalone.labels"),
disableComponents: stringArray(standalone.disableComponents, "k3s.standalone.disableComponents"),
},
registryMirror: {
source: text(mirror, "source", "registryMirror"),
endpoint: httpsUrl(mirror, "endpoint", "registryMirror"),
@@ -361,6 +378,13 @@ function stringMap(value: Record<string, unknown>, label: string): Record<string
}));
}
function stringArray(value: unknown, label: string): string[] {
if (!Array.isArray(value) || value.length === 0 || value.some((item) => typeof item !== "string" || item.length === 0)) {
throw new Error(`${label} must be a non-empty string array`);
}
return value as string[];
}
function hostProxyTargetId(ref: string): string {
const match = ref.match(/^config\/platform-infra\/host-proxy\.yaml#targets\.([A-Za-z0-9._-]+)$/u);
if (match === null) throw new Error(`worker hostProxyConfigRef must point at config/platform-infra/host-proxy.yaml#targets.<id>`);
+202 -5
View File
@@ -13,7 +13,7 @@ import {
} from "./platform-infra-host-proxy";
import { address, K3S_CLUSTER_CONFIG_LABEL, readK3sClusterConfig, type K3sClusterTarget } from "./platform-infra-k3s-cluster-config";
type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "drain" | "status" | "smoke" | "registry-smoke";
type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "separate" | "drain" | "status" | "smoke" | "registry-smoke";
interface Options {
action: Action;
@@ -39,18 +39,20 @@ export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig,
if (options.action === "plan") return plan(target);
if (options.action === "secret-init") return secretInit(target, options.confirm);
if (options.action === "registry-credential-init") return registryCredentialInit(target, options.confirm);
if (options.action === "separate") return separateCluster(target, options.confirm);
if (options.action === "drain") return drainWorker(target, options.confirm);
if (options.action === "status") return status(target);
if (options.action === "smoke") return smoke(target, options.confirm);
if (options.action === "status") return target.topologyMode === "separated" ? separatedStatus(target) : status(target);
if (options.action === "smoke") return target.topologyMode === "separated" ? separatedSmoke(target) : smoke(target, options.confirm);
if (options.action === "registry-smoke") return registrySmoke(target, options.confirm);
if (!options.confirm) return { ...plan(target), mode: "dry-run", mutation: false };
if (target.topologyMode === "separated") return separateCluster(target, true);
return apply(target);
}
function parseOptions(args: string[]): Options {
const action = args[0] ?? "plan";
if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "drain" && action !== "status" && action !== "smoke" && action !== "registry-smoke") {
throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|drain|status|smoke|registry-smoke [--target <id>] [--confirm]");
if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "separate" && action !== "drain" && action !== "status" && action !== "smoke" && action !== "registry-smoke") {
throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|separate|drain|status|smoke|registry-smoke [--target <id>] [--confirm]");
}
return { action, targetId: option(args, "--target") ?? undefined, confirm: args.includes("--confirm") };
}
@@ -61,6 +63,7 @@ function plan(target: K3sClusterTarget): Record<string, unknown> {
mutation: false,
targetId: target.id,
source: K3S_CLUSTER_CONFIG_LABEL,
topologyMode: target.topologyMode,
topology: {
clusterName: target.clusterName,
controlPlane: { id: target.controlPlane.id, nodeName: target.controlPlane.nodeName, route: target.controlPlane.route, kubeRoute: target.controlPlane.kubeRoute },
@@ -89,6 +92,7 @@ function plan(target: K3sClusterTarget): Record<string, unknown> {
secretInit: `bun scripts/cli.ts platform-infra k3s-cluster secret-init --target ${target.id} --confirm`,
registryCredentialInit: `bun scripts/cli.ts platform-infra k3s-cluster registry-credential-init --target ${target.id} --confirm`,
apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm`,
separate: `bun scripts/cli.ts platform-infra k3s-cluster separate --target ${target.id} --confirm`,
drain: `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm`,
status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`,
smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`,
@@ -282,6 +286,105 @@ function apply(target: K3sClusterTarget): Record<string, unknown> {
};
}
function separateCluster(target: K3sClusterTarget, confirm: boolean): Record<string, unknown> {
if (target.topologyMode !== "separated") throw new Error("separate requires topologyMode=separated in the owning YAML");
const workloads = workerWorkloadStatus(target);
if (workloads.ok !== true) throw new Error(`worker must be empty before separation: ${bounded(JSON.stringify(workloads))}`);
if (!confirm) {
return {
ok: true,
mutation: false,
mode: "dry-run",
targetId: target.id,
topologyMode: target.topologyMode,
workerWorkloads: workloads,
standalone: target.k3s.standalone,
};
}
const secret = readSecrets(target);
const standalone = runWorker(target, standaloneApplyScript(target, secret), true, 300_000);
requireSuccess(standalone, "worker-standalone-k3s-apply");
const localStatus = waitForStandalone(target, 180_000);
requireSuccess(localStatus, "worker-standalone-k3s-status");
const localData = parseFirstJson(localStatus.stdout);
if (localData.ok !== true) throw new Error(`standalone cluster did not converge: ${bounded(JSON.stringify(localData))}`);
const deleteOldNode = runCommand([
"trans", target.controlPlane.kubeRoute, "kubectl", "delete", "node", target.worker.nodeName, "--ignore-not-found=true",
], rootPath(), { timeoutMs: 60_000 });
requireSuccess(deleteOldNode, "old-cluster-worker-delete");
const retirePeer = runControlPlane(target, controlPlaneRetireWorkerPeerScript(target, secret), 60_000);
requireSuccess(retirePeer, "control-plane-wireguard-peer-retire");
const result = separatedStatus(target);
if (result.ok !== true) throw new Error(`cluster separation final status failed: ${bounded(JSON.stringify(result))}`);
return {
ok: true,
mutation: true,
targetId: target.id,
topologyMode: target.topologyMode,
standalone: localData,
oldClusterNodeDeleted: true,
wireguardPeerRetired: true,
stages: {
standalone: compact(standalone),
oldClusterNode: compact(deleteOldNode),
controlPlaneWireguard: compact(retirePeer),
},
next: {
status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`,
smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`,
},
};
}
function separatedStatus(target: K3sClusterTarget): Record<string, unknown> {
const standalone = runWorker(target, standaloneStatusScript(target), false, 60_000);
const standaloneData = parseFirstJson(standalone.stdout);
const oldNode = runCommand([
"trans", target.controlPlane.kubeRoute, "kubectl", "get", "node", target.worker.nodeName, "-o", "name",
], rootPath(), { timeoutMs: 30_000 });
const oldNodeAbsent = oldNode.exitCode !== 0;
return {
ok: standalone.exitCode === 0 && standaloneData.ok === true && oldNodeAbsent,
mutation: false,
targetId: target.id,
topologyMode: target.topologyMode,
standalone: standaloneData,
oldCluster: {
controlPlane: target.controlPlane.id,
workerNodeAbsent: oldNodeAbsent,
},
result: {
standalone: { exitCode: standalone.exitCode },
oldClusterNode: { exitCode: oldNode.exitCode },
},
};
}
function waitForStandalone(target: K3sClusterTarget, timeoutMs: number): ReturnType<typeof runCommand> {
const deadline = Date.now() + timeoutMs;
let current = runWorker(target, standaloneStatusScript(target), false, 55_000);
while ((current.exitCode !== 0 || parseFirstJson(current.stdout).ok !== true) && Date.now() < deadline) {
runCommand(["sleep", "5"], rootPath(), { timeoutMs: 10_000 });
current = runWorker(target, standaloneStatusScript(target), false, 55_000);
}
if (current.exitCode === 0 && parseFirstJson(current.stdout).ok === true) return current;
return { ...current, exitCode: 1, stderr: current.stderr || `standalone cluster did not become ready within ${timeoutMs}ms` };
}
function separatedSmoke(target: K3sClusterTarget): Record<string, unknown> {
const status = separatedStatus(target);
return {
...status,
action: "separated-cluster-smoke",
checks: {
standaloneSingleNodeReady: (status.standalone as Record<string, unknown>)?.ok === true,
oldClusterNodeAbsent: (status.oldCluster as Record<string, unknown>)?.workerNodeAbsent === true,
businessPodCountZero: (status.standalone as Record<string, unknown>)?.businessPodCount === 0,
},
};
}
function status(target: K3sClusterTarget): Record<string, unknown> {
const control = runControlPlane(target, statusScript(target), 60_000);
const workerProxy = runWorker(target, hostProxyGuestStatusScript(target.worker.hostProxyTargetId), false, 60_000);
@@ -1013,6 +1116,100 @@ wg show ${wg.interfaceName} >/dev/null
`;
}
function controlPlaneRetireWorkerPeerScript(target: K3sClusterTarget, secret: SecretMaterial): string {
const wg = target.wireguard;
return `set -eu
umask 077
candidate="$(mktemp)"
trap 'rm -f "$candidate"' EXIT
cat >"$candidate" <<'EOF'
[Interface]
Address = ${wg.controlPlaneAddress}
ListenPort = ${wg.listenPort}
PrivateKey = ${secret.controlPrivate}
MTU = ${wg.mtu}
EOF
if ! cmp -s "$candidate" /etc/wireguard/${wg.interfaceName}.conf; then
install -m 0600 "$candidate" /etc/wireguard/${wg.interfaceName}.conf
systemctl restart wg-quick@${wg.interfaceName}
fi
wg show ${wg.interfaceName} peers | grep -q . && { echo "retired worker peer still present" >&2; exit 1; }
`;
}
function standaloneApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string {
const standalone = target.k3s.standalone;
const disableArgs = standalone.disableComponents.flatMap((component) => ["--disable", component]);
const labels = Object.entries(standalone.labels).flatMap(([key, value]) => ["--node-label", `${key}=${value}`]);
const exec = [
"server",
...disableArgs,
"--cluster-init",
"--node-name", standalone.nodeName,
...labels,
"--write-kubeconfig-mode", standalone.writeKubeconfigMode,
"--kubelet-arg", `max-pods=${target.k3s.maxPods}`,
].map(shellToken).join(" ");
const desiredHash = hash(JSON.stringify({
version: target.worker.k3sVersion,
exec,
registryEndpoint: target.k3s.registryMirror.endpoint,
registryUsername: secret.registryUsername,
registryPassword: secret.registryPassword,
}));
return `set -eu
desired_hash=${shQuote(desiredHash)}
desired_hash_file=/etc/unidesk/k3s-standalone-desired.sha256
if [ "$(cat "$desired_hash_file" 2>/dev/null || true)" = "$desired_hash" ] && systemctl is-active --quiet k3s; then
exit 0
fi
systemctl disable --now wg-quick@${target.wireguard.interfaceName} >/dev/null 2>&1 || true
if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh; fi
if [ -x /usr/local/bin/k3s-uninstall.sh ]; then /usr/local/bin/k3s-uninstall.sh; fi
rm -rf /var/lib/rancher/k3s /etc/rancher/k3s /var/lib/kubelet /var/lib/cni /etc/cni/net.d /run/k3s /run/flannel
ip link delete cni0 >/dev/null 2>&1 || true
ip link delete flannel.1 >/dev/null 2>&1 || true
rm -f /etc/wireguard/${target.wireguard.interfaceName}.conf
install -d -m 0700 /etc/rancher/k3s /etc/unidesk
cat >/etc/rancher/k3s/registries.yaml <<'EOF'
mirrors:
"${target.k3s.registryMirror.source}":
endpoint:
- "${target.k3s.registryMirror.endpoint}"
configs:
"${new URL(target.k3s.registryMirror.endpoint).host}":
auth:
username: ${JSON.stringify(secret.registryUsername)}
password: ${JSON.stringify(secret.registryPassword)}
EOF
chmod 0600 /etc/rancher/k3s/registries.yaml
install -m 0755 /tmp/k3s /usr/local/bin/k3s
chmod 0755 /tmp/install-k3s.sh
INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_VERSION=${shQuote(target.worker.k3sVersion)} INSTALL_K3S_EXEC=${shQuote(exec)} sh /tmp/install-k3s.sh
printf '%s\n' "$desired_hash" >"$desired_hash_file"
chmod 0600 "$desired_hash_file"
systemctl enable k3s >/dev/null
systemctl restart --no-block k3s
`;
}
function standaloneStatusScript(target: K3sClusterTarget): string {
const nodeName = target.k3s.standalone.nodeName;
return `set -eu
active=false; systemctl is-active --quiet k3s && active=true
agent_active=false; systemctl is-active --quiet k3s-agent && agent_active=true
ready="$(k3s kubectl get node ${shQuote(nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
node_count="$(k3s kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ')"
business_pods="$(k3s kubectl get pods -A --no-headers 2>/dev/null | awk '$1 != "kube-system" {count++} END {print count+0}')"
python3 - "$active" "$agent_active" "$ready" "$node_count" "$business_pods" <<'PY'
import json, sys
active, agent, ready, nodes, business = sys.argv[1:]
ok = active == "true" and agent == "false" and ready == "True" and nodes == "1" and business == "0"
print(json.dumps({"ok":ok,"serverActive":active=="true","oldAgentActive":agent=="true","nodeReady":ready=="True","nodeCount":int(nodes or 0),"businessPodCount":int(business or 0)}, separators=(",",":")))
PY
`;
}
function tcpTuningApplyScript(target: K3sClusterTarget): string {
const tuning = target.tcpTuning;
const content = [