fix(platform-infra): split D601 from NC01 cluster
This commit is contained in:
@@ -374,6 +374,18 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help
|
||||
- 验收统一走 `platform-infra k3s-cluster registry-smoke`,不手工 patch Caddy 或 containerd。
|
||||
- worker 安装使用短启动并由 control-plane Node Ready 轮询收口,禁止让 `trans`/SSH 长挂等待 systemd notify。
|
||||
- worker 加入集群不改变 PaC consumer 的 source commit authority,也不授权人工 PipelineRun、Argo sync 或业务 rollout。
|
||||
- NC01 与 D601 是永久分离的调度域:
|
||||
- `config/platform-infra/k3s-clusters.yaml` 必须保持
|
||||
`topologyMode=separated`,D601 只作为独立单节点 k3s 集群存在;
|
||||
- CI/CD、PaC、Tekton、Argo、GitOps、VM bootstrap 和业务 renderer
|
||||
禁止把 D601 注册为 NC01 agent,禁止恢复跨地域 WireGuard peer,
|
||||
禁止把 NC01 workload、DaemonSet 或 namespace 默认复制到 D601;
|
||||
- 拆分和验收只走
|
||||
`platform-infra k3s-cluster separate|status|smoke`;
|
||||
- 跨地域 artifact 传输使用鉴权、只读的公网 HTTPS registry,
|
||||
不得通过 Kubernetes Pod 网络、Service、DNS 或 VXLAN;
|
||||
- 如未来确需跨地域计算,必须建立独立 workload 级交付规格并显式选择
|
||||
D601 集群,不得通过重新组成单一集群获得隐式调度。
|
||||
- 长命令用异步 job 或短轮询;不要长时间挂住 trans/ssh。
|
||||
- Web 哨兵 YAML cadence 变更不新增发布 orchestrator:提交前用 `web-probe sentinel validate --local`;合并后用只读 `web-probe sentinel control-plane status --wait`。线上 desired schedule 必须来自 GitOps 清单,不能用操作者旧 worktree 的本地渲染值做阻塞门禁;收敛差异统一为非阻塞 warning。
|
||||
|
||||
|
||||
@@ -112,6 +112,14 @@ description: UniDesk YAML-first 运维正规化技能。用户提到 ymal-first/
|
||||
- worker agent 使用 YAML 派生摘要决定是否重装,禁止每次 apply 先卸载再覆盖;
|
||||
- 禁止借用全量 infra apply 顺带覆盖 registry、CI/CD 或业务运行面。
|
||||
- 日常入口和验收判定统一见 `docs/reference/platform-infra.md#多主机-k3s-集群`。
|
||||
- NC01-D601 的 owning YAML 必须声明 `topologyMode=separated`:
|
||||
- D601 独立 server 的节点名、标签、禁用组件和 kubeconfig mode 必须
|
||||
位于同一 YAML 的 `k3s.standalone`;
|
||||
- Hyper-V YAML 只能用 `management.mode=standalone` 引用该片段;
|
||||
- parser、bootstrap、CI/CD 和 GitOps 禁止通过隐藏默认值把 D601
|
||||
恢复为 agent、恢复 WireGuard peer或共享调度域;
|
||||
- 独立集群间只共享显式声明的只读 artifact endpoint,不共享 Pod、
|
||||
Service、DNS、CNI、datastore 或默认 workload。
|
||||
- 新节点 node-local registry 必须支持从零部署:优先由 YAML 渲染为 k3s workload/PVC;host Docker registry 只能是 legacy/迁移前状态。需要证明从零能力时先停旧 registry,不从旧 registry seed,并在状态中披露 `seededFromOldRegistry`、`zeroSeeded`、PVC、workload、endpoint 与 runtime/tools image preload 结果。
|
||||
- YAML 配置必须可复用、可继承、可组合;节点/lane 只能作为变量或 YAML value 参与渲染,禁止把具体 node/lane 写入 YAML 文件名、parser/renderer/helper 函数名或长期 schema 名。发现不同节点复制大段配置时,先抽 baseline/configRef/override,再继续扩展节点。
|
||||
- CLI `plan/status` 应输出 redacted 配置引用图:每个 ref 的文件、path、presence、摘要 hash、缺失字段和下一步 drill-down 命令。不要默认 dump 展开后的完整 YAML 或 Secret。
|
||||
|
||||
@@ -73,8 +73,8 @@ targets:
|
||||
- https://docker.m.daocloud.io
|
||||
k3s:
|
||||
management:
|
||||
mode: external-cluster
|
||||
configRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.workers.D601-VM
|
||||
mode: standalone
|
||||
configRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.k3s.standalone
|
||||
version: v1.36.2+k3s1
|
||||
installMirror: cn
|
||||
binaryUrl: https://github.com/k3s-io/k3s/releases/download/v1.36.2%2Bk3s1/k3s
|
||||
|
||||
@@ -9,6 +9,7 @@ targets:
|
||||
nc01-d601:
|
||||
enabled: true
|
||||
clusterName: nc01
|
||||
topologyMode: separated
|
||||
controlPlane:
|
||||
id: NC01
|
||||
configRef: config/hwlab-node-control-plane.yaml#nodes.NC01
|
||||
@@ -57,6 +58,18 @@ targets:
|
||||
agentTokenFile: /etc/unidesk/k3s-agent-token
|
||||
flannelInterface: wg0
|
||||
maxPods: 500
|
||||
standalone:
|
||||
clusterName: d601
|
||||
nodeName: d601-vm
|
||||
writeKubeconfigMode: "0644"
|
||||
labels:
|
||||
unidesk.ai/node-id: D601-VM
|
||||
unidesk.ai/provider-id: D601-VM
|
||||
unidesk.ai/runtime-plane: standalone
|
||||
disableComponents:
|
||||
- traefik
|
||||
- servicelb
|
||||
- metrics-server
|
||||
registryMirror:
|
||||
source: 127.0.0.1:5000
|
||||
endpoint: https://registry.hwpod.com
|
||||
|
||||
@@ -10,16 +10,27 @@
|
||||
- UniDesk 拥有的 image version、namespace、endpoint、FRP port 与 profile 列表不得隐藏在 helper 常量里;外部工具要求的 TOML、JSON 和 env 仅是边缘渲染格式。
|
||||
- 新节点出网 bootstrap 遵循 `docs/reference/yaml-first-ops.md` 的零依赖 host proxy 边界,不得依赖尚未安装的 Docker、k3s、containerd 或包管理器网络能力。
|
||||
|
||||
## 多主机 k3s 集群
|
||||
## k3s 集群拓扑
|
||||
|
||||
- 权威配置:
|
||||
- 跨主机拓扑、WireGuard、agent token `sourceRef`、worker host proxy 引用、集群内 artifact registry 映射、ServiceLB 节点和 smoke 归 `config/platform-infra/k3s-clusters.yaml`。
|
||||
- 集群边界、拓扑模式、历史 WireGuard 迁移参数、独立 server 角色、
|
||||
registry endpoint、ServiceLB 节点和 smoke 归
|
||||
`config/platform-infra/k3s-clusters.yaml`。
|
||||
- worker host proxy 的服务端来源、客户端工件、Secret 引用、代理环境和 systemd drop-in 归 `config/platform-infra/host-proxy.yaml`。
|
||||
- Hyper-V VM 资源和宿主内网 SSH 归 `config/platform-infra/hyperv-vms.yaml`。
|
||||
- 既有 control-plane 的 k3s systemd drop-in 继续归 `config/hwlab-node-control-plane.yaml`,并通过 `clusterConfigRef` 消费集群网络参数。
|
||||
- 受控入口:
|
||||
- `platform-infra k3s-cluster secret-init --target <id> --confirm` 只在 Secret source 不存在时生成 WireGuard key pair 和 k3s agent token,存在时拒绝覆盖。
|
||||
- `platform-infra k3s-cluster plan|apply|drain|status|smoke --target <id>` 是部署、清空不可调度 worker 和验收入口。
|
||||
- `platform-infra k3s-cluster plan|apply|separate|drain|status|smoke
|
||||
--target <id>` 是部署、拓扑拆分、清空不可调度 worker和验收入口。
|
||||
- `topologyMode=separated` 时,`apply --confirm` 与
|
||||
`separate --confirm` 都只执行受控拆分:
|
||||
- 先证明旧 worker 的非 DaemonSet Pod 为零;
|
||||
- 再清理旧 agent datastore、CNI 和跨地域 WireGuard 配置;
|
||||
- 在 D601 安装并等待独立单节点 k3s server Ready;
|
||||
- 最后从 NC01 删除旧 Node 和对端 WireGuard peer。
|
||||
- 拆分不得先删除 NC01 Node 或隧道;D601 独立 server 未 Ready 时必须
|
||||
保留旧控制面侧事实并返回失败。
|
||||
- 无固定公网地址的 worker 由宿主内网 SSH 控制,主动连接 control-plane WireGuard endpoint;worker provider route 只作为部署后补充验收入口。
|
||||
- 集群 apply 先经宿主内网 SSH 分发零依赖 proxy client,确认它连接声明的 `vpn-server` 且外网 probe 成功,再启动或恢复 `k3s-agent`。
|
||||
- 集群 apply 在 proxy 前先收敛 worker WireGuard;内容一致时不强写,只有 unit 未运行或握手过期且隧道探针失败时才恢复 worker 接口。
|
||||
@@ -48,10 +59,25 @@
|
||||
- 同一入口等待节点 Ready;
|
||||
- 禁止调用 HWLAB 全量 `infra apply`,也不得触碰 Kubernetes manifests、registry、Tekton、Argo、git-mirror 或业务 rollout。
|
||||
- 运行边界:
|
||||
- control-plane 保持唯一 k3s server,同时承担 workload。
|
||||
- worker 只运行 `k3s-agent`;加入前必须卸载旧独立 server,禁止保留第二 datastore/control-plane。
|
||||
- worker 是否参与业务调度及其 taint 必须由集群 owning YAML 的
|
||||
`workers.<id>.scheduling` 声明:
|
||||
- NC01 与 D601 必须保持两个独立 k3s 集群,禁止把 D601 重新注册为
|
||||
NC01 agent,也禁止把 NC01 注册到 D601。
|
||||
- D601 独立集群只运行一个 server 节点,初始业务 Pod 数必须为零;
|
||||
是否承载新业务必须由新的 workload owning YAML 显式选择,不得继承
|
||||
NC01 的 GitOps、PipelineRun、DaemonSet 或默认 namespace。
|
||||
- NC01 保留自身 control-plane 与运行面;拆分后可保留无 peer 的本地
|
||||
WireGuard 接口以避免重写生产 control-plane 地址,但不得恢复 D601 peer。
|
||||
- 跨地域连接只用于受控 SSH/Provider 控制通道和公网 HTTPS
|
||||
artifact pull,不再承担 Kubernetes API、Pod、Service、DNS 或 VXLAN 数据面。
|
||||
- `topologyMode=separated` 是 owning YAML 的架构事实。CI/CD、GitOps、
|
||||
VM bootstrap 和普通集群 apply 禁止改回 joined、启动 `k3s-agent`、
|
||||
写入 agent token 或渲染跨地域 Flannel/WireGuard peer。
|
||||
- 下列 worker 规则只适用于仍由 owning YAML 明确声明为 joined 的其他
|
||||
集群,不得用于 NC01-D601:
|
||||
- control-plane 保持唯一 k3s server,同时承担 workload;
|
||||
- worker 只运行 `k3s-agent`,加入前卸载旧独立 server;
|
||||
- worker 是否参与业务调度及其 taint 必须由集群 owning YAML 的
|
||||
`workers.<id>.scheduling` 声明。
|
||||
- joined worker 的调度规则:
|
||||
- `schedulable=false` 时必须同时声明 `NoSchedule` 或 `NoExecute` taint;
|
||||
- agent 注册参数和 control-plane 的 `cordon`/taint 收敛必须消费同一声明,
|
||||
防止节点重装、重注册或 CI 自动覆盖后短暂接收业务 Pod;
|
||||
@@ -88,7 +114,13 @@
|
||||
- k3s API 暂时不可读属于 unknown,不得当作容量漂移触发 restart;
|
||||
- 集群 CLI 只组合各自 owning domain,不借用全量基础设施入口扩大写入范围。
|
||||
- 验收:
|
||||
- `status` 必须同时证明 WireGuard handshake、worker host proxy、server/agent systemd、两个 Ready node、worker InternalIP 和旧 server 退役。
|
||||
- `topologyMode=separated` 的 `status` 必须同时证明:
|
||||
- D601 只有一个 Ready 节点;
|
||||
- D601 `k3s` server active 且旧 `k3s-agent` inactive;
|
||||
- D601 非 `kube-system` Pod 为零;
|
||||
- NC01 不再存在 D601 Node。
|
||||
- joined 拓扑的 `status` 才验证 WireGuard handshake、host proxy、
|
||||
server/agent、两个 Ready node、worker InternalIP 和旧 server 退役。
|
||||
- 仅当 worker 的 `scheduling.schedulable=true` 时,
|
||||
`smoke --confirm` 才可把 Job 定向调度到 worker并验证集群 DNS与
|
||||
Kubernetes ClusterIP;不可调度 worker 的 smoke 只读验证 taint、
|
||||
|
||||
@@ -18,7 +18,7 @@
|
||||
- Hyper-V 虚拟机、VHDX、处理器、内存和自动启动策略。
|
||||
- 独立内部交换机、NAT、固定地址和出网连通性。
|
||||
- Ubuntu cloud-init、SSH、Docker 和受控 k3s 运行角色初始化。
|
||||
- 无固定公网地址 worker 通过原生 WireGuard 主动接入既有 k3s 控制面。
|
||||
- D601-VM 运行 YAML 声明的独立单节点 k3s 空集群。
|
||||
- provider-gateway 的节点身份、常驻恢复和 trans 验收。
|
||||
- 后台安装的阶段、日志、错误和终态可见性。
|
||||
|
||||
@@ -57,8 +57,9 @@ Hyper-V 虚拟机必须配置为宿主启动后自动启动、宿主关机时保
|
||||
- VM 为运行状态且资源与 YAML 一致。
|
||||
- Ubuntu 固定地址可达并能正常出网。
|
||||
- Docker daemon 和 k3s 节点健康。
|
||||
- worker 以 agent 身份加入声明的控制面,且独立 server 不再运行。
|
||||
- 定向调度到 worker 的 smoke 能完成集群 DNS 与 ClusterIP 访问。
|
||||
- D601 只运行独立 k3s server,旧 agent 已停止。
|
||||
- 独立集群只有一个 Ready 节点,且非 `kube-system` Pod 为零。
|
||||
- NC01 集群内不存在 D601 Node。
|
||||
- provider-gateway 以 YAML 声明的 Provider ID 在线。
|
||||
- `trans <provider-id> hostname` 和 k3s route 均可用。
|
||||
|
||||
@@ -74,24 +75,27 @@ Provider TCP 数据池恢复必须满足:
|
||||
- 修复范围只包含 Provider 服务,不得连带重启 VM、Docker 或 k3s。
|
||||
- 同一入口必须在修复后重新执行 host 和 k3s route 验收。
|
||||
|
||||
### 3.8 HYPERV-VM-REQ-008 多主机 k3s 运行面
|
||||
### 3.8 HYPERV-VM-REQ-008 独立 k3s 运行面
|
||||
|
||||
既有控制面与 Hyper-V worker 组成集群时必须满足:
|
||||
D601-VM 与 NC01 必须作为两个独立集群运行:
|
||||
|
||||
- 控制面保持唯一 server,并继续承担 workload。
|
||||
- Hyper-V 节点只能安装为 agent,不得保留独立 server。
|
||||
- 无固定公网地址的 worker 通过主动拨号的原生 WireGuard 三层隧道接入控制面。
|
||||
- WireGuard 私钥和 k3s agent token 只能来自 YAML `sourceRef`,不得从运行面反解或打印。
|
||||
- k3s 节点地址、Flannel 接口、API TLS SAN、集群内 artifact registry 映射和 ServiceLB 节点选择必须由 owning YAML 渲染。
|
||||
- worker 的业务调度资格、cordon 和 taint 必须由 owning YAML 声明并由
|
||||
agent 注册参数与受控集群入口共同收敛。声明为不可调度的 worker 不得承载
|
||||
非 DaemonSet 业务 Pod;CI/CD、GitOps 和 smoke 不得通过 toleration、
|
||||
`nodeName` 或删除 taint 绕过该隔离。
|
||||
- worker 必须先受控部署 host proxy client 并连接 YAML 引用的 `vpn-server`,再启动 agent;containerd 直接通过该 proxy 访问公共 registry,不得依赖公共镜像站或新增 registry mirror。
|
||||
- `config/platform-infra/k3s-clusters.yaml` 必须显式声明
|
||||
`topologyMode=separated` 和 D601 `k3s.standalone` 参数。
|
||||
- `config/platform-infra/hyperv-vms.yaml` 必须以
|
||||
`management.mode=standalone` 引用独立集群配置。
|
||||
- D601 只运行单节点 k3s server;旧 `k3s-agent`、agent datastore、CNI
|
||||
和跨地域 WireGuard peer 必须由受控拆分入口退役。
|
||||
- 拆分前必须证明 D601 非 DaemonSet Pod 为零;D601 独立 server Ready
|
||||
后,才能从 NC01 删除旧 Node 和 WireGuard peer。
|
||||
- 拆分后 D601 非 `kube-system` Pod 必须为零,NC01 不得存在 D601 Node。
|
||||
- CI/CD、GitOps、VM bootstrap 和业务 renderer 禁止重新安装 agent、
|
||||
恢复 agent token、重建跨地域 CNI 或把 NC01 workload 自动投递到 D601。
|
||||
- D601 host proxy 可继续为独立集群提供公共依赖出网,不得依赖公共镜像站
|
||||
或新增公共 registry mirror。
|
||||
- 集群自有 artifact registry 可以通过公网可信 TLS 向异地 worker 提供鉴权只读拉取:
|
||||
- 内部 CI/publish 继续使用集群内写入 endpoint,公网入口不得成为第二个 push authority。
|
||||
- 公网边缘只允许 Registry V2 拉取所需的 `GET` 与 `HEAD`,其他方法必须在反向代理层拒绝。
|
||||
- endpoint、DNS、上游、允许方法、凭据 `sourceRef`、镜像摘要和拉取验收参数必须来自集群 owning YAML。
|
||||
- worker containerd 继续消费既有镜像引用,只把该内部引用映射到声明的公网 HTTPS endpoint。
|
||||
- D601 containerd 继续消费既有镜像引用,只把该内部引用映射到声明的公网 HTTPS endpoint。
|
||||
- artifact 大文件不得经过 WireGuard、Flannel/VXLAN 或受传输时长和对象大小限制的 CDN。
|
||||
- 宿主内网 SSH 可以作为 worker 的稳定部署控制通道,provider route 只作为部署后验收入口。
|
||||
- 宿主内网 SSH 可以作为 D601 的稳定部署控制通道,provider route 只作为部署后验收入口。
|
||||
|
||||
@@ -492,12 +492,14 @@ try {
|
||||
Wait-For-Ssh -KeyPath $key.private
|
||||
if ([string]$Config.k3s.management.mode -eq 'external-cluster') {
|
||||
Write-State -Stage 'staging-k3s-agent' -Message 'Pinned k3s artifacts are staged; the YAML-controlled cluster manager owns the agent role'
|
||||
} elseif ([string]$Config.k3s.management.mode -eq 'standalone') {
|
||||
Write-State -Stage 'staging-k3s-standalone' -Message 'Pinned k3s artifacts are staged; the YAML-controlled cluster manager owns the standalone role'
|
||||
} else {
|
||||
throw "unsupported-k3s-management-mode: $($Config.k3s.management.mode)"
|
||||
}
|
||||
Install-Provider -KeyPath $key.private
|
||||
|
||||
Write-State -Stage 'completed' -State 'completed' -Message 'Ubuntu VM, Docker, pinned k3s artifacts and provider-gateway are ready for the external cluster manager' -Extra @{
|
||||
Write-State -Stage 'completed' -State 'completed' -Message 'Ubuntu VM, Docker, YAML-managed k3s and provider-gateway are ready' -Extra @{
|
||||
address = [string]$Config.network.address
|
||||
providerId = [string]$Config.provider.id
|
||||
processorCount = [int]$Config.vm.processorCount
|
||||
|
||||
@@ -100,7 +100,9 @@ function validateTarget(id: string, target: HyperVTarget): void {
|
||||
if (!/^[a-f0-9]{64}$/u.test(target.image.sha256)) throw new Error(`${configLabel}.targets.${id}.image.sha256 must be lowercase SHA-256`);
|
||||
if (target.image.sources.length < 2) throw new Error(`${configLabel}.targets.${id}.image.sources must include mirror and canonical fallback`);
|
||||
const k3sManagement = record(target.k3s.management, `${configLabel}.targets.${id}.k3s.management`);
|
||||
if (k3sManagement.mode !== "external-cluster") throw new Error(`${configLabel}.targets.${id}.k3s.management.mode must be external-cluster`);
|
||||
if (k3sManagement.mode !== "external-cluster" && k3sManagement.mode !== "standalone") {
|
||||
throw new Error(`${configLabel}.targets.${id}.k3s.management.mode must be external-cluster or standalone`);
|
||||
}
|
||||
if (typeof k3sManagement.configRef !== "string" || !k3sManagement.configRef.startsWith("config/platform-infra/k3s-clusters.yaml#")) {
|
||||
throw new Error(`${configLabel}.targets.${id}.k3s.management.configRef must reference the k3s cluster owning YAML`);
|
||||
}
|
||||
|
||||
@@ -6,6 +6,7 @@ export const K3S_CLUSTER_CONFIG_LABEL = "config/platform-infra/k3s-clusters.yaml
|
||||
export interface K3sClusterTarget {
|
||||
id: string;
|
||||
clusterName: string;
|
||||
topologyMode: "joined" | "separated";
|
||||
controlPlane: {
|
||||
id: string;
|
||||
configRef: string;
|
||||
@@ -61,6 +62,13 @@ export interface K3sClusterTarget {
|
||||
agentTokenFile: string;
|
||||
flannelInterface: string;
|
||||
maxPods: number;
|
||||
standalone: {
|
||||
clusterName: string;
|
||||
nodeName: string;
|
||||
writeKubeconfigMode: string;
|
||||
labels: Record<string, string>;
|
||||
disableComponents: string[];
|
||||
};
|
||||
registryMirror: {
|
||||
source: string;
|
||||
endpoint: string;
|
||||
@@ -112,6 +120,7 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri
|
||||
const tcpTuning = record(raw.tcpTuning, `targets.${id}.tcpTuning`);
|
||||
const secrets = record(raw.secrets, `targets.${id}.secrets`);
|
||||
const k3s = record(raw.k3s, `targets.${id}.k3s`);
|
||||
const standalone = record(k3s.standalone, `targets.${id}.k3s.standalone`);
|
||||
const mirror = record(k3s.registryMirror, `targets.${id}.k3s.registryMirror`);
|
||||
const mirrorAuth = record(mirror.auth, `targets.${id}.k3s.registryMirror.auth`);
|
||||
const registryPullSmoke = record(k3s.registryPullSmoke, `targets.${id}.k3s.registryPullSmoke`);
|
||||
@@ -134,6 +143,7 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri
|
||||
const target: K3sClusterTarget = {
|
||||
id,
|
||||
clusterName: text(raw, "clusterName", `targets.${id}`),
|
||||
topologyMode: enumText(raw, "topologyMode", `targets.${id}`, ["joined", "separated"]),
|
||||
controlPlane: {
|
||||
id: text(control, "id", `targets.${id}.controlPlane`),
|
||||
configRef: text(control, "configRef", `targets.${id}.controlPlane`),
|
||||
@@ -189,6 +199,13 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri
|
||||
agentTokenFile: text(k3s, "agentTokenFile", `targets.${id}.k3s`),
|
||||
flannelInterface: text(k3s, "flannelInterface", `targets.${id}.k3s`),
|
||||
maxPods: integer(k3s, "maxPods", `targets.${id}.k3s`),
|
||||
standalone: {
|
||||
clusterName: kubernetesName(standalone, "clusterName", "k3s.standalone"),
|
||||
nodeName: kubernetesName(standalone, "nodeName", "k3s.standalone"),
|
||||
writeKubeconfigMode: enumText(standalone, "writeKubeconfigMode", "k3s.standalone", ["0600", "0640", "0644"]),
|
||||
labels: stringMap(record(standalone.labels, "k3s.standalone.labels"), "k3s.standalone.labels"),
|
||||
disableComponents: stringArray(standalone.disableComponents, "k3s.standalone.disableComponents"),
|
||||
},
|
||||
registryMirror: {
|
||||
source: text(mirror, "source", "registryMirror"),
|
||||
endpoint: httpsUrl(mirror, "endpoint", "registryMirror"),
|
||||
@@ -361,6 +378,13 @@ function stringMap(value: Record<string, unknown>, label: string): Record<string
|
||||
}));
|
||||
}
|
||||
|
||||
function stringArray(value: unknown, label: string): string[] {
|
||||
if (!Array.isArray(value) || value.length === 0 || value.some((item) => typeof item !== "string" || item.length === 0)) {
|
||||
throw new Error(`${label} must be a non-empty string array`);
|
||||
}
|
||||
return value as string[];
|
||||
}
|
||||
|
||||
function hostProxyTargetId(ref: string): string {
|
||||
const match = ref.match(/^config\/platform-infra\/host-proxy\.yaml#targets\.([A-Za-z0-9._-]+)$/u);
|
||||
if (match === null) throw new Error(`worker hostProxyConfigRef must point at config/platform-infra/host-proxy.yaml#targets.<id>`);
|
||||
|
||||
@@ -13,7 +13,7 @@ import {
|
||||
} from "./platform-infra-host-proxy";
|
||||
import { address, K3S_CLUSTER_CONFIG_LABEL, readK3sClusterConfig, type K3sClusterTarget } from "./platform-infra-k3s-cluster-config";
|
||||
|
||||
type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "drain" | "status" | "smoke" | "registry-smoke";
|
||||
type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "separate" | "drain" | "status" | "smoke" | "registry-smoke";
|
||||
|
||||
interface Options {
|
||||
action: Action;
|
||||
@@ -39,18 +39,20 @@ export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig,
|
||||
if (options.action === "plan") return plan(target);
|
||||
if (options.action === "secret-init") return secretInit(target, options.confirm);
|
||||
if (options.action === "registry-credential-init") return registryCredentialInit(target, options.confirm);
|
||||
if (options.action === "separate") return separateCluster(target, options.confirm);
|
||||
if (options.action === "drain") return drainWorker(target, options.confirm);
|
||||
if (options.action === "status") return status(target);
|
||||
if (options.action === "smoke") return smoke(target, options.confirm);
|
||||
if (options.action === "status") return target.topologyMode === "separated" ? separatedStatus(target) : status(target);
|
||||
if (options.action === "smoke") return target.topologyMode === "separated" ? separatedSmoke(target) : smoke(target, options.confirm);
|
||||
if (options.action === "registry-smoke") return registrySmoke(target, options.confirm);
|
||||
if (!options.confirm) return { ...plan(target), mode: "dry-run", mutation: false };
|
||||
if (target.topologyMode === "separated") return separateCluster(target, true);
|
||||
return apply(target);
|
||||
}
|
||||
|
||||
function parseOptions(args: string[]): Options {
|
||||
const action = args[0] ?? "plan";
|
||||
if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "drain" && action !== "status" && action !== "smoke" && action !== "registry-smoke") {
|
||||
throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|drain|status|smoke|registry-smoke [--target <id>] [--confirm]");
|
||||
if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "separate" && action !== "drain" && action !== "status" && action !== "smoke" && action !== "registry-smoke") {
|
||||
throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|separate|drain|status|smoke|registry-smoke [--target <id>] [--confirm]");
|
||||
}
|
||||
return { action, targetId: option(args, "--target") ?? undefined, confirm: args.includes("--confirm") };
|
||||
}
|
||||
@@ -61,6 +63,7 @@ function plan(target: K3sClusterTarget): Record<string, unknown> {
|
||||
mutation: false,
|
||||
targetId: target.id,
|
||||
source: K3S_CLUSTER_CONFIG_LABEL,
|
||||
topologyMode: target.topologyMode,
|
||||
topology: {
|
||||
clusterName: target.clusterName,
|
||||
controlPlane: { id: target.controlPlane.id, nodeName: target.controlPlane.nodeName, route: target.controlPlane.route, kubeRoute: target.controlPlane.kubeRoute },
|
||||
@@ -89,6 +92,7 @@ function plan(target: K3sClusterTarget): Record<string, unknown> {
|
||||
secretInit: `bun scripts/cli.ts platform-infra k3s-cluster secret-init --target ${target.id} --confirm`,
|
||||
registryCredentialInit: `bun scripts/cli.ts platform-infra k3s-cluster registry-credential-init --target ${target.id} --confirm`,
|
||||
apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm`,
|
||||
separate: `bun scripts/cli.ts platform-infra k3s-cluster separate --target ${target.id} --confirm`,
|
||||
drain: `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm`,
|
||||
status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`,
|
||||
smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`,
|
||||
@@ -282,6 +286,105 @@ function apply(target: K3sClusterTarget): Record<string, unknown> {
|
||||
};
|
||||
}
|
||||
|
||||
function separateCluster(target: K3sClusterTarget, confirm: boolean): Record<string, unknown> {
|
||||
if (target.topologyMode !== "separated") throw new Error("separate requires topologyMode=separated in the owning YAML");
|
||||
const workloads = workerWorkloadStatus(target);
|
||||
if (workloads.ok !== true) throw new Error(`worker must be empty before separation: ${bounded(JSON.stringify(workloads))}`);
|
||||
if (!confirm) {
|
||||
return {
|
||||
ok: true,
|
||||
mutation: false,
|
||||
mode: "dry-run",
|
||||
targetId: target.id,
|
||||
topologyMode: target.topologyMode,
|
||||
workerWorkloads: workloads,
|
||||
standalone: target.k3s.standalone,
|
||||
};
|
||||
}
|
||||
const secret = readSecrets(target);
|
||||
const standalone = runWorker(target, standaloneApplyScript(target, secret), true, 300_000);
|
||||
requireSuccess(standalone, "worker-standalone-k3s-apply");
|
||||
const localStatus = waitForStandalone(target, 180_000);
|
||||
requireSuccess(localStatus, "worker-standalone-k3s-status");
|
||||
const localData = parseFirstJson(localStatus.stdout);
|
||||
if (localData.ok !== true) throw new Error(`standalone cluster did not converge: ${bounded(JSON.stringify(localData))}`);
|
||||
|
||||
const deleteOldNode = runCommand([
|
||||
"trans", target.controlPlane.kubeRoute, "kubectl", "delete", "node", target.worker.nodeName, "--ignore-not-found=true",
|
||||
], rootPath(), { timeoutMs: 60_000 });
|
||||
requireSuccess(deleteOldNode, "old-cluster-worker-delete");
|
||||
const retirePeer = runControlPlane(target, controlPlaneRetireWorkerPeerScript(target, secret), 60_000);
|
||||
requireSuccess(retirePeer, "control-plane-wireguard-peer-retire");
|
||||
const result = separatedStatus(target);
|
||||
if (result.ok !== true) throw new Error(`cluster separation final status failed: ${bounded(JSON.stringify(result))}`);
|
||||
return {
|
||||
ok: true,
|
||||
mutation: true,
|
||||
targetId: target.id,
|
||||
topologyMode: target.topologyMode,
|
||||
standalone: localData,
|
||||
oldClusterNodeDeleted: true,
|
||||
wireguardPeerRetired: true,
|
||||
stages: {
|
||||
standalone: compact(standalone),
|
||||
oldClusterNode: compact(deleteOldNode),
|
||||
controlPlaneWireguard: compact(retirePeer),
|
||||
},
|
||||
next: {
|
||||
status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`,
|
||||
smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function separatedStatus(target: K3sClusterTarget): Record<string, unknown> {
|
||||
const standalone = runWorker(target, standaloneStatusScript(target), false, 60_000);
|
||||
const standaloneData = parseFirstJson(standalone.stdout);
|
||||
const oldNode = runCommand([
|
||||
"trans", target.controlPlane.kubeRoute, "kubectl", "get", "node", target.worker.nodeName, "-o", "name",
|
||||
], rootPath(), { timeoutMs: 30_000 });
|
||||
const oldNodeAbsent = oldNode.exitCode !== 0;
|
||||
return {
|
||||
ok: standalone.exitCode === 0 && standaloneData.ok === true && oldNodeAbsent,
|
||||
mutation: false,
|
||||
targetId: target.id,
|
||||
topologyMode: target.topologyMode,
|
||||
standalone: standaloneData,
|
||||
oldCluster: {
|
||||
controlPlane: target.controlPlane.id,
|
||||
workerNodeAbsent: oldNodeAbsent,
|
||||
},
|
||||
result: {
|
||||
standalone: { exitCode: standalone.exitCode },
|
||||
oldClusterNode: { exitCode: oldNode.exitCode },
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function waitForStandalone(target: K3sClusterTarget, timeoutMs: number): ReturnType<typeof runCommand> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let current = runWorker(target, standaloneStatusScript(target), false, 55_000);
|
||||
while ((current.exitCode !== 0 || parseFirstJson(current.stdout).ok !== true) && Date.now() < deadline) {
|
||||
runCommand(["sleep", "5"], rootPath(), { timeoutMs: 10_000 });
|
||||
current = runWorker(target, standaloneStatusScript(target), false, 55_000);
|
||||
}
|
||||
if (current.exitCode === 0 && parseFirstJson(current.stdout).ok === true) return current;
|
||||
return { ...current, exitCode: 1, stderr: current.stderr || `standalone cluster did not become ready within ${timeoutMs}ms` };
|
||||
}
|
||||
|
||||
function separatedSmoke(target: K3sClusterTarget): Record<string, unknown> {
|
||||
const status = separatedStatus(target);
|
||||
return {
|
||||
...status,
|
||||
action: "separated-cluster-smoke",
|
||||
checks: {
|
||||
standaloneSingleNodeReady: (status.standalone as Record<string, unknown>)?.ok === true,
|
||||
oldClusterNodeAbsent: (status.oldCluster as Record<string, unknown>)?.workerNodeAbsent === true,
|
||||
businessPodCountZero: (status.standalone as Record<string, unknown>)?.businessPodCount === 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function status(target: K3sClusterTarget): Record<string, unknown> {
|
||||
const control = runControlPlane(target, statusScript(target), 60_000);
|
||||
const workerProxy = runWorker(target, hostProxyGuestStatusScript(target.worker.hostProxyTargetId), false, 60_000);
|
||||
@@ -1013,6 +1116,100 @@ wg show ${wg.interfaceName} >/dev/null
|
||||
`;
|
||||
}
|
||||
|
||||
function controlPlaneRetireWorkerPeerScript(target: K3sClusterTarget, secret: SecretMaterial): string {
|
||||
const wg = target.wireguard;
|
||||
return `set -eu
|
||||
umask 077
|
||||
candidate="$(mktemp)"
|
||||
trap 'rm -f "$candidate"' EXIT
|
||||
cat >"$candidate" <<'EOF'
|
||||
[Interface]
|
||||
Address = ${wg.controlPlaneAddress}
|
||||
ListenPort = ${wg.listenPort}
|
||||
PrivateKey = ${secret.controlPrivate}
|
||||
MTU = ${wg.mtu}
|
||||
EOF
|
||||
if ! cmp -s "$candidate" /etc/wireguard/${wg.interfaceName}.conf; then
|
||||
install -m 0600 "$candidate" /etc/wireguard/${wg.interfaceName}.conf
|
||||
systemctl restart wg-quick@${wg.interfaceName}
|
||||
fi
|
||||
wg show ${wg.interfaceName} peers | grep -q . && { echo "retired worker peer still present" >&2; exit 1; }
|
||||
`;
|
||||
}
|
||||
|
||||
function standaloneApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string {
|
||||
const standalone = target.k3s.standalone;
|
||||
const disableArgs = standalone.disableComponents.flatMap((component) => ["--disable", component]);
|
||||
const labels = Object.entries(standalone.labels).flatMap(([key, value]) => ["--node-label", `${key}=${value}`]);
|
||||
const exec = [
|
||||
"server",
|
||||
...disableArgs,
|
||||
"--cluster-init",
|
||||
"--node-name", standalone.nodeName,
|
||||
...labels,
|
||||
"--write-kubeconfig-mode", standalone.writeKubeconfigMode,
|
||||
"--kubelet-arg", `max-pods=${target.k3s.maxPods}`,
|
||||
].map(shellToken).join(" ");
|
||||
const desiredHash = hash(JSON.stringify({
|
||||
version: target.worker.k3sVersion,
|
||||
exec,
|
||||
registryEndpoint: target.k3s.registryMirror.endpoint,
|
||||
registryUsername: secret.registryUsername,
|
||||
registryPassword: secret.registryPassword,
|
||||
}));
|
||||
return `set -eu
|
||||
desired_hash=${shQuote(desiredHash)}
|
||||
desired_hash_file=/etc/unidesk/k3s-standalone-desired.sha256
|
||||
if [ "$(cat "$desired_hash_file" 2>/dev/null || true)" = "$desired_hash" ] && systemctl is-active --quiet k3s; then
|
||||
exit 0
|
||||
fi
|
||||
systemctl disable --now wg-quick@${target.wireguard.interfaceName} >/dev/null 2>&1 || true
|
||||
if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh; fi
|
||||
if [ -x /usr/local/bin/k3s-uninstall.sh ]; then /usr/local/bin/k3s-uninstall.sh; fi
|
||||
rm -rf /var/lib/rancher/k3s /etc/rancher/k3s /var/lib/kubelet /var/lib/cni /etc/cni/net.d /run/k3s /run/flannel
|
||||
ip link delete cni0 >/dev/null 2>&1 || true
|
||||
ip link delete flannel.1 >/dev/null 2>&1 || true
|
||||
rm -f /etc/wireguard/${target.wireguard.interfaceName}.conf
|
||||
install -d -m 0700 /etc/rancher/k3s /etc/unidesk
|
||||
cat >/etc/rancher/k3s/registries.yaml <<'EOF'
|
||||
mirrors:
|
||||
"${target.k3s.registryMirror.source}":
|
||||
endpoint:
|
||||
- "${target.k3s.registryMirror.endpoint}"
|
||||
configs:
|
||||
"${new URL(target.k3s.registryMirror.endpoint).host}":
|
||||
auth:
|
||||
username: ${JSON.stringify(secret.registryUsername)}
|
||||
password: ${JSON.stringify(secret.registryPassword)}
|
||||
EOF
|
||||
chmod 0600 /etc/rancher/k3s/registries.yaml
|
||||
install -m 0755 /tmp/k3s /usr/local/bin/k3s
|
||||
chmod 0755 /tmp/install-k3s.sh
|
||||
INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_SKIP_START=true INSTALL_K3S_VERSION=${shQuote(target.worker.k3sVersion)} INSTALL_K3S_EXEC=${shQuote(exec)} sh /tmp/install-k3s.sh
|
||||
printf '%s\n' "$desired_hash" >"$desired_hash_file"
|
||||
chmod 0600 "$desired_hash_file"
|
||||
systemctl enable k3s >/dev/null
|
||||
systemctl restart --no-block k3s
|
||||
`;
|
||||
}
|
||||
|
||||
function standaloneStatusScript(target: K3sClusterTarget): string {
|
||||
const nodeName = target.k3s.standalone.nodeName;
|
||||
return `set -eu
|
||||
active=false; systemctl is-active --quiet k3s && active=true
|
||||
agent_active=false; systemctl is-active --quiet k3s-agent && agent_active=true
|
||||
ready="$(k3s kubectl get node ${shQuote(nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
|
||||
node_count="$(k3s kubectl get nodes --no-headers 2>/dev/null | wc -l | tr -d ' ')"
|
||||
business_pods="$(k3s kubectl get pods -A --no-headers 2>/dev/null | awk '$1 != "kube-system" {count++} END {print count+0}')"
|
||||
python3 - "$active" "$agent_active" "$ready" "$node_count" "$business_pods" <<'PY'
|
||||
import json, sys
|
||||
active, agent, ready, nodes, business = sys.argv[1:]
|
||||
ok = active == "true" and agent == "false" and ready == "True" and nodes == "1" and business == "0"
|
||||
print(json.dumps({"ok":ok,"serverActive":active=="true","oldAgentActive":agent=="true","nodeReady":ready=="True","nodeCount":int(nodes or 0),"businessPodCount":int(business or 0)}, separators=(",",":")))
|
||||
PY
|
||||
`;
|
||||
}
|
||||
|
||||
function tcpTuningApplyScript(target: K3sClusterTarget): string {
|
||||
const tuning = target.tcpTuning;
|
||||
const content = [
|
||||
|
||||
Reference in New Issue
Block a user