From 66675a5726f5b16e577275061a7b152f4bfca70e Mon Sep 17 00:00:00 2001 From: pikastech Date: Mon, 20 Jul 2026 09:02:58 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20=E9=9A=94=E7=A6=BB=20D601=20=E4=B8=9A?= =?UTF-8?q?=E5=8A=A1=E8=B0=83=E5=BA=A6?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .agents/skills/unidesk-cicd/SKILL.md | 6 + .agents/skills/unidesk-ymalops/SKILL.md | 3 + config/platform-infra/k3s-clusters.yaml | 6 + docs/reference/platform-infra.md | 20 +- .../specs/PJ2026-01060314-hyperv-ubuntu-vm.md | 4 + .../src/platform-infra-k3s-cluster-config.ts | 44 ++++ scripts/src/platform-infra-k3s-cluster.ts | 193 +++++++++++++++++- 7 files changed, 265 insertions(+), 11 deletions(-) diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index a5e87c5a..cbdf957b 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -355,6 +355,12 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - k3s drop-in 未变化且节点容量已收敛时必须 `mutation=false`；禁止无条件重启、重装、强写或通过 CI 自动覆盖同一节点配置。 - WireGuard 配置一致时不得 restart；k3s API/容量暂时不可读时保持 unknown 并停止 mutation，禁止把控制通道抖动解释为配置漂移。 - worker agent 只在 YAML 派生摘要变化或 service 未运行时安装/恢复，不得在每次集群 apply 中先卸载再重装。 + - worker 调度资格和 taint 只认集群 owning YAML： + - `schedulable=false` 必须由 agent 注册参数与受控 `cordon`/taint 双重收敛； + - CI、PipelineRun、GitOps 和业务 renderer 禁止 `uncordon`、删除该 taint， + 或添加 toleration/`nodeName` 将业务 Pod 强制放回不可调度 worker； + - 清空不可调度 worker 只走 `platform-infra k3s-cluster drain --confirm`， + 保留 DaemonSet/节点系统 Pod，且节点重注册后必须先恢复调度隔离。 - `Type=notify` agent 的 `activating` 且 MainPID 存在属于运行中；节点 Ready 时不得仅因 systemd 未进入 `active` 而重复重装。 - worker 必须先由集群入口按 `hostProxyConfigRef` 部署并验证 proxy client，再启动 agent；containerd 通过该 proxy 直接访问上游 registry，禁止用公共 registry mirror 代替出网链路。 - 集群自有 artifact registry 的异地拉取是明确例外： diff --git a/.agents/skills/unidesk-ymalops/SKILL.md b/.agents/skills/unidesk-ymalops/SKILL.md index 435659b0..654d14b6 100644 --- a/.agents/skills/unidesk-ymalops/SKILL.md +++ b/.agents/skills/unidesk-ymalops/SKILL.md @@ -166,6 +166,9 @@ description: UniDesk YAML-first 运维正规化技能。用户提到 ymal-first/ 当前倾向消除的代码特例： - 在代码里硬编码 `G14`、`D601`、`PK01`、`platform-infra`、`sub2api`、`sub2api-secrets` 等本应来自 YAML 的默认目标。 +- 在 CI、业务 manifest 或一次性 `kubectl` 中长期维护 worker 的 cordon、taint、 + toleration 或调度资格。节点是否承载业务必须由集群 owning YAML 声明， + 由节点注册参数和受控集群 CLI 双重收敛；CI/GitOps 不得反向覆盖。 - `defaultTargets()`、未披露的 runtime fallback、隐藏 sourceRef、隐藏 probe、 隐藏 NO_PROXY 等没有 YAML 归属或 warning 的默认值。 - 重复的 `asRecord`、`stringField`、`parseEnvFile`、`fingerprintValues`、compact capture、field extraction 等 helper。 diff --git a/config/platform-infra/k3s-clusters.yaml b/config/platform-infra/k3s-clusters.yaml index 24926d67..d89ed446 100644 --- a/config/platform-infra/k3s-clusters.yaml +++ b/config/platform-infra/k3s-clusters.yaml @@ -25,6 +25,12 @@ targets: unidesk.ai/node-id: D601-VM unidesk.ai/provider-id: D601-VM unidesk.ai/runtime-plane: worker + scheduling: + schedulable: false + taints: + - key: unidesk.ai/business-scheduling + value: disabled + effect: NoSchedule wireguard: interfaceName: wg0 packageName: wireguard-tools diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index b9b0ffd0..d5476ca9 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -19,7 +19,7 @@ - 既有 control-plane 的 k3s systemd drop-in 继续归 `config/hwlab-node-control-plane.yaml`，并通过 `clusterConfigRef` 消费集群网络参数。 - 受控入口： - `platform-infra k3s-cluster secret-init --target --confirm` 只在 Secret source 不存在时生成 WireGuard key pair 和 k3s agent token，存在时拒绝覆盖。 - - `platform-infra k3s-cluster plan|apply|status|smoke --target ` 是部署和验收入口。 + - `platform-infra k3s-cluster plan|apply|drain|status|smoke --target ` 是部署、清空不可调度 worker 和验收入口。 - 无固定公网地址的 worker 由宿主内网 SSH 控制，主动连接 control-plane WireGuard endpoint；worker provider route 只作为部署后补充验收入口。 - 集群 apply 先经宿主内网 SSH 分发零依赖 proxy client，确认它连接声明的 `vpn-server` 且外网 probe 成功，再启动或恢复 `k3s-agent`。 - 集群 apply 在 proxy 前先收敛 worker WireGuard；内容一致时不强写，只有 unit 未运行或握手过期且隧道探针失败时才恢复 worker 接口。 @@ -50,8 +50,18 @@ - 运行边界： - control-plane 保持唯一 k3s server，同时承担 workload。 - worker 只运行 `k3s-agent`；加入前必须卸载旧独立 server，禁止保留第二 datastore/control-plane。 + - worker 是否参与业务调度及其 taint 必须由集群 owning YAML 的 + `workers..scheduling` 声明： + - `schedulable=false` 时必须同时声明 `NoSchedule` 或 `NoExecute` taint； + - agent 注册参数和 control-plane 的 `cordon`/taint 收敛必须消费同一声明， + 防止节点重装、重注册或 CI 自动覆盖后短暂接收业务 Pod； + - `drain --confirm` 只允许清空声明为不可调度的 worker，并保留 + DaemonSet/节点系统 Pod；普通 apply、CI 和业务 renderer 禁止隐式 + `uncordon` 或移除 owning YAML 声明的 taint。 - Flannel 使用 WireGuard 接口承载 VXLAN，不能只用 FRP 转发 Kubernetes API。 - - 跨地域 worker DNS 使用集群 YAML 声明的独立 CoreDNS Deployment 和固定 ClusterIP： + - 参与业务调度的跨地域 worker DNS 使用集群 YAML 声明的独立 + CoreDNS Deployment 和固定 ClusterIP；不可调度 worker 必须移除该 + 非 DaemonSet DNS Pod： - Deployment 固定调度到 worker，Service 使用 `internalTrafficPolicy: Local`； - worker kubelet 只在本地 DNS Ready 后通过 `cluster-dns` 参数切换； - 禁止 patch k3s Addon 拥有的 `coredns` Deployment 或 `kube-dns` Service； @@ -73,7 +83,11 @@ - 集群 CLI 只组合各自 owning domain，不借用全量基础设施入口扩大写入范围。 - 验收： - `status` 必须同时证明 WireGuard handshake、worker host proxy、server/agent systemd、两个 Ready node、worker InternalIP 和旧 server 退役。 - - `smoke --confirm` 必须把 Job 定向调度到 worker，并验证集群 DNS 和 Kubernetes ClusterIP；成功后删除 Job。 + - 仅当 worker 的 `scheduling.schedulable=true` 时， + `smoke --confirm` 才可把 Job 定向调度到 worker并验证集群 DNS与 + Kubernetes ClusterIP；不可调度 worker 的 smoke 只读验证 taint、 + cordon、Node Ready 和非 DaemonSet Pod 为零，不得创建带 toleration + 的例外 Job。 - smoke 等待必须使用短 `kubectl get` 轮询，禁止让单次 `trans` 挂住等待 Job 条件。 - 基础 smoke 要求完整服务 FQDN 的解析结果包含 Pod 注入的 Kubernetes Service IP，并通过 API `ClusterIP:443` TCP connect 验证 Service 数据面；不得接受 `nslookup` 的空应答，也不把容器 TLS 客户端能力混入判定。 - Secret 输出只显示 sourceRef、presence、key 名和 fingerprint，不显示值。 diff --git a/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md b/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md index 7e2552a6..2163fc34 100644 --- a/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md +++ b/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md @@ -83,6 +83,10 @@ Provider TCP 数据池恢复必须满足： - 无固定公网地址的 worker 通过主动拨号的原生 WireGuard 三层隧道接入控制面。 - WireGuard 私钥和 k3s agent token 只能来自 YAML `sourceRef`，不得从运行面反解或打印。 - k3s 节点地址、Flannel 接口、API TLS SAN、集群内 artifact registry 映射和 ServiceLB 节点选择必须由 owning YAML 渲染。 +- worker 的业务调度资格、cordon 和 taint 必须由 owning YAML 声明并由 + agent 注册参数与受控集群入口共同收敛。声明为不可调度的 worker 不得承载 + 非 DaemonSet 业务 Pod；CI/CD、GitOps 和 smoke 不得通过 toleration、 + `nodeName` 或删除 taint 绕过该隔离。 - worker 必须先受控部署 host proxy client 并连接 YAML 引用的 `vpn-server`，再启动 agent；containerd 直接通过该 proxy 访问公共 registry，不得依赖公共镜像站或新增 registry mirror。 - 集群自有 artifact registry 可以通过公网可信 TLS 向异地 worker 提供鉴权只读拉取： - 内部 CI/publish 继续使用集群内写入 endpoint，公网入口不得成为第二个 push authority。 diff --git a/scripts/src/platform-infra-k3s-cluster-config.ts b/scripts/src/platform-infra-k3s-cluster-config.ts index aeddeb29..39fa807d 100644 --- a/scripts/src/platform-infra-k3s-cluster-config.ts +++ b/scripts/src/platform-infra-k3s-cluster-config.ts @@ -22,6 +22,10 @@ export interface K3sClusterTarget { hostProxyTargetId: string; nodeName: string; labels: Record; + scheduling: { + schedulable: boolean; + taints: Array<{ key: string; value: string; effect: "NoSchedule" | "PreferNoSchedule" | "NoExecute" }>; + }; windowsRoute: string; guestAddress: string; guestUser: string; @@ -121,6 +125,12 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri const ubuntu = record(bootstrapSource.ubuntu, "worker bootstrap ubuntu"); const bootstrapK3s = record(bootstrapSource.k3s, "worker bootstrap k3s"); const labels = stringMap(record(worker.labels, `targets.${id}.workers.${workerId}.labels`), `targets.${id}.workers.${workerId}.labels`); + const scheduling = record(worker.scheduling, `targets.${id}.workers.${workerId}.scheduling`); + const taints = objectArray(scheduling.taints, `targets.${id}.workers.${workerId}.scheduling.taints`).map((item, index) => ({ + key: kubernetesLabelKey(item, "key", `targets.${id}.workers.${workerId}.scheduling.taints[${index}]`), + value: kubernetesLabelValue(item, "value", `targets.${id}.workers.${workerId}.scheduling.taints[${index}]`), + effect: enumText(item, "effect", `targets.${id}.workers.${workerId}.scheduling.taints[${index}]`, ["NoSchedule", "PreferNoSchedule", "NoExecute"]), + })); const target: K3sClusterTarget = { id, clusterName: text(raw, "clusterName", `targets.${id}`), @@ -140,6 +150,10 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri hostProxyTargetId: hostProxyTargetId(text(worker, "hostProxyConfigRef", `targets.${id}.workers.${workerId}`)), nodeName: text(worker, "nodeName", `targets.${id}.workers.${workerId}`), labels, + scheduling: { + schedulable: boolean(scheduling, "schedulable", `targets.${id}.workers.${workerId}.scheduling`), + taints, + }, windowsRoute: text(host, "route", "worker bootstrap host"), guestAddress: text(network, "address", "worker bootstrap network"), guestUser: text(ubuntu, "user", "worker bootstrap ubuntu"), @@ -214,6 +228,9 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri }; if (target.k3s.flannelInterface !== target.wireguard.interfaceName) throw new Error(`targets.${id}.k3s.flannelInterface must match wireguard.interfaceName`); if (target.k3s.serviceLb.enabledNode !== target.controlPlane.nodeName) throw new Error(`targets.${id}.k3s.serviceLb.enabledNode must match controlPlane.nodeName`); + if (!target.worker.scheduling.schedulable && !target.worker.scheduling.taints.some((taint) => taint.effect === "NoSchedule" || taint.effect === "NoExecute")) { + throw new Error(`targets.${id}.workers.${workerId}.scheduling must declare a NoSchedule or NoExecute taint when schedulable=false`); + } if (!target.tcpTuning.sysctlPath.startsWith("/etc/sysctl.d/")) throw new Error(`targets.${id}.tcpTuning.sysctlPath must stay under /etc/sysctl.d`); return { defaultTargetId, target }; } @@ -261,6 +278,17 @@ function integer(value: Record, key: string, label: string): nu return Number(result); } +function boolean(value: Record, key: string, label: string): boolean { + const result = value[key]; + if (typeof result !== "boolean") throw new Error(`${label}.${key} must be a boolean`); + return result; +} + +function objectArray(value: unknown, label: string): Array> { + if (!Array.isArray(value)) throw new Error(`${label} must be an array`); + return value.map((item, index) => record(item, `${label}[${index}]`)); +} + function httpsUrl(value: Record, key: string, label: string): string { const result = text(value, key, label); const parsed = new URL(result); @@ -310,6 +338,22 @@ function kubernetesName(value: Record, key: string, label: stri return result; } +function kubernetesLabelKey(value: Record, key: string, label: string): string { + const result = text(value, key, label); + if (!/^(?:[a-z0-9](?:[-a-z0-9.]*[a-z0-9])?\/)?[A-Za-z0-9](?:[-A-Za-z0-9_.]*[A-Za-z0-9])?$/u.test(result) || result.length > 253) { + throw new Error(`${label}.${key} must be a Kubernetes label key`); + } + return result; +} + +function kubernetesLabelValue(value: Record, key: string, label: string): string { + const result = text(value, key, label); + if (!/^[A-Za-z0-9](?:[-A-Za-z0-9_.]*[A-Za-z0-9])?$/u.test(result) || result.length > 63) { + throw new Error(`${label}.${key} must be a Kubernetes label value`); + } + return result; +} + function stringMap(value: Record, label: string): Record { return Object.fromEntries(Object.entries(value).map(([key, item]) => { if (typeof item !== "string" || item.length === 0) throw new Error(`${label}.${key} must be a non-empty string`); diff --git a/scripts/src/platform-infra-k3s-cluster.ts b/scripts/src/platform-infra-k3s-cluster.ts index 6a5bc97f..732cc507 100644 --- a/scripts/src/platform-infra-k3s-cluster.ts +++ b/scripts/src/platform-infra-k3s-cluster.ts @@ -13,7 +13,7 @@ import { } from "./platform-infra-host-proxy"; import { address, K3S_CLUSTER_CONFIG_LABEL, readK3sClusterConfig, type K3sClusterTarget } from "./platform-infra-k3s-cluster-config"; -type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "status" | "smoke" | "registry-smoke"; +type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "drain" | "status" | "smoke" | "registry-smoke"; interface Options { action: Action; @@ -39,6 +39,7 @@ export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig, if (options.action === "plan") return plan(target); if (options.action === "secret-init") return secretInit(target, options.confirm); if (options.action === "registry-credential-init") return registryCredentialInit(target, options.confirm); + if (options.action === "drain") return drainWorker(target, options.confirm); if (options.action === "status") return status(target); if (options.action === "smoke") return smoke(target, options.confirm); if (options.action === "registry-smoke") return registrySmoke(target, options.confirm); @@ -48,8 +49,8 @@ export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig, function parseOptions(args: string[]): Options { const action = args[0] ?? "plan"; - if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "status" && action !== "smoke" && action !== "registry-smoke") { - throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|status|smoke|registry-smoke [--target ] [--confirm]"); + if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "drain" && action !== "status" && action !== "smoke" && action !== "registry-smoke") { + throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|drain|status|smoke|registry-smoke [--target ] [--confirm]"); } return { action, targetId: option(args, "--target") ?? undefined, confirm: args.includes("--confirm") }; } @@ -69,6 +70,7 @@ function plan(target: K3sClusterTarget): Record { bootstrapConfigRef: target.worker.bootstrapConfigRef, hostProxyConfigRef: target.worker.hostProxyConfigRef, controlRoute: target.worker.windowsRoute, + scheduling: target.worker.scheduling, }, }, wireguard: { ...target.wireguard, endpoint: target.wireguard.endpoint }, @@ -87,6 +89,7 @@ function plan(target: K3sClusterTarget): Record { secretInit: `bun scripts/cli.ts platform-infra k3s-cluster secret-init --target ${target.id} --confirm`, registryCredentialInit: `bun scripts/cli.ts platform-infra k3s-cluster registry-credential-init --target ${target.id} --confirm`, apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm`, + drain: `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm`, status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`, smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`, registrySmoke: `bun scripts/cli.ts platform-infra k3s-cluster registry-smoke --target ${target.id} --confirm`, @@ -206,10 +209,12 @@ function apply(target: K3sClusterTarget): Record { ]) as Record; if (controlPlane.ok !== true) throw new Error(`control-plane-k3s-apply failed: ${bounded(JSON.stringify(compactDomain(controlPlane)))}`); + const workerSchedulingBeforeAgent = reconcileWorkerScheduling(target); const workerDns = reconcileWorkerDns(target); const worker = runWorker(target, workerApplyScript(target, secret), true, 300_000); requireSuccess(worker, "worker-agent-apply"); + const workerSchedulingAfterAgent = reconcileWorkerScheduling(target); const labels = [ "label", "node", target.k3s.serviceLb.enabledNode, @@ -237,12 +242,15 @@ function apply(target: K3sClusterTarget): Record { controlPlaneK3s: compactDomain(controlPlane), workerDns, workerAgent: compact(worker), + workerSchedulingBeforeAgent, + workerSchedulingAfterAgent, serviceLbPlacement: compact(labelResult), nodeReadiness: compact(ready), }, secret: { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: secret.fingerprint, valuesPrinted: false }, registryAuth: registryAuthSummary(target), next: { + drain: target.worker.scheduling.schedulable ? null : `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm`, status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`, smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`, }, @@ -257,13 +265,17 @@ function status(target: K3sClusterTarget): Record { const controlData = parseFirstJson(control.stdout); const workerProxyData = parseFirstJson(workerProxy.stdout); const workerData = parseFirstJson(worker.stdout); + const workerScheduling = workerSchedulingStatus(target); + const workerWorkloads = workerWorkloadStatus(target); const ok = control.exitCode === 0 && workerProxy.exitCode === 0 && worker.exitCode === 0 && controlData.ok === true && workerProxyData.ok === true && workerData.ok === true - && workerDns.data.ok === true; + && workerDns.data.ok === true + && workerScheduling.ok === true + && workerWorkloads.ok === true; return { ok, mutation: false, @@ -272,14 +284,38 @@ function status(target: K3sClusterTarget): Record { workerHostProxy: workerProxyData, worker: workerData, workerDns: workerDns.data, + workerScheduling, + workerWorkloads, secret: secretSummary(target), registryAuth: registryAuthSummary(target), - result: { controlPlane: compact(control), workerHostProxy: compact(workerProxy), worker: compact(worker), workerDns: compact(workerDns.result) }, - next: ok ? { smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm` } : { apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm` }, + result: { + controlPlane: { exitCode: control.exitCode }, + workerHostProxy: { exitCode: workerProxy.exitCode }, + worker: { exitCode: worker.exitCode }, + workerDns: { exitCode: workerDns.result.exitCode }, + }, + next: ok + ? { smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm` } + : !target.worker.scheduling.schedulable && workerScheduling.ok === true && workerWorkloads.ok !== true + ? { drain: `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm` } + : { apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm` }, }; } function smoke(target: K3sClusterTarget, confirm: boolean): Record { + if (!target.worker.scheduling.schedulable) { + const scheduling = workerSchedulingStatus(target); + const workloads = workerWorkloadStatus(target); + const readiness = parseFirstJson(runControlPlane(target, nodeReadinessScript(target), 30_000).stdout); + return { + ok: scheduling.ok === true && workloads.ok === true && readiness.ok === true, + mutation: false, + mode: confirm ? "confirmed-read-only" : "dry-run", + targetId: target.id, + workerNode: target.worker.nodeName, + checks: { scheduling, workloads, readiness }, + }; + } const manifest = smokeManifest(target); if (!confirm) return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, workerNode: target.worker.nodeName, manifestSha256: hash(manifest) }; const namespace = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "create", "namespace", target.smoke.namespace, "--dry-run=client", "-o", "yaml"], rootPath(), { timeoutMs: 30_000 }); @@ -495,6 +531,20 @@ PY function reconcileWorkerDns(target: K3sClusterTarget): Record { const before = workerDnsStatus(target); + if (!target.worker.scheduling.schedulable) { + if (before.data.presence !== true) { + return { ok: true, mutation: false, enabled: false, reason: "worker-unschedulable", presence: false }; + } + const deletion = runCommand([ + "trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.k3s.workerDns.namespace, + "delete", "deployment", target.k3s.workerDns.deploymentName, + "service", target.k3s.workerDns.serviceName, "--ignore-not-found=true", "--wait=true", + ], rootPath(), { timeoutMs: 120_000 }); + requireSuccess(deletion, "worker-dns-delete"); + const after = workerDnsStatus(target); + if (after.data.ok !== true) throw new Error(`worker-dns-removal failed: ${bounded(JSON.stringify(after.data))}`); + return { ok: true, mutation: true, enabled: false, reason: "worker-unschedulable", presence: false, deletion: compact(deletion) }; + } const mutation = before.data.configurationCurrent !== true; let applyResult: ReturnType | null = null; if (mutation) { @@ -521,6 +571,116 @@ function reconcileWorkerDns(target: K3sClusterTarget): Record { }; } +function workerSchedulingStatus(target: K3sClusterTarget): Record { + const result = runCommand([ + "trans", target.controlPlane.kubeRoute, "kubectl", "get", "node", target.worker.nodeName, + "-o", String.raw`jsonpath={.spec.unschedulable}|{range .spec.taints[*]}{.key}={.value}:{.effect},{end}`, + ], rootPath(), { timeoutMs: 60_000 }); + if (result.exitCode !== 0) return { ok: false, nodePresent: false, result: compact(result) }; + const [unschedulableRaw = "", taintsRaw = ""] = result.stdout.trim().split("|", 2); + const actualTaints = taintsRaw.split(",").filter(Boolean); + const desiredTaintsPresent = target.worker.scheduling.taints.every((desired) => + actualTaints.includes(`${desired.key}=${desired.value}:${desired.effect}`)); + const unschedulable = unschedulableRaw === "true"; + return { + ok: desiredTaintsPresent && unschedulable === !target.worker.scheduling.schedulable, + nodePresent: true, + schedulable: !unschedulable, + desiredSchedulable: target.worker.scheduling.schedulable, + desiredTaintsPresent, + desiredTaints: target.worker.scheduling.taints, + }; +} + +function reconcileWorkerScheduling(target: K3sClusterTarget): Record { + const before = workerSchedulingStatus(target); + if (before.nodePresent !== true) throw new Error(`worker node is absent: ${target.worker.nodeName}`); + const mutations: string[] = []; + if (before.schedulable !== target.worker.scheduling.schedulable) { + const action = target.worker.scheduling.schedulable ? "uncordon" : "cordon"; + const result = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", action, target.worker.nodeName], rootPath(), { timeoutMs: 60_000 }); + requireSuccess(result, `worker-${action}`); + mutations.push(action); + } + if (before.desiredTaintsPresent !== true) { + for (const taint of target.worker.scheduling.taints) { + const desired = `${taint.key}=${taint.value}:${taint.effect}`; + const result = runCommand([ + "trans", target.controlPlane.kubeRoute, "kubectl", "taint", "node", target.worker.nodeName, desired, "--overwrite", + ], rootPath(), { timeoutMs: 60_000 }); + requireSuccess(result, `worker-taint-${taint.key}`); + mutations.push(`taint:${taint.key}`); + } + } + const after = workerSchedulingStatus(target); + if (after.ok !== true) throw new Error(`worker scheduling did not converge: ${bounded(JSON.stringify(after))}`); + return { ok: true, mutation: mutations.length > 0, mutations, before, after }; +} + +function workerWorkloadStatus(target: K3sClusterTarget): Record { + const result = runCommand([ + "trans", target.controlPlane.kubeRoute, "kubectl", "get", "pods", "-A", + "--field-selector", `spec.nodeName=${target.worker.nodeName}`, + "-o", "custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,OWNER:.metadata.ownerReferences[0].kind,PHASE:.status.phase", + "--no-headers", + ], rootPath(), { timeoutMs: 60_000 }); + if (result.exitCode !== 0) return { ok: false, result: compact(result) }; + const items = result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => { + const [namespace = "", name = "", ownerKind = "", phase = ""] = line.split(/\s+/u, 4); + return { namespace, name, ownerKind, phase }; + }); + const retainedSystemPods = items.filter((pod) => pod.ownerKind === "DaemonSet" || pod.ownerKind === "Node"); + const nonDaemonSetPods = items.filter((pod) => !retainedSystemPods.includes(pod)); + const requiredEmpty = !target.worker.scheduling.schedulable; + return { + ok: !requiredEmpty || nonDaemonSetPods.length === 0, + requiredEmpty, + nonDaemonSetPodCount: nonDaemonSetPods.length, + retainedSystemPodCount: retainedSystemPods.length, + nonDaemonSetPods: nonDaemonSetPods.slice(0, 40), + valuesPrinted: false, + }; +} + +function drainWorker(target: K3sClusterTarget, confirm: boolean): Record { + if (target.worker.scheduling.schedulable) throw new Error("worker drain requires scheduling.schedulable=false in the owning YAML"); + if (!confirm) { + return { + ok: true, + mutation: false, + mode: "dry-run", + targetId: target.id, + workerNode: target.worker.nodeName, + scheduling: workerSchedulingStatus(target), + workloads: workerWorkloadStatus(target), + }; + } + const scheduling = reconcileWorkerScheduling(target); + const dns = reconcileWorkerDns(target); + const before = workerWorkloadStatus(target); + let drain: ReturnType | null = null; + if (before.nonDaemonSetPodCount !== 0) { + drain = runCommand([ + "trans", target.controlPlane.kubeRoute, "kubectl", "drain", target.worker.nodeName, + "--ignore-daemonsets", "--delete-emptydir-data", "--force", "--grace-period=60", "--timeout=10m", + ], rootPath(), { timeoutMs: 660_000 }); + requireSuccess(drain, "worker-drain"); + } + const after = workerWorkloadStatus(target); + if (after.ok !== true) throw new Error(`worker drain did not converge: ${bounded(JSON.stringify(after))}`); + return { + ok: true, + mutation: before.nonDaemonSetPodCount !== 0 || scheduling.mutation === true || dns.mutation === true, + targetId: target.id, + workerNode: target.worker.nodeName, + scheduling, + workerDns: dns, + before, + after, + drain: drain === null ? { skipped: true, reason: "no-non-daemonset-pods" } : compact(drain), + }; +} + function workerDnsStatus(target: K3sClusterTarget): { data: Record; result: ReturnType } { const dns = target.k3s.workerDns; const result = runCommand([ @@ -547,8 +707,23 @@ function workerDnsStatus(target: K3sClusterTarget): { data: Record ["--node-label", `${key}=${value}`]); + const taints = target.worker.scheduling.taints.flatMap((taint) => ["--node-taint", `${taint.key}=${taint.value}:${taint.effect}`]); + const dnsArgs = target.worker.scheduling.schedulable ? ["--kubelet-arg", `cluster-dns=${target.k3s.workerDns.clusterIp}`] : []; const exec = [ "agent", "--server", target.k3s.serverUrl, "--token-file", target.k3s.agentTokenFile, "--node-name", target.worker.nodeName, "--node-ip", address(wg.workerAddress), - "--flannel-iface", target.k3s.flannelInterface, ...labels, + "--flannel-iface", target.k3s.flannelInterface, ...labels, ...taints, "--kubelet-arg", `max-pods=${target.k3s.maxPods}`, - "--kubelet-arg", `cluster-dns=${target.k3s.workerDns.clusterIp}`, + ...dnsArgs, ].map(shellToken).join(" "); const desiredHash = createHash("sha256").update(JSON.stringify({ version: target.worker.k3sVersion,