diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index d5476ca9..77ed5797 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -57,7 +57,10 @@ 防止节点重装、重注册或 CI 自动覆盖后短暂接收业务 Pod; - `drain --confirm` 只允许清空声明为不可调度的 worker,并保留 DaemonSet/节点系统 Pod;普通 apply、CI 和业务 renderer 禁止隐式 - `uncordon` 或移除 owning YAML 声明的 taint。 + `uncordon` 或移除 owning YAML 声明的 taint; + - drain 与 Node Ready 等待必须由本地 CLI 组合短于 `trans` 上限的 + 目标侧操作和短轮询;节点已不可调度且用户要求彻底清空时允许 + `--disable-eviction` 直接删除非 DaemonSet Pod,但不得扩大到其他节点。 - Flannel 使用 WireGuard 接口承载 VXLAN,不能只用 FRP 转发 Kubernetes API。 - 参与业务调度的跨地域 worker DNS 使用集群 YAML 声明的独立 CoreDNS Deployment 和固定 ClusterIP;不可调度 worker 必须移除该 diff --git a/scripts/src/platform-infra-k3s-cluster.ts b/scripts/src/platform-infra-k3s-cluster.ts index 732cc507..073024fd 100644 --- a/scripts/src/platform-infra-k3s-cluster.ts +++ b/scripts/src/platform-infra-k3s-cluster.ts @@ -223,7 +223,7 @@ function apply(target: K3sClusterTarget): Record { const labelResult = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", ...labels], rootPath(), { timeoutMs: 60_000 }); requireSuccess(labelResult, "servicelb-node-label"); - const ready = runControlPlane(target, waitForNodesScript(target), 180_000); + const ready = waitForNodes(target, 180_000); requireSuccess(ready, "cluster-node-readiness"); return { ok: true, @@ -541,7 +541,12 @@ function reconcileWorkerDns(target: K3sClusterTarget): Record { "service", target.k3s.workerDns.serviceName, "--ignore-not-found=true", "--wait=true", ], rootPath(), { timeoutMs: 120_000 }); requireSuccess(deletion, "worker-dns-delete"); - const after = workerDnsStatus(target); + let after = workerDnsStatus(target); + const deadline = Date.now() + 30_000; + while (after.data.ok !== true && Date.now() < deadline) { + runCommand(["sleep", "2"], rootPath(), { timeoutMs: 5_000 }); + after = workerDnsStatus(target); + } if (after.data.ok !== true) throw new Error(`worker-dns-removal failed: ${bounded(JSON.stringify(after.data))}`); return { ok: true, mutation: true, enabled: false, reason: "worker-unschedulable", presence: false, deletion: compact(deletion) }; } @@ -658,15 +663,28 @@ function drainWorker(target: K3sClusterTarget, confirm: boolean): Record | null = null; - if (before.nonDaemonSetPodCount !== 0) { - drain = runCommand([ + const drainAttempts: Array> = []; + let current = before; + const drainDeadline = Date.now() + 10 * 60_000; + while (current.nonDaemonSetPodCount !== 0 && Date.now() < drainDeadline) { + const drain = runCommand([ "trans", target.controlPlane.kubeRoute, "kubectl", "drain", target.worker.nodeName, - "--ignore-daemonsets", "--delete-emptydir-data", "--force", "--grace-period=60", "--timeout=10m", - ], rootPath(), { timeoutMs: 660_000 }); - requireSuccess(drain, "worker-drain"); + "--ignore-daemonsets", "--delete-emptydir-data", "--force", "--disable-eviction", + "--grace-period=10", "--skip-wait-for-delete-timeout=10", "--timeout=45s", + ], rootPath(), { timeoutMs: 55_000 }); + const next = workerWorkloadStatus(target); + drainAttempts.push({ + exitCode: drain.exitCode, + beforeCount: current.nonDaemonSetPodCount, + afterCount: next.nonDaemonSetPodCount, + stderr: bounded(drain.stderr, 1_000), + }); + if (next.nonDaemonSetPodCount === current.nonDaemonSetPodCount && drain.exitCode !== 0) { + throw new Error(`worker drain made no progress: ${bounded(JSON.stringify(drainAttempts.at(-1)))}`); + } + current = next; } - const after = workerWorkloadStatus(target); + const after = current; if (after.ok !== true) throw new Error(`worker drain did not converge: ${bounded(JSON.stringify(after))}`); return { ok: true, @@ -677,7 +695,7 @@ function drainWorker(target: K3sClusterTarget, confirm: boolean): Record/dev/null || true) - [ "$ready" = True ] && exit 0 - sleep 5 -done -/usr/local/bin/kubectl get nodes -o wide >&2 -exit 1 -`; +function waitForNodes(target: K3sClusterTarget, timeoutMs: number): ReturnType { + const deadline = Date.now() + timeoutMs; + let current = runControlPlane(target, nodeReadinessScript(target), 30_000); + while ((current.exitCode !== 0 || parseFirstJson(current.stdout).ok !== true) && Date.now() < deadline) { + runCommand(["sleep", "5"], rootPath(), { timeoutMs: 10_000 }); + current = runControlPlane(target, nodeReadinessScript(target), 30_000); + } + if (current.exitCode === 0 && parseFirstJson(current.stdout).ok === true) return current; + return { ...current, exitCode: 1, stderr: current.stderr || `nodes did not become Ready within ${timeoutMs}ms` }; } function statusScript(target: K3sClusterTarget): string {