From 620fe5b6f85b768e4089e2f21b0b87a137468cb8 Mon Sep 17 00:00:00 2001 From: pikastech Date: Mon, 20 Jul 2026 06:28:44 +0200 Subject: [PATCH] =?UTF-8?q?fix:=20=E7=A8=B3=E5=AE=9A=E8=B7=A8=E5=9C=B0?= =?UTF-8?q?=E5=9F=9F=20worker=20DNS?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-Authored-By: Claude Opus 4.7 --- config/platform-infra/k3s-clusters.yaml | 12 +- docs/reference/platform-infra.md | 6 + .../src/platform-infra-k3s-cluster-config.ts | 39 ++++ scripts/src/platform-infra-k3s-cluster.ts | 203 +++++++++++++++++- 4 files changed, 257 insertions(+), 3 deletions(-) diff --git a/config/platform-infra/k3s-clusters.yaml b/config/platform-infra/k3s-clusters.yaml index b1a29acd..d2bcdef3 100644 --- a/config/platform-infra/k3s-clusters.yaml +++ b/config/platform-infra/k3s-clusters.yaml @@ -33,7 +33,7 @@ targets: controlPlaneAddress: 10.89.0.1/24 workerAddress: 10.89.0.2/24 endpoint: 152.53.229.148:51820 - persistentKeepaliveSeconds: 25 + persistentKeepaliveSeconds: 10 secrets: sourceRef: /root/.unidesk/.state/secrets/platform-infra/k3s-clusters/nc01-d601.env controlPlanePrivateKey: NC01_WIREGUARD_PRIVATE_KEY @@ -52,6 +52,16 @@ targets: serviceLb: enabledNodeLabel: svccontroller.k3s.cattle.io/enablelb enabledNode: v2202607378382480008 + workerDns: + namespace: kube-system + deploymentName: unidesk-worker-coredns + serviceName: unidesk-worker-kube-dns + clusterIp: 10.43.0.53 + image: rancher/mirrored-coredns-coredns:1.14.4 + configMapName: coredns + customConfigMapName: coredns-custom + replicas: 1 + timeoutSeconds: 120 smoke: namespace: platform-infra-smoke jobName: d601-runtime-plane-smoke diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 079979ed..007f3c2f 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -24,6 +24,7 @@ - 集群 apply 先经宿主内网 SSH 分发零依赖 proxy client，确认它连接声明的 `vpn-server` 且外网 probe 成功，再启动或恢复 `k3s-agent`。 - 集群 apply 在 proxy 前先收敛 worker WireGuard；内容一致时不强写，只有 unit 未运行或握手过期且隧道探针失败时才恢复 worker 接口。 - worker WireGuard 的业务探针固定使用 control-plane API `:6443`，不以可能被策略丢弃的 ICMP 判断隧道健康。 + - NAT 后 worker 的 WireGuard `PersistentKeepalive` 周期由集群 YAML 声明；跨地域节点优先用较短周期维持映射，但不得据此宣称降低物理 RTT。 - 大型 proxy 工件由 Windows 宿主按 YAML `upstreamUrl` 获取并校验 SHA-256，Provider/trans 只承载控制命令，不作为大二进制数据面。 - WireGuard 只承载集群三层网络；NAT 或链路不稳定的 worker proxy 使用 `vpn-server` 输出的 Hysteria2 client YAML，不把应用出网 TCP 再套入 WireGuard TCP 路径。 - proxy client 协议实现版本由 host-proxy YAML 声明，并与所连接的 `vpn-server` 协议实现版本保持一致。 @@ -36,6 +37,11 @@ - control-plane 保持唯一 k3s server，同时承担 workload。 - worker 只运行 `k3s-agent`；加入前必须卸载旧独立 server，禁止保留第二 datastore/control-plane。 - Flannel 使用 WireGuard 接口承载 VXLAN，不能只用 FRP 转发 Kubernetes API。 + - 跨地域 worker DNS 使用集群 YAML 声明的独立 CoreDNS Deployment 和固定 ClusterIP： + - Deployment 固定调度到 worker，Service 使用 `internalTrafficPolicy: Local`； + - worker kubelet 只在本地 DNS Ready 后通过 `cluster-dns` 参数切换； + - 禁止 patch k3s Addon 拥有的 `coredns` Deployment 或 `kube-dns` Service； + - 受控 apply 先比较声明态哈希，内容一致时跳过 `kubectl apply`，不得触发无意义 rollout。 - ServiceLB 只在 owning YAML 指定的公网节点启用，防止无公网地址 worker 接管 hostPort。 - worker 的 containerd 外网访问继承 host proxy systemd 环境，直接访问上游 registry；禁止为公共依赖新增 registry mirror。 - `127.0.0.1:5000` 只表示 UniDesk 集群内 artifact registry，worker 的 `registries.yaml` 可把该本地引用映射到集群 endpoint，不得把它扩展为公共镜像代理。 diff --git a/scripts/src/platform-infra-k3s-cluster-config.ts b/scripts/src/platform-infra-k3s-cluster-config.ts index 019aa2ef..dad555e3 100644 --- a/scripts/src/platform-infra-k3s-cluster-config.ts +++ b/scripts/src/platform-infra-k3s-cluster-config.ts @@ -53,6 +53,17 @@ export interface K3sClusterTarget { maxPods: number; registryMirror: { source: string; endpoint: string }; serviceLb: { enabledNodeLabel: string; enabledNode: string }; + workerDns: { + namespace: string; + deploymentName: string; + serviceName: string; + clusterIp: string; + image: string; + configMapName: string; + customConfigMapName: string; + replicas: number; + timeoutSeconds: number; + }; }; smoke: { namespace: string; jobName: string; image: string; timeoutSeconds: number }; } @@ -76,6 +87,7 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri const k3s = record(raw.k3s, `targets.${id}.k3s`); const mirror = record(k3s.registryMirror, `targets.${id}.k3s.registryMirror`); const serviceLb = record(k3s.serviceLb, `targets.${id}.k3s.serviceLb`); + const workerDns = record(k3s.workerDns, `targets.${id}.k3s.workerDns`); const smoke = record(raw.smoke, `targets.${id}.smoke`); const controlSource = resolveRef(text(control, "configRef", `targets.${id}.controlPlane`)); const bootstrapSource = resolveRef(text(worker, "bootstrapConfigRef", `targets.${id}.workers[0]`)); @@ -134,6 +146,17 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri maxPods: integer(k3s, "maxPods", `targets.${id}.k3s`), registryMirror: { source: text(mirror, "source", "registryMirror"), endpoint: text(mirror, "endpoint", "registryMirror") }, serviceLb: { enabledNodeLabel: text(serviceLb, "enabledNodeLabel", "serviceLb"), enabledNode: text(serviceLb, "enabledNode", "serviceLb") }, + workerDns: { + namespace: kubernetesName(workerDns, "namespace", "workerDns"), + deploymentName: kubernetesName(workerDns, "deploymentName", "workerDns"), + serviceName: kubernetesName(workerDns, "serviceName", "workerDns"), + clusterIp: ipv4Address(workerDns, "clusterIp", "workerDns"), + image: text(workerDns, "image", "workerDns"), + configMapName: kubernetesName(workerDns, "configMapName", "workerDns"), + customConfigMapName: kubernetesName(workerDns, "customConfigMapName", "workerDns"), + replicas: integer(workerDns, "replicas", "workerDns"), + timeoutSeconds: integer(workerDns, "timeoutSeconds", "workerDns"), + }, }, smoke: { namespace: text(smoke, "namespace", `targets.${id}.smoke`), @@ -196,6 +219,22 @@ function cidr(value: Record, key: string, label: string): strin return result; } +function ipv4Address(value: Record, key: string, label: string): string { + const result = text(value, key, label); + if (!/^\d{1,3}(?:\.\d{1,3}){3}$/u.test(result) || result.split(".").some((part) => Number(part) > 255)) { + throw new Error(`${label}.${key} must be an IPv4 address`); + } + return result; +} + +function kubernetesName(value: Record, key: string, label: string): string { + const result = text(value, key, label); + if (!/^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/u.test(result) || result.length > 63) { + throw new Error(`${label}.${key} must be a Kubernetes DNS label`); + } + return result; +} + function stringMap(value: Record, label: string): Record { return Object.fromEntries(Object.entries(value).map(([key, item]) => { if (typeof item !== "string" || item.length === 0) throw new Error(`${label}.${key} must be a non-empty string`); diff --git a/scripts/src/platform-infra-k3s-cluster.ts b/scripts/src/platform-infra-k3s-cluster.ts index 087b8a60..1b9d21be 100644 --- a/scripts/src/platform-infra-k3s-cluster.ts +++ b/scripts/src/platform-infra-k3s-cluster.ts @@ -72,6 +72,7 @@ function plan(target: K3sClusterTarget): Record { agentTokenFile: target.k3s.agentTokenFile, registryMirror: target.k3s.registryMirror, serviceLb: target.k3s.serviceLb, + workerDns: target.k3s.workerDns, }, secret: secretSummary(target), next: { @@ -119,6 +120,8 @@ function apply(target: K3sClusterTarget): Record { ]) as Record; if (controlPlane.ok !== true) throw new Error(`control-plane-k3s-apply failed: ${bounded(JSON.stringify(compactDomain(controlPlane)))}`); + const workerDns = reconcileWorkerDns(target); + const worker = runWorker(target, workerApplyScript(target, secret), true, 300_000); requireSuccess(worker, "worker-agent-apply"); @@ -146,6 +149,7 @@ function apply(target: K3sClusterTarget): Record { }, controlPlaneWireguard: compact(controlWireguard), controlPlaneK3s: compactDomain(controlPlane), + workerDns, workerAgent: compact(worker), serviceLbPlacement: compact(labelResult), nodeReadiness: compact(ready), @@ -162,6 +166,7 @@ function status(target: K3sClusterTarget): Record { const control = runControlPlane(target, statusScript(target), 60_000); const workerProxy = runWorker(target, hostProxyGuestStatusScript(target.worker.hostProxyTargetId), false, 60_000); const worker = runWorker(target, workerStatusScript(target), false, 60_000); + const workerDns = workerDnsStatus(target); const controlData = parseFirstJson(control.stdout); const workerProxyData = parseFirstJson(workerProxy.stdout); const workerData = parseFirstJson(worker.stdout); @@ -170,7 +175,8 @@ function status(target: K3sClusterTarget): Record { && worker.exitCode === 0 && controlData.ok === true && workerProxyData.ok === true - && workerData.ok === true; + && workerData.ok === true + && workerDns.data.ok === true; return { ok, mutation: false, @@ -178,8 +184,9 @@ function status(target: K3sClusterTarget): Record { controlPlane: controlData, workerHostProxy: workerProxyData, worker: workerData, + workerDns: workerDns.data, secret: secretSummary(target), - result: { controlPlane: compact(control), workerHostProxy: compact(workerProxy), worker: compact(worker) }, + result: { controlPlane: compact(control), workerHostProxy: compact(workerProxy), worker: compact(worker), workerDns: compact(workerDns.result) }, next: ok ? { smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm` } : { apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm` }, }; } @@ -229,6 +236,196 @@ function smoke(target: K3sClusterTarget, confirm: boolean): Record { + const before = workerDnsStatus(target); + const mutation = before.data.configurationCurrent !== true; + let applyResult: ReturnType | null = null; + if (mutation) { + applyResult = runCommand( + ["trans", target.controlPlane.kubeRoute, "kubectl", "apply", "-f", "-"], + rootPath(), + { input: workerDnsManifest(target), timeoutMs: 60_000 }, + ); + requireSuccess(applyResult, "worker-dns-apply"); + } + + const deadline = Date.now() + target.k3s.workerDns.timeoutSeconds * 1000; + let current = workerDnsStatus(target); + while (current.data.ok !== true && Date.now() < deadline) { + runCommand(["sleep", "3"], rootPath(), { timeoutMs: 5_000 }); + current = workerDnsStatus(target); + } + if (current.data.ok !== true) throw new Error(`worker-dns-readiness failed: ${bounded(JSON.stringify(current.data))}`); + return { + mutation, + desiredHash: workerDnsDesiredHash(target), + ...current.data, + apply: applyResult === null ? { skipped: true, reason: "configuration-current" } : compact(applyResult), + }; +} + +function workerDnsStatus(target: K3sClusterTarget): { data: Record; result: ReturnType } { + const dns = target.k3s.workerDns; + const result = runCommand([ + "trans", target.controlPlane.kubeRoute, "kubectl", "-n", dns.namespace, "get", "deployment,service,endpoints", + "-l", `app.kubernetes.io/name=${dns.deploymentName}`, "-o", "json", + ], rootPath(), { timeoutMs: 30_000 }); + if (result.exitCode !== 0) { + return { + data: { ok: false, presence: false, configurationCurrent: false, error: bounded(result.stderr || result.stdout) }, + result, + }; + } + let items: Array> = []; + try { + const parsed = JSON.parse(result.stdout) as { items?: Array> }; + items = Array.isArray(parsed.items) ? parsed.items : []; + } catch { + return { data: { ok: false, presence: false, configurationCurrent: false, error: "worker-dns-json-invalid" }, result }; + } + const deployment = items.find((item) => item.kind === "Deployment"); + const service = items.find((item) => item.kind === "Service"); + const endpoints = items.find((item) => item.kind === "Endpoints"); + const desiredHash = workerDnsDesiredHash(target); + const deploymentHash = deployment?.metadata?.annotations?.["unidesk.ai/config-sha256"] ?? null; + const serviceHash = service?.metadata?.annotations?.["unidesk.ai/config-sha256"] ?? null; + const readyReplicas = Number(deployment?.status?.readyReplicas ?? 0); + const endpointNodes = (endpoints?.subsets ?? []).flatMap((subset: Record) => subset.addresses ?? []) + .map((entry: Record) => entry.nodeName) + .filter((nodeName: unknown): nodeName is string => typeof nodeName === "string"); + const configurationCurrent = deploymentHash === desiredHash && serviceHash === desiredHash; + const deploymentReady = readyReplicas === dns.replicas; + const endpointLocal = endpointNodes.includes(target.worker.nodeName); + const serviceIp = service?.spec?.clusterIP ?? null; + const localTrafficPolicy = service?.spec?.internalTrafficPolicy ?? null; + return { + data: { + ok: configurationCurrent && deploymentReady && endpointLocal && serviceIp === dns.clusterIp && localTrafficPolicy === "Local", + presence: deployment !== undefined && service !== undefined && endpoints !== undefined, + configurationCurrent, + deploymentReady, + readyReplicas, + desiredReplicas: dns.replicas, + serviceIp, + localTrafficPolicy, + endpointNodes, + workerNode: target.worker.nodeName, + desiredHash, + }, + result, + }; +} + +function workerDnsDesiredHash(target: K3sClusterTarget): string { + return hash(JSON.stringify({ workerNode: target.worker.nodeName, ...target.k3s.workerDns })); +} + +function workerDnsManifest(target: K3sClusterTarget): string { + const dns = target.k3s.workerDns; + const desiredHash = workerDnsDesiredHash(target); + const labels = `app.kubernetes.io/name: ${dns.deploymentName}\n app.kubernetes.io/managed-by: unidesk`; + return `apiVersion: apps/v1 +kind: Deployment +metadata: + name: ${dns.deploymentName} + namespace: ${dns.namespace} + annotations: + unidesk.ai/config-sha256: ${desiredHash} + labels: + ${labels} +spec: + replicas: ${dns.replicas} + revisionHistoryLimit: 1 + strategy: + type: RollingUpdate + rollingUpdate: + maxUnavailable: 0 + maxSurge: 1 + selector: + matchLabels: + app.kubernetes.io/name: ${dns.deploymentName} + template: + metadata: + annotations: + unidesk.ai/config-sha256: ${desiredHash} + labels: + app.kubernetes.io/name: ${dns.deploymentName} + app.kubernetes.io/managed-by: unidesk + spec: + serviceAccountName: coredns + priorityClassName: system-cluster-critical + dnsPolicy: Default + nodeSelector: + kubernetes.io/hostname: ${target.worker.nodeName} + tolerations: + - key: CriticalAddonsOnly + operator: Exists + containers: + - name: coredns + image: ${dns.image} + imagePullPolicy: IfNotPresent + args: ["-conf", "/etc/coredns/Corefile"] + ports: + - { name: dns, containerPort: 53, protocol: UDP } + - { name: dns-tcp, containerPort: 53, protocol: TCP } + - { name: metrics, containerPort: 9153, protocol: TCP } + readinessProbe: + httpGet: { path: /ready, port: 8181 } + periodSeconds: 2 + timeoutSeconds: 1 + failureThreshold: 3 + livenessProbe: + httpGet: { path: /health, port: 8080 } + initialDelaySeconds: 60 + periodSeconds: 10 + timeoutSeconds: 1 + failureThreshold: 3 + resources: + requests: { cpu: 100m, memory: 70Mi } + limits: { memory: 170Mi } + securityContext: + allowPrivilegeEscalation: false + readOnlyRootFilesystem: true + capabilities: + add: [NET_BIND_SERVICE] + drop: [ALL] + volumeMounts: + - { name: config-volume, mountPath: /etc/coredns, readOnly: true } + - { name: custom-config-volume, mountPath: /etc/coredns/custom, readOnly: true } + volumes: + - name: config-volume + configMap: + name: ${dns.configMapName} + items: + - { key: Corefile, path: Corefile } + - { key: NodeHosts, path: NodeHosts } + - name: custom-config-volume + configMap: + name: ${dns.customConfigMapName} + optional: true +--- +apiVersion: v1 +kind: Service +metadata: + name: ${dns.serviceName} + namespace: ${dns.namespace} + annotations: + unidesk.ai/config-sha256: ${desiredHash} + labels: + app.kubernetes.io/name: ${dns.deploymentName} + app.kubernetes.io/managed-by: unidesk +spec: + clusterIP: ${dns.clusterIp} + internalTrafficPolicy: Local + selector: + app.kubernetes.io/name: ${dns.deploymentName} + ports: + - { name: dns, port: 53, targetPort: 53, protocol: UDP } + - { name: dns-tcp, port: 53, targetPort: 53, protocol: TCP } + - { name: metrics, port: 9153, targetPort: 9153, protocol: TCP } +`; +} + function workerWireguardApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string { const wg = target.wireguard; return `set -eu @@ -320,6 +517,7 @@ function workerApplyScript(target: K3sClusterTarget, secret: SecretMaterial): st "--node-name", target.worker.nodeName, "--node-ip", address(wg.workerAddress), "--flannel-iface", target.k3s.flannelInterface, ...labels, "--kubelet-arg", `max-pods=${target.k3s.maxPods}`, + "--kubelet-arg", `cluster-dns=${target.k3s.workerDns.clusterIp}`, ].map(shellToken).join(" "); const desiredHash = createHash("sha256").update(JSON.stringify({ version: target.worker.k3sVersion, @@ -467,6 +665,7 @@ spec: - sh - -ec - | + grep -F ${shQuote(`nameserver ${target.k3s.workerDns.clusterIp}`)} /etc/resolv.conf dns_output="$(nslookup kubernetes.default.svc.cluster.local 2>&1)" printf '%s\\n' "$dns_output" printf '%s\\n' "$dns_output" | grep -F "$KUBERNETES_SERVICE_HOST"