Merge remote-tracking branch 'origin/master'

config/platform-infra/k3s-clusters.yaml

@@ -33,7 +33,7 @@ targets:
       controlPlaneAddress: 10.89.0.1/24
       workerAddress: 10.89.0.2/24
       endpoint: 152.53.229.148:51820
-      persistentKeepaliveSeconds: 25
+      persistentKeepaliveSeconds: 10
     secrets:
       sourceRef: /root/.unidesk/.state/secrets/platform-infra/k3s-clusters/nc01-d601.env
       controlPlanePrivateKey: NC01_WIREGUARD_PRIVATE_KEY
@@ -52,6 +52,16 @@ targets:
       serviceLb:
         enabledNodeLabel: svccontroller.k3s.cattle.io/enablelb
         enabledNode: v2202607378382480008
+      workerDns:
+        namespace: kube-system
+        deploymentName: unidesk-worker-coredns
+        serviceName: unidesk-worker-kube-dns
+        clusterIp: 10.43.0.53
+        image: rancher/mirrored-coredns-coredns:1.14.4
+        configMapName: coredns
+        customConfigMapName: coredns-custom
+        replicas: 1
+        timeoutSeconds: 120
     smoke:
       namespace: platform-infra-smoke
       jobName: d601-runtime-plane-smoke

docs/reference/platform-infra.md

@@ -24,6 +24,7 @@
   - 集群 apply 先经宿主内网 SSH 分发零依赖 proxy client，确认它连接声明的 `vpn-server` 且外网 probe 成功，再启动或恢复 `k3s-agent`。
   - 集群 apply 在 proxy 前先收敛 worker WireGuard；内容一致时不强写，只有 unit 未运行或握手过期且隧道探针失败时才恢复 worker 接口。
   - worker WireGuard 的业务探针固定使用 control-plane API `:6443`，不以可能被策略丢弃的 ICMP 判断隧道健康。
+  - NAT 后 worker 的 WireGuard `PersistentKeepalive` 周期由集群 YAML 声明；跨地域节点优先用较短周期维持映射，但不得据此宣称降低物理 RTT。
   - 大型 proxy 工件由 Windows 宿主按 YAML `upstreamUrl` 获取并校验 SHA-256，Provider/trans 只承载控制命令，不作为大二进制数据面。
   - WireGuard 只承载集群三层网络；NAT 或链路不稳定的 worker proxy 使用 `vpn-server` 输出的 Hysteria2 client YAML，不把应用出网 TCP 再套入 WireGuard TCP 路径。
   - proxy client 协议实现版本由 host-proxy YAML 声明，并与所连接的 `vpn-server` 协议实现版本保持一致。
@@ -36,6 +37,11 @@
   - control-plane 保持唯一 k3s server，同时承担 workload。
   - worker 只运行 `k3s-agent`；加入前必须卸载旧独立 server，禁止保留第二 datastore/control-plane。
   - Flannel 使用 WireGuard 接口承载 VXLAN，不能只用 FRP 转发 Kubernetes API。
+  - 跨地域 worker DNS 使用集群 YAML 声明的独立 CoreDNS Deployment 和固定 ClusterIP：
+    - Deployment 固定调度到 worker，Service 使用 `internalTrafficPolicy: Local`；
+    - worker kubelet 只在本地 DNS Ready 后通过 `cluster-dns` 参数切换；
+    - 禁止 patch k3s Addon 拥有的 `coredns` Deployment 或 `kube-dns` Service；
+    - 受控 apply 先比较声明态哈希，内容一致时跳过 `kubectl apply`，不得触发无意义 rollout。
   - ServiceLB 只在 owning YAML 指定的公网节点启用，防止无公网地址 worker 接管 hostPort。
   - worker 的 containerd 外网访问继承 host proxy systemd 环境，直接访问上游 registry；禁止为公共依赖新增 registry mirror。
   - `127.0.0.1:5000` 只表示 UniDesk 集群内 artifact registry，worker 的 `registries.yaml` 可把该本地引用映射到集群 endpoint，不得把它扩展为公共镜像代理。

scripts/src/platform-infra-k3s-cluster-config.ts

@@ -53,6 +53,17 @@ export interface K3sClusterTarget {
     maxPods: number;
     registryMirror: { source: string; endpoint: string };
     serviceLb: { enabledNodeLabel: string; enabledNode: string };
+    workerDns: {
+      namespace: string;
+      deploymentName: string;
+      serviceName: string;
+      clusterIp: string;
+      image: string;
+      configMapName: string;
+      customConfigMapName: string;
+      replicas: number;
+      timeoutSeconds: number;
+    };
   };
   smoke: { namespace: string; jobName: string; image: string; timeoutSeconds: number };
 }
@@ -76,6 +87,7 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri
   const k3s = record(raw.k3s, `targets.${id}.k3s`);
   const mirror = record(k3s.registryMirror, `targets.${id}.k3s.registryMirror`);
   const serviceLb = record(k3s.serviceLb, `targets.${id}.k3s.serviceLb`);
+  const workerDns = record(k3s.workerDns, `targets.${id}.k3s.workerDns`);
   const smoke = record(raw.smoke, `targets.${id}.smoke`);
   const controlSource = resolveRef(text(control, "configRef", `targets.${id}.controlPlane`));
   const bootstrapSource = resolveRef(text(worker, "bootstrapConfigRef", `targets.${id}.workers[0]`));
@@ -134,6 +146,17 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri
       maxPods: integer(k3s, "maxPods", `targets.${id}.k3s`),
       registryMirror: { source: text(mirror, "source", "registryMirror"), endpoint: text(mirror, "endpoint", "registryMirror") },
       serviceLb: { enabledNodeLabel: text(serviceLb, "enabledNodeLabel", "serviceLb"), enabledNode: text(serviceLb, "enabledNode", "serviceLb") },
+      workerDns: {
+        namespace: kubernetesName(workerDns, "namespace", "workerDns"),
+        deploymentName: kubernetesName(workerDns, "deploymentName", "workerDns"),
+        serviceName: kubernetesName(workerDns, "serviceName", "workerDns"),
+        clusterIp: ipv4Address(workerDns, "clusterIp", "workerDns"),
+        image: text(workerDns, "image", "workerDns"),
+        configMapName: kubernetesName(workerDns, "configMapName", "workerDns"),
+        customConfigMapName: kubernetesName(workerDns, "customConfigMapName", "workerDns"),
+        replicas: integer(workerDns, "replicas", "workerDns"),
+        timeoutSeconds: integer(workerDns, "timeoutSeconds", "workerDns"),
+      },
     },
     smoke: {
       namespace: text(smoke, "namespace", `targets.${id}.smoke`),
@@ -196,6 +219,22 @@ function cidr(value: Record<string, unknown>, key: string, label: string): strin
   return result;
 }
 
+function ipv4Address(value: Record<string, unknown>, key: string, label: string): string {
+  const result = text(value, key, label);
+  if (!/^\d{1,3}(?:\.\d{1,3}){3}$/u.test(result) || result.split(".").some((part) => Number(part) > 255)) {
+    throw new Error(`${label}.${key} must be an IPv4 address`);
+  }
+  return result;
+}
+
+function kubernetesName(value: Record<string, unknown>, key: string, label: string): string {
+  const result = text(value, key, label);
+  if (!/^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/u.test(result) || result.length > 63) {
+    throw new Error(`${label}.${key} must be a Kubernetes DNS label`);
+  }
+  return result;
+}
+
 function stringMap(value: Record<string, unknown>, label: string): Record<string, string> {
   return Object.fromEntries(Object.entries(value).map(([key, item]) => {
     if (typeof item !== "string" || item.length === 0) throw new Error(`${label}.${key} must be a non-empty string`);

scripts/src/platform-infra-k3s-cluster.ts

@@ -72,6 +72,7 @@ function plan(target: K3sClusterTarget): Record<string, unknown> {
       agentTokenFile: target.k3s.agentTokenFile,
       registryMirror: target.k3s.registryMirror,
       serviceLb: target.k3s.serviceLb,
+      workerDns: target.k3s.workerDns,
     },
     secret: secretSummary(target),
     next: {
@@ -119,6 +120,8 @@ function apply(target: K3sClusterTarget): Record<string, unknown> {
   ]) as Record<string, unknown>;
   if (controlPlane.ok !== true) throw new Error(`control-plane-k3s-apply failed: ${bounded(JSON.stringify(compactDomain(controlPlane)))}`);
 
+  const workerDns = reconcileWorkerDns(target);
+
   const worker = runWorker(target, workerApplyScript(target, secret), true, 300_000);
   requireSuccess(worker, "worker-agent-apply");
 
@@ -146,6 +149,7 @@ function apply(target: K3sClusterTarget): Record<string, unknown> {
       },
       controlPlaneWireguard: compact(controlWireguard),
       controlPlaneK3s: compactDomain(controlPlane),
+      workerDns,
       workerAgent: compact(worker),
       serviceLbPlacement: compact(labelResult),
       nodeReadiness: compact(ready),
@@ -162,6 +166,7 @@ function status(target: K3sClusterTarget): Record<string, unknown> {
   const control = runControlPlane(target, statusScript(target), 60_000);
   const workerProxy = runWorker(target, hostProxyGuestStatusScript(target.worker.hostProxyTargetId), false, 60_000);
   const worker = runWorker(target, workerStatusScript(target), false, 60_000);
+  const workerDns = workerDnsStatus(target);
   const controlData = parseFirstJson(control.stdout);
   const workerProxyData = parseFirstJson(workerProxy.stdout);
   const workerData = parseFirstJson(worker.stdout);
@@ -170,7 +175,8 @@ function status(target: K3sClusterTarget): Record<string, unknown> {
     && worker.exitCode === 0
     && controlData.ok === true
     && workerProxyData.ok === true
-    && workerData.ok === true;
+    && workerData.ok === true
+    && workerDns.data.ok === true;
   return {
     ok,
     mutation: false,
@@ -178,8 +184,9 @@ function status(target: K3sClusterTarget): Record<string, unknown> {
     controlPlane: controlData,
     workerHostProxy: workerProxyData,
     worker: workerData,
+    workerDns: workerDns.data,
     secret: secretSummary(target),
-    result: { controlPlane: compact(control), workerHostProxy: compact(workerProxy), worker: compact(worker) },
+    result: { controlPlane: compact(control), workerHostProxy: compact(workerProxy), worker: compact(worker), workerDns: compact(workerDns.result) },
     next: ok ? { smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm` } : { apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm` },
   };
 }
@@ -229,6 +236,196 @@ function smoke(target: K3sClusterTarget, confirm: boolean): Record<string, unkno
   };
 }
 
+function reconcileWorkerDns(target: K3sClusterTarget): Record<string, unknown> {
+  const before = workerDnsStatus(target);
+  const mutation = before.data.configurationCurrent !== true;
+  let applyResult: ReturnType<typeof runCommand> | null = null;
+  if (mutation) {
+    applyResult = runCommand(
+      ["trans", target.controlPlane.kubeRoute, "kubectl", "apply", "-f", "-"],
+      rootPath(),
+      { input: workerDnsManifest(target), timeoutMs: 60_000 },
+    );
+    requireSuccess(applyResult, "worker-dns-apply");
+  }
+
+  const deadline = Date.now() + target.k3s.workerDns.timeoutSeconds * 1000;
+  let current = workerDnsStatus(target);
+  while (current.data.ok !== true && Date.now() < deadline) {
+    runCommand(["sleep", "3"], rootPath(), { timeoutMs: 5_000 });
+    current = workerDnsStatus(target);
+  }
+  if (current.data.ok !== true) throw new Error(`worker-dns-readiness failed: ${bounded(JSON.stringify(current.data))}`);
+  return {
+    mutation,
+    desiredHash: workerDnsDesiredHash(target),
+    ...current.data,
+    apply: applyResult === null ? { skipped: true, reason: "configuration-current" } : compact(applyResult),
+  };
+}
+
+function workerDnsStatus(target: K3sClusterTarget): { data: Record<string, unknown>; result: ReturnType<typeof runCommand> } {
+  const dns = target.k3s.workerDns;
+  const result = runCommand([
+    "trans", target.controlPlane.kubeRoute, "kubectl", "-n", dns.namespace, "get", "deployment,service,endpoints",
+    "-l", `app.kubernetes.io/name=${dns.deploymentName}`, "-o", "json",
+  ], rootPath(), { timeoutMs: 30_000 });
+  if (result.exitCode !== 0) {
+    return {
+      data: { ok: false, presence: false, configurationCurrent: false, error: bounded(result.stderr || result.stdout) },
+      result,
+    };
+  }
+  let items: Array<Record<string, any>> = [];
+  try {
+    const parsed = JSON.parse(result.stdout) as { items?: Array<Record<string, any>> };
+    items = Array.isArray(parsed.items) ? parsed.items : [];
+  } catch {
+    return { data: { ok: false, presence: false, configurationCurrent: false, error: "worker-dns-json-invalid" }, result };
+  }
+  const deployment = items.find((item) => item.kind === "Deployment");
+  const service = items.find((item) => item.kind === "Service");
+  const endpoints = items.find((item) => item.kind === "Endpoints");
+  const desiredHash = workerDnsDesiredHash(target);
+  const deploymentHash = deployment?.metadata?.annotations?.["unidesk.ai/config-sha256"] ?? null;
+  const serviceHash = service?.metadata?.annotations?.["unidesk.ai/config-sha256"] ?? null;
+  const readyReplicas = Number(deployment?.status?.readyReplicas ?? 0);
+  const endpointNodes = (endpoints?.subsets ?? []).flatMap((subset: Record<string, any>) => subset.addresses ?? [])
+    .map((entry: Record<string, any>) => entry.nodeName)
+    .filter((nodeName: unknown): nodeName is string => typeof nodeName === "string");
+  const configurationCurrent = deploymentHash === desiredHash && serviceHash === desiredHash;
+  const deploymentReady = readyReplicas === dns.replicas;
+  const endpointLocal = endpointNodes.includes(target.worker.nodeName);
+  const serviceIp = service?.spec?.clusterIP ?? null;
+  const localTrafficPolicy = service?.spec?.internalTrafficPolicy ?? null;
+  return {
+    data: {
+      ok: configurationCurrent && deploymentReady && endpointLocal && serviceIp === dns.clusterIp && localTrafficPolicy === "Local",
+      presence: deployment !== undefined && service !== undefined && endpoints !== undefined,
+      configurationCurrent,
+      deploymentReady,
+      readyReplicas,
+      desiredReplicas: dns.replicas,
+      serviceIp,
+      localTrafficPolicy,
+      endpointNodes,
+      workerNode: target.worker.nodeName,
+      desiredHash,
+    },
+    result,
+  };
+}
+
+function workerDnsDesiredHash(target: K3sClusterTarget): string {
+  return hash(JSON.stringify({ workerNode: target.worker.nodeName, ...target.k3s.workerDns }));
+}
+
+function workerDnsManifest(target: K3sClusterTarget): string {
+  const dns = target.k3s.workerDns;
+  const desiredHash = workerDnsDesiredHash(target);
+  const labels = `app.kubernetes.io/name: ${dns.deploymentName}\n    app.kubernetes.io/managed-by: unidesk`;
+  return `apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: ${dns.deploymentName}
+  namespace: ${dns.namespace}
+  annotations:
+    unidesk.ai/config-sha256: ${desiredHash}
+  labels:
+    ${labels}
+spec:
+  replicas: ${dns.replicas}
+  revisionHistoryLimit: 1
+  strategy:
+    type: RollingUpdate
+    rollingUpdate:
+      maxUnavailable: 0
+      maxSurge: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/name: ${dns.deploymentName}
+  template:
+    metadata:
+      annotations:
+        unidesk.ai/config-sha256: ${desiredHash}
+      labels:
+        app.kubernetes.io/name: ${dns.deploymentName}
+        app.kubernetes.io/managed-by: unidesk
+    spec:
+      serviceAccountName: coredns
+      priorityClassName: system-cluster-critical
+      dnsPolicy: Default
+      nodeSelector:
+        kubernetes.io/hostname: ${target.worker.nodeName}
+      tolerations:
+        - key: CriticalAddonsOnly
+          operator: Exists
+      containers:
+        - name: coredns
+          image: ${dns.image}
+          imagePullPolicy: IfNotPresent
+          args: ["-conf", "/etc/coredns/Corefile"]
+          ports:
+            - { name: dns, containerPort: 53, protocol: UDP }
+            - { name: dns-tcp, containerPort: 53, protocol: TCP }
+            - { name: metrics, containerPort: 9153, protocol: TCP }
+          readinessProbe:
+            httpGet: { path: /ready, port: 8181 }
+            periodSeconds: 2
+            timeoutSeconds: 1
+            failureThreshold: 3
+          livenessProbe:
+            httpGet: { path: /health, port: 8080 }
+            initialDelaySeconds: 60
+            periodSeconds: 10
+            timeoutSeconds: 1
+            failureThreshold: 3
+          resources:
+            requests: { cpu: 100m, memory: 70Mi }
+            limits: { memory: 170Mi }
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities:
+              add: [NET_BIND_SERVICE]
+              drop: [ALL]
+          volumeMounts:
+            - { name: config-volume, mountPath: /etc/coredns, readOnly: true }
+            - { name: custom-config-volume, mountPath: /etc/coredns/custom, readOnly: true }
+      volumes:
+        - name: config-volume
+          configMap:
+            name: ${dns.configMapName}
+            items:
+              - { key: Corefile, path: Corefile }
+              - { key: NodeHosts, path: NodeHosts }
+        - name: custom-config-volume
+          configMap:
+            name: ${dns.customConfigMapName}
+            optional: true
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: ${dns.serviceName}
+  namespace: ${dns.namespace}
+  annotations:
+    unidesk.ai/config-sha256: ${desiredHash}
+  labels:
+    app.kubernetes.io/name: ${dns.deploymentName}
+    app.kubernetes.io/managed-by: unidesk
+spec:
+  clusterIP: ${dns.clusterIp}
+  internalTrafficPolicy: Local
+  selector:
+    app.kubernetes.io/name: ${dns.deploymentName}
+  ports:
+    - { name: dns, port: 53, targetPort: 53, protocol: UDP }
+    - { name: dns-tcp, port: 53, targetPort: 53, protocol: TCP }
+    - { name: metrics, port: 9153, targetPort: 9153, protocol: TCP }
+`;
+}
+
 function workerWireguardApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string {
   const wg = target.wireguard;
   return `set -eu
@@ -320,6 +517,7 @@ function workerApplyScript(target: K3sClusterTarget, secret: SecretMaterial): st
     "--node-name", target.worker.nodeName, "--node-ip", address(wg.workerAddress),
     "--flannel-iface", target.k3s.flannelInterface, ...labels,
     "--kubelet-arg", `max-pods=${target.k3s.maxPods}`,
+    "--kubelet-arg", `cluster-dns=${target.k3s.workerDns.clusterIp}`,
   ].map(shellToken).join(" ");
   const desiredHash = createHash("sha256").update(JSON.stringify({
     version: target.worker.k3sVersion,
@@ -467,6 +665,7 @@ spec:
             - sh
             - -ec
             - |
+              grep -F ${shQuote(`nameserver ${target.k3s.workerDns.clusterIp}`)} /etc/resolv.conf
               dns_output="$(nslookup kubernetes.default.svc.cluster.local 2>&1)"
               printf '%s\\n' "$dns_output"
               printf '%s\\n' "$dns_output" | grep -F "$KUBERNETES_SERVICE_HOST"