From 43bd89c151574a5089eef977b30530c911d78f80 Mon Sep 17 00:00:00 2001 From: pikastech Date: Mon, 20 Jul 2026 07:58:04 +0200 Subject: [PATCH] perf(k3s): tune cross-region TCP for loss --- config/platform-infra/k3s-clusters.yaml | 5 ++ docs/reference/platform-infra.md | 6 +++ .../src/platform-infra-k3s-cluster-config.ts | 26 ++++++++++ scripts/src/platform-infra-k3s-cluster.ts | 50 ++++++++++++++++--- 4 files changed, 79 insertions(+), 8 deletions(-) diff --git a/config/platform-infra/k3s-clusters.yaml b/config/platform-infra/k3s-clusters.yaml index 027fae40..76b8c415 100644 --- a/config/platform-infra/k3s-clusters.yaml +++ b/config/platform-infra/k3s-clusters.yaml @@ -34,6 +34,11 @@ targets: workerAddress: 10.89.0.2/24 endpoint: 152.53.229.148:51820 persistentKeepaliveSeconds: 10 + tcpTuning: + congestionControl: bbr + defaultQdisc: fq + mtuProbing: 1 + sysctlPath: /etc/sysctl.d/90-unidesk-k3s-cross-region.conf secrets: sourceRef: /root/.unidesk/.state/secrets/platform-infra/k3s-clusters/nc01-d601.env controlPlanePrivateKey: NC01_WIREGUARD_PRIVATE_KEY diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 15786da6..4a79ba14 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -25,6 +25,12 @@ - 集群 apply 在 proxy 前先收敛 worker WireGuard；内容一致时不强写，只有 unit 未运行或握手过期且隧道探针失败时才恢复 worker 接口。 - worker WireGuard 的业务探针固定使用 control-plane API `:6443`，不以可能被策略丢弃的 ICMP 判断隧道健康。 - NAT 后 worker 的 WireGuard `PersistentKeepalive` 周期由集群 YAML 声明；跨地域节点优先用较短周期维持映射，但不得据此宣称降低物理 RTT。 + - 跨地域 TCP 调优由集群 YAML 的 `tcpTuning` 统一声明并受控应用到两端： + - 高 RTT、持续丢包链路优先使用内核支持的 BBR、`fq` 与 + `tcp_mtu_probing=1`，同时保留实际吞吐复测； + - sysctl 文件和 live 值均先比较后收敛，不得靠裸 `sysctl`、 + 预拉镜像、导入镜像或反复重启 Pod 掩盖数据面问题； + - status 必须披露两端 live tuning 是否与 YAML 一致。 - 大型 proxy 工件由 Windows 宿主按 YAML `upstreamUrl` 获取并校验 SHA-256，Provider/trans 只承载控制命令，不作为大二进制数据面。 - WireGuard 只承载集群三层网络；D601 worker proxy 使用 `vpn-server` 的 NC01 公网 Shadowsocks TCP 入口，不把应用出网 TCP 再套入 WireGuard， diff --git a/scripts/src/platform-infra-k3s-cluster-config.ts b/scripts/src/platform-infra-k3s-cluster-config.ts index dad555e3..1dec9947 100644 --- a/scripts/src/platform-infra-k3s-cluster-config.ts +++ b/scripts/src/platform-infra-k3s-cluster-config.ts @@ -38,6 +38,12 @@ export interface K3sClusterTarget { endpoint: string; persistentKeepaliveSeconds: number; }; + tcpTuning: { + congestionControl: "bbr" | "cubic"; + defaultQdisc: "fq" | "fq_codel"; + mtuProbing: 1 | 2; + sysctlPath: string; + }; secrets: { sourceRef: string; controlPlanePrivateKey: string; @@ -83,6 +89,7 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri const [workerId, workerRaw] = workerEntries[0] ?? []; const worker = record(workerRaw, `targets.${id}.workers.${workerId}`); const wireguard = record(raw.wireguard, `targets.${id}.wireguard`); + const tcpTuning = record(raw.tcpTuning, `targets.${id}.tcpTuning`); const secrets = record(raw.secrets, `targets.${id}.secrets`); const k3s = record(raw.k3s, `targets.${id}.k3s`); const mirror = record(k3s.registryMirror, `targets.${id}.k3s.registryMirror`); @@ -131,6 +138,12 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri endpoint: text(wireguard, "endpoint", `targets.${id}.wireguard`), persistentKeepaliveSeconds: integer(wireguard, "persistentKeepaliveSeconds", `targets.${id}.wireguard`), }, + tcpTuning: { + congestionControl: enumText(tcpTuning, "congestionControl", `targets.${id}.tcpTuning`, ["bbr", "cubic"]), + defaultQdisc: enumText(tcpTuning, "defaultQdisc", `targets.${id}.tcpTuning`, ["fq", "fq_codel"]), + mtuProbing: oneOfInteger(tcpTuning, "mtuProbing", `targets.${id}.tcpTuning`, [1, 2]), + sysctlPath: text(tcpTuning, "sysctlPath", `targets.${id}.tcpTuning`), + }, secrets: { sourceRef: text(secrets, "sourceRef", `targets.${id}.secrets`), controlPlanePrivateKey: text(secrets, "controlPlanePrivateKey", `targets.${id}.secrets`), @@ -167,6 +180,7 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri }; if (target.k3s.flannelInterface !== target.wireguard.interfaceName) throw new Error(`targets.${id}.k3s.flannelInterface must match wireguard.interfaceName`); if (target.k3s.serviceLb.enabledNode !== target.controlPlane.nodeName) throw new Error(`targets.${id}.k3s.serviceLb.enabledNode must match controlPlane.nodeName`); + if (!target.tcpTuning.sysctlPath.startsWith("/etc/sysctl.d/")) throw new Error(`targets.${id}.tcpTuning.sysctlPath must stay under /etc/sysctl.d`); return { defaultTargetId, target }; } @@ -213,6 +227,18 @@ function integer(value: Record, key: string, label: string): nu return Number(result); } +function enumText(value: Record, key: string, label: string, allowed: readonly T[]): T { + const result = text(value, key, label); + if (!allowed.includes(result as T)) throw new Error(`${label}.${key} must be one of ${allowed.join(", ")}`); + return result as T; +} + +function oneOfInteger(value: Record, key: string, label: string, allowed: readonly T[]): T { + const result = integer(value, key, label); + if (!allowed.includes(result as T)) throw new Error(`${label}.${key} must be one of ${allowed.join(", ")}`); + return result as T; +} + function cidr(value: Record, key: string, label: string): string { const result = text(value, key, label); if (!/^\d{1,3}(?:\.\d{1,3}){3}\/\d{1,2}$/u.test(result)) throw new Error(`${label}.${key} must be an IPv4 CIDR`); diff --git a/scripts/src/platform-infra-k3s-cluster.ts b/scripts/src/platform-infra-k3s-cluster.ts index e61ebaac..08847f9e 100644 --- a/scripts/src/platform-infra-k3s-cluster.ts +++ b/scripts/src/platform-infra-k3s-cluster.ts @@ -67,6 +67,7 @@ function plan(target: K3sClusterTarget): Record { }, }, wireguard: { ...target.wireguard, endpoint: target.wireguard.endpoint }, + tcpTuning: target.tcpTuning, k3s: { serverUrl: target.k3s.serverUrl, agentTokenFile: target.k3s.agentTokenFile, @@ -433,6 +434,7 @@ spec: function workerWireguardApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string { const wg = target.wireguard; return `set -eu +${tcpTuningApplyScript(target)} if ! command -v wg >/dev/null 2>&1; then DEBIAN_FRONTEND=noninteractive apt-get update -qq DEBIAN_FRONTEND=noninteractive apt-get install -y -qq ${shellToken(wg.packageName)} @@ -477,6 +479,7 @@ function controlPlaneApplyScript(target: K3sClusterTarget, secret: SecretMateria const wg = target.wireguard; return `set -eu export DEBIAN_FRONTEND=noninteractive +${tcpTuningApplyScript(target)} if ! command -v wg >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq ${shQuote(wg.packageName)}; fi umask 077 install -d -m 700 /etc/wireguard /etc/unidesk @@ -513,6 +516,33 @@ wg show ${wg.interfaceName} >/dev/null `; } +function tcpTuningApplyScript(target: K3sClusterTarget): string { + const tuning = target.tcpTuning; + const content = [ + `net.core.default_qdisc=${tuning.defaultQdisc}`, + `net.ipv4.tcp_congestion_control=${tuning.congestionControl}`, + `net.ipv4.tcp_mtu_probing=${tuning.mtuProbing}`, + "", + ].join("\n"); + const encoded = Buffer.from(content, "utf8").toString("base64"); + return `modprobe tcp_${tuning.congestionControl} 2>/dev/null || true +available="$(sysctl -n net.ipv4.tcp_available_congestion_control)" +case " $available " in + *" ${tuning.congestionControl} "*) ;; + *) echo "configured TCP congestion control is unavailable: ${tuning.congestionControl}" >&2; exit 1 ;; +esac +sysctl_candidate="$(mktemp)" +printf %s ${shQuote(encoded)} | base64 -d >"$sysctl_candidate" +if ! cmp -s "$sysctl_candidate" ${shQuote(tuning.sysctlPath)} 2>/dev/null; then + install -D -m 0644 "$sysctl_candidate" ${shQuote(tuning.sysctlPath)} +fi +rm -f "$sysctl_candidate" +[ "$(sysctl -n net.core.default_qdisc)" = ${shQuote(tuning.defaultQdisc)} ] || sysctl -q -w net.core.default_qdisc=${shellToken(tuning.defaultQdisc)} +[ "$(sysctl -n net.ipv4.tcp_congestion_control)" = ${shQuote(tuning.congestionControl)} ] || sysctl -q -w net.ipv4.tcp_congestion_control=${shellToken(tuning.congestionControl)} +[ "$(sysctl -n net.ipv4.tcp_mtu_probing)" = ${shQuote(String(tuning.mtuProbing))} ] || sysctl -q -w net.ipv4.tcp_mtu_probing=${tuning.mtuProbing} +`; +} + function workerApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string { const wg = target.wireguard; const labels = Object.entries(target.worker.labels).flatMap(([key, value]) => ["--node-label", `${key}=${value}`]); @@ -617,13 +647,15 @@ cp_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.controlPlane.nodeNam worker_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) worker_ip=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null || true) handshake=$(wg show ${target.wireguard.interfaceName} latest-handshakes 2>/dev/null | awk '{print $2}' | head -n1) -python3 - "$wg_active" "$k3s_active" "$cp_ready" "$worker_ready" "$worker_ip" "$handshake" <<'PY' +tuning="$(sysctl -n net.ipv4.tcp_congestion_control)/$(sysctl -n net.core.default_qdisc)/$(sysctl -n net.ipv4.tcp_mtu_probing)" +python3 - "$wg_active" "$k3s_active" "$cp_ready" "$worker_ready" "$worker_ip" "$handshake" "$tuning" <<'PY' import json, sys, time -wg, k3s, cp, worker, worker_ip, handshake = sys.argv[1:] +wg, k3s, cp, worker, worker_ip, handshake, tuning = sys.argv[1:] stamp = int(handshake or 0) age = None if stamp == 0 else max(0, int(time.time()) - stamp) -ok = wg == 'true' and k3s == 'true' and cp == 'True' and worker == 'True' and worker_ip == '${address(target.wireguard.workerAddress)}' and age is not None and age < 180 -print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'k3sActive': k3s == 'true', 'controlPlaneReady': cp == 'True', 'workerReady': worker == 'True', 'workerInternalIp': worker_ip, 'handshakeAgeSeconds': age, 'controlPlaneAddress': '${cp}'}, separators=(',', ':'))) +tuning_ready = tuning == '${target.tcpTuning.congestionControl}/${target.tcpTuning.defaultQdisc}/${target.tcpTuning.mtuProbing}' +ok = wg == 'true' and k3s == 'true' and cp == 'True' and worker == 'True' and worker_ip == '${address(target.wireguard.workerAddress)}' and age is not None and age < 180 and tuning_ready +print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'k3sActive': k3s == 'true', 'controlPlaneReady': cp == 'True', 'workerReady': worker == 'True', 'workerInternalIp': worker_ip, 'handshakeAgeSeconds': age, 'controlPlaneAddress': '${cp}', 'tcpTuningReady': tuning_ready, 'tcpTuning': tuning}, separators=(',', ':'))) PY `; } @@ -635,12 +667,14 @@ agent_state="$(systemctl show k3s-agent -p ActiveState --value 2>/dev/null || tr agent_pid="$(systemctl show k3s-agent -p MainPID --value 2>/dev/null || true)" old_server=false; systemctl is-active --quiet k3s && old_server=true ip_present=false; ip -4 address show dev ${target.wireguard.interfaceName} | grep -F ${shQuote(address(target.wireguard.workerAddress))} >/dev/null 2>&1 && ip_present=true -python3 - "$wg_active" "$agent_state" "$agent_pid" "$old_server" "$ip_present" <<'PY' +tuning="$(sysctl -n net.ipv4.tcp_congestion_control)/$(sysctl -n net.core.default_qdisc)/$(sysctl -n net.ipv4.tcp_mtu_probing)" +python3 - "$wg_active" "$agent_state" "$agent_pid" "$old_server" "$ip_present" "$tuning" <<'PY' import json, sys -wg, agent_state, agent_pid, old_server, ip_present = sys.argv[1:] +wg, agent_state, agent_pid, old_server, ip_present, tuning = sys.argv[1:] agent_running = agent_state in ('active', 'activating') and int(agent_pid or 0) > 0 -ok = wg == 'true' and agent_running and old_server == 'false' and ip_present == 'true' -print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'agentActive': agent_state == 'active', 'agentRunning': agent_running, 'agentState': agent_state, 'agentMainPidPresent': int(agent_pid or 0) > 0, 'standaloneServerActive': old_server == 'true', 'wireguardAddressPresent': ip_present == 'true'}, separators=(',', ':'))) +tuning_ready = tuning == '${target.tcpTuning.congestionControl}/${target.tcpTuning.defaultQdisc}/${target.tcpTuning.mtuProbing}' +ok = wg == 'true' and agent_running and old_server == 'false' and ip_present == 'true' and tuning_ready +print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'agentActive': agent_state == 'active', 'agentRunning': agent_running, 'agentState': agent_state, 'agentMainPidPresent': int(agent_pid or 0) > 0, 'standaloneServerActive': old_server == 'true', 'wireguardAddressPresent': ip_present == 'true', 'tcpTuningReady': tuning_ready, 'tcpTuning': tuning}, separators=(',', ':'))) PY `; }