diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 64b46b68..3813545e 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -352,6 +352,8 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - 节点加入、状态和调度 smoke 只走 `platform-infra k3s-cluster plan|apply|status|smoke`,具体 authority 和控制通道见 `docs/reference/platform-infra.md#多主机-k3s-集群`。 - 集群 apply 的 control-plane 阶段只允许组合 `hwlab nodes control-plane infra k3s-config` 窄域;禁止调用全量 `infra apply`,避免节点加入顺带覆盖 registry、Tekton、Argo、git-mirror 或运行面对象。 - k3s drop-in 未变化且节点容量已收敛时必须 `mutation=false`;禁止无条件重启、重装、强写或通过 CI 自动覆盖同一节点配置。 + - WireGuard 配置一致时不得 restart;k3s API/容量暂时不可读时保持 unknown 并停止 mutation,禁止把控制通道抖动解释为配置漂移。 + - worker agent 只在 YAML 派生摘要变化或 service 未运行时安装/恢复,不得在每次集群 apply 中先卸载再重装。 - worker 加入集群不改变 PaC consumer 的 source commit authority,也不授权人工 PipelineRun、Argo sync 或业务 rollout。 - 长命令用异步 job 或短轮询;不要长时间挂住 trans/ssh。 - Web 哨兵 YAML cadence 变更不新增发布 orchestrator:提交前用 `web-probe sentinel validate --local`;合并后用只读 `web-probe sentinel control-plane status --wait`。线上 desired schedule 必须来自 GitOps 清单,不能用操作者旧 worktree 的本地渲染值做阻塞门禁;收敛差异统一为非阻塞 warning。 diff --git a/.agents/skills/unidesk-ymalops/SKILL.md b/.agents/skills/unidesk-ymalops/SKILL.md index bd9cb4ce..f53ca9b2 100644 --- a/.agents/skills/unidesk-ymalops/SKILL.md +++ b/.agents/skills/unidesk-ymalops/SKILL.md @@ -95,6 +95,8 @@ description: UniDesk YAML-first 运维正规化技能。用户提到 ymal-first/ - control-plane drop-in 只走 `hwlab nodes control-plane infra k3s-config` 窄域: - 渲染结果一致且节点容量已收敛时返回 `mutation=false`; - 只在必要时写入 drop-in、reload 和重启 k3s; + - WireGuard 内容一致时不重启,API/容量读取暂时失败时保持 unknown 并停止 mutation; + - worker agent 使用 YAML 派生摘要决定是否重装,禁止每次 apply 先卸载再覆盖; - 禁止借用全量 infra apply 顺带覆盖 registry、CI/CD 或业务运行面。 - 日常入口和验收判定统一见 `docs/reference/platform-infra.md#多主机-k3s-集群`。 - 新节点 node-local registry 必须支持从零部署:优先由 YAML 渲染为 k3s workload/PVC;host Docker registry 只能是 legacy/迁移前状态。需要证明从零能力时先停旧 registry,不从旧 registry seed,并在状态中披露 `seededFromOldRegistry`、`zeroSeeded`、PVC、workload、endpoint 与 runtime/tools image preload 结果。 diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 109c1bea..1e94ccc4 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -34,6 +34,9 @@ - 幂等重复执行不得强写: - WireGuard、systemd drop-in、agent unit、token 文件和节点标签均先比较声明态与运行态; - 内容一致时保持 `mutation=false`,不得无条件重启、重装或覆盖; + - WireGuard 只在配置变化或 unit 未运行时重启/启动,避免短暂断网触发 control-plane 级联重启; + - worker agent 以 owning YAML、版本、参数、registry mirror 和 token 的脱敏摘要判断是否需要重装; + - k3s API 暂时不可读属于 unknown,不得当作容量漂移触发 restart; - 集群 CLI 只组合各自 owning domain,不借用全量基础设施入口扩大写入范围。 - 验收: - `status` 必须同时证明 WireGuard handshake、server/agent systemd、两个 Ready node、worker InternalIP 和旧 server 退役。 diff --git a/scripts/src/hwlab-node-control-plane-runtime.ts b/scripts/src/hwlab-node-control-plane-runtime.ts index c6b3fa44..64560a2d 100644 --- a/scripts/src/hwlab-node-control-plane-runtime.ts +++ b/scripts/src/hwlab-node-control-plane-runtime.ts @@ -223,12 +223,24 @@ expected=$(mktemp /tmp/unidesk-k3s-dropin.XXXXXX) printf %s ${shQuote(encoded)} | base64 -d >"$expected" before_sha= if [ -f "$dropin" ]; then before_sha=$(sha256sum "$dropin" | awk '{print "sha256:"$1}'); fi -before_capacity=$(kubectl get node "$node" -o 'jsonpath={.status.capacity.pods}' 2>/dev/null || true) -before_allocatable=$(kubectl get node "$node" -o 'jsonpath={.status.allocatable.pods}' 2>/dev/null || true) +service_before=$(systemctl is-active "$service" 2>/dev/null || true) +before_capacity= +before_allocatable= +if [ "$service_before" = active ]; then + for _ in $(seq 1 10); do + before_capacity=$(kubectl get node "$node" -o 'jsonpath={.status.capacity.pods}' 2>/dev/null || true) + before_allocatable=$(kubectl get node "$node" -o 'jsonpath={.status.allocatable.pods}' 2>/dev/null || true) + [ -n "$before_capacity" ] && [ -n "$before_allocatable" ] && break + sleep 2 + done +fi changed=false restart_required=false if ! cmp -s "$expected" "$dropin" 2>/dev/null; then changed=true; fi -if [ "$changed" = true ] || [ "$before_capacity" != "$desired_max_pods" ] || [ "$before_allocatable" != "$desired_max_pods" ]; then +if [ "$changed" = true ] || [ "$service_before" != active ]; then + restart_required=true +elif [ -n "$before_capacity" ] && [ -n "$before_allocatable" ] && \ + { [ "$before_capacity" != "$desired_max_pods" ] || [ "$before_allocatable" != "$desired_max_pods" ]; }; then restart_required=true fi mutation=false @@ -242,7 +254,14 @@ if [ "$apply" = true ] && [ "$restart_required" = true ]; then chmod 0644 "$dropin" fi systemctl daemon-reload || daemon_reload_rc=$? - if [ "$daemon_reload_rc" = 0 ]; then systemctl restart "$service" || restart_rc=$?; fi + if [ "$daemon_reload_rc" = 0 ]; then + state_db=/var/lib/rancher/k3s/server/db/state.db + derived_passwd=/var/lib/rancher/k3s/server/cred/passwd + if [ -f "$state_db" ] && [ -f "$derived_passwd" ] && [ "$derived_passwd" -nt "$state_db" ]; then + mv "$derived_passwd" "$derived_passwd.unidesk-pre-restart-$(date -u +%Y%m%dT%H%M%SZ)" + fi + systemctl restart "$service" || restart_rc=$? + fi mutation=true fi rm -f "$expected" diff --git a/scripts/src/platform-infra-k3s-cluster.ts b/scripts/src/platform-infra-k3s-cluster.ts index 79a4e655..d60f17cc 100644 --- a/scripts/src/platform-infra-k3s-cluster.ts +++ b/scripts/src/platform-infra-k3s-cluster.ts @@ -184,7 +184,10 @@ export DEBIAN_FRONTEND=noninteractive if ! command -v wg >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq ${shQuote(wg.packageName)}; fi umask 077 install -d -m 700 /etc/wireguard /etc/unidesk -cat > /etc/wireguard/${wg.interfaceName}.conf <<'EOF' +wg_candidate=$(mktemp /tmp/unidesk-${wg.interfaceName}.XXXXXX) +token_candidate=$(mktemp /tmp/unidesk-k3s-agent-token.XXXXXX) +trap 'rm -f "$wg_candidate" "$token_candidate"' EXIT +cat >"$wg_candidate" <<'EOF' [Interface] Address = ${wg.controlPlaneAddress} ListenPort = ${wg.listenPort} @@ -195,11 +198,21 @@ MTU = ${wg.mtu} PublicKey = ${secret.workerPublic} AllowedIPs = ${address(wg.workerAddress)}/32 EOF -chmod 600 /etc/wireguard/${wg.interfaceName}.conf -printf '%s\n' ${shQuote(secret.agentToken)} > ${shQuote(target.k3s.agentTokenFile)} -chmod 600 ${shQuote(target.k3s.agentTokenFile)} -systemctl enable wg-quick@${wg.interfaceName} >/dev/null -systemctl restart wg-quick@${wg.interfaceName} +printf '%s\n' ${shQuote(secret.agentToken)} >"$token_candidate" +wg_changed=false +if ! cmp -s "$wg_candidate" /etc/wireguard/${wg.interfaceName}.conf 2>/dev/null; then + install -m 0600 "$wg_candidate" /etc/wireguard/${wg.interfaceName}.conf + wg_changed=true +fi +if ! cmp -s "$token_candidate" ${shQuote(target.k3s.agentTokenFile)} 2>/dev/null; then + install -m 0600 "$token_candidate" ${shQuote(target.k3s.agentTokenFile)} +fi +systemctl enable wg-quick@${wg.interfaceName} >/dev/null 2>&1 || true +if [ "$wg_changed" = true ]; then + systemctl restart wg-quick@${wg.interfaceName} +elif ! systemctl is-active --quiet wg-quick@${wg.interfaceName}; then + systemctl start wg-quick@${wg.interfaceName} +fi wg show ${wg.interfaceName} >/dev/null `; } @@ -213,12 +226,22 @@ function workerApplyScript(target: K3sClusterTarget, secret: SecretMaterial): st "--flannel-iface", target.k3s.flannelInterface, ...labels, "--kubelet-arg", `max-pods=${target.k3s.maxPods}`, ].map(shellToken).join(" "); + const desiredHash = createHash("sha256").update(JSON.stringify({ + version: target.worker.k3sVersion, + exec, + registryMirror: target.k3s.registryMirror, + agentToken: secret.agentToken, + })).digest("hex"); return `set -eu export DEBIAN_FRONTEND=noninteractive if ! command -v wg >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq ${shQuote(wg.packageName)}; fi umask 077 install -d -m 700 /etc/wireguard /etc/unidesk /etc/rancher/k3s -cat > /etc/wireguard/${wg.interfaceName}.conf <<'EOF' +wg_candidate=$(mktemp /tmp/unidesk-${wg.interfaceName}.XXXXXX) +token_candidate=$(mktemp /tmp/unidesk-k3s-agent-token.XXXXXX) +registry_candidate=$(mktemp /tmp/unidesk-k3s-registries.XXXXXX) +trap 'rm -f "$wg_candidate" "$token_candidate" "$registry_candidate"' EXIT +cat >"$wg_candidate" <<'EOF' [Interface] Address = ${wg.workerAddress} PrivateKey = ${secret.workerPrivate} @@ -230,27 +253,47 @@ Endpoint = ${wg.endpoint} AllowedIPs = ${address(wg.controlPlaneAddress)}/32 PersistentKeepalive = ${wg.persistentKeepaliveSeconds} EOF -chmod 600 /etc/wireguard/${wg.interfaceName}.conf -systemctl enable wg-quick@${wg.interfaceName} >/dev/null -systemctl restart wg-quick@${wg.interfaceName} -ping -c 3 -W 5 ${address(wg.controlPlaneAddress)} >/dev/null -printf '%s\n' ${shQuote(secret.agentToken)} > ${shQuote(target.k3s.agentTokenFile)} -chmod 600 ${shQuote(target.k3s.agentTokenFile)} -if [ -x /usr/local/bin/k3s-uninstall.sh ]; then /usr/local/bin/k3s-uninstall.sh; fi -if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh; fi -install -d -m 700 /etc/rancher/k3s -cat > /etc/rancher/k3s/registries.yaml <<'EOF' +printf '%s\n' ${shQuote(secret.agentToken)} >"$token_candidate" +cat >"$registry_candidate" <<'EOF' mirrors: "${target.k3s.registryMirror.source}": endpoint: - "${target.k3s.registryMirror.endpoint}" EOF +wg_changed=false +if ! cmp -s "$wg_candidate" /etc/wireguard/${wg.interfaceName}.conf 2>/dev/null; then + install -m 0600 "$wg_candidate" /etc/wireguard/${wg.interfaceName}.conf + wg_changed=true +fi +if ! cmp -s "$token_candidate" ${shQuote(target.k3s.agentTokenFile)} 2>/dev/null; then + install -m 0600 "$token_candidate" ${shQuote(target.k3s.agentTokenFile)} +fi +if ! cmp -s "$registry_candidate" /etc/rancher/k3s/registries.yaml 2>/dev/null; then + install -m 0600 "$registry_candidate" /etc/rancher/k3s/registries.yaml +fi +systemctl enable wg-quick@${wg.interfaceName} >/dev/null 2>&1 || true +if [ "$wg_changed" = true ]; then + systemctl restart wg-quick@${wg.interfaceName} +elif ! systemctl is-active --quiet wg-quick@${wg.interfaceName}; then + systemctl start wg-quick@${wg.interfaceName} +fi +ping -c 3 -W 5 ${address(wg.controlPlaneAddress)} >/dev/null +desired_hash=${shQuote(desiredHash)} +desired_hash_file=/etc/unidesk/k3s-agent-desired.sha256 +current_hash=$(cat "$desired_hash_file" 2>/dev/null || true) +if [ "$current_hash" = "$desired_hash" ] && systemctl is-active --quiet k3s-agent; then exit 0; fi +if [ -x /usr/local/bin/k3s-uninstall.sh ]; then /usr/local/bin/k3s-uninstall.sh; fi +if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh; fi +install -d -m 700 /etc/rancher/k3s +install -m 0600 "$registry_candidate" /etc/rancher/k3s/registries.yaml install -m 0755 /tmp/k3s /usr/local/bin/k3s chmod 0755 /tmp/install-k3s.sh INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION=${shQuote(target.worker.k3sVersion)} INSTALL_K3S_EXEC=${shQuote(exec)} sh /tmp/install-k3s.sh systemctl enable k3s-agent >/dev/null systemctl restart k3s-agent systemctl is-active --quiet k3s-agent +printf '%s\n' "$desired_hash" >"$desired_hash_file" +chmod 0600 "$desired_hash_file" `; }