diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 664495de..64b46b68 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -350,6 +350,8 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - 多主机 k3s 节点变更: - CI/CD、业务 renderer、PipelineRun 和普通 GitOps workload 不得直接写 WireGuard 配置、k3s systemd unit、agent token 文件或节点 ServiceLB 标签。 - 节点加入、状态和调度 smoke 只走 `platform-infra k3s-cluster plan|apply|status|smoke`,具体 authority 和控制通道见 `docs/reference/platform-infra.md#多主机-k3s-集群`。 + - 集群 apply 的 control-plane 阶段只允许组合 `hwlab nodes control-plane infra k3s-config` 窄域;禁止调用全量 `infra apply`,避免节点加入顺带覆盖 registry、Tekton、Argo、git-mirror 或运行面对象。 + - k3s drop-in 未变化且节点容量已收敛时必须 `mutation=false`;禁止无条件重启、重装、强写或通过 CI 自动覆盖同一节点配置。 - worker 加入集群不改变 PaC consumer 的 source commit authority,也不授权人工 PipelineRun、Argo sync 或业务 rollout。 - 长命令用异步 job 或短轮询;不要长时间挂住 trans/ssh。 - Web 哨兵 YAML cadence 变更不新增发布 orchestrator:提交前用 `web-probe sentinel validate --local`;合并后用只读 `web-probe sentinel control-plane status --wait`。线上 desired schedule 必须来自 GitOps 清单,不能用操作者旧 worktree 的本地渲染值做阻塞门禁;收敛差异统一为非阻塞 warning。 diff --git a/.agents/skills/unidesk-ymalops/SKILL.md b/.agents/skills/unidesk-ymalops/SKILL.md index 0d3fd052..bd9cb4ce 100644 --- a/.agents/skills/unidesk-ymalops/SKILL.md +++ b/.agents/skills/unidesk-ymalops/SKILL.md @@ -92,6 +92,10 @@ description: UniDesk YAML-first 运维正规化技能。用户提到 ymal-first/ - 跨主机拓扑、WireGuard、agent token、registry mirror、ServiceLB 节点和调度 smoke 归独立集群 owning YAML。 - 既有 control-plane 的 k3s renderer 通过 `clusterConfigRef` 消费集群网络参数,不得由集群 CLI 再写第二个 systemd drop-in。 - 无固定公网地址 worker 使用宿主内网 SSH 作为部署控制通道并主动连接 WireGuard endpoint,不使用 FRP 代替三层集群网络。 + - control-plane drop-in 只走 `hwlab nodes control-plane infra k3s-config` 窄域: + - 渲染结果一致且节点容量已收敛时返回 `mutation=false`; + - 只在必要时写入 drop-in、reload 和重启 k3s; + - 禁止借用全量 infra apply 顺带覆盖 registry、CI/CD 或业务运行面。 - 日常入口和验收判定统一见 `docs/reference/platform-infra.md#多主机-k3s-集群`。 - 新节点 node-local registry 必须支持从零部署:优先由 YAML 渲染为 k3s workload/PVC;host Docker registry 只能是 legacy/迁移前状态。需要证明从零能力时先停旧 registry,不从旧 registry seed,并在状态中披露 `seededFromOldRegistry`、`zeroSeeded`、PVC、workload、endpoint 与 runtime/tools image preload 结果。 - YAML 配置必须可复用、可继承、可组合;节点/lane 只能作为变量或 YAML value 参与渲染,禁止把具体 node/lane 写入 YAML 文件名、parser/renderer/helper 函数名或长期 schema 名。发现不同节点复制大段配置时,先抽 baseline/configRef/override,再继续扩展节点。 diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index 6da52ab9..109c1bea 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -20,12 +20,21 @@ - `platform-infra k3s-cluster secret-init --target --confirm` 只在 Secret source 不存在时生成 WireGuard key pair 和 k3s agent token,存在时拒绝覆盖。 - `platform-infra k3s-cluster plan|apply|status|smoke --target ` 是部署和验收入口。 - 无固定公网地址的 worker 由宿主内网 SSH 控制,主动连接 control-plane WireGuard endpoint;worker provider route 只作为部署后补充验收入口。 + - control-plane 节点配置只组合 `hwlab nodes control-plane infra k3s-config plan|status|apply`: + - 该窄域只比较并安装 owning YAML 渲染的 k3s systemd drop-in; + - 只在 drop-in 或节点 pod 容量需要收敛时重启 k3s; + - 同一入口等待节点 Ready; + - 禁止调用 HWLAB 全量 `infra apply`,也不得触碰 Kubernetes manifests、registry、Tekton、Argo、git-mirror 或业务 rollout。 - 运行边界: - control-plane 保持唯一 k3s server,同时承担 workload。 - worker 只运行 `k3s-agent`;加入前必须卸载旧独立 server,禁止保留第二 datastore/control-plane。 - Flannel 使用 WireGuard 接口承载 VXLAN,不能只用 FRP 转发 Kubernetes API。 - ServiceLB 只在 owning YAML 指定的公网节点启用,防止无公网地址 worker 接管 hostPort。 - `127.0.0.1:5000` 镜像引用必须通过 worker 的 `registries.yaml` 映射到集群可达 registry endpoint。 + - 幂等重复执行不得强写: + - WireGuard、systemd drop-in、agent unit、token 文件和节点标签均先比较声明态与运行态; + - 内容一致时保持 `mutation=false`,不得无条件重启、重装或覆盖; + - 集群 CLI 只组合各自 owning domain,不借用全量基础设施入口扩大写入范围。 - 验收: - `status` 必须同时证明 WireGuard handshake、server/agent systemd、两个 Ready node、worker InternalIP 和旧 server 退役。 - `smoke --confirm` 必须把 Job 定向调度到 worker,并验证集群 DNS 和 Kubernetes ClusterIP;成功后删除 Job。 diff --git a/scripts/src/hwlab-node-control-plane-runtime.ts b/scripts/src/hwlab-node-control-plane-runtime.ts index 58de2d82..c6b3fa44 100644 --- a/scripts/src/hwlab-node-control-plane-runtime.ts +++ b/scripts/src/hwlab-node-control-plane-runtime.ts @@ -208,6 +208,121 @@ export function k3sDropInContent(spec: ControlPlaneK3sNodeSpec): string { ].join("\n"); } +export function k3sNodeConfigHostScript(spec: ControlPlaneK3sNodeSpec, apply: boolean): string { + const content = k3sDropInContent(spec); + const encoded = Buffer.from(content, "utf8").toString("base64"); + return ` +set +e +service=${shQuote(spec.serviceName)} +dropin=${shQuote(spec.dropInPath)} +node=${shQuote(spec.nodeStatusName)} +desired_max_pods=${shQuote(String(spec.kubelet.maxPods))} +expected_sha=${shQuote(sha256Short(content))} +apply=${apply ? "true" : "false"} +expected=$(mktemp /tmp/unidesk-k3s-dropin.XXXXXX) +printf %s ${shQuote(encoded)} | base64 -d >"$expected" +before_sha= +if [ -f "$dropin" ]; then before_sha=$(sha256sum "$dropin" | awk '{print "sha256:"$1}'); fi +before_capacity=$(kubectl get node "$node" -o 'jsonpath={.status.capacity.pods}' 2>/dev/null || true) +before_allocatable=$(kubectl get node "$node" -o 'jsonpath={.status.allocatable.pods}' 2>/dev/null || true) +changed=false +restart_required=false +if ! cmp -s "$expected" "$dropin" 2>/dev/null; then changed=true; fi +if [ "$changed" = true ] || [ "$before_capacity" != "$desired_max_pods" ] || [ "$before_allocatable" != "$desired_max_pods" ]; then + restart_required=true +fi +mutation=false +daemon_reload_rc=0 +restart_rc=0 +if [ "$apply" = true ] && [ "$restart_required" = true ]; then + mkdir -p "$(dirname "$dropin")" + if [ "$changed" = true ]; then + cp "$expected" "$dropin" + chown 0:0 "$dropin" 2>/dev/null || true + chmod 0644 "$dropin" + fi + systemctl daemon-reload || daemon_reload_rc=$? + if [ "$daemon_reload_rc" = 0 ]; then systemctl restart "$service" || restart_rc=$?; fi + mutation=true +fi +rm -f "$expected" +ready=false +if [ "$apply" = true ] && [ "$restart_required" = true ] && [ "$daemon_reload_rc" = 0 ] && [ "$restart_rc" = 0 ]; then + for _ in $(seq 1 60); do + if [ "$(systemctl is-active "$service" 2>/dev/null || true)" = active ] && \ + [ "$(kubectl get node "$node" -o 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" = True ]; then + ready=true + break + fi + sleep 2 + done +else + for _ in $(seq 1 5); do + if [ "$(systemctl is-active "$service" 2>/dev/null || true)" = active ] && \ + [ "$(kubectl get node "$node" -o 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" = True ]; then + ready=true + break + fi + sleep 2 + done +fi +after_sha= +if [ -f "$dropin" ]; then after_sha=$(sha256sum "$dropin" | awk '{print "sha256:"$1}'); fi +after_capacity=$(kubectl get node "$node" -o 'jsonpath={.status.capacity.pods}' 2>/dev/null || true) +after_allocatable=$(kubectl get node "$node" -o 'jsonpath={.status.allocatable.pods}' 2>/dev/null || true) +service_active=$(systemctl is-active "$service" 2>/dev/null || true) +python3 - "$apply" "$mutation" "$changed" "$restart_required" "$daemon_reload_rc" "$restart_rc" "$before_sha" "$after_sha" "$expected_sha" "$service_active" "$ready" "$before_capacity" "$before_allocatable" "$after_capacity" "$after_allocatable" "$desired_max_pods" <<'PY' +import json, sys +apply, mutation, changed, restart_required = [value == "true" for value in sys.argv[1:5]] +daemon_reload_rc, restart_rc = int(sys.argv[5]), int(sys.argv[6]) +before_sha, after_sha, expected_sha = sys.argv[7:10] +service_active, ready = sys.argv[10], sys.argv[11] == "true" +before_capacity, before_allocatable, after_capacity, after_allocatable, desired = sys.argv[12:17] +def number(value): + return int(value) if value.isdigit() else None +ok = ( + after_sha == expected_sha + and service_active == "active" + and ready + and after_capacity == desired + and after_allocatable == desired + and daemon_reload_rc == 0 + and restart_rc == 0 +) +print(json.dumps({ + "ok": ok, + "mode": "apply" if apply else "status", + "mutation": mutation, + "changed": changed, + "restartRequired": restart_required, + "daemonReloadExitCode": daemon_reload_rc, + "restartExitCode": restart_rc, + "beforeDropInSha256": before_sha or None, + "dropInSha256": after_sha or None, + "expectedDropInSha256": expected_sha, + "dropInMatches": after_sha == expected_sha, + "serviceActive": service_active == "active", + "nodeReady": ready, + "desiredMaxPods": int(desired), + "beforeCapacityPods": number(before_capacity), + "beforeAllocatablePods": number(before_allocatable), + "liveCapacityPods": number(after_capacity), + "liveAllocatablePods": number(after_allocatable), + "valuesPrinted": False, +}, ensure_ascii=False)) +PY +rc=$? +if [ "$rc" != 0 ]; then exit "$rc"; fi +if [ "$apply" = true ]; then + [ "$after_sha" = "$expected_sha" ] && [ "$service_active" = active ] && [ "$ready" = true ] && \ + [ "$after_capacity" = "$desired_max_pods" ] && [ "$after_allocatable" = "$desired_max_pods" ] && \ + [ "$daemon_reload_rc" = 0 ] && [ "$restart_rc" = 0 ] + exit $? +fi +exit 0 +`; +} + export function k3sServiceUnitContent(spec: ControlPlaneK3sNodeSpec, install: ControlPlaneK3sInstallSpec): string { return [ "[Unit]", diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 414bb70d..102e7a40 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -59,6 +59,7 @@ import { k3sInstallPlan, k3sInstallStatusScript, k3sInstallSubmitScript, + k3sNodeConfigHostScript, k3sNodeConfigPlan, manifestObjectSummary, parseRemoteJson, @@ -86,6 +87,11 @@ import { fingerprintSecretValues, readEnvSourceFile, requiredEnvValue } from "./ export { HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH } from "./hwlab-node-control-plane-model"; export function runHwlabNodeControlPlaneInfra(args: string[]): Record | RenderedCliResult { + if (args[0] === "k3s-config") { + const options = parseInfraOptions(args.slice(1)); + const { node, target } = controlPlaneContext(options.node, options.lane); + return runK3sNodeConfigCommand(node, target, options); + } if (args[0] === "k3s") { const options = parseK3sInstallOptions(args.slice(1)); const { config, node, target } = controlPlaneContext(options.node, options.lane); @@ -173,6 +179,9 @@ export function hwlabNodeControlPlaneInfraHelp(): Record { "bun scripts/cli.ts hwlab nodes control-plane infra k3s plan --node JD01 --lane v03", "bun scripts/cli.ts hwlab nodes control-plane infra k3s install --node JD01 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra k3s status --node JD01 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane infra k3s-config plan --node NC01 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane infra k3s-config status --node NC01 --lane v03", + "bun scripts/cli.ts hwlab nodes control-plane infra k3s-config apply --node NC01 --lane v03 --confirm", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image status --node --lane ", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node --lane --dry-run", "bun scripts/cli.ts hwlab nodes control-plane infra tools-image build --node --lane --confirm", @@ -195,6 +204,52 @@ export function hwlabNodeControlPlaneInfraHelp(): Record { }; } +function runK3sNodeConfigCommand( + node: ControlPlaneNodeSpec, + target: ControlPlaneTargetSpec, + options: InfraOptions, +): Record { + if (node.k3s === null) throw new Error(`nodes.${node.id}.k3s is not configured in ${HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH}`); + const plan = k3sNodeConfigPlan(node); + if (options.action === "plan" || (options.action === "apply" && options.dryRun)) { + return { + ok: true, + command: `hwlab nodes control-plane infra k3s-config ${options.action}`, + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + mode: options.action === "apply" ? "dry-run" : "plan", + mutation: false, + plan, + scope: ["k3s-systemd-drop-in", "k3s-service-restart-if-required", "node-ready-wait"], + excluded: ["kubernetes-manifests", "registry", "tekton", "argo", "git-mirror", "runtime-rollout"], + next: { + apply: `bun scripts/cli.ts hwlab nodes control-plane infra k3s-config apply --node ${node.id} --lane ${target.lane} --confirm`, + status: `bun scripts/cli.ts hwlab nodes control-plane infra k3s-config status --node ${node.id} --lane ${target.lane}`, + }, + }; + } + const result = runTransHost(node.route, k3sNodeConfigHostScript(node.k3s, options.action === "apply"), options.timeoutSeconds); + const parsed = parseRemoteJson(result.stdout); + const status = typeof parsed === "object" && parsed !== null ? parsed as Record : { stdoutPreview: result.stdout.slice(0, 2000) }; + const ok = result.exitCode === 0 && status.ok === true; + return { + ok, + command: `hwlab nodes control-plane infra k3s-config ${options.action}`, + configPath: HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH, + node: node.id, + lane: target.lane, + mode: options.action, + mutation: options.action === "apply" && status.mutation === true, + plan, + status, + result: compactCommandResult(result), + next: ok + ? { status: `bun scripts/cli.ts hwlab nodes control-plane infra k3s-config status --node ${node.id} --lane ${target.lane}` } + : { apply: `bun scripts/cli.ts hwlab nodes control-plane infra k3s-config apply --node ${node.id} --lane ${target.lane} --confirm` }, + }; +} + function infraPlan(_config: ControlPlaneConfig, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, options: InfraOptions): Record { return { ok: true, diff --git a/scripts/src/platform-infra-k3s-cluster.ts b/scripts/src/platform-infra-k3s-cluster.ts index e9c2e67b..79a4e655 100644 --- a/scripts/src/platform-infra-k3s-cluster.ts +++ b/scripts/src/platform-infra-k3s-cluster.ts @@ -97,7 +97,7 @@ function apply(target: K3sClusterTarget): Record { requireSuccess(controlWireguard, "control-plane-wireguard-apply"); const controlPlane = runHwlabNodeControlPlaneInfra([ - "apply", "--node", target.controlPlane.id, "--lane", target.controlPlane.lane, "--confirm", + "k3s-config", "apply", "--node", target.controlPlane.id, "--lane", target.controlPlane.lane, "--confirm", ]) as Record; if (controlPlane.ok !== true) throw new Error(`control-plane-k3s-apply failed: ${bounded(JSON.stringify(compactDomain(controlPlane)))}`);