diff --git a/.agents/skills/unidesk-cicd/SKILL.md b/.agents/skills/unidesk-cicd/SKILL.md index 5ed41486..664495de 100644 --- a/.agents/skills/unidesk-cicd/SKILL.md +++ b/.agents/skills/unidesk-cicd/SKILL.md @@ -347,6 +347,10 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help - 一旦发现 CI/CD CLI 被误用且可能写入错误状态、产生伪证据或绕过目标运行面,必须立刻先把用法改成更符合直觉的公开入口并更新本 skill/reference,再继续验证或交付;不要只靠口头记忆、隐藏 flag、手动约定或后续小心来避免复发。内部 in-cluster 模式必须只由目标 k8s Job/Pod 调用,操作者从本机只能用公开入口提交目标侧 Job 或读取目标侧摘要。 - PaC migrated consumer 的旧手动 publish/debug 命令不得作为正式 closeout 引导;调查命令默认只能指向只读的 `platform-infra pipelines-as-code status|history`。`closeout` 仅保留只读兼容能力,不得作为主代理、子代理或操作者的 PR 合并后步骤。需要保留的旧 mutation 命令只能属于 YAML 明确的 legacy authority 或独立平台维护,不得标记或引导 manual recovery;单步确认的自动链缺陷必须回到 YAML/controller/源码修复。 - Secret 只通过 YAML sourceRef/targetKey 和受控 CLI 下发;输出只披露 presence/fingerprint。 +- 多主机 k3s 节点变更: + - CI/CD、业务 renderer、PipelineRun 和普通 GitOps workload 不得直接写 WireGuard 配置、k3s systemd unit、agent token 文件或节点 ServiceLB 标签。 + - 节点加入、状态和调度 smoke 只走 `platform-infra k3s-cluster plan|apply|status|smoke`,具体 authority 和控制通道见 `docs/reference/platform-infra.md#多主机-k3s-集群`。 + - worker 加入集群不改变 PaC consumer 的 source commit authority,也不授权人工 PipelineRun、Argo sync 或业务 rollout。 - 长命令用异步 job 或短轮询;不要长时间挂住 trans/ssh。 - Web 哨兵 YAML cadence 变更不新增发布 orchestrator:提交前用 `web-probe sentinel validate --local`;合并后用只读 `web-probe sentinel control-plane status --wait`。线上 desired schedule 必须来自 GitOps 清单,不能用操作者旧 worktree 的本地渲染值做阻塞门禁;收敛差异统一为非阻塞 warning。 diff --git a/.agents/skills/unidesk-ymalops/SKILL.md b/.agents/skills/unidesk-ymalops/SKILL.md index beed899e..0d3fd052 100644 --- a/.agents/skills/unidesk-ymalops/SKILL.md +++ b/.agents/skills/unidesk-ymalops/SKILL.md @@ -88,6 +88,11 @@ description: UniDesk YAML-first 运维正规化技能。用户提到 ymal-first/ - 非核心字段可以使用显式领域默认值,但必须同时生成 `blocking=false` warning; - 不得生成不可见默认值,也不得把合并后的大对象写成新的 source of truth。 - 新 VPS 或裸节点 bootstrap 的出网必须优先走 YAML-first 声明的 0 依赖 host proxy client:通过 `trans` 分发零依赖客户端,先让 host route 能出网,再安装 k3s、containerd、Docker、包管理器依赖和 Git/gitrepo 同步;不得把 provider-gateway WebSocket egress 当作大二进制或依赖下载的数据面。 +- 多主机 k3s 集群: + - 跨主机拓扑、WireGuard、agent token、registry mirror、ServiceLB 节点和调度 smoke 归独立集群 owning YAML。 + - 既有 control-plane 的 k3s renderer 通过 `clusterConfigRef` 消费集群网络参数,不得由集群 CLI 再写第二个 systemd drop-in。 + - 无固定公网地址 worker 使用宿主内网 SSH 作为部署控制通道并主动连接 WireGuard endpoint,不使用 FRP 代替三层集群网络。 + - 日常入口和验收判定统一见 `docs/reference/platform-infra.md#多主机-k3s-集群`。 - 新节点 node-local registry 必须支持从零部署:优先由 YAML 渲染为 k3s workload/PVC;host Docker registry 只能是 legacy/迁移前状态。需要证明从零能力时先停旧 registry,不从旧 registry seed,并在状态中披露 `seededFromOldRegistry`、`zeroSeeded`、PVC、workload、endpoint 与 runtime/tools image preload 结果。 - YAML 配置必须可复用、可继承、可组合;节点/lane 只能作为变量或 YAML value 参与渲染,禁止把具体 node/lane 写入 YAML 文件名、parser/renderer/helper 函数名或长期 schema 名。发现不同节点复制大段配置时,先抽 baseline/configRef/override,再继续扩展节点。 - CLI `plan/status` 应输出 redacted 配置引用图:每个 ref 的文件、path、presence、摘要 hash、缺失字段和下一步 drill-down 命令。不要默认 dump 展开后的完整 YAML 或 Secret。 diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index 268d0b06..1dcd7068 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -316,6 +316,7 @@ nodes: serviceName: k3s dropInPath: /etc/systemd/system/k3s.service.d/20-unidesk-node-config.conf nodeStatusName: v2202607378382480008 + clusterConfigRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.controlPlane execStartPre: [] install: enabled: true diff --git a/config/platform-infra/hyperv-vms.yaml b/config/platform-infra/hyperv-vms.yaml index de792a16..d9396039 100644 --- a/config/platform-infra/hyperv-vms.yaml +++ b/config/platform-infra/hyperv-vms.yaml @@ -72,15 +72,15 @@ targets: registryMirrors: - https://docker.m.daocloud.io k3s: + management: + mode: external-cluster + configRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.workers.D601-VM version: v1.36.2+k3s1 installMirror: cn binaryUrl: https://github.com/k3s-io/k3s/releases/download/v1.36.2%2Bk3s1/k3s binarySha256: 65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de installerUrl: https://get.k3s.io nodeName: d601-vm - disable: - - traefik - - servicelb maxPods: 500 provider: id: D601-VM diff --git a/config/platform-infra/k3s-clusters.yaml b/config/platform-infra/k3s-clusters.yaml new file mode 100644 index 00000000..de56deac --- /dev/null +++ b/config/platform-infra/k3s-clusters.yaml @@ -0,0 +1,58 @@ +version: 1 +kind: platform-infra-k3s-clusters +metadata: + owner: unidesk + specRef: project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md +defaults: + targetId: nc01-d601 +targets: + nc01-d601: + enabled: true + clusterName: nc01 + controlPlane: + id: NC01 + configRef: config/hwlab-node-control-plane.yaml#nodes.NC01 + lane: v03 + nodeName: v2202607378382480008 + publicAddress: 152.53.229.148 + workers: + D601-VM: + id: D601-VM + bootstrapConfigRef: config/platform-infra/hyperv-vms.yaml#targets.D601-VM + nodeName: d601-vm + labels: + unidesk.ai/node-id: D601-VM + unidesk.ai/provider-id: D601-VM + unidesk.ai/runtime-plane: worker + wireguard: + interfaceName: wg0 + packageName: wireguard-tools + mtu: 1380 + listenPort: 51820 + controlPlaneAddress: 10.89.0.1/24 + workerAddress: 10.89.0.2/24 + endpoint: 152.53.229.148:51820 + persistentKeepaliveSeconds: 25 + secrets: + sourceRef: /root/.unidesk/.state/secrets/platform-infra/k3s-clusters/nc01-d601.env + controlPlanePrivateKey: NC01_WIREGUARD_PRIVATE_KEY + controlPlanePublicKey: NC01_WIREGUARD_PUBLIC_KEY + workerPrivateKey: D601_VM_WIREGUARD_PRIVATE_KEY + workerPublicKey: D601_VM_WIREGUARD_PUBLIC_KEY + agentToken: K3S_AGENT_TOKEN + k3s: + serverUrl: https://10.89.0.1:6443 + agentTokenFile: /etc/unidesk/k3s-agent-token + flannelInterface: wg0 + maxPods: 500 + registryMirror: + source: 127.0.0.1:5000 + endpoint: http://10.43.249.83:5000 + serviceLb: + enabledNodeLabel: svccontroller.k3s.cattle.io/enablelb + enabledNode: v2202607378382480008 + smoke: + namespace: platform-infra-smoke + jobName: d601-runtime-plane-smoke + image: docker.m.daocloud.io/library/busybox:1.36.1 + timeoutSeconds: 180 diff --git a/docs/reference/platform-infra.md b/docs/reference/platform-infra.md index ea6ba8b2..253c505b 100644 --- a/docs/reference/platform-infra.md +++ b/docs/reference/platform-infra.md @@ -10,6 +10,27 @@ - UniDesk 拥有的 image version、namespace、endpoint、FRP port 与 profile 列表不得隐藏在 helper 常量里;外部工具要求的 TOML、JSON 和 env 仅是边缘渲染格式。 - 新节点出网 bootstrap 遵循 `docs/reference/yaml-first-ops.md` 的零依赖 host proxy 边界,不得依赖尚未安装的 Docker、k3s、containerd 或包管理器网络能力。 +## 多主机 k3s 集群 + +- 权威配置: + - 跨主机拓扑、WireGuard、agent token `sourceRef`、registry mirror、ServiceLB 节点和 smoke 归 `config/platform-infra/k3s-clusters.yaml`。 + - Hyper-V VM 资源和宿主内网 SSH 归 `config/platform-infra/hyperv-vms.yaml`。 + - 既有 control-plane 的 k3s systemd drop-in 继续归 `config/hwlab-node-control-plane.yaml`,并通过 `clusterConfigRef` 消费集群网络参数。 +- 受控入口: + - `platform-infra k3s-cluster secret-init --target --confirm` 只在 Secret source 不存在时生成 WireGuard key pair 和 k3s agent token,存在时拒绝覆盖。 + - `platform-infra k3s-cluster plan|apply|status|smoke --target ` 是部署和验收入口。 + - 无固定公网地址的 worker 由宿主内网 SSH 控制,主动连接 control-plane WireGuard endpoint;worker provider route 只作为部署后补充验收入口。 +- 运行边界: + - control-plane 保持唯一 k3s server,同时承担 workload。 + - worker 只运行 `k3s-agent`;加入前必须卸载旧独立 server,禁止保留第二 datastore/control-plane。 + - Flannel 使用 WireGuard 接口承载 VXLAN,不能只用 FRP 转发 Kubernetes API。 + - ServiceLB 只在 owning YAML 指定的公网节点启用,防止无公网地址 worker 接管 hostPort。 + - `127.0.0.1:5000` 镜像引用必须通过 worker 的 `registries.yaml` 映射到集群可达 registry endpoint。 +- 验收: + - `status` 必须同时证明 WireGuard handshake、server/agent systemd、两个 Ready node、worker InternalIP 和旧 server 退役。 + - `smoke --confirm` 必须把 Job 定向调度到 worker,并验证集群 DNS 和 Kubernetes ClusterIP;成功后删除 Job。 + - Secret 输出只显示 sourceRef、presence、key 名和 fingerprint,不显示值。 + ## systemd socket-pair 收敛 - Host-native `.socket` 与 `.service` 配对收敛统一复用 `scripts/src/platform-infra-systemd-socket-pair.ts`: diff --git a/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md b/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md index 9a83caea..59db6e29 100644 --- a/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md +++ b/project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md @@ -17,7 +17,8 @@ - Hyper-V 虚拟机、VHDX、处理器、内存和自动启动策略。 - 独立内部交换机、NAT、固定地址和出网连通性。 -- Ubuntu cloud-init、SSH、Docker 和单节点 k3s 初始化。 +- Ubuntu cloud-init、SSH、Docker 和受控 k3s 运行角色初始化。 +- 无固定公网地址 worker 通过原生 WireGuard 主动接入既有 k3s 控制面。 - provider-gateway 的节点身份、常驻恢复和 trans 验收。 - 后台安装的阶段、日志、错误和终态可见性。 @@ -31,7 +32,7 @@ ### 3.1 HYPERV-VM-REQ-001 YAML 真相 -虚拟机名称、宿主 route、CPU、内存、磁盘、镜像 URL 与摘要、网络、Ubuntu 用户、k3s 版本和 Provider ID 必须来自 `config/platform-infra/hyperv-vms.yaml`。实现只能校验和渲染,不能维护隐藏运行参数。 +虚拟机名称、宿主 route、CPU、内存、磁盘、镜像 URL 与摘要、网络、Ubuntu 用户、k3s 版本和 Provider ID 必须来自 `config/platform-infra/hyperv-vms.yaml`。跨主机集群拓扑、WireGuard、k3s server/agent 角色、SecretRef、registry mirror 和调度 smoke 必须来自独立的集群 owning YAML。实现只能校验、解析引用和渲染,不能维护隐藏运行参数。 ### 3.2 HYPERV-VM-REQ-002 稳定资源边界 @@ -56,5 +57,18 @@ Hyper-V 虚拟机必须配置为宿主启动后自动启动、宿主关机时保 - VM 为运行状态且资源与 YAML 一致。 - Ubuntu 固定地址可达并能正常出网。 - Docker daemon 和 k3s 节点健康。 +- worker 以 agent 身份加入声明的控制面,且独立 server 不再运行。 +- 定向调度到 worker 的 smoke 能完成集群 DNS 与 ClusterIP 访问。 - provider-gateway 以 YAML 声明的 Provider ID 在线。 - `trans hostname` 和 k3s route 均可用。 + +### 3.7 HYPERV-VM-REQ-007 多主机 k3s 运行面 + +既有控制面与 Hyper-V worker 组成集群时必须满足: + +- 控制面保持唯一 server,并继续承担 workload。 +- Hyper-V 节点只能安装为 agent,不得保留独立 server。 +- 无固定公网地址的 worker 通过主动拨号的原生 WireGuard 三层隧道接入控制面。 +- WireGuard 私钥和 k3s agent token 只能来自 YAML `sourceRef`,不得从运行面反解或打印。 +- k3s 节点地址、Flannel 接口、API TLS SAN、registry mirror 和 ServiceLB 节点选择必须由 owning YAML 渲染。 +- 宿主内网 SSH 可以作为 worker 的稳定部署控制通道,provider route 只作为部署后验收入口。 diff --git a/scripts/assets/platform-infra/hyperv-vm-bootstrap.ps1 b/scripts/assets/platform-infra/hyperv-vm-bootstrap.ps1 index aebab98d..0c2e99da 100644 --- a/scripts/assets/platform-infra/hyperv-vm-bootstrap.ps1 +++ b/scripts/assets/platform-infra/hyperv-vm-bootstrap.ps1 @@ -490,10 +490,14 @@ try { Ensure-VirtualMachine -DiskPath $diskPath -SeedPath $seedPath } Wait-For-Ssh -KeyPath $key.private - Install-K3s -KeyPath $key.private + if ([string]$Config.k3s.management.mode -eq 'external-cluster') { + Write-State -Stage 'staging-k3s-agent' -Message 'Pinned k3s artifacts are staged; the YAML-controlled cluster manager owns the agent role' + } else { + throw "unsupported-k3s-management-mode: $($Config.k3s.management.mode)" + } Install-Provider -KeyPath $key.private - Write-State -Stage 'completed' -State 'completed' -Message 'Ubuntu VM, Docker, k3s and provider-gateway are installed' -Extra @{ + Write-State -Stage 'completed' -State 'completed' -Message 'Ubuntu VM, Docker, pinned k3s artifacts and provider-gateway are ready for the external cluster manager' -Extra @{ address = [string]$Config.network.address providerId = [string]$Config.provider.id processorCount = [int]$Config.vm.processorCount diff --git a/scripts/src/help.ts b/scripts/src/help.ts index d06b66db..16893eec 100644 --- a/scripts/src/help.ts +++ b/scripts/src/help.ts @@ -77,7 +77,7 @@ export function rootHelp(): unknown { { command: "hwlab nodes control-plane|git-mirror|hwpod-preinstall|secret|test-accounts --node --lane ", description: "Manage HWLAB node/lane runtime prerequisites, including NC01 YAML-declared k3s infra/tools-image/Argo bootstrap, HWPOD preinstall configRefs, redacted test-account preparation, and NC01 v0.3 runtime lane, with the node identity passed as data." }, { command: "hwlab nodes control-plane|git-mirror|hwpod-preinstall|secret|test-accounts --node NC01 --lane v03", description: "Operate the NC01 HWLAB v0.3 development and deployment lane from YAML source truth." }, { command: "agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|send|control-plane|git-mirror", description: "Use AgentRun v0.1 resource primitives with low-noise human output by default; session follow-up uses send only and the server decides internal steer vs turn." }, - { command: "platform-infra sub2api|langbot|n8n|webterm|wechat-archive|hyperv-vm ...", description: "Deploy YAML-controlled platform services and Hyper-V Ubuntu nodes, and inspect status/logs without printing secrets." }, + { command: "platform-infra sub2api|langbot|n8n|webterm|wechat-archive|hyperv-vm|k3s-cluster ...", description: "Deploy YAML-controlled platform services, Hyper-V Ubuntu nodes and multi-host k3s clusters without printing secrets." }, { command: "pikaoa attachments-backup ...", description: "管理 PikaOA 生产附件离机备份控制面;开发与生产运行面由 PaC 自动交付。" }, { command: "secrets plan|sync|status", description: "Plan, push and inspect YAML-declared local secret source keys to Kubernetes Secrets without printing or reverse-engineering values." }, { command: "platform-db postgres plan|status|export-secrets|apply|swap", description: "Manage YAML-declared host-native PostgreSQL and its node swap capacity through bounded plans, asynchronous mutation, status, TLS and redacted secret output." }, @@ -894,6 +894,9 @@ function platformInfraHelpSummary(): unknown { "bun scripts/cli.ts platform-infra hyperv-vm plan --target D601-VM", "bun scripts/cli.ts platform-infra hyperv-vm apply --target D601-VM --confirm", "bun scripts/cli.ts platform-infra hyperv-vm status --target D601-VM", + "bun scripts/cli.ts platform-infra k3s-cluster plan --target nc01-d601", + "bun scripts/cli.ts platform-infra k3s-cluster apply --target nc01-d601 --confirm", + "bun scripts/cli.ts platform-infra k3s-cluster smoke --target nc01-d601 --confirm", "bun scripts/cli.ts platform-infra langbot plan", "bun scripts/cli.ts platform-infra langbot apply --confirm", "bun scripts/cli.ts platform-infra langbot status", diff --git a/scripts/src/hwlab-node-control-plane-model.ts b/scripts/src/hwlab-node-control-plane-model.ts index e441c189..3e526743 100644 --- a/scripts/src/hwlab-node-control-plane-model.ts +++ b/scripts/src/hwlab-node-control-plane-model.ts @@ -240,6 +240,7 @@ export interface ControlPlaneK3sNodeSpec { serviceName: string; dropInPath: string; nodeStatusName: string; + clusterConfigRef: string | null; execStartPre: readonly (readonly string[])[]; install: ControlPlaneK3sInstallSpec | null; serverArgs: readonly string[]; diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index aaa0c4c9..414bb70d 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -4,6 +4,7 @@ import { rootPath } from "./config"; import { runCommand, type CommandResult } from "./command"; import { egressBenchmarkCompactResult, egressBenchmarkDryRun, egressBenchmarkStartScript, egressBenchmarkStatusScript, type EgressBenchmarkSpec } from "./egress-proxy-benchmark"; import { resolveEgressProxySourceRef } from "./egress-proxy-sources"; +import { clusterServerArgs } from "./platform-infra-k3s-cluster-config"; import { hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes"; import { registryInfraManifest } from "./hwlab-node-control-plane-registry"; import { @@ -1582,7 +1583,9 @@ function k3sNodeSpec(raw: Record, path: string): ControlPlaneK3 if (!/^[A-Za-z0-9_.-]+$/u.test(nodeStatusName)) throw new Error(`${path}.nodeStatusName has an unsupported Kubernetes node name`); const execStartPre = execStartPreField(raw.execStartPre, `${path}.execStartPre`); const install = raw.install === undefined ? null : k3sInstallSpec(asRecord(raw.install, `${path}.install`), `${path}.install`); - const serverArgs = stringArrayField(raw, "serverArgs", path); + const clusterConfigRef = optionalStringField(raw, "clusterConfigRef", path) ?? null; + const baseServerArgs = stringArrayField(raw, "serverArgs", path); + const serverArgs = clusterConfigRef === null ? baseServerArgs : [...baseServerArgs, ...clusterServerArgs(clusterConfigRef)]; if (serverArgs.length === 0 || serverArgs[0] !== "server") throw new Error(`${path}.serverArgs must start with k3s server`); for (const [index, arg] of serverArgs.entries()) { if (arg.includes("\n") || arg.includes("\r") || arg.length === 0) throw new Error(`${path}.serverArgs[${index}] must be a single non-empty argv token`); @@ -1598,6 +1601,7 @@ function k3sNodeSpec(raw: Record, path: string): ControlPlaneK3 serviceName, dropInPath, nodeStatusName, + clusterConfigRef, execStartPre, install, serverArgs, diff --git a/scripts/src/platform-infra-hyperv-vm.ts b/scripts/src/platform-infra-hyperv-vm.ts index e2dae763..d8a52c96 100644 --- a/scripts/src/platform-infra-hyperv-vm.ts +++ b/scripts/src/platform-infra-hyperv-vm.ts @@ -88,6 +88,11 @@ function validateTarget(id: string, target: HyperVTarget): void { if (target.vm.diskBytes !== 274_877_906_944) throw new Error(`${configLabel}.targets.${id}.vm.diskBytes must describe the requested 256 GiB disk`); if (!/^[a-f0-9]{64}$/u.test(target.image.sha256)) throw new Error(`${configLabel}.targets.${id}.image.sha256 must be lowercase SHA-256`); if (target.image.sources.length < 2) throw new Error(`${configLabel}.targets.${id}.image.sources must include mirror and canonical fallback`); + const k3sManagement = record(target.k3s.management, `${configLabel}.targets.${id}.k3s.management`); + if (k3sManagement.mode !== "external-cluster") throw new Error(`${configLabel}.targets.${id}.k3s.management.mode must be external-cluster`); + if (typeof k3sManagement.configRef !== "string" || !k3sManagement.configRef.startsWith("config/platform-infra/k3s-clusters.yaml#")) { + throw new Error(`${configLabel}.targets.${id}.k3s.management.configRef must reference the k3s cluster owning YAML`); + } for (const value of [target.provider.id, target.provider.name, target.provider.composeProject, target.provider.containerName, target.provider.image, target.ubuntu.user, target.k3s.nodeName]) { if (!/^[A-Za-z0-9._:/-]+$/u.test(value)) throw new Error(`${configLabel}.targets.${id} contains a shell-unsafe provider or user value`); } diff --git a/scripts/src/platform-infra-k3s-cluster-config.ts b/scripts/src/platform-infra-k3s-cluster-config.ts new file mode 100644 index 00000000..e871c116 --- /dev/null +++ b/scripts/src/platform-infra-k3s-cluster-config.ts @@ -0,0 +1,200 @@ +import { readFileSync } from "node:fs"; +import { rootPath } from "./config"; + +export const K3S_CLUSTER_CONFIG_LABEL = "config/platform-infra/k3s-clusters.yaml"; + +export interface K3sClusterTarget { + id: string; + clusterName: string; + controlPlane: { + id: string; + configRef: string; + lane: string; + nodeName: string; + publicAddress: string; + route: string; + kubeRoute: string; + }; + worker: { + id: string; + bootstrapConfigRef: string; + nodeName: string; + labels: Record; + windowsRoute: string; + guestAddress: string; + guestUser: string; + stateDir: string; + k3sVersion: string; + }; + wireguard: { + interfaceName: string; + packageName: string; + mtu: number; + listenPort: number; + controlPlaneAddress: string; + workerAddress: string; + endpoint: string; + persistentKeepaliveSeconds: number; + }; + secrets: { + sourceRef: string; + controlPlanePrivateKey: string; + controlPlanePublicKey: string; + workerPrivateKey: string; + workerPublicKey: string; + agentToken: string; + }; + k3s: { + serverUrl: string; + agentTokenFile: string; + flannelInterface: string; + maxPods: number; + registryMirror: { source: string; endpoint: string }; + serviceLb: { enabledNodeLabel: string; enabledNode: string }; + }; + smoke: { namespace: string; jobName: string; image: string; timeoutSeconds: number }; +} + +export function readK3sClusterConfig(targetId?: string): { defaultTargetId: string; target: K3sClusterTarget } { + const root = record(Bun.YAML.parse(readFileSync(rootPath(K3S_CLUSTER_CONFIG_LABEL), "utf8")), K3S_CLUSTER_CONFIG_LABEL); + if (root.kind !== "platform-infra-k3s-clusters") throw new Error(`${K3S_CLUSTER_CONFIG_LABEL}.kind must be platform-infra-k3s-clusters`); + const defaults = record(root.defaults, `${K3S_CLUSTER_CONFIG_LABEL}.defaults`); + const defaultTargetId = text(defaults, "targetId", `${K3S_CLUSTER_CONFIG_LABEL}.defaults`); + const id = targetId ?? defaultTargetId; + const raw = record(record(root.targets, `${K3S_CLUSTER_CONFIG_LABEL}.targets`)[id], `${K3S_CLUSTER_CONFIG_LABEL}.targets.${id}`); + if (raw.enabled !== true) throw new Error(`${K3S_CLUSTER_CONFIG_LABEL}.targets.${id}.enabled must be true`); + const control = record(raw.controlPlane, `targets.${id}.controlPlane`); + const workers = record(raw.workers, `targets.${id}.workers`); + const workerEntries = Object.entries(workers); + if (workerEntries.length !== 1) throw new Error(`targets.${id}.workers must contain exactly one worker`); + const [workerId, workerRaw] = workerEntries[0] ?? []; + const worker = record(workerRaw, `targets.${id}.workers.${workerId}`); + const wireguard = record(raw.wireguard, `targets.${id}.wireguard`); + const secrets = record(raw.secrets, `targets.${id}.secrets`); + const k3s = record(raw.k3s, `targets.${id}.k3s`); + const mirror = record(k3s.registryMirror, `targets.${id}.k3s.registryMirror`); + const serviceLb = record(k3s.serviceLb, `targets.${id}.k3s.serviceLb`); + const smoke = record(raw.smoke, `targets.${id}.smoke`); + const controlSource = resolveRef(text(control, "configRef", `targets.${id}.controlPlane`)); + const bootstrapSource = resolveRef(text(worker, "bootstrapConfigRef", `targets.${id}.workers[0]`)); + const host = record(bootstrapSource.host, "worker bootstrap host"); + const network = record(bootstrapSource.network, "worker bootstrap network"); + const ubuntu = record(bootstrapSource.ubuntu, "worker bootstrap ubuntu"); + const bootstrapK3s = record(bootstrapSource.k3s, "worker bootstrap k3s"); + const labels = stringMap(record(worker.labels, `targets.${id}.workers.${workerId}.labels`), `targets.${id}.workers.${workerId}.labels`); + const target: K3sClusterTarget = { + id, + clusterName: text(raw, "clusterName", `targets.${id}`), + controlPlane: { + id: text(control, "id", `targets.${id}.controlPlane`), + configRef: text(control, "configRef", `targets.${id}.controlPlane`), + lane: text(control, "lane", `targets.${id}.controlPlane`), + nodeName: text(control, "nodeName", `targets.${id}.controlPlane`), + publicAddress: text(control, "publicAddress", `targets.${id}.controlPlane`), + route: text(controlSource, "route", "control plane configRef"), + kubeRoute: text(controlSource, "kubeRoute", "control plane configRef"), + }, + worker: { + id: text(worker, "id", `targets.${id}.workers.${workerId}`), + bootstrapConfigRef: text(worker, "bootstrapConfigRef", `targets.${id}.workers.${workerId}`), + nodeName: text(worker, "nodeName", `targets.${id}.workers.${workerId}`), + labels, + windowsRoute: text(host, "route", "worker bootstrap host"), + guestAddress: text(network, "address", "worker bootstrap network"), + guestUser: text(ubuntu, "user", "worker bootstrap ubuntu"), + stateDir: text(host, "stateDir", "worker bootstrap host"), + k3sVersion: text(bootstrapK3s, "version", "worker bootstrap k3s"), + }, + wireguard: { + interfaceName: text(wireguard, "interfaceName", `targets.${id}.wireguard`), + packageName: text(wireguard, "packageName", `targets.${id}.wireguard`), + mtu: integer(wireguard, "mtu", `targets.${id}.wireguard`), + listenPort: integer(wireguard, "listenPort", `targets.${id}.wireguard`), + controlPlaneAddress: cidr(wireguard, "controlPlaneAddress", `targets.${id}.wireguard`), + workerAddress: cidr(wireguard, "workerAddress", `targets.${id}.wireguard`), + endpoint: text(wireguard, "endpoint", `targets.${id}.wireguard`), + persistentKeepaliveSeconds: integer(wireguard, "persistentKeepaliveSeconds", `targets.${id}.wireguard`), + }, + secrets: { + sourceRef: text(secrets, "sourceRef", `targets.${id}.secrets`), + controlPlanePrivateKey: text(secrets, "controlPlanePrivateKey", `targets.${id}.secrets`), + controlPlanePublicKey: text(secrets, "controlPlanePublicKey", `targets.${id}.secrets`), + workerPrivateKey: text(secrets, "workerPrivateKey", `targets.${id}.secrets`), + workerPublicKey: text(secrets, "workerPublicKey", `targets.${id}.secrets`), + agentToken: text(secrets, "agentToken", `targets.${id}.secrets`), + }, + k3s: { + serverUrl: text(k3s, "serverUrl", `targets.${id}.k3s`), + agentTokenFile: text(k3s, "agentTokenFile", `targets.${id}.k3s`), + flannelInterface: text(k3s, "flannelInterface", `targets.${id}.k3s`), + maxPods: integer(k3s, "maxPods", `targets.${id}.k3s`), + registryMirror: { source: text(mirror, "source", "registryMirror"), endpoint: text(mirror, "endpoint", "registryMirror") }, + serviceLb: { enabledNodeLabel: text(serviceLb, "enabledNodeLabel", "serviceLb"), enabledNode: text(serviceLb, "enabledNode", "serviceLb") }, + }, + smoke: { + namespace: text(smoke, "namespace", `targets.${id}.smoke`), + jobName: text(smoke, "jobName", `targets.${id}.smoke`), + image: text(smoke, "image", `targets.${id}.smoke`), + timeoutSeconds: integer(smoke, "timeoutSeconds", `targets.${id}.smoke`), + }, + }; + if (target.k3s.flannelInterface !== target.wireguard.interfaceName) throw new Error(`targets.${id}.k3s.flannelInterface must match wireguard.interfaceName`); + if (target.k3s.serviceLb.enabledNode !== target.controlPlane.nodeName) throw new Error(`targets.${id}.k3s.serviceLb.enabledNode must match controlPlane.nodeName`); + return { defaultTargetId, target }; +} + +export function clusterServerArgs(configRef: string): string[] { + const match = configRef.match(/^config\/platform-infra\/k3s-clusters\.yaml#targets\.([A-Za-z0-9._-]+)\.controlPlane$/u); + if (match === null) throw new Error(`unsupported k3s clusterConfigRef: ${configRef}`); + const { target } = readK3sClusterConfig(match[1]); + return [ + "--node-ip", address(target.wireguard.controlPlaneAddress), + "--node-external-ip", target.controlPlane.publicAddress, + "--flannel-iface", target.k3s.flannelInterface, + "--tls-san", address(target.wireguard.controlPlaneAddress), + "--tls-san", target.controlPlane.publicAddress, + "--agent-token-file", target.k3s.agentTokenFile, + ]; +} + +export function address(value: string): string { + return value.split("/", 1)[0] ?? value; +} + +function resolveRef(value: string): Record { + const [file, path] = value.split("#", 2); + if (!file || !path) throw new Error(`configRef must contain file#path: ${value}`); + let current: unknown = Bun.YAML.parse(readFileSync(rootPath(file), "utf8")); + for (const segment of path.split(".")) current = record(current, value)[segment]; + return record(current, value); +} + +function record(value: unknown, label: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${label} must be an object`); + return value as Record; +} + +function text(value: Record, key: string, label: string): string { + const result = value[key]; + if (typeof result !== "string" || result.length === 0) throw new Error(`${label}.${key} must be a non-empty string`); + return result; +} + +function integer(value: Record, key: string, label: string): number { + const result = value[key]; + if (!Number.isInteger(result) || Number(result) <= 0) throw new Error(`${label}.${key} must be a positive integer`); + return Number(result); +} + +function cidr(value: Record, key: string, label: string): string { + const result = text(value, key, label); + if (!/^\d{1,3}(?:\.\d{1,3}){3}\/\d{1,2}$/u.test(result)) throw new Error(`${label}.${key} must be an IPv4 CIDR`); + return result; +} + +function stringMap(value: Record, label: string): Record { + return Object.fromEntries(Object.entries(value).map(([key, item]) => { + if (typeof item !== "string" || item.length === 0) throw new Error(`${label}.${key} must be a non-empty string`); + return [key, item]; + })); +} diff --git a/scripts/src/platform-infra-k3s-cluster.ts b/scripts/src/platform-infra-k3s-cluster.ts new file mode 100644 index 00000000..29c74b5c --- /dev/null +++ b/scripts/src/platform-infra-k3s-cluster.ts @@ -0,0 +1,455 @@ +import { createHash, generateKeyPairSync, randomBytes } from "node:crypto"; +import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { dirname } from "node:path"; +import type { UniDeskConfig } from "./config"; +import { rootPath } from "./config"; +import { runCommand } from "./command"; +import { parseEnvFile } from "./secrets"; +import { address, K3S_CLUSTER_CONFIG_LABEL, readK3sClusterConfig, type K3sClusterTarget } from "./platform-infra-k3s-cluster-config"; + +type Action = "plan" | "secret-init" | "apply" | "status" | "smoke"; + +interface Options { + action: Action; + targetId?: string; + confirm: boolean; +} + +interface SecretMaterial { + controlPrivate: string; + controlPublic: string; + workerPrivate: string; + workerPublic: string; + agentToken: string; + fingerprint: string; +} + +export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig, args: string[]): Promise> { + const options = parseOptions(args); + const { target } = readK3sClusterConfig(options.targetId); + if (options.action === "plan") return plan(target); + if (options.action === "secret-init") return secretInit(target, options.confirm); + if (options.action === "status") return status(target); + if (options.action === "smoke") return smoke(target, options.confirm); + if (!options.confirm) return { ...plan(target), mode: "dry-run", mutation: false }; + return apply(target); +} + +function parseOptions(args: string[]): Options { + const action = args[0] ?? "plan"; + if (action !== "plan" && action !== "secret-init" && action !== "apply" && action !== "status" && action !== "smoke") { + throw new Error("platform-infra k3s-cluster usage: plan|secret-init|apply|status|smoke [--target ] [--confirm]"); + } + return { action, targetId: option(args, "--target") ?? undefined, confirm: args.includes("--confirm") }; +} + +function plan(target: K3sClusterTarget): Record { + return { + ok: true, + mutation: false, + targetId: target.id, + source: K3S_CLUSTER_CONFIG_LABEL, + topology: { + clusterName: target.clusterName, + controlPlane: { id: target.controlPlane.id, nodeName: target.controlPlane.nodeName, route: target.controlPlane.route, kubeRoute: target.controlPlane.kubeRoute }, + worker: { id: target.worker.id, nodeName: target.worker.nodeName, bootstrapConfigRef: target.worker.bootstrapConfigRef, controlRoute: target.worker.windowsRoute }, + }, + wireguard: { ...target.wireguard, endpoint: target.wireguard.endpoint }, + k3s: { + serverUrl: target.k3s.serverUrl, + agentTokenFile: target.k3s.agentTokenFile, + registryMirror: target.k3s.registryMirror, + serviceLb: target.k3s.serviceLb, + }, + secret: secretSummary(target), + next: { + secretInit: `bun scripts/cli.ts platform-infra k3s-cluster secret-init --target ${target.id} --confirm`, + apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm`, + status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`, + smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`, + }, + }; +} + +function secretInit(target: K3sClusterTarget, confirm: boolean): Record { + if (!confirm) return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, secret: secretSummary(target) }; + const path = target.secrets.sourceRef; + if (existsSync(path)) throw new Error(`${path} already exists; secret-init never overwrites cluster credentials`); + const control = wireguardKeyPair(); + const worker = wireguardKeyPair(); + const values: Record = { + [target.secrets.controlPlanePrivateKey]: control.privateKey, + [target.secrets.controlPlanePublicKey]: control.publicKey, + [target.secrets.workerPrivateKey]: worker.privateKey, + [target.secrets.workerPublicKey]: worker.publicKey, + [target.secrets.agentToken]: randomBytes(32).toString("base64url"), + }; + mkdirSync(dirname(path), { recursive: true, mode: 0o700 }); + writeFileSync(path, Object.entries(values).map(([key, value]) => `${key}=${value}`).join("\n") + "\n", { encoding: "utf8", mode: 0o600 }); + chmodSync(path, 0o600); + return { ok: true, mutation: true, targetId: target.id, secret: secretSummary(target), valuesPrinted: false }; +} + +function apply(target: K3sClusterTarget): Record { + const secret = readSecrets(target); + const controlWireguard = runControlPlane(target, controlPlaneApplyScript(target, secret), 180_000); + requireSuccess(controlWireguard, "control-plane-wireguard-apply"); + + const controlPlane = runCommand([ + "bun", "scripts/cli.ts", "hwlab", "nodes", "control-plane", "infra", "apply", + "--node", target.controlPlane.id, "--lane", target.controlPlane.lane, "--confirm", + ], rootPath(), { timeoutMs: 180_000 }); + requireSuccess(controlPlane, "control-plane-k3s-apply"); + + const worker = runWorker(target, workerApplyScript(target, secret), true, 300_000); + requireSuccess(worker, "worker-agent-apply"); + + const labels = [ + "label", "node", target.k3s.serviceLb.enabledNode, + `${target.k3s.serviceLb.enabledNodeLabel}=true`, "--overwrite", + ]; + const labelResult = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", ...labels], rootPath(), { timeoutMs: 60_000 }); + requireSuccess(labelResult, "servicelb-node-label"); + + const ready = runControlPlane(target, waitForNodesScript(target), 180_000); + requireSuccess(ready, "cluster-node-readiness"); + return { + ok: true, + mutation: true, + targetId: target.id, + stages: { + controlPlaneWireguard: compact(controlWireguard), + controlPlaneK3s: compact(controlPlane), + workerAgent: compact(worker), + serviceLbPlacement: compact(labelResult), + nodeReadiness: compact(ready), + }, + secret: { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: secret.fingerprint, valuesPrinted: false }, + next: { + status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`, + smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`, + }, + }; +} + +function status(target: K3sClusterTarget): Record { + const control = runControlPlane(target, statusScript(target), 60_000); + const worker = runWorker(target, workerStatusScript(target), false, 60_000); + const controlData = parseFirstJson(control.stdout); + const workerData = parseFirstJson(worker.stdout); + const ok = control.exitCode === 0 && worker.exitCode === 0 && controlData.ok === true && workerData.ok === true; + return { + ok, + mutation: false, + targetId: target.id, + controlPlane: controlData, + worker: workerData, + secret: secretSummary(target), + result: { controlPlane: compact(control), worker: compact(worker) }, + next: ok ? { smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm` } : { apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm` }, + }; +} + +function smoke(target: K3sClusterTarget, confirm: boolean): Record { + const manifest = smokeManifest(target); + if (!confirm) return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, workerNode: target.worker.nodeName, manifestSha256: hash(manifest) }; + const namespace = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "create", "namespace", target.smoke.namespace, "--dry-run=client", "-o", "yaml"], rootPath(), { timeoutMs: 30_000 }); + requireSuccess(namespace, "smoke-namespace-render"); + const applyNamespace = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "apply", "-f", "-"], rootPath(), { input: namespace.stdout, timeoutMs: 30_000 }); + requireSuccess(applyNamespace, "smoke-namespace-apply"); + runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "delete", "job", target.smoke.jobName, "--ignore-not-found=true", "--wait=true"], rootPath(), { timeoutMs: 60_000 }); + const applyJob = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "apply", "-f", "-"], rootPath(), { input: manifest, timeoutMs: 30_000 }); + requireSuccess(applyJob, "smoke-job-apply"); + const wait = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "wait", `job/${target.smoke.jobName}`, "--for=condition=complete", `--timeout=${target.smoke.timeoutSeconds}s`], rootPath(), { timeoutMs: (target.smoke.timeoutSeconds + 20) * 1000 }); + const logs = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "logs", `job/${target.smoke.jobName}`], rootPath(), { timeoutMs: 30_000 }); + const placement = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "get", "pod", "-l", `job-name=${target.smoke.jobName}`, "-o", "jsonpath={.items[0].spec.nodeName}"], rootPath(), { timeoutMs: 30_000 }); + const ok = wait.exitCode === 0 && logs.exitCode === 0 && logs.stdout.includes("D601_RUNTIME_SMOKE_OK") && placement.stdout.trim() === target.worker.nodeName; + if (ok) runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "delete", "job", target.smoke.jobName, "--wait=true"], rootPath(), { timeoutMs: 60_000 }); + return { + ok, + mutation: true, + targetId: target.id, + workerNode: target.worker.nodeName, + scheduledNode: placement.stdout.trim(), + checks: { jobCompleted: wait.exitCode === 0, dnsAndClusterIp: logs.stdout.includes("D601_RUNTIME_SMOKE_OK"), workerPlacement: placement.stdout.trim() === target.worker.nodeName }, + logs: bounded(logs.stdout, 4_000), + result: { apply: compact(applyJob), wait: compact(wait), placement: compact(placement), cleanedUp: ok }, + }; +} + +function controlPlaneApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string { + const wg = target.wireguard; + return `set -eu +export DEBIAN_FRONTEND=noninteractive +if ! command -v wg >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq ${shQuote(wg.packageName)}; fi +umask 077 +install -d -m 700 /etc/wireguard /etc/unidesk +cat > /etc/wireguard/${wg.interfaceName}.conf <<'EOF' +[Interface] +Address = ${wg.controlPlaneAddress} +ListenPort = ${wg.listenPort} +PrivateKey = ${secret.controlPrivate} +MTU = ${wg.mtu} + +[Peer] +PublicKey = ${secret.workerPublic} +AllowedIPs = ${address(wg.workerAddress)}/32 +EOF +chmod 600 /etc/wireguard/${wg.interfaceName}.conf +printf '%s\n' ${shQuote(secret.agentToken)} > ${shQuote(target.k3s.agentTokenFile)} +chmod 600 ${shQuote(target.k3s.agentTokenFile)} +systemctl enable wg-quick@${wg.interfaceName} >/dev/null +systemctl restart wg-quick@${wg.interfaceName} +wg show ${wg.interfaceName} >/dev/null +`; +} + +function workerApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string { + const wg = target.wireguard; + const labels = Object.entries(target.worker.labels).flatMap(([key, value]) => ["--node-label", `${key}=${value}`]); + const exec = [ + "agent", "--server", target.k3s.serverUrl, "--token-file", target.k3s.agentTokenFile, + "--node-name", target.worker.nodeName, "--node-ip", address(wg.workerAddress), + "--flannel-iface", target.k3s.flannelInterface, ...labels, + "--kubelet-arg", `max-pods=${target.k3s.maxPods}`, + ].map(shellToken).join(" "); + return `set -eu +export DEBIAN_FRONTEND=noninteractive +if ! command -v wg >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq ${shQuote(wg.packageName)}; fi +umask 077 +install -d -m 700 /etc/wireguard /etc/unidesk /etc/rancher/k3s +cat > /etc/wireguard/${wg.interfaceName}.conf <<'EOF' +[Interface] +Address = ${wg.workerAddress} +PrivateKey = ${secret.workerPrivate} +MTU = ${wg.mtu} + +[Peer] +PublicKey = ${secret.controlPublic} +Endpoint = ${wg.endpoint} +AllowedIPs = ${address(wg.controlPlaneAddress)}/32 +PersistentKeepalive = ${wg.persistentKeepaliveSeconds} +EOF +chmod 600 /etc/wireguard/${wg.interfaceName}.conf +systemctl enable wg-quick@${wg.interfaceName} >/dev/null +systemctl restart wg-quick@${wg.interfaceName} +ping -c 3 -W 5 ${address(wg.controlPlaneAddress)} >/dev/null +printf '%s\n' ${shQuote(secret.agentToken)} > ${shQuote(target.k3s.agentTokenFile)} +chmod 600 ${shQuote(target.k3s.agentTokenFile)} +cat > /etc/rancher/k3s/registries.yaml <<'EOF' +mirrors: + "${target.k3s.registryMirror.source}": + endpoint: + - "${target.k3s.registryMirror.endpoint}" +EOF +if [ -x /usr/local/bin/k3s-uninstall.sh ]; then /usr/local/bin/k3s-uninstall.sh; fi +if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh; fi +install -m 0755 /tmp/k3s /usr/local/bin/k3s +chmod 0755 /tmp/install-k3s.sh +INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION=${shQuote(target.worker.k3sVersion)} INSTALL_K3S_EXEC=${shQuote(exec)} sh /tmp/install-k3s.sh +systemctl enable k3s-agent >/dev/null +systemctl restart k3s-agent +systemctl is-active --quiet k3s-agent +`; +} + +function waitForNodesScript(target: K3sClusterTarget): string { + return `set -eu +deadline=$(( $(date +%s) + 150 )) +while [ "$(date +%s)" -lt "$deadline" ]; do + ready=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) + [ "$ready" = True ] && exit 0 + sleep 5 +done +/usr/local/bin/kubectl get nodes -o wide >&2 +exit 1 +`; +} + +function statusScript(target: K3sClusterTarget): string { + const cp = address(target.wireguard.controlPlaneAddress); + return `set -eu +wg_active=false; systemctl is-active --quiet wg-quick@${target.wireguard.interfaceName} && wg_active=true +k3s_active=false; systemctl is-active --quiet k3s && k3s_active=true +cp_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.controlPlane.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) +worker_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true) +worker_ip=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null || true) +handshake=$(wg show ${target.wireguard.interfaceName} latest-handshakes 2>/dev/null | awk '{print $2}' | head -n1) +python3 - "$wg_active" "$k3s_active" "$cp_ready" "$worker_ready" "$worker_ip" "$handshake" <<'PY' +import json, sys, time +wg, k3s, cp, worker, worker_ip, handshake = sys.argv[1:] +stamp = int(handshake or 0) +age = None if stamp == 0 else max(0, int(time.time()) - stamp) +ok = wg == 'true' and k3s == 'true' and cp == 'True' and worker == 'True' and worker_ip == '${address(target.wireguard.workerAddress)}' and age is not None and age < 180 +print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'k3sActive': k3s == 'true', 'controlPlaneReady': cp == 'True', 'workerReady': worker == 'True', 'workerInternalIp': worker_ip, 'handshakeAgeSeconds': age, 'controlPlaneAddress': '${cp}'}, separators=(',', ':'))) +PY +`; +} + +function workerStatusScript(target: K3sClusterTarget): string { + return `set -eu +wg_active=false; systemctl is-active --quiet wg-quick@${target.wireguard.interfaceName} && wg_active=true +agent_active=false; systemctl is-active --quiet k3s-agent && agent_active=true +old_server=false; systemctl is-active --quiet k3s && old_server=true +ip_present=false; ip -4 address show dev ${target.wireguard.interfaceName} | grep -F ${shQuote(address(target.wireguard.workerAddress))} >/dev/null 2>&1 && ip_present=true +python3 - "$wg_active" "$agent_active" "$old_server" "$ip_present" <<'PY' +import json, sys +wg, agent, old_server, ip_present = sys.argv[1:] +ok = wg == 'true' and agent == 'true' and old_server == 'false' and ip_present == 'true' +print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'agentActive': agent == 'true', 'standaloneServerActive': old_server == 'true', 'wireguardAddressPresent': ip_present == 'true'}, separators=(',', ':'))) +PY +`; +} + +function smokeManifest(target: K3sClusterTarget): string { + return `apiVersion: batch/v1 +kind: Job +metadata: + name: ${target.smoke.jobName} + namespace: ${target.smoke.namespace} +spec: + backoffLimit: 0 + template: + metadata: + labels: + app.kubernetes.io/name: ${target.smoke.jobName} + spec: + restartPolicy: Never + nodeSelector: + kubernetes.io/hostname: ${target.worker.nodeName} + containers: + - name: smoke + image: ${target.smoke.image} + imagePullPolicy: IfNotPresent + command: + - sh + - -ec + - | + nslookup kubernetes.default.svc >/tmp/dns.txt + wget -qO- --no-check-certificate --timeout=15 https://kubernetes.default.svc/healthz | grep -q '^ok$' + echo D601_RUNTIME_SMOKE_OK +`; +} + +function runControlPlane(target: K3sClusterTarget, script: string, timeoutMs: number) { + return runCommand(["trans", target.controlPlane.route, "sh"], rootPath(), { input: script, timeoutMs }); +} + +function runWorker(target: K3sClusterTarget, script: string, copyArtifacts: boolean, timeoutMs: number) { + const encoded = Buffer.from(script, "utf8").toString("base64"); + const keyPath = `${target.worker.stateDir}\\id_ed25519`; + const remoteScript = `/tmp/unidesk-k3s-cluster-${target.id}.sh`; + const lines = [ + "$ErrorActionPreference='Stop'", + `$key=${psQuote(keyPath)}`, + `$guest=${psQuote(`${target.worker.guestUser}@${target.worker.guestAddress}`)}`, + `$temp=Join-Path $env:TEMP ${psQuote(`unidesk-k3s-cluster-${target.id}.sh`)}`, + `[IO.File]::WriteAllBytes($temp,[Convert]::FromBase64String(${psQuote(encoded)}))`, + ]; + if (copyArtifacts) { + lines.push( + `$binary=${psQuote(`${target.worker.stateDir}\\k3s`)}`, + `$installer=${psQuote(`${target.worker.stateDir}\\install-k3s.sh`)}`, + "& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $binary \"${guest}:/tmp/k3s\"", + "if($LASTEXITCODE -ne 0){throw 'worker-k3s-binary-scp-failed'}", + "& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $installer \"${guest}:/tmp/install-k3s.sh\"", + "if($LASTEXITCODE -ne 0){throw 'worker-k3s-installer-scp-failed'}", + ); + } + lines.push( + `& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $temp \"${guest}:${remoteScript}\"`, + "if($LASTEXITCODE -ne 0){throw 'worker-script-scp-failed'}", + `& ssh.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 $guest ${psQuote(`sudo sh ${remoteScript}`)}`, + "$rc=$LASTEXITCODE", + "Remove-Item -Force -LiteralPath $temp -ErrorAction SilentlyContinue", + `& ssh.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $guest ${psQuote(`rm -f ${remoteScript}`)} | Out-Null`, + "exit $rc", + ); + return runCommand(["trans", target.worker.windowsRoute, "ps"], rootPath(), { input: lines.join("\n") + "\n", timeoutMs }); +} + +function readSecrets(target: K3sClusterTarget): SecretMaterial { + if (!existsSync(target.secrets.sourceRef)) throw new Error(`${target.secrets.sourceRef} is missing; run platform-infra k3s-cluster secret-init --confirm`); + const values = parseEnvFile(readFileSync(target.secrets.sourceRef, "utf8")); + const required = (key: string) => { + const value = values[key]; + if (typeof value !== "string" || value.length < 32) throw new Error(`${target.secrets.sourceRef}.${key} is missing or invalid`); + return value; + }; + const material = { + controlPrivate: required(target.secrets.controlPlanePrivateKey), + controlPublic: required(target.secrets.controlPlanePublicKey), + workerPrivate: required(target.secrets.workerPrivateKey), + workerPublic: required(target.secrets.workerPublicKey), + agentToken: required(target.secrets.agentToken), + }; + return { ...material, fingerprint: hash(Object.entries(material).sort().map(([key, value]) => `${key}=${value}`).join("\n")) }; +} + +function secretSummary(target: K3sClusterTarget): Record { + if (!existsSync(target.secrets.sourceRef)) return { sourceRef: target.secrets.sourceRef, presence: false, fingerprint: null, valuesPrinted: false }; + try { + const secret = readSecrets(target); + return { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: secret.fingerprint, keys: Object.values(target.secrets).filter((value) => value !== target.secrets.sourceRef), valuesPrinted: false }; + } catch (error) { + return { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: null, error: error instanceof Error ? error.message : String(error), valuesPrinted: false }; + } +} + +function wireguardKeyPair(): { privateKey: string; publicKey: string } { + const pair = generateKeyPairSync("x25519"); + const privateJwk = pair.privateKey.export({ format: "jwk" }) as JsonWebKey; + const publicJwk = pair.publicKey.export({ format: "jwk" }) as JsonWebKey; + if (!privateJwk.d || !publicJwk.x) throw new Error("x25519 key generation did not produce raw JWK keys"); + return { privateKey: base64UrlToBase64(privateJwk.d), publicKey: base64UrlToBase64(publicJwk.x) }; +} + +function base64UrlToBase64(value: string): string { + return Buffer.from(value, "base64url").toString("base64"); +} + +function requireSuccess(result: ReturnType, stage: string): void { + if (result.exitCode !== 0) throw new Error(`${stage} failed: ${bounded(result.stderr || result.stdout)}`); +} + +function compact(result: ReturnType): Record { + return { exitCode: result.exitCode, timedOut: result.timedOut, stdout: bounded(result.stdout, 2_000), stderr: bounded(result.stderr, 2_000) }; +} + +function parseFirstJson(value: string): Record { + for (const line of value.split(/\r?\n/u)) { + const text = line.trim(); + if (!text.startsWith("{")) continue; + try { return JSON.parse(text) as Record; } catch { /* continue */ } + } + return { ok: false, error: "remote-json-missing", output: bounded(value) }; +} + +function option(args: string[], name: string): string | null { + const index = args.indexOf(name); + if (index < 0) return null; + const value = args[index + 1]; + if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`); + return value; +} + +function shQuote(value: string): string { + return `'${value.replaceAll("'", `'"'"'`)}'`; +} + +function shellToken(value: string): string { + return /^[A-Za-z0-9._:/=@+-]+$/u.test(value) ? value : shQuote(value); +} + +function psQuote(value: string): string { + return `'${value.replaceAll("'", "''")}'`; +} + +function hash(value: string): string { + return createHash("sha256").update(value).digest("hex"); +} + +function bounded(value: string, max = 4_000): string { + const text = value.trim(); + return text.length <= max ? text : `${text.slice(0, max)}\n...[truncated ${text.length - max} chars]`; +} diff --git a/scripts/src/platform-infra/options.ts b/scripts/src/platform-infra/options.ts index e1a5f817..2ec62164 100644 --- a/scripts/src/platform-infra/options.ts +++ b/scripts/src/platform-infra/options.ts @@ -99,6 +99,10 @@ export async function runPlatformInfraCommand(config: UniDeskConfig, args: strin const { runPlatformInfraHyperVVmCommand } = await import("../platform-infra-hyperv-vm"); return await runPlatformInfraHyperVVmCommand(config, args.slice(1)); } + if (target === "k3s-cluster") { + const { runPlatformInfraK3sClusterCommand } = await import("../platform-infra-k3s-cluster"); + return await runPlatformInfraK3sClusterCommand(config, args.slice(1)); + } if (target === "pipelines-as-code" || target === "pac") { const { runPlatformInfraPipelinesAsCodeCommand } = await import("../platform-infra-pipelines-as-code"); return await runPlatformInfraPipelinesAsCodeCommand(config, args.slice(1));