feat: add controlled NC01 D601 k3s cluster
This commit is contained in:
@@ -347,6 +347,10 @@ bun scripts/cli.ts hwlab nodes control-plane legacy-cicd --help
|
||||
- 一旦发现 CI/CD CLI 被误用且可能写入错误状态、产生伪证据或绕过目标运行面,必须立刻先把用法改成更符合直觉的公开入口并更新本 skill/reference,再继续验证或交付;不要只靠口头记忆、隐藏 flag、手动约定或后续小心来避免复发。内部 in-cluster 模式必须只由目标 k8s Job/Pod 调用,操作者从本机只能用公开入口提交目标侧 Job 或读取目标侧摘要。
|
||||
- PaC migrated consumer 的旧手动 publish/debug 命令不得作为正式 closeout 引导;调查命令默认只能指向只读的 `platform-infra pipelines-as-code status|history`。`closeout` 仅保留只读兼容能力,不得作为主代理、子代理或操作者的 PR 合并后步骤。需要保留的旧 mutation 命令只能属于 YAML 明确的 legacy authority 或独立平台维护,不得标记或引导 manual recovery;单步确认的自动链缺陷必须回到 YAML/controller/源码修复。
|
||||
- Secret 只通过 YAML sourceRef/targetKey 和受控 CLI 下发;输出只披露 presence/fingerprint。
|
||||
- 多主机 k3s 节点变更:
|
||||
- CI/CD、业务 renderer、PipelineRun 和普通 GitOps workload 不得直接写 WireGuard 配置、k3s systemd unit、agent token 文件或节点 ServiceLB 标签。
|
||||
- 节点加入、状态和调度 smoke 只走 `platform-infra k3s-cluster plan|apply|status|smoke`,具体 authority 和控制通道见 `docs/reference/platform-infra.md#多主机-k3s-集群`。
|
||||
- worker 加入集群不改变 PaC consumer 的 source commit authority,也不授权人工 PipelineRun、Argo sync 或业务 rollout。
|
||||
- 长命令用异步 job 或短轮询;不要长时间挂住 trans/ssh。
|
||||
- Web 哨兵 YAML cadence 变更不新增发布 orchestrator:提交前用 `web-probe sentinel validate --local`;合并后用只读 `web-probe sentinel control-plane status --wait`。线上 desired schedule 必须来自 GitOps 清单,不能用操作者旧 worktree 的本地渲染值做阻塞门禁;收敛差异统一为非阻塞 warning。
|
||||
|
||||
|
||||
@@ -88,6 +88,11 @@ description: UniDesk YAML-first 运维正规化技能。用户提到 ymal-first/
|
||||
- 非核心字段可以使用显式领域默认值,但必须同时生成 `blocking=false` warning;
|
||||
- 不得生成不可见默认值,也不得把合并后的大对象写成新的 source of truth。
|
||||
- 新 VPS 或裸节点 bootstrap 的出网必须优先走 YAML-first 声明的 0 依赖 host proxy client:通过 `trans` 分发零依赖客户端,先让 host route 能出网,再安装 k3s、containerd、Docker、包管理器依赖和 Git/gitrepo 同步;不得把 provider-gateway WebSocket egress 当作大二进制或依赖下载的数据面。
|
||||
- 多主机 k3s 集群:
|
||||
- 跨主机拓扑、WireGuard、agent token、registry mirror、ServiceLB 节点和调度 smoke 归独立集群 owning YAML。
|
||||
- 既有 control-plane 的 k3s renderer 通过 `clusterConfigRef` 消费集群网络参数,不得由集群 CLI 再写第二个 systemd drop-in。
|
||||
- 无固定公网地址 worker 使用宿主内网 SSH 作为部署控制通道并主动连接 WireGuard endpoint,不使用 FRP 代替三层集群网络。
|
||||
- 日常入口和验收判定统一见 `docs/reference/platform-infra.md#多主机-k3s-集群`。
|
||||
- 新节点 node-local registry 必须支持从零部署:优先由 YAML 渲染为 k3s workload/PVC;host Docker registry 只能是 legacy/迁移前状态。需要证明从零能力时先停旧 registry,不从旧 registry seed,并在状态中披露 `seededFromOldRegistry`、`zeroSeeded`、PVC、workload、endpoint 与 runtime/tools image preload 结果。
|
||||
- YAML 配置必须可复用、可继承、可组合;节点/lane 只能作为变量或 YAML value 参与渲染,禁止把具体 node/lane 写入 YAML 文件名、parser/renderer/helper 函数名或长期 schema 名。发现不同节点复制大段配置时,先抽 baseline/configRef/override,再继续扩展节点。
|
||||
- CLI `plan/status` 应输出 redacted 配置引用图:每个 ref 的文件、path、presence、摘要 hash、缺失字段和下一步 drill-down 命令。不要默认 dump 展开后的完整 YAML 或 Secret。
|
||||
|
||||
@@ -316,6 +316,7 @@ nodes:
|
||||
serviceName: k3s
|
||||
dropInPath: /etc/systemd/system/k3s.service.d/20-unidesk-node-config.conf
|
||||
nodeStatusName: v2202607378382480008
|
||||
clusterConfigRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.controlPlane
|
||||
execStartPre: []
|
||||
install:
|
||||
enabled: true
|
||||
|
||||
@@ -72,15 +72,15 @@ targets:
|
||||
registryMirrors:
|
||||
- https://docker.m.daocloud.io
|
||||
k3s:
|
||||
management:
|
||||
mode: external-cluster
|
||||
configRef: config/platform-infra/k3s-clusters.yaml#targets.nc01-d601.workers.D601-VM
|
||||
version: v1.36.2+k3s1
|
||||
installMirror: cn
|
||||
binaryUrl: https://github.com/k3s-io/k3s/releases/download/v1.36.2%2Bk3s1/k3s
|
||||
binarySha256: 65a55ec56c24eab44383086166ec620a491952b7e23941a49ddca6e8a4c4b4de
|
||||
installerUrl: https://get.k3s.io
|
||||
nodeName: d601-vm
|
||||
disable:
|
||||
- traefik
|
||||
- servicelb
|
||||
maxPods: 500
|
||||
provider:
|
||||
id: D601-VM
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
version: 1
|
||||
kind: platform-infra-k3s-clusters
|
||||
metadata:
|
||||
owner: unidesk
|
||||
specRef: project-management/PJ2026-01/specs/PJ2026-01060314-hyperv-ubuntu-vm.md
|
||||
defaults:
|
||||
targetId: nc01-d601
|
||||
targets:
|
||||
nc01-d601:
|
||||
enabled: true
|
||||
clusterName: nc01
|
||||
controlPlane:
|
||||
id: NC01
|
||||
configRef: config/hwlab-node-control-plane.yaml#nodes.NC01
|
||||
lane: v03
|
||||
nodeName: v2202607378382480008
|
||||
publicAddress: 152.53.229.148
|
||||
workers:
|
||||
D601-VM:
|
||||
id: D601-VM
|
||||
bootstrapConfigRef: config/platform-infra/hyperv-vms.yaml#targets.D601-VM
|
||||
nodeName: d601-vm
|
||||
labels:
|
||||
unidesk.ai/node-id: D601-VM
|
||||
unidesk.ai/provider-id: D601-VM
|
||||
unidesk.ai/runtime-plane: worker
|
||||
wireguard:
|
||||
interfaceName: wg0
|
||||
packageName: wireguard-tools
|
||||
mtu: 1380
|
||||
listenPort: 51820
|
||||
controlPlaneAddress: 10.89.0.1/24
|
||||
workerAddress: 10.89.0.2/24
|
||||
endpoint: 152.53.229.148:51820
|
||||
persistentKeepaliveSeconds: 25
|
||||
secrets:
|
||||
sourceRef: /root/.unidesk/.state/secrets/platform-infra/k3s-clusters/nc01-d601.env
|
||||
controlPlanePrivateKey: NC01_WIREGUARD_PRIVATE_KEY
|
||||
controlPlanePublicKey: NC01_WIREGUARD_PUBLIC_KEY
|
||||
workerPrivateKey: D601_VM_WIREGUARD_PRIVATE_KEY
|
||||
workerPublicKey: D601_VM_WIREGUARD_PUBLIC_KEY
|
||||
agentToken: K3S_AGENT_TOKEN
|
||||
k3s:
|
||||
serverUrl: https://10.89.0.1:6443
|
||||
agentTokenFile: /etc/unidesk/k3s-agent-token
|
||||
flannelInterface: wg0
|
||||
maxPods: 500
|
||||
registryMirror:
|
||||
source: 127.0.0.1:5000
|
||||
endpoint: http://10.43.249.83:5000
|
||||
serviceLb:
|
||||
enabledNodeLabel: svccontroller.k3s.cattle.io/enablelb
|
||||
enabledNode: v2202607378382480008
|
||||
smoke:
|
||||
namespace: platform-infra-smoke
|
||||
jobName: d601-runtime-plane-smoke
|
||||
image: docker.m.daocloud.io/library/busybox:1.36.1
|
||||
timeoutSeconds: 180
|
||||
@@ -10,6 +10,27 @@
|
||||
- UniDesk 拥有的 image version、namespace、endpoint、FRP port 与 profile 列表不得隐藏在 helper 常量里;外部工具要求的 TOML、JSON 和 env 仅是边缘渲染格式。
|
||||
- 新节点出网 bootstrap 遵循 `docs/reference/yaml-first-ops.md` 的零依赖 host proxy 边界,不得依赖尚未安装的 Docker、k3s、containerd 或包管理器网络能力。
|
||||
|
||||
## 多主机 k3s 集群
|
||||
|
||||
- 权威配置:
|
||||
- 跨主机拓扑、WireGuard、agent token `sourceRef`、registry mirror、ServiceLB 节点和 smoke 归 `config/platform-infra/k3s-clusters.yaml`。
|
||||
- Hyper-V VM 资源和宿主内网 SSH 归 `config/platform-infra/hyperv-vms.yaml`。
|
||||
- 既有 control-plane 的 k3s systemd drop-in 继续归 `config/hwlab-node-control-plane.yaml`,并通过 `clusterConfigRef` 消费集群网络参数。
|
||||
- 受控入口:
|
||||
- `platform-infra k3s-cluster secret-init --target <id> --confirm` 只在 Secret source 不存在时生成 WireGuard key pair 和 k3s agent token,存在时拒绝覆盖。
|
||||
- `platform-infra k3s-cluster plan|apply|status|smoke --target <id>` 是部署和验收入口。
|
||||
- 无固定公网地址的 worker 由宿主内网 SSH 控制,主动连接 control-plane WireGuard endpoint;worker provider route 只作为部署后补充验收入口。
|
||||
- 运行边界:
|
||||
- control-plane 保持唯一 k3s server,同时承担 workload。
|
||||
- worker 只运行 `k3s-agent`;加入前必须卸载旧独立 server,禁止保留第二 datastore/control-plane。
|
||||
- Flannel 使用 WireGuard 接口承载 VXLAN,不能只用 FRP 转发 Kubernetes API。
|
||||
- ServiceLB 只在 owning YAML 指定的公网节点启用,防止无公网地址 worker 接管 hostPort。
|
||||
- `127.0.0.1:5000` 镜像引用必须通过 worker 的 `registries.yaml` 映射到集群可达 registry endpoint。
|
||||
- 验收:
|
||||
- `status` 必须同时证明 WireGuard handshake、server/agent systemd、两个 Ready node、worker InternalIP 和旧 server 退役。
|
||||
- `smoke --confirm` 必须把 Job 定向调度到 worker,并验证集群 DNS 和 Kubernetes ClusterIP;成功后删除 Job。
|
||||
- Secret 输出只显示 sourceRef、presence、key 名和 fingerprint,不显示值。
|
||||
|
||||
## systemd socket-pair 收敛
|
||||
|
||||
- Host-native `.socket` 与 `.service` 配对收敛统一复用 `scripts/src/platform-infra-systemd-socket-pair.ts`:
|
||||
|
||||
@@ -17,7 +17,8 @@
|
||||
|
||||
- Hyper-V 虚拟机、VHDX、处理器、内存和自动启动策略。
|
||||
- 独立内部交换机、NAT、固定地址和出网连通性。
|
||||
- Ubuntu cloud-init、SSH、Docker 和单节点 k3s 初始化。
|
||||
- Ubuntu cloud-init、SSH、Docker 和受控 k3s 运行角色初始化。
|
||||
- 无固定公网地址 worker 通过原生 WireGuard 主动接入既有 k3s 控制面。
|
||||
- provider-gateway 的节点身份、常驻恢复和 trans 验收。
|
||||
- 后台安装的阶段、日志、错误和终态可见性。
|
||||
|
||||
@@ -31,7 +32,7 @@
|
||||
|
||||
### 3.1 HYPERV-VM-REQ-001 YAML 真相
|
||||
|
||||
虚拟机名称、宿主 route、CPU、内存、磁盘、镜像 URL 与摘要、网络、Ubuntu 用户、k3s 版本和 Provider ID 必须来自 `config/platform-infra/hyperv-vms.yaml`。实现只能校验和渲染,不能维护隐藏运行参数。
|
||||
虚拟机名称、宿主 route、CPU、内存、磁盘、镜像 URL 与摘要、网络、Ubuntu 用户、k3s 版本和 Provider ID 必须来自 `config/platform-infra/hyperv-vms.yaml`。跨主机集群拓扑、WireGuard、k3s server/agent 角色、SecretRef、registry mirror 和调度 smoke 必须来自独立的集群 owning YAML。实现只能校验、解析引用和渲染,不能维护隐藏运行参数。
|
||||
|
||||
### 3.2 HYPERV-VM-REQ-002 稳定资源边界
|
||||
|
||||
@@ -56,5 +57,18 @@ Hyper-V 虚拟机必须配置为宿主启动后自动启动、宿主关机时保
|
||||
- VM 为运行状态且资源与 YAML 一致。
|
||||
- Ubuntu 固定地址可达并能正常出网。
|
||||
- Docker daemon 和 k3s 节点健康。
|
||||
- worker 以 agent 身份加入声明的控制面,且独立 server 不再运行。
|
||||
- 定向调度到 worker 的 smoke 能完成集群 DNS 与 ClusterIP 访问。
|
||||
- provider-gateway 以 YAML 声明的 Provider ID 在线。
|
||||
- `trans <provider-id> hostname` 和 k3s route 均可用。
|
||||
|
||||
### 3.7 HYPERV-VM-REQ-007 多主机 k3s 运行面
|
||||
|
||||
既有控制面与 Hyper-V worker 组成集群时必须满足:
|
||||
|
||||
- 控制面保持唯一 server,并继续承担 workload。
|
||||
- Hyper-V 节点只能安装为 agent,不得保留独立 server。
|
||||
- 无固定公网地址的 worker 通过主动拨号的原生 WireGuard 三层隧道接入控制面。
|
||||
- WireGuard 私钥和 k3s agent token 只能来自 YAML `sourceRef`,不得从运行面反解或打印。
|
||||
- k3s 节点地址、Flannel 接口、API TLS SAN、registry mirror 和 ServiceLB 节点选择必须由 owning YAML 渲染。
|
||||
- 宿主内网 SSH 可以作为 worker 的稳定部署控制通道,provider route 只作为部署后验收入口。
|
||||
|
||||
@@ -490,10 +490,14 @@ try {
|
||||
Ensure-VirtualMachine -DiskPath $diskPath -SeedPath $seedPath
|
||||
}
|
||||
Wait-For-Ssh -KeyPath $key.private
|
||||
Install-K3s -KeyPath $key.private
|
||||
if ([string]$Config.k3s.management.mode -eq 'external-cluster') {
|
||||
Write-State -Stage 'staging-k3s-agent' -Message 'Pinned k3s artifacts are staged; the YAML-controlled cluster manager owns the agent role'
|
||||
} else {
|
||||
throw "unsupported-k3s-management-mode: $($Config.k3s.management.mode)"
|
||||
}
|
||||
Install-Provider -KeyPath $key.private
|
||||
|
||||
Write-State -Stage 'completed' -State 'completed' -Message 'Ubuntu VM, Docker, k3s and provider-gateway are installed' -Extra @{
|
||||
Write-State -Stage 'completed' -State 'completed' -Message 'Ubuntu VM, Docker, pinned k3s artifacts and provider-gateway are ready for the external cluster manager' -Extra @{
|
||||
address = [string]$Config.network.address
|
||||
providerId = [string]$Config.provider.id
|
||||
processorCount = [int]$Config.vm.processorCount
|
||||
|
||||
+4
-1
@@ -77,7 +77,7 @@ export function rootHelp(): unknown {
|
||||
{ command: "hwlab nodes control-plane|git-mirror|hwpod-preinstall|secret|test-accounts --node <node> --lane <lane>", description: "Manage HWLAB node/lane runtime prerequisites, including NC01 YAML-declared k3s infra/tools-image/Argo bootstrap, HWPOD preinstall configRefs, redacted test-account preparation, and NC01 v0.3 runtime lane, with the node identity passed as data." },
|
||||
{ command: "hwlab nodes control-plane|git-mirror|hwpod-preinstall|secret|test-accounts --node NC01 --lane v03", description: "Operate the NC01 HWLAB v0.3 development and deployment lane from YAML source truth." },
|
||||
{ command: "agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|send|control-plane|git-mirror", description: "Use AgentRun v0.1 resource primitives with low-noise human output by default; session follow-up uses send only and the server decides internal steer vs turn." },
|
||||
{ command: "platform-infra sub2api|langbot|n8n|webterm|wechat-archive|hyperv-vm ...", description: "Deploy YAML-controlled platform services and Hyper-V Ubuntu nodes, and inspect status/logs without printing secrets." },
|
||||
{ command: "platform-infra sub2api|langbot|n8n|webterm|wechat-archive|hyperv-vm|k3s-cluster ...", description: "Deploy YAML-controlled platform services, Hyper-V Ubuntu nodes and multi-host k3s clusters without printing secrets." },
|
||||
{ command: "pikaoa attachments-backup ...", description: "管理 PikaOA 生产附件离机备份控制面;开发与生产运行面由 PaC 自动交付。" },
|
||||
{ command: "secrets plan|sync|status", description: "Plan, push and inspect YAML-declared local secret source keys to Kubernetes Secrets without printing or reverse-engineering values." },
|
||||
{ command: "platform-db postgres plan|status|export-secrets|apply|swap", description: "Manage YAML-declared host-native PostgreSQL and its node swap capacity through bounded plans, asynchronous mutation, status, TLS and redacted secret output." },
|
||||
@@ -894,6 +894,9 @@ function platformInfraHelpSummary(): unknown {
|
||||
"bun scripts/cli.ts platform-infra hyperv-vm plan --target D601-VM",
|
||||
"bun scripts/cli.ts platform-infra hyperv-vm apply --target D601-VM --confirm",
|
||||
"bun scripts/cli.ts platform-infra hyperv-vm status --target D601-VM",
|
||||
"bun scripts/cli.ts platform-infra k3s-cluster plan --target nc01-d601",
|
||||
"bun scripts/cli.ts platform-infra k3s-cluster apply --target nc01-d601 --confirm",
|
||||
"bun scripts/cli.ts platform-infra k3s-cluster smoke --target nc01-d601 --confirm",
|
||||
"bun scripts/cli.ts platform-infra langbot plan",
|
||||
"bun scripts/cli.ts platform-infra langbot apply --confirm",
|
||||
"bun scripts/cli.ts platform-infra langbot status",
|
||||
|
||||
@@ -240,6 +240,7 @@ export interface ControlPlaneK3sNodeSpec {
|
||||
serviceName: string;
|
||||
dropInPath: string;
|
||||
nodeStatusName: string;
|
||||
clusterConfigRef: string | null;
|
||||
execStartPre: readonly (readonly string[])[];
|
||||
install: ControlPlaneK3sInstallSpec | null;
|
||||
serverArgs: readonly string[];
|
||||
|
||||
@@ -4,6 +4,7 @@ import { rootPath } from "./config";
|
||||
import { runCommand, type CommandResult } from "./command";
|
||||
import { egressBenchmarkCompactResult, egressBenchmarkDryRun, egressBenchmarkStartScript, egressBenchmarkStatusScript, type EgressBenchmarkSpec } from "./egress-proxy-benchmark";
|
||||
import { resolveEgressProxySourceRef } from "./egress-proxy-sources";
|
||||
import { clusterServerArgs } from "./platform-infra-k3s-cluster-config";
|
||||
import { hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
|
||||
import { registryInfraManifest } from "./hwlab-node-control-plane-registry";
|
||||
import {
|
||||
@@ -1582,7 +1583,9 @@ function k3sNodeSpec(raw: Record<string, unknown>, path: string): ControlPlaneK3
|
||||
if (!/^[A-Za-z0-9_.-]+$/u.test(nodeStatusName)) throw new Error(`${path}.nodeStatusName has an unsupported Kubernetes node name`);
|
||||
const execStartPre = execStartPreField(raw.execStartPre, `${path}.execStartPre`);
|
||||
const install = raw.install === undefined ? null : k3sInstallSpec(asRecord(raw.install, `${path}.install`), `${path}.install`);
|
||||
const serverArgs = stringArrayField(raw, "serverArgs", path);
|
||||
const clusterConfigRef = optionalStringField(raw, "clusterConfigRef", path) ?? null;
|
||||
const baseServerArgs = stringArrayField(raw, "serverArgs", path);
|
||||
const serverArgs = clusterConfigRef === null ? baseServerArgs : [...baseServerArgs, ...clusterServerArgs(clusterConfigRef)];
|
||||
if (serverArgs.length === 0 || serverArgs[0] !== "server") throw new Error(`${path}.serverArgs must start with k3s server`);
|
||||
for (const [index, arg] of serverArgs.entries()) {
|
||||
if (arg.includes("\n") || arg.includes("\r") || arg.length === 0) throw new Error(`${path}.serverArgs[${index}] must be a single non-empty argv token`);
|
||||
@@ -1598,6 +1601,7 @@ function k3sNodeSpec(raw: Record<string, unknown>, path: string): ControlPlaneK3
|
||||
serviceName,
|
||||
dropInPath,
|
||||
nodeStatusName,
|
||||
clusterConfigRef,
|
||||
execStartPre,
|
||||
install,
|
||||
serverArgs,
|
||||
|
||||
@@ -88,6 +88,11 @@ function validateTarget(id: string, target: HyperVTarget): void {
|
||||
if (target.vm.diskBytes !== 274_877_906_944) throw new Error(`${configLabel}.targets.${id}.vm.diskBytes must describe the requested 256 GiB disk`);
|
||||
if (!/^[a-f0-9]{64}$/u.test(target.image.sha256)) throw new Error(`${configLabel}.targets.${id}.image.sha256 must be lowercase SHA-256`);
|
||||
if (target.image.sources.length < 2) throw new Error(`${configLabel}.targets.${id}.image.sources must include mirror and canonical fallback`);
|
||||
const k3sManagement = record(target.k3s.management, `${configLabel}.targets.${id}.k3s.management`);
|
||||
if (k3sManagement.mode !== "external-cluster") throw new Error(`${configLabel}.targets.${id}.k3s.management.mode must be external-cluster`);
|
||||
if (typeof k3sManagement.configRef !== "string" || !k3sManagement.configRef.startsWith("config/platform-infra/k3s-clusters.yaml#")) {
|
||||
throw new Error(`${configLabel}.targets.${id}.k3s.management.configRef must reference the k3s cluster owning YAML`);
|
||||
}
|
||||
for (const value of [target.provider.id, target.provider.name, target.provider.composeProject, target.provider.containerName, target.provider.image, target.ubuntu.user, target.k3s.nodeName]) {
|
||||
if (!/^[A-Za-z0-9._:/-]+$/u.test(value)) throw new Error(`${configLabel}.targets.${id} contains a shell-unsafe provider or user value`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,200 @@
|
||||
import { readFileSync } from "node:fs";
|
||||
import { rootPath } from "./config";
|
||||
|
||||
export const K3S_CLUSTER_CONFIG_LABEL = "config/platform-infra/k3s-clusters.yaml";
|
||||
|
||||
export interface K3sClusterTarget {
|
||||
id: string;
|
||||
clusterName: string;
|
||||
controlPlane: {
|
||||
id: string;
|
||||
configRef: string;
|
||||
lane: string;
|
||||
nodeName: string;
|
||||
publicAddress: string;
|
||||
route: string;
|
||||
kubeRoute: string;
|
||||
};
|
||||
worker: {
|
||||
id: string;
|
||||
bootstrapConfigRef: string;
|
||||
nodeName: string;
|
||||
labels: Record<string, string>;
|
||||
windowsRoute: string;
|
||||
guestAddress: string;
|
||||
guestUser: string;
|
||||
stateDir: string;
|
||||
k3sVersion: string;
|
||||
};
|
||||
wireguard: {
|
||||
interfaceName: string;
|
||||
packageName: string;
|
||||
mtu: number;
|
||||
listenPort: number;
|
||||
controlPlaneAddress: string;
|
||||
workerAddress: string;
|
||||
endpoint: string;
|
||||
persistentKeepaliveSeconds: number;
|
||||
};
|
||||
secrets: {
|
||||
sourceRef: string;
|
||||
controlPlanePrivateKey: string;
|
||||
controlPlanePublicKey: string;
|
||||
workerPrivateKey: string;
|
||||
workerPublicKey: string;
|
||||
agentToken: string;
|
||||
};
|
||||
k3s: {
|
||||
serverUrl: string;
|
||||
agentTokenFile: string;
|
||||
flannelInterface: string;
|
||||
maxPods: number;
|
||||
registryMirror: { source: string; endpoint: string };
|
||||
serviceLb: { enabledNodeLabel: string; enabledNode: string };
|
||||
};
|
||||
smoke: { namespace: string; jobName: string; image: string; timeoutSeconds: number };
|
||||
}
|
||||
|
||||
export function readK3sClusterConfig(targetId?: string): { defaultTargetId: string; target: K3sClusterTarget } {
|
||||
const root = record(Bun.YAML.parse(readFileSync(rootPath(K3S_CLUSTER_CONFIG_LABEL), "utf8")), K3S_CLUSTER_CONFIG_LABEL);
|
||||
if (root.kind !== "platform-infra-k3s-clusters") throw new Error(`${K3S_CLUSTER_CONFIG_LABEL}.kind must be platform-infra-k3s-clusters`);
|
||||
const defaults = record(root.defaults, `${K3S_CLUSTER_CONFIG_LABEL}.defaults`);
|
||||
const defaultTargetId = text(defaults, "targetId", `${K3S_CLUSTER_CONFIG_LABEL}.defaults`);
|
||||
const id = targetId ?? defaultTargetId;
|
||||
const raw = record(record(root.targets, `${K3S_CLUSTER_CONFIG_LABEL}.targets`)[id], `${K3S_CLUSTER_CONFIG_LABEL}.targets.${id}`);
|
||||
if (raw.enabled !== true) throw new Error(`${K3S_CLUSTER_CONFIG_LABEL}.targets.${id}.enabled must be true`);
|
||||
const control = record(raw.controlPlane, `targets.${id}.controlPlane`);
|
||||
const workers = record(raw.workers, `targets.${id}.workers`);
|
||||
const workerEntries = Object.entries(workers);
|
||||
if (workerEntries.length !== 1) throw new Error(`targets.${id}.workers must contain exactly one worker`);
|
||||
const [workerId, workerRaw] = workerEntries[0] ?? [];
|
||||
const worker = record(workerRaw, `targets.${id}.workers.${workerId}`);
|
||||
const wireguard = record(raw.wireguard, `targets.${id}.wireguard`);
|
||||
const secrets = record(raw.secrets, `targets.${id}.secrets`);
|
||||
const k3s = record(raw.k3s, `targets.${id}.k3s`);
|
||||
const mirror = record(k3s.registryMirror, `targets.${id}.k3s.registryMirror`);
|
||||
const serviceLb = record(k3s.serviceLb, `targets.${id}.k3s.serviceLb`);
|
||||
const smoke = record(raw.smoke, `targets.${id}.smoke`);
|
||||
const controlSource = resolveRef(text(control, "configRef", `targets.${id}.controlPlane`));
|
||||
const bootstrapSource = resolveRef(text(worker, "bootstrapConfigRef", `targets.${id}.workers[0]`));
|
||||
const host = record(bootstrapSource.host, "worker bootstrap host");
|
||||
const network = record(bootstrapSource.network, "worker bootstrap network");
|
||||
const ubuntu = record(bootstrapSource.ubuntu, "worker bootstrap ubuntu");
|
||||
const bootstrapK3s = record(bootstrapSource.k3s, "worker bootstrap k3s");
|
||||
const labels = stringMap(record(worker.labels, `targets.${id}.workers.${workerId}.labels`), `targets.${id}.workers.${workerId}.labels`);
|
||||
const target: K3sClusterTarget = {
|
||||
id,
|
||||
clusterName: text(raw, "clusterName", `targets.${id}`),
|
||||
controlPlane: {
|
||||
id: text(control, "id", `targets.${id}.controlPlane`),
|
||||
configRef: text(control, "configRef", `targets.${id}.controlPlane`),
|
||||
lane: text(control, "lane", `targets.${id}.controlPlane`),
|
||||
nodeName: text(control, "nodeName", `targets.${id}.controlPlane`),
|
||||
publicAddress: text(control, "publicAddress", `targets.${id}.controlPlane`),
|
||||
route: text(controlSource, "route", "control plane configRef"),
|
||||
kubeRoute: text(controlSource, "kubeRoute", "control plane configRef"),
|
||||
},
|
||||
worker: {
|
||||
id: text(worker, "id", `targets.${id}.workers.${workerId}`),
|
||||
bootstrapConfigRef: text(worker, "bootstrapConfigRef", `targets.${id}.workers.${workerId}`),
|
||||
nodeName: text(worker, "nodeName", `targets.${id}.workers.${workerId}`),
|
||||
labels,
|
||||
windowsRoute: text(host, "route", "worker bootstrap host"),
|
||||
guestAddress: text(network, "address", "worker bootstrap network"),
|
||||
guestUser: text(ubuntu, "user", "worker bootstrap ubuntu"),
|
||||
stateDir: text(host, "stateDir", "worker bootstrap host"),
|
||||
k3sVersion: text(bootstrapK3s, "version", "worker bootstrap k3s"),
|
||||
},
|
||||
wireguard: {
|
||||
interfaceName: text(wireguard, "interfaceName", `targets.${id}.wireguard`),
|
||||
packageName: text(wireguard, "packageName", `targets.${id}.wireguard`),
|
||||
mtu: integer(wireguard, "mtu", `targets.${id}.wireguard`),
|
||||
listenPort: integer(wireguard, "listenPort", `targets.${id}.wireguard`),
|
||||
controlPlaneAddress: cidr(wireguard, "controlPlaneAddress", `targets.${id}.wireguard`),
|
||||
workerAddress: cidr(wireguard, "workerAddress", `targets.${id}.wireguard`),
|
||||
endpoint: text(wireguard, "endpoint", `targets.${id}.wireguard`),
|
||||
persistentKeepaliveSeconds: integer(wireguard, "persistentKeepaliveSeconds", `targets.${id}.wireguard`),
|
||||
},
|
||||
secrets: {
|
||||
sourceRef: text(secrets, "sourceRef", `targets.${id}.secrets`),
|
||||
controlPlanePrivateKey: text(secrets, "controlPlanePrivateKey", `targets.${id}.secrets`),
|
||||
controlPlanePublicKey: text(secrets, "controlPlanePublicKey", `targets.${id}.secrets`),
|
||||
workerPrivateKey: text(secrets, "workerPrivateKey", `targets.${id}.secrets`),
|
||||
workerPublicKey: text(secrets, "workerPublicKey", `targets.${id}.secrets`),
|
||||
agentToken: text(secrets, "agentToken", `targets.${id}.secrets`),
|
||||
},
|
||||
k3s: {
|
||||
serverUrl: text(k3s, "serverUrl", `targets.${id}.k3s`),
|
||||
agentTokenFile: text(k3s, "agentTokenFile", `targets.${id}.k3s`),
|
||||
flannelInterface: text(k3s, "flannelInterface", `targets.${id}.k3s`),
|
||||
maxPods: integer(k3s, "maxPods", `targets.${id}.k3s`),
|
||||
registryMirror: { source: text(mirror, "source", "registryMirror"), endpoint: text(mirror, "endpoint", "registryMirror") },
|
||||
serviceLb: { enabledNodeLabel: text(serviceLb, "enabledNodeLabel", "serviceLb"), enabledNode: text(serviceLb, "enabledNode", "serviceLb") },
|
||||
},
|
||||
smoke: {
|
||||
namespace: text(smoke, "namespace", `targets.${id}.smoke`),
|
||||
jobName: text(smoke, "jobName", `targets.${id}.smoke`),
|
||||
image: text(smoke, "image", `targets.${id}.smoke`),
|
||||
timeoutSeconds: integer(smoke, "timeoutSeconds", `targets.${id}.smoke`),
|
||||
},
|
||||
};
|
||||
if (target.k3s.flannelInterface !== target.wireguard.interfaceName) throw new Error(`targets.${id}.k3s.flannelInterface must match wireguard.interfaceName`);
|
||||
if (target.k3s.serviceLb.enabledNode !== target.controlPlane.nodeName) throw new Error(`targets.${id}.k3s.serviceLb.enabledNode must match controlPlane.nodeName`);
|
||||
return { defaultTargetId, target };
|
||||
}
|
||||
|
||||
export function clusterServerArgs(configRef: string): string[] {
|
||||
const match = configRef.match(/^config\/platform-infra\/k3s-clusters\.yaml#targets\.([A-Za-z0-9._-]+)\.controlPlane$/u);
|
||||
if (match === null) throw new Error(`unsupported k3s clusterConfigRef: ${configRef}`);
|
||||
const { target } = readK3sClusterConfig(match[1]);
|
||||
return [
|
||||
"--node-ip", address(target.wireguard.controlPlaneAddress),
|
||||
"--node-external-ip", target.controlPlane.publicAddress,
|
||||
"--flannel-iface", target.k3s.flannelInterface,
|
||||
"--tls-san", address(target.wireguard.controlPlaneAddress),
|
||||
"--tls-san", target.controlPlane.publicAddress,
|
||||
"--agent-token-file", target.k3s.agentTokenFile,
|
||||
];
|
||||
}
|
||||
|
||||
export function address(value: string): string {
|
||||
return value.split("/", 1)[0] ?? value;
|
||||
}
|
||||
|
||||
function resolveRef(value: string): Record<string, unknown> {
|
||||
const [file, path] = value.split("#", 2);
|
||||
if (!file || !path) throw new Error(`configRef must contain file#path: ${value}`);
|
||||
let current: unknown = Bun.YAML.parse(readFileSync(rootPath(file), "utf8"));
|
||||
for (const segment of path.split(".")) current = record(current, value)[segment];
|
||||
return record(current, value);
|
||||
}
|
||||
|
||||
function record(value: unknown, label: string): Record<string, any> {
|
||||
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${label} must be an object`);
|
||||
return value as Record<string, any>;
|
||||
}
|
||||
|
||||
function text(value: Record<string, unknown>, key: string, label: string): string {
|
||||
const result = value[key];
|
||||
if (typeof result !== "string" || result.length === 0) throw new Error(`${label}.${key} must be a non-empty string`);
|
||||
return result;
|
||||
}
|
||||
|
||||
function integer(value: Record<string, unknown>, key: string, label: string): number {
|
||||
const result = value[key];
|
||||
if (!Number.isInteger(result) || Number(result) <= 0) throw new Error(`${label}.${key} must be a positive integer`);
|
||||
return Number(result);
|
||||
}
|
||||
|
||||
function cidr(value: Record<string, unknown>, key: string, label: string): string {
|
||||
const result = text(value, key, label);
|
||||
if (!/^\d{1,3}(?:\.\d{1,3}){3}\/\d{1,2}$/u.test(result)) throw new Error(`${label}.${key} must be an IPv4 CIDR`);
|
||||
return result;
|
||||
}
|
||||
|
||||
function stringMap(value: Record<string, unknown>, label: string): Record<string, string> {
|
||||
return Object.fromEntries(Object.entries(value).map(([key, item]) => {
|
||||
if (typeof item !== "string" || item.length === 0) throw new Error(`${label}.${key} must be a non-empty string`);
|
||||
return [key, item];
|
||||
}));
|
||||
}
|
||||
@@ -0,0 +1,455 @@
|
||||
import { createHash, generateKeyPairSync, randomBytes } from "node:crypto";
|
||||
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
||||
import { dirname } from "node:path";
|
||||
import type { UniDeskConfig } from "./config";
|
||||
import { rootPath } from "./config";
|
||||
import { runCommand } from "./command";
|
||||
import { parseEnvFile } from "./secrets";
|
||||
import { address, K3S_CLUSTER_CONFIG_LABEL, readK3sClusterConfig, type K3sClusterTarget } from "./platform-infra-k3s-cluster-config";
|
||||
|
||||
type Action = "plan" | "secret-init" | "apply" | "status" | "smoke";
|
||||
|
||||
interface Options {
|
||||
action: Action;
|
||||
targetId?: string;
|
||||
confirm: boolean;
|
||||
}
|
||||
|
||||
interface SecretMaterial {
|
||||
controlPrivate: string;
|
||||
controlPublic: string;
|
||||
workerPrivate: string;
|
||||
workerPublic: string;
|
||||
agentToken: string;
|
||||
fingerprint: string;
|
||||
}
|
||||
|
||||
export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig, args: string[]): Promise<Record<string, unknown>> {
|
||||
const options = parseOptions(args);
|
||||
const { target } = readK3sClusterConfig(options.targetId);
|
||||
if (options.action === "plan") return plan(target);
|
||||
if (options.action === "secret-init") return secretInit(target, options.confirm);
|
||||
if (options.action === "status") return status(target);
|
||||
if (options.action === "smoke") return smoke(target, options.confirm);
|
||||
if (!options.confirm) return { ...plan(target), mode: "dry-run", mutation: false };
|
||||
return apply(target);
|
||||
}
|
||||
|
||||
function parseOptions(args: string[]): Options {
|
||||
const action = args[0] ?? "plan";
|
||||
if (action !== "plan" && action !== "secret-init" && action !== "apply" && action !== "status" && action !== "smoke") {
|
||||
throw new Error("platform-infra k3s-cluster usage: plan|secret-init|apply|status|smoke [--target <id>] [--confirm]");
|
||||
}
|
||||
return { action, targetId: option(args, "--target") ?? undefined, confirm: args.includes("--confirm") };
|
||||
}
|
||||
|
||||
function plan(target: K3sClusterTarget): Record<string, unknown> {
|
||||
return {
|
||||
ok: true,
|
||||
mutation: false,
|
||||
targetId: target.id,
|
||||
source: K3S_CLUSTER_CONFIG_LABEL,
|
||||
topology: {
|
||||
clusterName: target.clusterName,
|
||||
controlPlane: { id: target.controlPlane.id, nodeName: target.controlPlane.nodeName, route: target.controlPlane.route, kubeRoute: target.controlPlane.kubeRoute },
|
||||
worker: { id: target.worker.id, nodeName: target.worker.nodeName, bootstrapConfigRef: target.worker.bootstrapConfigRef, controlRoute: target.worker.windowsRoute },
|
||||
},
|
||||
wireguard: { ...target.wireguard, endpoint: target.wireguard.endpoint },
|
||||
k3s: {
|
||||
serverUrl: target.k3s.serverUrl,
|
||||
agentTokenFile: target.k3s.agentTokenFile,
|
||||
registryMirror: target.k3s.registryMirror,
|
||||
serviceLb: target.k3s.serviceLb,
|
||||
},
|
||||
secret: secretSummary(target),
|
||||
next: {
|
||||
secretInit: `bun scripts/cli.ts platform-infra k3s-cluster secret-init --target ${target.id} --confirm`,
|
||||
apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm`,
|
||||
status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`,
|
||||
smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function secretInit(target: K3sClusterTarget, confirm: boolean): Record<string, unknown> {
|
||||
if (!confirm) return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, secret: secretSummary(target) };
|
||||
const path = target.secrets.sourceRef;
|
||||
if (existsSync(path)) throw new Error(`${path} already exists; secret-init never overwrites cluster credentials`);
|
||||
const control = wireguardKeyPair();
|
||||
const worker = wireguardKeyPair();
|
||||
const values: Record<string, string> = {
|
||||
[target.secrets.controlPlanePrivateKey]: control.privateKey,
|
||||
[target.secrets.controlPlanePublicKey]: control.publicKey,
|
||||
[target.secrets.workerPrivateKey]: worker.privateKey,
|
||||
[target.secrets.workerPublicKey]: worker.publicKey,
|
||||
[target.secrets.agentToken]: randomBytes(32).toString("base64url"),
|
||||
};
|
||||
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
|
||||
writeFileSync(path, Object.entries(values).map(([key, value]) => `${key}=${value}`).join("\n") + "\n", { encoding: "utf8", mode: 0o600 });
|
||||
chmodSync(path, 0o600);
|
||||
return { ok: true, mutation: true, targetId: target.id, secret: secretSummary(target), valuesPrinted: false };
|
||||
}
|
||||
|
||||
function apply(target: K3sClusterTarget): Record<string, unknown> {
|
||||
const secret = readSecrets(target);
|
||||
const controlWireguard = runControlPlane(target, controlPlaneApplyScript(target, secret), 180_000);
|
||||
requireSuccess(controlWireguard, "control-plane-wireguard-apply");
|
||||
|
||||
const controlPlane = runCommand([
|
||||
"bun", "scripts/cli.ts", "hwlab", "nodes", "control-plane", "infra", "apply",
|
||||
"--node", target.controlPlane.id, "--lane", target.controlPlane.lane, "--confirm",
|
||||
], rootPath(), { timeoutMs: 180_000 });
|
||||
requireSuccess(controlPlane, "control-plane-k3s-apply");
|
||||
|
||||
const worker = runWorker(target, workerApplyScript(target, secret), true, 300_000);
|
||||
requireSuccess(worker, "worker-agent-apply");
|
||||
|
||||
const labels = [
|
||||
"label", "node", target.k3s.serviceLb.enabledNode,
|
||||
`${target.k3s.serviceLb.enabledNodeLabel}=true`, "--overwrite",
|
||||
];
|
||||
const labelResult = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", ...labels], rootPath(), { timeoutMs: 60_000 });
|
||||
requireSuccess(labelResult, "servicelb-node-label");
|
||||
|
||||
const ready = runControlPlane(target, waitForNodesScript(target), 180_000);
|
||||
requireSuccess(ready, "cluster-node-readiness");
|
||||
return {
|
||||
ok: true,
|
||||
mutation: true,
|
||||
targetId: target.id,
|
||||
stages: {
|
||||
controlPlaneWireguard: compact(controlWireguard),
|
||||
controlPlaneK3s: compact(controlPlane),
|
||||
workerAgent: compact(worker),
|
||||
serviceLbPlacement: compact(labelResult),
|
||||
nodeReadiness: compact(ready),
|
||||
},
|
||||
secret: { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: secret.fingerprint, valuesPrinted: false },
|
||||
next: {
|
||||
status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`,
|
||||
smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function status(target: K3sClusterTarget): Record<string, unknown> {
|
||||
const control = runControlPlane(target, statusScript(target), 60_000);
|
||||
const worker = runWorker(target, workerStatusScript(target), false, 60_000);
|
||||
const controlData = parseFirstJson(control.stdout);
|
||||
const workerData = parseFirstJson(worker.stdout);
|
||||
const ok = control.exitCode === 0 && worker.exitCode === 0 && controlData.ok === true && workerData.ok === true;
|
||||
return {
|
||||
ok,
|
||||
mutation: false,
|
||||
targetId: target.id,
|
||||
controlPlane: controlData,
|
||||
worker: workerData,
|
||||
secret: secretSummary(target),
|
||||
result: { controlPlane: compact(control), worker: compact(worker) },
|
||||
next: ok ? { smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm` } : { apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm` },
|
||||
};
|
||||
}
|
||||
|
||||
function smoke(target: K3sClusterTarget, confirm: boolean): Record<string, unknown> {
|
||||
const manifest = smokeManifest(target);
|
||||
if (!confirm) return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, workerNode: target.worker.nodeName, manifestSha256: hash(manifest) };
|
||||
const namespace = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "create", "namespace", target.smoke.namespace, "--dry-run=client", "-o", "yaml"], rootPath(), { timeoutMs: 30_000 });
|
||||
requireSuccess(namespace, "smoke-namespace-render");
|
||||
const applyNamespace = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "apply", "-f", "-"], rootPath(), { input: namespace.stdout, timeoutMs: 30_000 });
|
||||
requireSuccess(applyNamespace, "smoke-namespace-apply");
|
||||
runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "delete", "job", target.smoke.jobName, "--ignore-not-found=true", "--wait=true"], rootPath(), { timeoutMs: 60_000 });
|
||||
const applyJob = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "apply", "-f", "-"], rootPath(), { input: manifest, timeoutMs: 30_000 });
|
||||
requireSuccess(applyJob, "smoke-job-apply");
|
||||
const wait = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "wait", `job/${target.smoke.jobName}`, "--for=condition=complete", `--timeout=${target.smoke.timeoutSeconds}s`], rootPath(), { timeoutMs: (target.smoke.timeoutSeconds + 20) * 1000 });
|
||||
const logs = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "logs", `job/${target.smoke.jobName}`], rootPath(), { timeoutMs: 30_000 });
|
||||
const placement = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "get", "pod", "-l", `job-name=${target.smoke.jobName}`, "-o", "jsonpath={.items[0].spec.nodeName}"], rootPath(), { timeoutMs: 30_000 });
|
||||
const ok = wait.exitCode === 0 && logs.exitCode === 0 && logs.stdout.includes("D601_RUNTIME_SMOKE_OK") && placement.stdout.trim() === target.worker.nodeName;
|
||||
if (ok) runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "delete", "job", target.smoke.jobName, "--wait=true"], rootPath(), { timeoutMs: 60_000 });
|
||||
return {
|
||||
ok,
|
||||
mutation: true,
|
||||
targetId: target.id,
|
||||
workerNode: target.worker.nodeName,
|
||||
scheduledNode: placement.stdout.trim(),
|
||||
checks: { jobCompleted: wait.exitCode === 0, dnsAndClusterIp: logs.stdout.includes("D601_RUNTIME_SMOKE_OK"), workerPlacement: placement.stdout.trim() === target.worker.nodeName },
|
||||
logs: bounded(logs.stdout, 4_000),
|
||||
result: { apply: compact(applyJob), wait: compact(wait), placement: compact(placement), cleanedUp: ok },
|
||||
};
|
||||
}
|
||||
|
||||
function controlPlaneApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string {
|
||||
const wg = target.wireguard;
|
||||
return `set -eu
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
if ! command -v wg >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq ${shQuote(wg.packageName)}; fi
|
||||
umask 077
|
||||
install -d -m 700 /etc/wireguard /etc/unidesk
|
||||
cat > /etc/wireguard/${wg.interfaceName}.conf <<'EOF'
|
||||
[Interface]
|
||||
Address = ${wg.controlPlaneAddress}
|
||||
ListenPort = ${wg.listenPort}
|
||||
PrivateKey = ${secret.controlPrivate}
|
||||
MTU = ${wg.mtu}
|
||||
|
||||
[Peer]
|
||||
PublicKey = ${secret.workerPublic}
|
||||
AllowedIPs = ${address(wg.workerAddress)}/32
|
||||
EOF
|
||||
chmod 600 /etc/wireguard/${wg.interfaceName}.conf
|
||||
printf '%s\n' ${shQuote(secret.agentToken)} > ${shQuote(target.k3s.agentTokenFile)}
|
||||
chmod 600 ${shQuote(target.k3s.agentTokenFile)}
|
||||
systemctl enable wg-quick@${wg.interfaceName} >/dev/null
|
||||
systemctl restart wg-quick@${wg.interfaceName}
|
||||
wg show ${wg.interfaceName} >/dev/null
|
||||
`;
|
||||
}
|
||||
|
||||
function workerApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string {
|
||||
const wg = target.wireguard;
|
||||
const labels = Object.entries(target.worker.labels).flatMap(([key, value]) => ["--node-label", `${key}=${value}`]);
|
||||
const exec = [
|
||||
"agent", "--server", target.k3s.serverUrl, "--token-file", target.k3s.agentTokenFile,
|
||||
"--node-name", target.worker.nodeName, "--node-ip", address(wg.workerAddress),
|
||||
"--flannel-iface", target.k3s.flannelInterface, ...labels,
|
||||
"--kubelet-arg", `max-pods=${target.k3s.maxPods}`,
|
||||
].map(shellToken).join(" ");
|
||||
return `set -eu
|
||||
export DEBIAN_FRONTEND=noninteractive
|
||||
if ! command -v wg >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq ${shQuote(wg.packageName)}; fi
|
||||
umask 077
|
||||
install -d -m 700 /etc/wireguard /etc/unidesk /etc/rancher/k3s
|
||||
cat > /etc/wireguard/${wg.interfaceName}.conf <<'EOF'
|
||||
[Interface]
|
||||
Address = ${wg.workerAddress}
|
||||
PrivateKey = ${secret.workerPrivate}
|
||||
MTU = ${wg.mtu}
|
||||
|
||||
[Peer]
|
||||
PublicKey = ${secret.controlPublic}
|
||||
Endpoint = ${wg.endpoint}
|
||||
AllowedIPs = ${address(wg.controlPlaneAddress)}/32
|
||||
PersistentKeepalive = ${wg.persistentKeepaliveSeconds}
|
||||
EOF
|
||||
chmod 600 /etc/wireguard/${wg.interfaceName}.conf
|
||||
systemctl enable wg-quick@${wg.interfaceName} >/dev/null
|
||||
systemctl restart wg-quick@${wg.interfaceName}
|
||||
ping -c 3 -W 5 ${address(wg.controlPlaneAddress)} >/dev/null
|
||||
printf '%s\n' ${shQuote(secret.agentToken)} > ${shQuote(target.k3s.agentTokenFile)}
|
||||
chmod 600 ${shQuote(target.k3s.agentTokenFile)}
|
||||
cat > /etc/rancher/k3s/registries.yaml <<'EOF'
|
||||
mirrors:
|
||||
"${target.k3s.registryMirror.source}":
|
||||
endpoint:
|
||||
- "${target.k3s.registryMirror.endpoint}"
|
||||
EOF
|
||||
if [ -x /usr/local/bin/k3s-uninstall.sh ]; then /usr/local/bin/k3s-uninstall.sh; fi
|
||||
if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh; fi
|
||||
install -m 0755 /tmp/k3s /usr/local/bin/k3s
|
||||
chmod 0755 /tmp/install-k3s.sh
|
||||
INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION=${shQuote(target.worker.k3sVersion)} INSTALL_K3S_EXEC=${shQuote(exec)} sh /tmp/install-k3s.sh
|
||||
systemctl enable k3s-agent >/dev/null
|
||||
systemctl restart k3s-agent
|
||||
systemctl is-active --quiet k3s-agent
|
||||
`;
|
||||
}
|
||||
|
||||
function waitForNodesScript(target: K3sClusterTarget): string {
|
||||
return `set -eu
|
||||
deadline=$(( $(date +%s) + 150 ))
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
ready=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
|
||||
[ "$ready" = True ] && exit 0
|
||||
sleep 5
|
||||
done
|
||||
/usr/local/bin/kubectl get nodes -o wide >&2
|
||||
exit 1
|
||||
`;
|
||||
}
|
||||
|
||||
function statusScript(target: K3sClusterTarget): string {
|
||||
const cp = address(target.wireguard.controlPlaneAddress);
|
||||
return `set -eu
|
||||
wg_active=false; systemctl is-active --quiet wg-quick@${target.wireguard.interfaceName} && wg_active=true
|
||||
k3s_active=false; systemctl is-active --quiet k3s && k3s_active=true
|
||||
cp_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.controlPlane.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
|
||||
worker_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
|
||||
worker_ip=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null || true)
|
||||
handshake=$(wg show ${target.wireguard.interfaceName} latest-handshakes 2>/dev/null | awk '{print $2}' | head -n1)
|
||||
python3 - "$wg_active" "$k3s_active" "$cp_ready" "$worker_ready" "$worker_ip" "$handshake" <<'PY'
|
||||
import json, sys, time
|
||||
wg, k3s, cp, worker, worker_ip, handshake = sys.argv[1:]
|
||||
stamp = int(handshake or 0)
|
||||
age = None if stamp == 0 else max(0, int(time.time()) - stamp)
|
||||
ok = wg == 'true' and k3s == 'true' and cp == 'True' and worker == 'True' and worker_ip == '${address(target.wireguard.workerAddress)}' and age is not None and age < 180
|
||||
print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'k3sActive': k3s == 'true', 'controlPlaneReady': cp == 'True', 'workerReady': worker == 'True', 'workerInternalIp': worker_ip, 'handshakeAgeSeconds': age, 'controlPlaneAddress': '${cp}'}, separators=(',', ':')))
|
||||
PY
|
||||
`;
|
||||
}
|
||||
|
||||
function workerStatusScript(target: K3sClusterTarget): string {
|
||||
return `set -eu
|
||||
wg_active=false; systemctl is-active --quiet wg-quick@${target.wireguard.interfaceName} && wg_active=true
|
||||
agent_active=false; systemctl is-active --quiet k3s-agent && agent_active=true
|
||||
old_server=false; systemctl is-active --quiet k3s && old_server=true
|
||||
ip_present=false; ip -4 address show dev ${target.wireguard.interfaceName} | grep -F ${shQuote(address(target.wireguard.workerAddress))} >/dev/null 2>&1 && ip_present=true
|
||||
python3 - "$wg_active" "$agent_active" "$old_server" "$ip_present" <<'PY'
|
||||
import json, sys
|
||||
wg, agent, old_server, ip_present = sys.argv[1:]
|
||||
ok = wg == 'true' and agent == 'true' and old_server == 'false' and ip_present == 'true'
|
||||
print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'agentActive': agent == 'true', 'standaloneServerActive': old_server == 'true', 'wireguardAddressPresent': ip_present == 'true'}, separators=(',', ':')))
|
||||
PY
|
||||
`;
|
||||
}
|
||||
|
||||
function smokeManifest(target: K3sClusterTarget): string {
|
||||
return `apiVersion: batch/v1
|
||||
kind: Job
|
||||
metadata:
|
||||
name: ${target.smoke.jobName}
|
||||
namespace: ${target.smoke.namespace}
|
||||
spec:
|
||||
backoffLimit: 0
|
||||
template:
|
||||
metadata:
|
||||
labels:
|
||||
app.kubernetes.io/name: ${target.smoke.jobName}
|
||||
spec:
|
||||
restartPolicy: Never
|
||||
nodeSelector:
|
||||
kubernetes.io/hostname: ${target.worker.nodeName}
|
||||
containers:
|
||||
- name: smoke
|
||||
image: ${target.smoke.image}
|
||||
imagePullPolicy: IfNotPresent
|
||||
command:
|
||||
- sh
|
||||
- -ec
|
||||
- |
|
||||
nslookup kubernetes.default.svc >/tmp/dns.txt
|
||||
wget -qO- --no-check-certificate --timeout=15 https://kubernetes.default.svc/healthz | grep -q '^ok$'
|
||||
echo D601_RUNTIME_SMOKE_OK
|
||||
`;
|
||||
}
|
||||
|
||||
function runControlPlane(target: K3sClusterTarget, script: string, timeoutMs: number) {
|
||||
return runCommand(["trans", target.controlPlane.route, "sh"], rootPath(), { input: script, timeoutMs });
|
||||
}
|
||||
|
||||
function runWorker(target: K3sClusterTarget, script: string, copyArtifacts: boolean, timeoutMs: number) {
|
||||
const encoded = Buffer.from(script, "utf8").toString("base64");
|
||||
const keyPath = `${target.worker.stateDir}\\id_ed25519`;
|
||||
const remoteScript = `/tmp/unidesk-k3s-cluster-${target.id}.sh`;
|
||||
const lines = [
|
||||
"$ErrorActionPreference='Stop'",
|
||||
`$key=${psQuote(keyPath)}`,
|
||||
`$guest=${psQuote(`${target.worker.guestUser}@${target.worker.guestAddress}`)}`,
|
||||
`$temp=Join-Path $env:TEMP ${psQuote(`unidesk-k3s-cluster-${target.id}.sh`)}`,
|
||||
`[IO.File]::WriteAllBytes($temp,[Convert]::FromBase64String(${psQuote(encoded)}))`,
|
||||
];
|
||||
if (copyArtifacts) {
|
||||
lines.push(
|
||||
`$binary=${psQuote(`${target.worker.stateDir}\\k3s`)}`,
|
||||
`$installer=${psQuote(`${target.worker.stateDir}\\install-k3s.sh`)}`,
|
||||
"& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $binary \"${guest}:/tmp/k3s\"",
|
||||
"if($LASTEXITCODE -ne 0){throw 'worker-k3s-binary-scp-failed'}",
|
||||
"& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $installer \"${guest}:/tmp/install-k3s.sh\"",
|
||||
"if($LASTEXITCODE -ne 0){throw 'worker-k3s-installer-scp-failed'}",
|
||||
);
|
||||
}
|
||||
lines.push(
|
||||
`& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $temp \"${guest}:${remoteScript}\"`,
|
||||
"if($LASTEXITCODE -ne 0){throw 'worker-script-scp-failed'}",
|
||||
`& ssh.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 $guest ${psQuote(`sudo sh ${remoteScript}`)}`,
|
||||
"$rc=$LASTEXITCODE",
|
||||
"Remove-Item -Force -LiteralPath $temp -ErrorAction SilentlyContinue",
|
||||
`& ssh.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $guest ${psQuote(`rm -f ${remoteScript}`)} | Out-Null`,
|
||||
"exit $rc",
|
||||
);
|
||||
return runCommand(["trans", target.worker.windowsRoute, "ps"], rootPath(), { input: lines.join("\n") + "\n", timeoutMs });
|
||||
}
|
||||
|
||||
function readSecrets(target: K3sClusterTarget): SecretMaterial {
|
||||
if (!existsSync(target.secrets.sourceRef)) throw new Error(`${target.secrets.sourceRef} is missing; run platform-infra k3s-cluster secret-init --confirm`);
|
||||
const values = parseEnvFile(readFileSync(target.secrets.sourceRef, "utf8"));
|
||||
const required = (key: string) => {
|
||||
const value = values[key];
|
||||
if (typeof value !== "string" || value.length < 32) throw new Error(`${target.secrets.sourceRef}.${key} is missing or invalid`);
|
||||
return value;
|
||||
};
|
||||
const material = {
|
||||
controlPrivate: required(target.secrets.controlPlanePrivateKey),
|
||||
controlPublic: required(target.secrets.controlPlanePublicKey),
|
||||
workerPrivate: required(target.secrets.workerPrivateKey),
|
||||
workerPublic: required(target.secrets.workerPublicKey),
|
||||
agentToken: required(target.secrets.agentToken),
|
||||
};
|
||||
return { ...material, fingerprint: hash(Object.entries(material).sort().map(([key, value]) => `${key}=${value}`).join("\n")) };
|
||||
}
|
||||
|
||||
function secretSummary(target: K3sClusterTarget): Record<string, unknown> {
|
||||
if (!existsSync(target.secrets.sourceRef)) return { sourceRef: target.secrets.sourceRef, presence: false, fingerprint: null, valuesPrinted: false };
|
||||
try {
|
||||
const secret = readSecrets(target);
|
||||
return { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: secret.fingerprint, keys: Object.values(target.secrets).filter((value) => value !== target.secrets.sourceRef), valuesPrinted: false };
|
||||
} catch (error) {
|
||||
return { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: null, error: error instanceof Error ? error.message : String(error), valuesPrinted: false };
|
||||
}
|
||||
}
|
||||
|
||||
function wireguardKeyPair(): { privateKey: string; publicKey: string } {
|
||||
const pair = generateKeyPairSync("x25519");
|
||||
const privateJwk = pair.privateKey.export({ format: "jwk" }) as JsonWebKey;
|
||||
const publicJwk = pair.publicKey.export({ format: "jwk" }) as JsonWebKey;
|
||||
if (!privateJwk.d || !publicJwk.x) throw new Error("x25519 key generation did not produce raw JWK keys");
|
||||
return { privateKey: base64UrlToBase64(privateJwk.d), publicKey: base64UrlToBase64(publicJwk.x) };
|
||||
}
|
||||
|
||||
function base64UrlToBase64(value: string): string {
|
||||
return Buffer.from(value, "base64url").toString("base64");
|
||||
}
|
||||
|
||||
function requireSuccess(result: ReturnType<typeof runCommand>, stage: string): void {
|
||||
if (result.exitCode !== 0) throw new Error(`${stage} failed: ${bounded(result.stderr || result.stdout)}`);
|
||||
}
|
||||
|
||||
function compact(result: ReturnType<typeof runCommand>): Record<string, unknown> {
|
||||
return { exitCode: result.exitCode, timedOut: result.timedOut, stdout: bounded(result.stdout, 2_000), stderr: bounded(result.stderr, 2_000) };
|
||||
}
|
||||
|
||||
function parseFirstJson(value: string): Record<string, unknown> {
|
||||
for (const line of value.split(/\r?\n/u)) {
|
||||
const text = line.trim();
|
||||
if (!text.startsWith("{")) continue;
|
||||
try { return JSON.parse(text) as Record<string, unknown>; } catch { /* continue */ }
|
||||
}
|
||||
return { ok: false, error: "remote-json-missing", output: bounded(value) };
|
||||
}
|
||||
|
||||
function option(args: string[], name: string): string | null {
|
||||
const index = args.indexOf(name);
|
||||
if (index < 0) return null;
|
||||
const value = args[index + 1];
|
||||
if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`);
|
||||
return value;
|
||||
}
|
||||
|
||||
function shQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", `'"'"'`)}'`;
|
||||
}
|
||||
|
||||
function shellToken(value: string): string {
|
||||
return /^[A-Za-z0-9._:/=@+-]+$/u.test(value) ? value : shQuote(value);
|
||||
}
|
||||
|
||||
function psQuote(value: string): string {
|
||||
return `'${value.replaceAll("'", "''")}'`;
|
||||
}
|
||||
|
||||
function hash(value: string): string {
|
||||
return createHash("sha256").update(value).digest("hex");
|
||||
}
|
||||
|
||||
function bounded(value: string, max = 4_000): string {
|
||||
const text = value.trim();
|
||||
return text.length <= max ? text : `${text.slice(0, max)}\n...[truncated ${text.length - max} chars]`;
|
||||
}
|
||||
@@ -99,6 +99,10 @@ export async function runPlatformInfraCommand(config: UniDeskConfig, args: strin
|
||||
const { runPlatformInfraHyperVVmCommand } = await import("../platform-infra-hyperv-vm");
|
||||
return await runPlatformInfraHyperVVmCommand(config, args.slice(1));
|
||||
}
|
||||
if (target === "k3s-cluster") {
|
||||
const { runPlatformInfraK3sClusterCommand } = await import("../platform-infra-k3s-cluster");
|
||||
return await runPlatformInfraK3sClusterCommand(config, args.slice(1));
|
||||
}
|
||||
if (target === "pipelines-as-code" || target === "pac") {
|
||||
const { runPlatformInfraPipelinesAsCodeCommand } = await import("../platform-infra-pipelines-as-code");
|
||||
return await runPlatformInfraPipelinesAsCodeCommand(config, args.slice(1));
|
||||
|
||||
Reference in New Issue
Block a user