feat: add controlled NC01 D601 k3s cluster

This commit is contained in:
pikastech
2026-07-20 03:17:57 +02:00
parent b89f48f9cc
commit 2cfbe8bedc
15 changed files with 788 additions and 9 deletions
+4 -1
View File
@@ -77,7 +77,7 @@ export function rootHelp(): unknown {
{ command: "hwlab nodes control-plane|git-mirror|hwpod-preinstall|secret|test-accounts --node <node> --lane <lane>", description: "Manage HWLAB node/lane runtime prerequisites, including NC01 YAML-declared k3s infra/tools-image/Argo bootstrap, HWPOD preinstall configRefs, redacted test-account preparation, and NC01 v0.3 runtime lane, with the node identity passed as data." },
{ command: "hwlab nodes control-plane|git-mirror|hwpod-preinstall|secret|test-accounts --node NC01 --lane v03", description: "Operate the NC01 HWLAB v0.3 development and deployment lane from YAML source truth." },
{ command: "agentrun get|describe|events|logs|result|ack|cancel|dispatch|create|apply|send|control-plane|git-mirror", description: "Use AgentRun v0.1 resource primitives with low-noise human output by default; session follow-up uses send only and the server decides internal steer vs turn." },
{ command: "platform-infra sub2api|langbot|n8n|webterm|wechat-archive|hyperv-vm ...", description: "Deploy YAML-controlled platform services and Hyper-V Ubuntu nodes, and inspect status/logs without printing secrets." },
{ command: "platform-infra sub2api|langbot|n8n|webterm|wechat-archive|hyperv-vm|k3s-cluster ...", description: "Deploy YAML-controlled platform services, Hyper-V Ubuntu nodes and multi-host k3s clusters without printing secrets." },
{ command: "pikaoa attachments-backup ...", description: "管理 PikaOA 生产附件离机备份控制面;开发与生产运行面由 PaC 自动交付。" },
{ command: "secrets plan|sync|status", description: "Plan, push and inspect YAML-declared local secret source keys to Kubernetes Secrets without printing or reverse-engineering values." },
{ command: "platform-db postgres plan|status|export-secrets|apply|swap", description: "Manage YAML-declared host-native PostgreSQL and its node swap capacity through bounded plans, asynchronous mutation, status, TLS and redacted secret output." },
@@ -894,6 +894,9 @@ function platformInfraHelpSummary(): unknown {
"bun scripts/cli.ts platform-infra hyperv-vm plan --target D601-VM",
"bun scripts/cli.ts platform-infra hyperv-vm apply --target D601-VM --confirm",
"bun scripts/cli.ts platform-infra hyperv-vm status --target D601-VM",
"bun scripts/cli.ts platform-infra k3s-cluster plan --target nc01-d601",
"bun scripts/cli.ts platform-infra k3s-cluster apply --target nc01-d601 --confirm",
"bun scripts/cli.ts platform-infra k3s-cluster smoke --target nc01-d601 --confirm",
"bun scripts/cli.ts platform-infra langbot plan",
"bun scripts/cli.ts platform-infra langbot apply --confirm",
"bun scripts/cli.ts platform-infra langbot status",
@@ -240,6 +240,7 @@ export interface ControlPlaneK3sNodeSpec {
serviceName: string;
dropInPath: string;
nodeStatusName: string;
clusterConfigRef: string | null;
execStartPre: readonly (readonly string[])[];
install: ControlPlaneK3sInstallSpec | null;
serverArgs: readonly string[];
+5 -1
View File
@@ -4,6 +4,7 @@ import { rootPath } from "./config";
import { runCommand, type CommandResult } from "./command";
import { egressBenchmarkCompactResult, egressBenchmarkDryRun, egressBenchmarkStartScript, egressBenchmarkStatusScript, type EgressBenchmarkSpec } from "./egress-proxy-benchmark";
import { resolveEgressProxySourceRef } from "./egress-proxy-sources";
import { clusterServerArgs } from "./platform-infra-k3s-cluster-config";
import { hwlabRuntimeLaneSpecForNode, isHwlabRuntimeLane, type HwlabRuntimeLaneSpec } from "./hwlab-node-lanes";
import { registryInfraManifest } from "./hwlab-node-control-plane-registry";
import {
@@ -1582,7 +1583,9 @@ function k3sNodeSpec(raw: Record<string, unknown>, path: string): ControlPlaneK3
if (!/^[A-Za-z0-9_.-]+$/u.test(nodeStatusName)) throw new Error(`${path}.nodeStatusName has an unsupported Kubernetes node name`);
const execStartPre = execStartPreField(raw.execStartPre, `${path}.execStartPre`);
const install = raw.install === undefined ? null : k3sInstallSpec(asRecord(raw.install, `${path}.install`), `${path}.install`);
const serverArgs = stringArrayField(raw, "serverArgs", path);
const clusterConfigRef = optionalStringField(raw, "clusterConfigRef", path) ?? null;
const baseServerArgs = stringArrayField(raw, "serverArgs", path);
const serverArgs = clusterConfigRef === null ? baseServerArgs : [...baseServerArgs, ...clusterServerArgs(clusterConfigRef)];
if (serverArgs.length === 0 || serverArgs[0] !== "server") throw new Error(`${path}.serverArgs must start with k3s server`);
for (const [index, arg] of serverArgs.entries()) {
if (arg.includes("\n") || arg.includes("\r") || arg.length === 0) throw new Error(`${path}.serverArgs[${index}] must be a single non-empty argv token`);
@@ -1598,6 +1601,7 @@ function k3sNodeSpec(raw: Record<string, unknown>, path: string): ControlPlaneK3
serviceName,
dropInPath,
nodeStatusName,
clusterConfigRef,
execStartPre,
install,
serverArgs,
+5
View File
@@ -88,6 +88,11 @@ function validateTarget(id: string, target: HyperVTarget): void {
if (target.vm.diskBytes !== 274_877_906_944) throw new Error(`${configLabel}.targets.${id}.vm.diskBytes must describe the requested 256 GiB disk`);
if (!/^[a-f0-9]{64}$/u.test(target.image.sha256)) throw new Error(`${configLabel}.targets.${id}.image.sha256 must be lowercase SHA-256`);
if (target.image.sources.length < 2) throw new Error(`${configLabel}.targets.${id}.image.sources must include mirror and canonical fallback`);
const k3sManagement = record(target.k3s.management, `${configLabel}.targets.${id}.k3s.management`);
if (k3sManagement.mode !== "external-cluster") throw new Error(`${configLabel}.targets.${id}.k3s.management.mode must be external-cluster`);
if (typeof k3sManagement.configRef !== "string" || !k3sManagement.configRef.startsWith("config/platform-infra/k3s-clusters.yaml#")) {
throw new Error(`${configLabel}.targets.${id}.k3s.management.configRef must reference the k3s cluster owning YAML`);
}
for (const value of [target.provider.id, target.provider.name, target.provider.composeProject, target.provider.containerName, target.provider.image, target.ubuntu.user, target.k3s.nodeName]) {
if (!/^[A-Za-z0-9._:/-]+$/u.test(value)) throw new Error(`${configLabel}.targets.${id} contains a shell-unsafe provider or user value`);
}
@@ -0,0 +1,200 @@
import { readFileSync } from "node:fs";
import { rootPath } from "./config";
export const K3S_CLUSTER_CONFIG_LABEL = "config/platform-infra/k3s-clusters.yaml";
export interface K3sClusterTarget {
id: string;
clusterName: string;
controlPlane: {
id: string;
configRef: string;
lane: string;
nodeName: string;
publicAddress: string;
route: string;
kubeRoute: string;
};
worker: {
id: string;
bootstrapConfigRef: string;
nodeName: string;
labels: Record<string, string>;
windowsRoute: string;
guestAddress: string;
guestUser: string;
stateDir: string;
k3sVersion: string;
};
wireguard: {
interfaceName: string;
packageName: string;
mtu: number;
listenPort: number;
controlPlaneAddress: string;
workerAddress: string;
endpoint: string;
persistentKeepaliveSeconds: number;
};
secrets: {
sourceRef: string;
controlPlanePrivateKey: string;
controlPlanePublicKey: string;
workerPrivateKey: string;
workerPublicKey: string;
agentToken: string;
};
k3s: {
serverUrl: string;
agentTokenFile: string;
flannelInterface: string;
maxPods: number;
registryMirror: { source: string; endpoint: string };
serviceLb: { enabledNodeLabel: string; enabledNode: string };
};
smoke: { namespace: string; jobName: string; image: string; timeoutSeconds: number };
}
export function readK3sClusterConfig(targetId?: string): { defaultTargetId: string; target: K3sClusterTarget } {
const root = record(Bun.YAML.parse(readFileSync(rootPath(K3S_CLUSTER_CONFIG_LABEL), "utf8")), K3S_CLUSTER_CONFIG_LABEL);
if (root.kind !== "platform-infra-k3s-clusters") throw new Error(`${K3S_CLUSTER_CONFIG_LABEL}.kind must be platform-infra-k3s-clusters`);
const defaults = record(root.defaults, `${K3S_CLUSTER_CONFIG_LABEL}.defaults`);
const defaultTargetId = text(defaults, "targetId", `${K3S_CLUSTER_CONFIG_LABEL}.defaults`);
const id = targetId ?? defaultTargetId;
const raw = record(record(root.targets, `${K3S_CLUSTER_CONFIG_LABEL}.targets`)[id], `${K3S_CLUSTER_CONFIG_LABEL}.targets.${id}`);
if (raw.enabled !== true) throw new Error(`${K3S_CLUSTER_CONFIG_LABEL}.targets.${id}.enabled must be true`);
const control = record(raw.controlPlane, `targets.${id}.controlPlane`);
const workers = record(raw.workers, `targets.${id}.workers`);
const workerEntries = Object.entries(workers);
if (workerEntries.length !== 1) throw new Error(`targets.${id}.workers must contain exactly one worker`);
const [workerId, workerRaw] = workerEntries[0] ?? [];
const worker = record(workerRaw, `targets.${id}.workers.${workerId}`);
const wireguard = record(raw.wireguard, `targets.${id}.wireguard`);
const secrets = record(raw.secrets, `targets.${id}.secrets`);
const k3s = record(raw.k3s, `targets.${id}.k3s`);
const mirror = record(k3s.registryMirror, `targets.${id}.k3s.registryMirror`);
const serviceLb = record(k3s.serviceLb, `targets.${id}.k3s.serviceLb`);
const smoke = record(raw.smoke, `targets.${id}.smoke`);
const controlSource = resolveRef(text(control, "configRef", `targets.${id}.controlPlane`));
const bootstrapSource = resolveRef(text(worker, "bootstrapConfigRef", `targets.${id}.workers[0]`));
const host = record(bootstrapSource.host, "worker bootstrap host");
const network = record(bootstrapSource.network, "worker bootstrap network");
const ubuntu = record(bootstrapSource.ubuntu, "worker bootstrap ubuntu");
const bootstrapK3s = record(bootstrapSource.k3s, "worker bootstrap k3s");
const labels = stringMap(record(worker.labels, `targets.${id}.workers.${workerId}.labels`), `targets.${id}.workers.${workerId}.labels`);
const target: K3sClusterTarget = {
id,
clusterName: text(raw, "clusterName", `targets.${id}`),
controlPlane: {
id: text(control, "id", `targets.${id}.controlPlane`),
configRef: text(control, "configRef", `targets.${id}.controlPlane`),
lane: text(control, "lane", `targets.${id}.controlPlane`),
nodeName: text(control, "nodeName", `targets.${id}.controlPlane`),
publicAddress: text(control, "publicAddress", `targets.${id}.controlPlane`),
route: text(controlSource, "route", "control plane configRef"),
kubeRoute: text(controlSource, "kubeRoute", "control plane configRef"),
},
worker: {
id: text(worker, "id", `targets.${id}.workers.${workerId}`),
bootstrapConfigRef: text(worker, "bootstrapConfigRef", `targets.${id}.workers.${workerId}`),
nodeName: text(worker, "nodeName", `targets.${id}.workers.${workerId}`),
labels,
windowsRoute: text(host, "route", "worker bootstrap host"),
guestAddress: text(network, "address", "worker bootstrap network"),
guestUser: text(ubuntu, "user", "worker bootstrap ubuntu"),
stateDir: text(host, "stateDir", "worker bootstrap host"),
k3sVersion: text(bootstrapK3s, "version", "worker bootstrap k3s"),
},
wireguard: {
interfaceName: text(wireguard, "interfaceName", `targets.${id}.wireguard`),
packageName: text(wireguard, "packageName", `targets.${id}.wireguard`),
mtu: integer(wireguard, "mtu", `targets.${id}.wireguard`),
listenPort: integer(wireguard, "listenPort", `targets.${id}.wireguard`),
controlPlaneAddress: cidr(wireguard, "controlPlaneAddress", `targets.${id}.wireguard`),
workerAddress: cidr(wireguard, "workerAddress", `targets.${id}.wireguard`),
endpoint: text(wireguard, "endpoint", `targets.${id}.wireguard`),
persistentKeepaliveSeconds: integer(wireguard, "persistentKeepaliveSeconds", `targets.${id}.wireguard`),
},
secrets: {
sourceRef: text(secrets, "sourceRef", `targets.${id}.secrets`),
controlPlanePrivateKey: text(secrets, "controlPlanePrivateKey", `targets.${id}.secrets`),
controlPlanePublicKey: text(secrets, "controlPlanePublicKey", `targets.${id}.secrets`),
workerPrivateKey: text(secrets, "workerPrivateKey", `targets.${id}.secrets`),
workerPublicKey: text(secrets, "workerPublicKey", `targets.${id}.secrets`),
agentToken: text(secrets, "agentToken", `targets.${id}.secrets`),
},
k3s: {
serverUrl: text(k3s, "serverUrl", `targets.${id}.k3s`),
agentTokenFile: text(k3s, "agentTokenFile", `targets.${id}.k3s`),
flannelInterface: text(k3s, "flannelInterface", `targets.${id}.k3s`),
maxPods: integer(k3s, "maxPods", `targets.${id}.k3s`),
registryMirror: { source: text(mirror, "source", "registryMirror"), endpoint: text(mirror, "endpoint", "registryMirror") },
serviceLb: { enabledNodeLabel: text(serviceLb, "enabledNodeLabel", "serviceLb"), enabledNode: text(serviceLb, "enabledNode", "serviceLb") },
},
smoke: {
namespace: text(smoke, "namespace", `targets.${id}.smoke`),
jobName: text(smoke, "jobName", `targets.${id}.smoke`),
image: text(smoke, "image", `targets.${id}.smoke`),
timeoutSeconds: integer(smoke, "timeoutSeconds", `targets.${id}.smoke`),
},
};
if (target.k3s.flannelInterface !== target.wireguard.interfaceName) throw new Error(`targets.${id}.k3s.flannelInterface must match wireguard.interfaceName`);
if (target.k3s.serviceLb.enabledNode !== target.controlPlane.nodeName) throw new Error(`targets.${id}.k3s.serviceLb.enabledNode must match controlPlane.nodeName`);
return { defaultTargetId, target };
}
export function clusterServerArgs(configRef: string): string[] {
const match = configRef.match(/^config\/platform-infra\/k3s-clusters\.yaml#targets\.([A-Za-z0-9._-]+)\.controlPlane$/u);
if (match === null) throw new Error(`unsupported k3s clusterConfigRef: ${configRef}`);
const { target } = readK3sClusterConfig(match[1]);
return [
"--node-ip", address(target.wireguard.controlPlaneAddress),
"--node-external-ip", target.controlPlane.publicAddress,
"--flannel-iface", target.k3s.flannelInterface,
"--tls-san", address(target.wireguard.controlPlaneAddress),
"--tls-san", target.controlPlane.publicAddress,
"--agent-token-file", target.k3s.agentTokenFile,
];
}
export function address(value: string): string {
return value.split("/", 1)[0] ?? value;
}
function resolveRef(value: string): Record<string, unknown> {
const [file, path] = value.split("#", 2);
if (!file || !path) throw new Error(`configRef must contain file#path: ${value}`);
let current: unknown = Bun.YAML.parse(readFileSync(rootPath(file), "utf8"));
for (const segment of path.split(".")) current = record(current, value)[segment];
return record(current, value);
}
function record(value: unknown, label: string): Record<string, any> {
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${label} must be an object`);
return value as Record<string, any>;
}
function text(value: Record<string, unknown>, key: string, label: string): string {
const result = value[key];
if (typeof result !== "string" || result.length === 0) throw new Error(`${label}.${key} must be a non-empty string`);
return result;
}
function integer(value: Record<string, unknown>, key: string, label: string): number {
const result = value[key];
if (!Number.isInteger(result) || Number(result) <= 0) throw new Error(`${label}.${key} must be a positive integer`);
return Number(result);
}
function cidr(value: Record<string, unknown>, key: string, label: string): string {
const result = text(value, key, label);
if (!/^\d{1,3}(?:\.\d{1,3}){3}\/\d{1,2}$/u.test(result)) throw new Error(`${label}.${key} must be an IPv4 CIDR`);
return result;
}
function stringMap(value: Record<string, unknown>, label: string): Record<string, string> {
return Object.fromEntries(Object.entries(value).map(([key, item]) => {
if (typeof item !== "string" || item.length === 0) throw new Error(`${label}.${key} must be a non-empty string`);
return [key, item];
}));
}
+455
View File
@@ -0,0 +1,455 @@
import { createHash, generateKeyPairSync, randomBytes } from "node:crypto";
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname } from "node:path";
import type { UniDeskConfig } from "./config";
import { rootPath } from "./config";
import { runCommand } from "./command";
import { parseEnvFile } from "./secrets";
import { address, K3S_CLUSTER_CONFIG_LABEL, readK3sClusterConfig, type K3sClusterTarget } from "./platform-infra-k3s-cluster-config";
type Action = "plan" | "secret-init" | "apply" | "status" | "smoke";
interface Options {
action: Action;
targetId?: string;
confirm: boolean;
}
interface SecretMaterial {
controlPrivate: string;
controlPublic: string;
workerPrivate: string;
workerPublic: string;
agentToken: string;
fingerprint: string;
}
export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig, args: string[]): Promise<Record<string, unknown>> {
const options = parseOptions(args);
const { target } = readK3sClusterConfig(options.targetId);
if (options.action === "plan") return plan(target);
if (options.action === "secret-init") return secretInit(target, options.confirm);
if (options.action === "status") return status(target);
if (options.action === "smoke") return smoke(target, options.confirm);
if (!options.confirm) return { ...plan(target), mode: "dry-run", mutation: false };
return apply(target);
}
function parseOptions(args: string[]): Options {
const action = args[0] ?? "plan";
if (action !== "plan" && action !== "secret-init" && action !== "apply" && action !== "status" && action !== "smoke") {
throw new Error("platform-infra k3s-cluster usage: plan|secret-init|apply|status|smoke [--target <id>] [--confirm]");
}
return { action, targetId: option(args, "--target") ?? undefined, confirm: args.includes("--confirm") };
}
function plan(target: K3sClusterTarget): Record<string, unknown> {
return {
ok: true,
mutation: false,
targetId: target.id,
source: K3S_CLUSTER_CONFIG_LABEL,
topology: {
clusterName: target.clusterName,
controlPlane: { id: target.controlPlane.id, nodeName: target.controlPlane.nodeName, route: target.controlPlane.route, kubeRoute: target.controlPlane.kubeRoute },
worker: { id: target.worker.id, nodeName: target.worker.nodeName, bootstrapConfigRef: target.worker.bootstrapConfigRef, controlRoute: target.worker.windowsRoute },
},
wireguard: { ...target.wireguard, endpoint: target.wireguard.endpoint },
k3s: {
serverUrl: target.k3s.serverUrl,
agentTokenFile: target.k3s.agentTokenFile,
registryMirror: target.k3s.registryMirror,
serviceLb: target.k3s.serviceLb,
},
secret: secretSummary(target),
next: {
secretInit: `bun scripts/cli.ts platform-infra k3s-cluster secret-init --target ${target.id} --confirm`,
apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm`,
status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`,
smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`,
},
};
}
function secretInit(target: K3sClusterTarget, confirm: boolean): Record<string, unknown> {
if (!confirm) return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, secret: secretSummary(target) };
const path = target.secrets.sourceRef;
if (existsSync(path)) throw new Error(`${path} already exists; secret-init never overwrites cluster credentials`);
const control = wireguardKeyPair();
const worker = wireguardKeyPair();
const values: Record<string, string> = {
[target.secrets.controlPlanePrivateKey]: control.privateKey,
[target.secrets.controlPlanePublicKey]: control.publicKey,
[target.secrets.workerPrivateKey]: worker.privateKey,
[target.secrets.workerPublicKey]: worker.publicKey,
[target.secrets.agentToken]: randomBytes(32).toString("base64url"),
};
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
writeFileSync(path, Object.entries(values).map(([key, value]) => `${key}=${value}`).join("\n") + "\n", { encoding: "utf8", mode: 0o600 });
chmodSync(path, 0o600);
return { ok: true, mutation: true, targetId: target.id, secret: secretSummary(target), valuesPrinted: false };
}
function apply(target: K3sClusterTarget): Record<string, unknown> {
const secret = readSecrets(target);
const controlWireguard = runControlPlane(target, controlPlaneApplyScript(target, secret), 180_000);
requireSuccess(controlWireguard, "control-plane-wireguard-apply");
const controlPlane = runCommand([
"bun", "scripts/cli.ts", "hwlab", "nodes", "control-plane", "infra", "apply",
"--node", target.controlPlane.id, "--lane", target.controlPlane.lane, "--confirm",
], rootPath(), { timeoutMs: 180_000 });
requireSuccess(controlPlane, "control-plane-k3s-apply");
const worker = runWorker(target, workerApplyScript(target, secret), true, 300_000);
requireSuccess(worker, "worker-agent-apply");
const labels = [
"label", "node", target.k3s.serviceLb.enabledNode,
`${target.k3s.serviceLb.enabledNodeLabel}=true`, "--overwrite",
];
const labelResult = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", ...labels], rootPath(), { timeoutMs: 60_000 });
requireSuccess(labelResult, "servicelb-node-label");
const ready = runControlPlane(target, waitForNodesScript(target), 180_000);
requireSuccess(ready, "cluster-node-readiness");
return {
ok: true,
mutation: true,
targetId: target.id,
stages: {
controlPlaneWireguard: compact(controlWireguard),
controlPlaneK3s: compact(controlPlane),
workerAgent: compact(worker),
serviceLbPlacement: compact(labelResult),
nodeReadiness: compact(ready),
},
secret: { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: secret.fingerprint, valuesPrinted: false },
next: {
status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`,
smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`,
},
};
}
function status(target: K3sClusterTarget): Record<string, unknown> {
const control = runControlPlane(target, statusScript(target), 60_000);
const worker = runWorker(target, workerStatusScript(target), false, 60_000);
const controlData = parseFirstJson(control.stdout);
const workerData = parseFirstJson(worker.stdout);
const ok = control.exitCode === 0 && worker.exitCode === 0 && controlData.ok === true && workerData.ok === true;
return {
ok,
mutation: false,
targetId: target.id,
controlPlane: controlData,
worker: workerData,
secret: secretSummary(target),
result: { controlPlane: compact(control), worker: compact(worker) },
next: ok ? { smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm` } : { apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm` },
};
}
function smoke(target: K3sClusterTarget, confirm: boolean): Record<string, unknown> {
const manifest = smokeManifest(target);
if (!confirm) return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, workerNode: target.worker.nodeName, manifestSha256: hash(manifest) };
const namespace = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "create", "namespace", target.smoke.namespace, "--dry-run=client", "-o", "yaml"], rootPath(), { timeoutMs: 30_000 });
requireSuccess(namespace, "smoke-namespace-render");
const applyNamespace = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "apply", "-f", "-"], rootPath(), { input: namespace.stdout, timeoutMs: 30_000 });
requireSuccess(applyNamespace, "smoke-namespace-apply");
runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "delete", "job", target.smoke.jobName, "--ignore-not-found=true", "--wait=true"], rootPath(), { timeoutMs: 60_000 });
const applyJob = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "apply", "-f", "-"], rootPath(), { input: manifest, timeoutMs: 30_000 });
requireSuccess(applyJob, "smoke-job-apply");
const wait = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "wait", `job/${target.smoke.jobName}`, "--for=condition=complete", `--timeout=${target.smoke.timeoutSeconds}s`], rootPath(), { timeoutMs: (target.smoke.timeoutSeconds + 20) * 1000 });
const logs = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "logs", `job/${target.smoke.jobName}`], rootPath(), { timeoutMs: 30_000 });
const placement = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "get", "pod", "-l", `job-name=${target.smoke.jobName}`, "-o", "jsonpath={.items[0].spec.nodeName}"], rootPath(), { timeoutMs: 30_000 });
const ok = wait.exitCode === 0 && logs.exitCode === 0 && logs.stdout.includes("D601_RUNTIME_SMOKE_OK") && placement.stdout.trim() === target.worker.nodeName;
if (ok) runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.smoke.namespace, "delete", "job", target.smoke.jobName, "--wait=true"], rootPath(), { timeoutMs: 60_000 });
return {
ok,
mutation: true,
targetId: target.id,
workerNode: target.worker.nodeName,
scheduledNode: placement.stdout.trim(),
checks: { jobCompleted: wait.exitCode === 0, dnsAndClusterIp: logs.stdout.includes("D601_RUNTIME_SMOKE_OK"), workerPlacement: placement.stdout.trim() === target.worker.nodeName },
logs: bounded(logs.stdout, 4_000),
result: { apply: compact(applyJob), wait: compact(wait), placement: compact(placement), cleanedUp: ok },
};
}
function controlPlaneApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string {
const wg = target.wireguard;
return `set -eu
export DEBIAN_FRONTEND=noninteractive
if ! command -v wg >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq ${shQuote(wg.packageName)}; fi
umask 077
install -d -m 700 /etc/wireguard /etc/unidesk
cat > /etc/wireguard/${wg.interfaceName}.conf <<'EOF'
[Interface]
Address = ${wg.controlPlaneAddress}
ListenPort = ${wg.listenPort}
PrivateKey = ${secret.controlPrivate}
MTU = ${wg.mtu}
[Peer]
PublicKey = ${secret.workerPublic}
AllowedIPs = ${address(wg.workerAddress)}/32
EOF
chmod 600 /etc/wireguard/${wg.interfaceName}.conf
printf '%s\n' ${shQuote(secret.agentToken)} > ${shQuote(target.k3s.agentTokenFile)}
chmod 600 ${shQuote(target.k3s.agentTokenFile)}
systemctl enable wg-quick@${wg.interfaceName} >/dev/null
systemctl restart wg-quick@${wg.interfaceName}
wg show ${wg.interfaceName} >/dev/null
`;
}
function workerApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string {
const wg = target.wireguard;
const labels = Object.entries(target.worker.labels).flatMap(([key, value]) => ["--node-label", `${key}=${value}`]);
const exec = [
"agent", "--server", target.k3s.serverUrl, "--token-file", target.k3s.agentTokenFile,
"--node-name", target.worker.nodeName, "--node-ip", address(wg.workerAddress),
"--flannel-iface", target.k3s.flannelInterface, ...labels,
"--kubelet-arg", `max-pods=${target.k3s.maxPods}`,
].map(shellToken).join(" ");
return `set -eu
export DEBIAN_FRONTEND=noninteractive
if ! command -v wg >/dev/null 2>&1; then apt-get update -qq && apt-get install -y -qq ${shQuote(wg.packageName)}; fi
umask 077
install -d -m 700 /etc/wireguard /etc/unidesk /etc/rancher/k3s
cat > /etc/wireguard/${wg.interfaceName}.conf <<'EOF'
[Interface]
Address = ${wg.workerAddress}
PrivateKey = ${secret.workerPrivate}
MTU = ${wg.mtu}
[Peer]
PublicKey = ${secret.controlPublic}
Endpoint = ${wg.endpoint}
AllowedIPs = ${address(wg.controlPlaneAddress)}/32
PersistentKeepalive = ${wg.persistentKeepaliveSeconds}
EOF
chmod 600 /etc/wireguard/${wg.interfaceName}.conf
systemctl enable wg-quick@${wg.interfaceName} >/dev/null
systemctl restart wg-quick@${wg.interfaceName}
ping -c 3 -W 5 ${address(wg.controlPlaneAddress)} >/dev/null
printf '%s\n' ${shQuote(secret.agentToken)} > ${shQuote(target.k3s.agentTokenFile)}
chmod 600 ${shQuote(target.k3s.agentTokenFile)}
cat > /etc/rancher/k3s/registries.yaml <<'EOF'
mirrors:
"${target.k3s.registryMirror.source}":
endpoint:
- "${target.k3s.registryMirror.endpoint}"
EOF
if [ -x /usr/local/bin/k3s-uninstall.sh ]; then /usr/local/bin/k3s-uninstall.sh; fi
if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh; fi
install -m 0755 /tmp/k3s /usr/local/bin/k3s
chmod 0755 /tmp/install-k3s.sh
INSTALL_K3S_SKIP_DOWNLOAD=true INSTALL_K3S_VERSION=${shQuote(target.worker.k3sVersion)} INSTALL_K3S_EXEC=${shQuote(exec)} sh /tmp/install-k3s.sh
systemctl enable k3s-agent >/dev/null
systemctl restart k3s-agent
systemctl is-active --quiet k3s-agent
`;
}
function waitForNodesScript(target: K3sClusterTarget): string {
return `set -eu
deadline=$(( $(date +%s) + 150 ))
while [ "$(date +%s)" -lt "$deadline" ]; do
ready=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
[ "$ready" = True ] && exit 0
sleep 5
done
/usr/local/bin/kubectl get nodes -o wide >&2
exit 1
`;
}
function statusScript(target: K3sClusterTarget): string {
const cp = address(target.wireguard.controlPlaneAddress);
return `set -eu
wg_active=false; systemctl is-active --quiet wg-quick@${target.wireguard.interfaceName} && wg_active=true
k3s_active=false; systemctl is-active --quiet k3s && k3s_active=true
cp_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.controlPlane.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
worker_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)
worker_ip=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.addresses[?(@.type=="InternalIP")].address}' 2>/dev/null || true)
handshake=$(wg show ${target.wireguard.interfaceName} latest-handshakes 2>/dev/null | awk '{print $2}' | head -n1)
python3 - "$wg_active" "$k3s_active" "$cp_ready" "$worker_ready" "$worker_ip" "$handshake" <<'PY'
import json, sys, time
wg, k3s, cp, worker, worker_ip, handshake = sys.argv[1:]
stamp = int(handshake or 0)
age = None if stamp == 0 else max(0, int(time.time()) - stamp)
ok = wg == 'true' and k3s == 'true' and cp == 'True' and worker == 'True' and worker_ip == '${address(target.wireguard.workerAddress)}' and age is not None and age < 180
print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'k3sActive': k3s == 'true', 'controlPlaneReady': cp == 'True', 'workerReady': worker == 'True', 'workerInternalIp': worker_ip, 'handshakeAgeSeconds': age, 'controlPlaneAddress': '${cp}'}, separators=(',', ':')))
PY
`;
}
function workerStatusScript(target: K3sClusterTarget): string {
return `set -eu
wg_active=false; systemctl is-active --quiet wg-quick@${target.wireguard.interfaceName} && wg_active=true
agent_active=false; systemctl is-active --quiet k3s-agent && agent_active=true
old_server=false; systemctl is-active --quiet k3s && old_server=true
ip_present=false; ip -4 address show dev ${target.wireguard.interfaceName} | grep -F ${shQuote(address(target.wireguard.workerAddress))} >/dev/null 2>&1 && ip_present=true
python3 - "$wg_active" "$agent_active" "$old_server" "$ip_present" <<'PY'
import json, sys
wg, agent, old_server, ip_present = sys.argv[1:]
ok = wg == 'true' and agent == 'true' and old_server == 'false' and ip_present == 'true'
print(json.dumps({'ok': ok, 'wireguardActive': wg == 'true', 'agentActive': agent == 'true', 'standaloneServerActive': old_server == 'true', 'wireguardAddressPresent': ip_present == 'true'}, separators=(',', ':')))
PY
`;
}
function smokeManifest(target: K3sClusterTarget): string {
return `apiVersion: batch/v1
kind: Job
metadata:
name: ${target.smoke.jobName}
namespace: ${target.smoke.namespace}
spec:
backoffLimit: 0
template:
metadata:
labels:
app.kubernetes.io/name: ${target.smoke.jobName}
spec:
restartPolicy: Never
nodeSelector:
kubernetes.io/hostname: ${target.worker.nodeName}
containers:
- name: smoke
image: ${target.smoke.image}
imagePullPolicy: IfNotPresent
command:
- sh
- -ec
- |
nslookup kubernetes.default.svc >/tmp/dns.txt
wget -qO- --no-check-certificate --timeout=15 https://kubernetes.default.svc/healthz | grep -q '^ok$'
echo D601_RUNTIME_SMOKE_OK
`;
}
function runControlPlane(target: K3sClusterTarget, script: string, timeoutMs: number) {
return runCommand(["trans", target.controlPlane.route, "sh"], rootPath(), { input: script, timeoutMs });
}
function runWorker(target: K3sClusterTarget, script: string, copyArtifacts: boolean, timeoutMs: number) {
const encoded = Buffer.from(script, "utf8").toString("base64");
const keyPath = `${target.worker.stateDir}\\id_ed25519`;
const remoteScript = `/tmp/unidesk-k3s-cluster-${target.id}.sh`;
const lines = [
"$ErrorActionPreference='Stop'",
`$key=${psQuote(keyPath)}`,
`$guest=${psQuote(`${target.worker.guestUser}@${target.worker.guestAddress}`)}`,
`$temp=Join-Path $env:TEMP ${psQuote(`unidesk-k3s-cluster-${target.id}.sh`)}`,
`[IO.File]::WriteAllBytes($temp,[Convert]::FromBase64String(${psQuote(encoded)}))`,
];
if (copyArtifacts) {
lines.push(
`$binary=${psQuote(`${target.worker.stateDir}\\k3s`)}`,
`$installer=${psQuote(`${target.worker.stateDir}\\install-k3s.sh`)}`,
"& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $binary \"${guest}:/tmp/k3s\"",
"if($LASTEXITCODE -ne 0){throw 'worker-k3s-binary-scp-failed'}",
"& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $installer \"${guest}:/tmp/install-k3s.sh\"",
"if($LASTEXITCODE -ne 0){throw 'worker-k3s-installer-scp-failed'}",
);
}
lines.push(
`& scp.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $temp \"${guest}:${remoteScript}\"`,
"if($LASTEXITCODE -ne 0){throw 'worker-script-scp-failed'}",
`& ssh.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new -o ServerAliveInterval=15 $guest ${psQuote(`sudo sh ${remoteScript}`)}`,
"$rc=$LASTEXITCODE",
"Remove-Item -Force -LiteralPath $temp -ErrorAction SilentlyContinue",
`& ssh.exe -i $key -o BatchMode=yes -o StrictHostKeyChecking=accept-new $guest ${psQuote(`rm -f ${remoteScript}`)} | Out-Null`,
"exit $rc",
);
return runCommand(["trans", target.worker.windowsRoute, "ps"], rootPath(), { input: lines.join("\n") + "\n", timeoutMs });
}
function readSecrets(target: K3sClusterTarget): SecretMaterial {
if (!existsSync(target.secrets.sourceRef)) throw new Error(`${target.secrets.sourceRef} is missing; run platform-infra k3s-cluster secret-init --confirm`);
const values = parseEnvFile(readFileSync(target.secrets.sourceRef, "utf8"));
const required = (key: string) => {
const value = values[key];
if (typeof value !== "string" || value.length < 32) throw new Error(`${target.secrets.sourceRef}.${key} is missing or invalid`);
return value;
};
const material = {
controlPrivate: required(target.secrets.controlPlanePrivateKey),
controlPublic: required(target.secrets.controlPlanePublicKey),
workerPrivate: required(target.secrets.workerPrivateKey),
workerPublic: required(target.secrets.workerPublicKey),
agentToken: required(target.secrets.agentToken),
};
return { ...material, fingerprint: hash(Object.entries(material).sort().map(([key, value]) => `${key}=${value}`).join("\n")) };
}
function secretSummary(target: K3sClusterTarget): Record<string, unknown> {
if (!existsSync(target.secrets.sourceRef)) return { sourceRef: target.secrets.sourceRef, presence: false, fingerprint: null, valuesPrinted: false };
try {
const secret = readSecrets(target);
return { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: secret.fingerprint, keys: Object.values(target.secrets).filter((value) => value !== target.secrets.sourceRef), valuesPrinted: false };
} catch (error) {
return { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: null, error: error instanceof Error ? error.message : String(error), valuesPrinted: false };
}
}
function wireguardKeyPair(): { privateKey: string; publicKey: string } {
const pair = generateKeyPairSync("x25519");
const privateJwk = pair.privateKey.export({ format: "jwk" }) as JsonWebKey;
const publicJwk = pair.publicKey.export({ format: "jwk" }) as JsonWebKey;
if (!privateJwk.d || !publicJwk.x) throw new Error("x25519 key generation did not produce raw JWK keys");
return { privateKey: base64UrlToBase64(privateJwk.d), publicKey: base64UrlToBase64(publicJwk.x) };
}
function base64UrlToBase64(value: string): string {
return Buffer.from(value, "base64url").toString("base64");
}
function requireSuccess(result: ReturnType<typeof runCommand>, stage: string): void {
if (result.exitCode !== 0) throw new Error(`${stage} failed: ${bounded(result.stderr || result.stdout)}`);
}
function compact(result: ReturnType<typeof runCommand>): Record<string, unknown> {
return { exitCode: result.exitCode, timedOut: result.timedOut, stdout: bounded(result.stdout, 2_000), stderr: bounded(result.stderr, 2_000) };
}
function parseFirstJson(value: string): Record<string, unknown> {
for (const line of value.split(/\r?\n/u)) {
const text = line.trim();
if (!text.startsWith("{")) continue;
try { return JSON.parse(text) as Record<string, unknown>; } catch { /* continue */ }
}
return { ok: false, error: "remote-json-missing", output: bounded(value) };
}
function option(args: string[], name: string): string | null {
const index = args.indexOf(name);
if (index < 0) return null;
const value = args[index + 1];
if (!value || value.startsWith("--")) throw new Error(`${name} requires a value`);
return value;
}
function shQuote(value: string): string {
return `'${value.replaceAll("'", `'"'"'`)}'`;
}
function shellToken(value: string): string {
return /^[A-Za-z0-9._:/=@+-]+$/u.test(value) ? value : shQuote(value);
}
function psQuote(value: string): string {
return `'${value.replaceAll("'", "''")}'`;
}
function hash(value: string): string {
return createHash("sha256").update(value).digest("hex");
}
function bounded(value: string, max = 4_000): string {
const text = value.trim();
return text.length <= max ? text : `${text.slice(0, max)}\n...[truncated ${text.length - max} chars]`;
}
+4
View File
@@ -99,6 +99,10 @@ export async function runPlatformInfraCommand(config: UniDeskConfig, args: strin
const { runPlatformInfraHyperVVmCommand } = await import("../platform-infra-hyperv-vm");
return await runPlatformInfraHyperVVmCommand(config, args.slice(1));
}
if (target === "k3s-cluster") {
const { runPlatformInfraK3sClusterCommand } = await import("../platform-infra-k3s-cluster");
return await runPlatformInfraK3sClusterCommand(config, args.slice(1));
}
if (target === "pipelines-as-code" || target === "pac") {
const { runPlatformInfraPipelinesAsCodeCommand } = await import("../platform-infra-pipelines-as-code");
return await runPlatformInfraPipelinesAsCodeCommand(config, args.slice(1));