|
|
|
@@ -13,7 +13,7 @@ import {
|
|
|
|
|
} from "./platform-infra-host-proxy";
|
|
|
|
|
import { address, K3S_CLUSTER_CONFIG_LABEL, readK3sClusterConfig, type K3sClusterTarget } from "./platform-infra-k3s-cluster-config";
|
|
|
|
|
|
|
|
|
|
type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "status" | "smoke" | "registry-smoke";
|
|
|
|
|
type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "drain" | "status" | "smoke" | "registry-smoke";
|
|
|
|
|
|
|
|
|
|
interface Options {
|
|
|
|
|
action: Action;
|
|
|
|
@@ -39,6 +39,7 @@ export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig,
|
|
|
|
|
if (options.action === "plan") return plan(target);
|
|
|
|
|
if (options.action === "secret-init") return secretInit(target, options.confirm);
|
|
|
|
|
if (options.action === "registry-credential-init") return registryCredentialInit(target, options.confirm);
|
|
|
|
|
if (options.action === "drain") return drainWorker(target, options.confirm);
|
|
|
|
|
if (options.action === "status") return status(target);
|
|
|
|
|
if (options.action === "smoke") return smoke(target, options.confirm);
|
|
|
|
|
if (options.action === "registry-smoke") return registrySmoke(target, options.confirm);
|
|
|
|
@@ -48,8 +49,8 @@ export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig,
|
|
|
|
|
|
|
|
|
|
function parseOptions(args: string[]): Options {
|
|
|
|
|
const action = args[0] ?? "plan";
|
|
|
|
|
if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "status" && action !== "smoke" && action !== "registry-smoke") {
|
|
|
|
|
throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|status|smoke|registry-smoke [--target <id>] [--confirm]");
|
|
|
|
|
if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "drain" && action !== "status" && action !== "smoke" && action !== "registry-smoke") {
|
|
|
|
|
throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|drain|status|smoke|registry-smoke [--target <id>] [--confirm]");
|
|
|
|
|
}
|
|
|
|
|
return { action, targetId: option(args, "--target") ?? undefined, confirm: args.includes("--confirm") };
|
|
|
|
|
}
|
|
|
|
@@ -69,6 +70,7 @@ function plan(target: K3sClusterTarget): Record<string, unknown> {
|
|
|
|
|
bootstrapConfigRef: target.worker.bootstrapConfigRef,
|
|
|
|
|
hostProxyConfigRef: target.worker.hostProxyConfigRef,
|
|
|
|
|
controlRoute: target.worker.windowsRoute,
|
|
|
|
|
scheduling: target.worker.scheduling,
|
|
|
|
|
},
|
|
|
|
|
},
|
|
|
|
|
wireguard: { ...target.wireguard, endpoint: target.wireguard.endpoint },
|
|
|
|
@@ -87,6 +89,7 @@ function plan(target: K3sClusterTarget): Record<string, unknown> {
|
|
|
|
|
secretInit: `bun scripts/cli.ts platform-infra k3s-cluster secret-init --target ${target.id} --confirm`,
|
|
|
|
|
registryCredentialInit: `bun scripts/cli.ts platform-infra k3s-cluster registry-credential-init --target ${target.id} --confirm`,
|
|
|
|
|
apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm`,
|
|
|
|
|
drain: `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm`,
|
|
|
|
|
status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`,
|
|
|
|
|
smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`,
|
|
|
|
|
registrySmoke: `bun scripts/cli.ts platform-infra k3s-cluster registry-smoke --target ${target.id} --confirm`,
|
|
|
|
@@ -206,10 +209,12 @@ function apply(target: K3sClusterTarget): Record<string, unknown> {
|
|
|
|
|
]) as Record<string, unknown>;
|
|
|
|
|
if (controlPlane.ok !== true) throw new Error(`control-plane-k3s-apply failed: ${bounded(JSON.stringify(compactDomain(controlPlane)))}`);
|
|
|
|
|
|
|
|
|
|
const workerSchedulingBeforeAgent = reconcileWorkerScheduling(target);
|
|
|
|
|
const workerDns = reconcileWorkerDns(target);
|
|
|
|
|
|
|
|
|
|
const worker = runWorker(target, workerApplyScript(target, secret), true, 300_000);
|
|
|
|
|
requireSuccess(worker, "worker-agent-apply");
|
|
|
|
|
const workerSchedulingAfterAgent = reconcileWorkerScheduling(target);
|
|
|
|
|
|
|
|
|
|
const labels = [
|
|
|
|
|
"label", "node", target.k3s.serviceLb.enabledNode,
|
|
|
|
@@ -237,12 +242,15 @@ function apply(target: K3sClusterTarget): Record<string, unknown> {
|
|
|
|
|
controlPlaneK3s: compactDomain(controlPlane),
|
|
|
|
|
workerDns,
|
|
|
|
|
workerAgent: compact(worker),
|
|
|
|
|
workerSchedulingBeforeAgent,
|
|
|
|
|
workerSchedulingAfterAgent,
|
|
|
|
|
serviceLbPlacement: compact(labelResult),
|
|
|
|
|
nodeReadiness: compact(ready),
|
|
|
|
|
},
|
|
|
|
|
secret: { sourceRef: target.secrets.sourceRef, presence: true, fingerprint: secret.fingerprint, valuesPrinted: false },
|
|
|
|
|
registryAuth: registryAuthSummary(target),
|
|
|
|
|
next: {
|
|
|
|
|
drain: target.worker.scheduling.schedulable ? null : `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm`,
|
|
|
|
|
status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`,
|
|
|
|
|
smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`,
|
|
|
|
|
},
|
|
|
|
@@ -257,13 +265,17 @@ function status(target: K3sClusterTarget): Record<string, unknown> {
|
|
|
|
|
const controlData = parseFirstJson(control.stdout);
|
|
|
|
|
const workerProxyData = parseFirstJson(workerProxy.stdout);
|
|
|
|
|
const workerData = parseFirstJson(worker.stdout);
|
|
|
|
|
const workerScheduling = workerSchedulingStatus(target);
|
|
|
|
|
const workerWorkloads = workerWorkloadStatus(target);
|
|
|
|
|
const ok = control.exitCode === 0
|
|
|
|
|
&& workerProxy.exitCode === 0
|
|
|
|
|
&& worker.exitCode === 0
|
|
|
|
|
&& controlData.ok === true
|
|
|
|
|
&& workerProxyData.ok === true
|
|
|
|
|
&& workerData.ok === true
|
|
|
|
|
&& workerDns.data.ok === true;
|
|
|
|
|
&& workerDns.data.ok === true
|
|
|
|
|
&& workerScheduling.ok === true
|
|
|
|
|
&& workerWorkloads.ok === true;
|
|
|
|
|
return {
|
|
|
|
|
ok,
|
|
|
|
|
mutation: false,
|
|
|
|
@@ -272,14 +284,38 @@ function status(target: K3sClusterTarget): Record<string, unknown> {
|
|
|
|
|
workerHostProxy: workerProxyData,
|
|
|
|
|
worker: workerData,
|
|
|
|
|
workerDns: workerDns.data,
|
|
|
|
|
workerScheduling,
|
|
|
|
|
workerWorkloads,
|
|
|
|
|
secret: secretSummary(target),
|
|
|
|
|
registryAuth: registryAuthSummary(target),
|
|
|
|
|
result: { controlPlane: compact(control), workerHostProxy: compact(workerProxy), worker: compact(worker), workerDns: compact(workerDns.result) },
|
|
|
|
|
next: ok ? { smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm` } : { apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm` },
|
|
|
|
|
result: {
|
|
|
|
|
controlPlane: { exitCode: control.exitCode },
|
|
|
|
|
workerHostProxy: { exitCode: workerProxy.exitCode },
|
|
|
|
|
worker: { exitCode: worker.exitCode },
|
|
|
|
|
workerDns: { exitCode: workerDns.result.exitCode },
|
|
|
|
|
},
|
|
|
|
|
next: ok
|
|
|
|
|
? { smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm` }
|
|
|
|
|
: !target.worker.scheduling.schedulable && workerScheduling.ok === true && workerWorkloads.ok !== true
|
|
|
|
|
? { drain: `bun scripts/cli.ts platform-infra k3s-cluster drain --target ${target.id} --confirm` }
|
|
|
|
|
: { apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm` },
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function smoke(target: K3sClusterTarget, confirm: boolean): Record<string, unknown> {
|
|
|
|
|
if (!target.worker.scheduling.schedulable) {
|
|
|
|
|
const scheduling = workerSchedulingStatus(target);
|
|
|
|
|
const workloads = workerWorkloadStatus(target);
|
|
|
|
|
const readiness = parseFirstJson(runControlPlane(target, nodeReadinessScript(target), 30_000).stdout);
|
|
|
|
|
return {
|
|
|
|
|
ok: scheduling.ok === true && workloads.ok === true && readiness.ok === true,
|
|
|
|
|
mutation: false,
|
|
|
|
|
mode: confirm ? "confirmed-read-only" : "dry-run",
|
|
|
|
|
targetId: target.id,
|
|
|
|
|
workerNode: target.worker.nodeName,
|
|
|
|
|
checks: { scheduling, workloads, readiness },
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
const manifest = smokeManifest(target);
|
|
|
|
|
if (!confirm) return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, workerNode: target.worker.nodeName, manifestSha256: hash(manifest) };
|
|
|
|
|
const namespace = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", "create", "namespace", target.smoke.namespace, "--dry-run=client", "-o", "yaml"], rootPath(), { timeoutMs: 30_000 });
|
|
|
|
@@ -495,6 +531,20 @@ PY
|
|
|
|
|
|
|
|
|
|
function reconcileWorkerDns(target: K3sClusterTarget): Record<string, unknown> {
|
|
|
|
|
const before = workerDnsStatus(target);
|
|
|
|
|
if (!target.worker.scheduling.schedulable) {
|
|
|
|
|
if (before.data.presence !== true) {
|
|
|
|
|
return { ok: true, mutation: false, enabled: false, reason: "worker-unschedulable", presence: false };
|
|
|
|
|
}
|
|
|
|
|
const deletion = runCommand([
|
|
|
|
|
"trans", target.controlPlane.kubeRoute, "kubectl", "-n", target.k3s.workerDns.namespace,
|
|
|
|
|
"delete", "deployment", target.k3s.workerDns.deploymentName,
|
|
|
|
|
"service", target.k3s.workerDns.serviceName, "--ignore-not-found=true", "--wait=true",
|
|
|
|
|
], rootPath(), { timeoutMs: 120_000 });
|
|
|
|
|
requireSuccess(deletion, "worker-dns-delete");
|
|
|
|
|
const after = workerDnsStatus(target);
|
|
|
|
|
if (after.data.ok !== true) throw new Error(`worker-dns-removal failed: ${bounded(JSON.stringify(after.data))}`);
|
|
|
|
|
return { ok: true, mutation: true, enabled: false, reason: "worker-unschedulable", presence: false, deletion: compact(deletion) };
|
|
|
|
|
}
|
|
|
|
|
const mutation = before.data.configurationCurrent !== true;
|
|
|
|
|
let applyResult: ReturnType<typeof runCommand> | null = null;
|
|
|
|
|
if (mutation) {
|
|
|
|
@@ -521,6 +571,116 @@ function reconcileWorkerDns(target: K3sClusterTarget): Record<string, unknown> {
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function workerSchedulingStatus(target: K3sClusterTarget): Record<string, unknown> {
|
|
|
|
|
const result = runCommand([
|
|
|
|
|
"trans", target.controlPlane.kubeRoute, "kubectl", "get", "node", target.worker.nodeName,
|
|
|
|
|
"-o", String.raw`jsonpath={.spec.unschedulable}|{range .spec.taints[*]}{.key}={.value}:{.effect},{end}`,
|
|
|
|
|
], rootPath(), { timeoutMs: 60_000 });
|
|
|
|
|
if (result.exitCode !== 0) return { ok: false, nodePresent: false, result: compact(result) };
|
|
|
|
|
const [unschedulableRaw = "", taintsRaw = ""] = result.stdout.trim().split("|", 2);
|
|
|
|
|
const actualTaints = taintsRaw.split(",").filter(Boolean);
|
|
|
|
|
const desiredTaintsPresent = target.worker.scheduling.taints.every((desired) =>
|
|
|
|
|
actualTaints.includes(`${desired.key}=${desired.value}:${desired.effect}`));
|
|
|
|
|
const unschedulable = unschedulableRaw === "true";
|
|
|
|
|
return {
|
|
|
|
|
ok: desiredTaintsPresent && unschedulable === !target.worker.scheduling.schedulable,
|
|
|
|
|
nodePresent: true,
|
|
|
|
|
schedulable: !unschedulable,
|
|
|
|
|
desiredSchedulable: target.worker.scheduling.schedulable,
|
|
|
|
|
desiredTaintsPresent,
|
|
|
|
|
desiredTaints: target.worker.scheduling.taints,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function reconcileWorkerScheduling(target: K3sClusterTarget): Record<string, unknown> {
|
|
|
|
|
const before = workerSchedulingStatus(target);
|
|
|
|
|
if (before.nodePresent !== true) throw new Error(`worker node is absent: ${target.worker.nodeName}`);
|
|
|
|
|
const mutations: string[] = [];
|
|
|
|
|
if (before.schedulable !== target.worker.scheduling.schedulable) {
|
|
|
|
|
const action = target.worker.scheduling.schedulable ? "uncordon" : "cordon";
|
|
|
|
|
const result = runCommand(["trans", target.controlPlane.kubeRoute, "kubectl", action, target.worker.nodeName], rootPath(), { timeoutMs: 60_000 });
|
|
|
|
|
requireSuccess(result, `worker-${action}`);
|
|
|
|
|
mutations.push(action);
|
|
|
|
|
}
|
|
|
|
|
if (before.desiredTaintsPresent !== true) {
|
|
|
|
|
for (const taint of target.worker.scheduling.taints) {
|
|
|
|
|
const desired = `${taint.key}=${taint.value}:${taint.effect}`;
|
|
|
|
|
const result = runCommand([
|
|
|
|
|
"trans", target.controlPlane.kubeRoute, "kubectl", "taint", "node", target.worker.nodeName, desired, "--overwrite",
|
|
|
|
|
], rootPath(), { timeoutMs: 60_000 });
|
|
|
|
|
requireSuccess(result, `worker-taint-${taint.key}`);
|
|
|
|
|
mutations.push(`taint:${taint.key}`);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
const after = workerSchedulingStatus(target);
|
|
|
|
|
if (after.ok !== true) throw new Error(`worker scheduling did not converge: ${bounded(JSON.stringify(after))}`);
|
|
|
|
|
return { ok: true, mutation: mutations.length > 0, mutations, before, after };
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function workerWorkloadStatus(target: K3sClusterTarget): Record<string, unknown> {
|
|
|
|
|
const result = runCommand([
|
|
|
|
|
"trans", target.controlPlane.kubeRoute, "kubectl", "get", "pods", "-A",
|
|
|
|
|
"--field-selector", `spec.nodeName=${target.worker.nodeName}`,
|
|
|
|
|
"-o", "custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,OWNER:.metadata.ownerReferences[0].kind,PHASE:.status.phase",
|
|
|
|
|
"--no-headers",
|
|
|
|
|
], rootPath(), { timeoutMs: 60_000 });
|
|
|
|
|
if (result.exitCode !== 0) return { ok: false, result: compact(result) };
|
|
|
|
|
const items = result.stdout.split(/\r?\n/u).map((line) => line.trim()).filter(Boolean).map((line) => {
|
|
|
|
|
const [namespace = "", name = "", ownerKind = "", phase = ""] = line.split(/\s+/u, 4);
|
|
|
|
|
return { namespace, name, ownerKind, phase };
|
|
|
|
|
});
|
|
|
|
|
const retainedSystemPods = items.filter((pod) => pod.ownerKind === "DaemonSet" || pod.ownerKind === "Node");
|
|
|
|
|
const nonDaemonSetPods = items.filter((pod) => !retainedSystemPods.includes(pod));
|
|
|
|
|
const requiredEmpty = !target.worker.scheduling.schedulable;
|
|
|
|
|
return {
|
|
|
|
|
ok: !requiredEmpty || nonDaemonSetPods.length === 0,
|
|
|
|
|
requiredEmpty,
|
|
|
|
|
nonDaemonSetPodCount: nonDaemonSetPods.length,
|
|
|
|
|
retainedSystemPodCount: retainedSystemPods.length,
|
|
|
|
|
nonDaemonSetPods: nonDaemonSetPods.slice(0, 40),
|
|
|
|
|
valuesPrinted: false,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function drainWorker(target: K3sClusterTarget, confirm: boolean): Record<string, unknown> {
|
|
|
|
|
if (target.worker.scheduling.schedulable) throw new Error("worker drain requires scheduling.schedulable=false in the owning YAML");
|
|
|
|
|
if (!confirm) {
|
|
|
|
|
return {
|
|
|
|
|
ok: true,
|
|
|
|
|
mutation: false,
|
|
|
|
|
mode: "dry-run",
|
|
|
|
|
targetId: target.id,
|
|
|
|
|
workerNode: target.worker.nodeName,
|
|
|
|
|
scheduling: workerSchedulingStatus(target),
|
|
|
|
|
workloads: workerWorkloadStatus(target),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
const scheduling = reconcileWorkerScheduling(target);
|
|
|
|
|
const dns = reconcileWorkerDns(target);
|
|
|
|
|
const before = workerWorkloadStatus(target);
|
|
|
|
|
let drain: ReturnType<typeof runCommand> | null = null;
|
|
|
|
|
if (before.nonDaemonSetPodCount !== 0) {
|
|
|
|
|
drain = runCommand([
|
|
|
|
|
"trans", target.controlPlane.kubeRoute, "kubectl", "drain", target.worker.nodeName,
|
|
|
|
|
"--ignore-daemonsets", "--delete-emptydir-data", "--force", "--grace-period=60", "--timeout=10m",
|
|
|
|
|
], rootPath(), { timeoutMs: 660_000 });
|
|
|
|
|
requireSuccess(drain, "worker-drain");
|
|
|
|
|
}
|
|
|
|
|
const after = workerWorkloadStatus(target);
|
|
|
|
|
if (after.ok !== true) throw new Error(`worker drain did not converge: ${bounded(JSON.stringify(after))}`);
|
|
|
|
|
return {
|
|
|
|
|
ok: true,
|
|
|
|
|
mutation: before.nonDaemonSetPodCount !== 0 || scheduling.mutation === true || dns.mutation === true,
|
|
|
|
|
targetId: target.id,
|
|
|
|
|
workerNode: target.worker.nodeName,
|
|
|
|
|
scheduling,
|
|
|
|
|
workerDns: dns,
|
|
|
|
|
before,
|
|
|
|
|
after,
|
|
|
|
|
drain: drain === null ? { skipped: true, reason: "no-non-daemonset-pods" } : compact(drain),
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function workerDnsStatus(target: K3sClusterTarget): { data: Record<string, unknown>; result: ReturnType<typeof runCommand> } {
|
|
|
|
|
const dns = target.k3s.workerDns;
|
|
|
|
|
const result = runCommand([
|
|
|
|
@@ -547,8 +707,23 @@ function workerDnsStatus(target: K3sClusterTarget): { data: Record<string, unkno
|
|
|
|
|
const service = rows.get("Service");
|
|
|
|
|
const endpoints = rows.get("Endpoints");
|
|
|
|
|
if (deployment === undefined || service === undefined || endpoints === undefined) {
|
|
|
|
|
if (!target.worker.scheduling.schedulable && deployment === undefined && service === undefined) {
|
|
|
|
|
return { data: { ok: true, presence: false, configurationCurrent: true, enabled: false, reason: "worker-unschedulable" }, result };
|
|
|
|
|
}
|
|
|
|
|
return { data: { ok: false, presence: false, configurationCurrent: false, error: "worker-dns-status-incomplete" }, result };
|
|
|
|
|
}
|
|
|
|
|
if (!target.worker.scheduling.schedulable) {
|
|
|
|
|
return {
|
|
|
|
|
data: {
|
|
|
|
|
ok: false,
|
|
|
|
|
presence: true,
|
|
|
|
|
configurationCurrent: false,
|
|
|
|
|
enabled: false,
|
|
|
|
|
reason: "worker-unschedulable-cleanup-required",
|
|
|
|
|
},
|
|
|
|
|
result,
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
const desiredHash = workerDnsDesiredHash(target);
|
|
|
|
|
const deploymentHash = deployment[1] || null;
|
|
|
|
|
const serviceHash = service[1] || null;
|
|
|
|
@@ -802,12 +977,14 @@ rm -f "$sysctl_candidate"
|
|
|
|
|
function workerApplyScript(target: K3sClusterTarget, secret: SecretMaterial): string {
|
|
|
|
|
const wg = target.wireguard;
|
|
|
|
|
const labels = Object.entries(target.worker.labels).flatMap(([key, value]) => ["--node-label", `${key}=${value}`]);
|
|
|
|
|
const taints = target.worker.scheduling.taints.flatMap((taint) => ["--node-taint", `${taint.key}=${taint.value}:${taint.effect}`]);
|
|
|
|
|
const dnsArgs = target.worker.scheduling.schedulable ? ["--kubelet-arg", `cluster-dns=${target.k3s.workerDns.clusterIp}`] : [];
|
|
|
|
|
const exec = [
|
|
|
|
|
"agent", "--server", target.k3s.serverUrl, "--token-file", target.k3s.agentTokenFile,
|
|
|
|
|
"--node-name", target.worker.nodeName, "--node-ip", address(wg.workerAddress),
|
|
|
|
|
"--flannel-iface", target.k3s.flannelInterface, ...labels,
|
|
|
|
|
"--flannel-iface", target.k3s.flannelInterface, ...labels, ...taints,
|
|
|
|
|
"--kubelet-arg", `max-pods=${target.k3s.maxPods}`,
|
|
|
|
|
"--kubelet-arg", `cluster-dns=${target.k3s.workerDns.clusterIp}`,
|
|
|
|
|
...dnsArgs,
|
|
|
|
|
].map(shellToken).join(" ");
|
|
|
|
|
const desiredHash = createHash("sha256").update(JSON.stringify({
|
|
|
|
|
version: target.worker.k3sVersion,
|
|
|
|
|