pikastech
1934 lines
93 KiB
TypeScript
1934 lines
93 KiB
TypeScript
import { createHash } from "node:crypto";
|
|
import { rootPath } from "./config";
|
|
import { runCommand, type CommandResult } from "./command";
|
|
import {
|
|
HWLAB_NODE_CONTROL_PLANE_CONFIG_PATH,
|
|
type ArgoOptions,
|
|
type CiBuildBenchmarkCachePolicy,
|
|
type CiBuildBenchmarkProfileSpec,
|
|
type ControlPlaneEgressProxySpec,
|
|
type ControlPlaneGitMirrorGithubTransportSpec,
|
|
type ControlPlaneHostRouteEgressProxySpec,
|
|
type ControlPlaneK3sInstallSpec,
|
|
type ControlPlaneK3sNodeSpec,
|
|
type ControlPlaneNodeSpec,
|
|
type ControlPlaneRegistrySpec,
|
|
type ControlPlaneRuntimeProxySpec,
|
|
type ControlPlaneTargetSpec,
|
|
type TektonInstallOptions,
|
|
type ToolsImageOptions,
|
|
} from "./hwlab-node-control-plane-model";
|
|
import { controlPlaneRegistrySummary } from "./hwlab-node-control-plane-registry";
|
|
import { argoInternalHttpIdentity } from "./hwlab-node-control-plane-argo-model";
|
|
import { argoReconciliationStatusPython } from "./hwlab-node-control-plane-argo-runtime";
|
|
import { detachedJobLaunchScript, manifestObjectSummary, remoteJobStateDir, remoteJobStatusScript, runTargetK3s } from "./hwlab-node-control-plane-executor";
|
|
|
|
export function planSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record<string, unknown> {
|
|
return {
|
|
id: target.id,
|
|
node: node.id,
|
|
kubeRoute: node.kubeRoute,
|
|
kubeExecution: node.kubeExecution,
|
|
lane: target.lane,
|
|
enabled: target.enabled,
|
|
ciNamespace: target.ciNamespace,
|
|
runtimeNamespace: target.runtimeNamespace,
|
|
k3sNodeConfig: k3sNodeConfigPlan(node),
|
|
registry: controlPlaneRegistrySummary(node.registry),
|
|
egressProxy: controlPlaneEgressProxySummary(node.egressProxy),
|
|
sourceBranch: target.source.branch,
|
|
gitopsBranch: target.gitops.branch,
|
|
gitopsPath: target.gitops.path,
|
|
gitMirrorNamespace: target.gitMirror.namespace,
|
|
readUrl: target.gitMirror.readUrl,
|
|
writeUrl: target.gitMirror.writeUrl,
|
|
pipeline: target.tekton.pipelineName,
|
|
pipelineRunPrefix: target.tekton.pipelineRunPrefix,
|
|
serviceAccount: target.tekton.serviceAccountName,
|
|
gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target),
|
|
runtimeObserverRbac: target.tekton.runtimeObserverRbac,
|
|
argoObserverRbac: target.tekton.argoObserverRbac,
|
|
tektonInstall: {
|
|
enabled: target.tekton.install.enabled,
|
|
version: target.tekton.install.version,
|
|
manifests: target.tekton.install.manifests,
|
|
requiredCrds: target.tekton.install.requiredCrds,
|
|
expectedDeploymentNamespaces: target.tekton.install.expectedDeploymentNamespaces,
|
|
},
|
|
toolsImage: target.tekton.toolsImage,
|
|
argoApplication: target.argo.applicationName,
|
|
argoInstall: {
|
|
enabled: target.argo.install.enabled,
|
|
version: target.argo.install.version,
|
|
manifestUrl: target.argo.install.manifestUrl,
|
|
imageRewrites: target.argo.install.imageRewrites,
|
|
internalHttp: target.argo.install.internalHttp,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function expectedSummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record<string, unknown> {
|
|
return {
|
|
sourceRepo: target.source.repository,
|
|
kubeExecution: node.kubeExecution,
|
|
branch: target.source.branch,
|
|
gitopsBranch: target.gitops.branch,
|
|
runtimePath: target.gitops.path,
|
|
runtimeNamespace: target.runtimeNamespace,
|
|
namespace: target.ciNamespace,
|
|
k3sNodeConfig: k3sNodeConfigPlan(node),
|
|
gitMirror: {
|
|
namespace: target.gitMirror.namespace,
|
|
readUrl: target.gitMirror.readUrl,
|
|
writeUrl: target.gitMirror.writeUrl,
|
|
cachePvc: target.gitMirror.cachePvcName,
|
|
cachePvcStorage: target.gitMirror.cachePvcStorage,
|
|
cacheHostPath: target.gitMirror.cacheHostPath,
|
|
servicePort: target.gitMirror.servicePort,
|
|
readContainerPort: target.gitMirror.readContainerPort,
|
|
writeContainerPort: target.gitMirror.writeContainerPort,
|
|
deploymentReplicas: target.gitMirror.deploymentReplicas,
|
|
syncConfigMap: target.gitMirror.syncConfigMapName,
|
|
egressProxy: target.gitMirror.egressProxy,
|
|
effectiveEgressProxy: gitMirrorEffectiveEgressProxySummary(node, target),
|
|
githubTransport: gitMirrorGithubTransportSummary(target.gitMirror.githubTransport),
|
|
statusSummaryKeys: ["localSource", "githubSource", "localGitops", "githubGitops", "pendingFlush", "flushNeeded", "githubInSync"],
|
|
},
|
|
pipeline: target.tekton.pipelineName,
|
|
pipelineRunPrefix: target.tekton.pipelineRunPrefix,
|
|
serviceAccount: target.tekton.serviceAccountName,
|
|
gitWorkspaceSecret: tektonGitWorkspaceSecretSummary(target),
|
|
runtimeObserverRbac: target.tekton.runtimeObserverRbac,
|
|
argoObserverRbac: target.tekton.argoObserverRbac,
|
|
tektonInstall: {
|
|
enabled: target.tekton.install.enabled,
|
|
sourceKind: target.tekton.install.sourceKind,
|
|
version: target.tekton.install.version,
|
|
manifests: target.tekton.install.manifests,
|
|
requiredCrds: target.tekton.install.requiredCrds,
|
|
expectedDeploymentNamespaces: target.tekton.install.expectedDeploymentNamespaces,
|
|
readinessTimeoutSeconds: target.tekton.install.readinessTimeoutSeconds,
|
|
},
|
|
toolsImage: target.tekton.toolsImage,
|
|
argoNamespace: target.argo.namespace,
|
|
argoApplication: target.argo.applicationName,
|
|
argoInstall: {
|
|
enabled: target.argo.install.enabled,
|
|
sourceKind: target.argo.install.sourceKind,
|
|
version: target.argo.install.version,
|
|
manifestUrl: target.argo.install.manifestUrl,
|
|
preloadImages: target.argo.install.preloadImages,
|
|
imageRewrites: target.argo.install.imageRewrites,
|
|
requiredCrds: target.argo.install.requiredCrds,
|
|
expectedDeployments: target.argo.install.expectedDeployments,
|
|
expectedStatefulSets: target.argo.install.expectedStatefulSets,
|
|
internalHttp: target.argo.install.internalHttp === null ? null : {
|
|
...target.argo.install.internalHttp,
|
|
rolloutIdentity: argoInternalHttpIdentity(target),
|
|
},
|
|
reconciliation: target.argo.install.reconciliation,
|
|
},
|
|
registry: controlPlaneRegistrySummary(node.registry),
|
|
imagePolicy: {
|
|
noPrivateInputImages: true,
|
|
buildInput: { sourceKind: target.tekton.toolsImage.sourceKind, context: target.tekton.toolsImage.context, dockerfile: target.tekton.toolsImage.dockerfile ?? null, dockerfileInline: target.tekton.toolsImage.dockerfileInline ?? null, composeFile: target.tekton.toolsImage.composeFile ?? null, publicBaseImages: target.tekton.toolsImage.publicBaseImages },
|
|
outputImage: target.tekton.toolsImage.output,
|
|
},
|
|
};
|
|
}
|
|
|
|
export function k3sNodeConfigPlan(node: ControlPlaneNodeSpec): Record<string, unknown> {
|
|
if (node.k3s === null) return { managed: false };
|
|
const dropIn = k3sDropInContent(node.k3s);
|
|
return {
|
|
managed: true,
|
|
serviceName: node.k3s.serviceName,
|
|
dropInPath: node.k3s.dropInPath,
|
|
nodeStatusName: node.k3s.nodeStatusName,
|
|
desiredMaxPods: node.k3s.kubelet.maxPods,
|
|
dropInSha256: sha256Short(dropIn),
|
|
execStartPreCount: node.k3s.execStartPre.length,
|
|
serverArgCount: node.k3s.serverArgs.length,
|
|
};
|
|
}
|
|
|
|
export function k3sInstallPlan(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, spec: ControlPlaneK3sInstallSpec): Record<string, unknown> {
|
|
return {
|
|
node: node.id,
|
|
route: node.route,
|
|
kubeRoute: node.kubeRoute,
|
|
lane: target.lane,
|
|
version: spec.version,
|
|
channel: spec.channel,
|
|
installScriptUrl: spec.installScriptUrl,
|
|
binaryUrl: spec.binaryUrl,
|
|
sha256Url: spec.sha256Url,
|
|
expectedSha256: `sha256:${spec.expectedSha256}`,
|
|
hostProxyConfigRef: spec.hostProxyConfigRef,
|
|
proxyEnvPath: spec.proxyEnvPath,
|
|
registriesYamlPath: spec.registriesYamlPath,
|
|
state: spec.state,
|
|
localRegistry: spec.localRegistry,
|
|
serviceName: node.k3s?.serviceName ?? null,
|
|
nodeStatusName: node.k3s?.nodeStatusName ?? null,
|
|
serverArgCount: node.k3s?.serverArgs.length ?? 0,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function k3sDropInContent(spec: ControlPlaneK3sNodeSpec): string {
|
|
return [
|
|
"# Managed by UniDesk. Source: config/hwlab-node-control-plane.yaml nodes.<node>.k3s",
|
|
"[Service]",
|
|
...spec.execStartPre.map((command) => `ExecStartPre=${command.map(systemdExecArg).join(" ")}`),
|
|
"ExecStart=",
|
|
`ExecStart=${["/usr/local/bin/k3s", ...spec.serverArgs].map(systemdExecArg).join(" ")}`,
|
|
"",
|
|
].join("\n");
|
|
}
|
|
|
|
export function k3sNodeConfigHostScript(spec: ControlPlaneK3sNodeSpec, apply: boolean): string {
|
|
const content = k3sDropInContent(spec);
|
|
const encoded = Buffer.from(content, "utf8").toString("base64");
|
|
return `
|
|
set +e
|
|
service=${shQuote(spec.serviceName)}
|
|
dropin=${shQuote(spec.dropInPath)}
|
|
node=${shQuote(spec.nodeStatusName)}
|
|
desired_max_pods=${shQuote(String(spec.kubelet.maxPods))}
|
|
expected_sha=${shQuote(sha256Short(content))}
|
|
apply=${apply ? "true" : "false"}
|
|
expected=$(mktemp /tmp/unidesk-k3s-dropin.XXXXXX)
|
|
printf %s ${shQuote(encoded)} | base64 -d >"$expected"
|
|
before_sha=
|
|
if [ -f "$dropin" ]; then before_sha=$(sha256sum "$dropin" | awk '{print "sha256:"$1}'); fi
|
|
service_before=$(systemctl is-active "$service" 2>/dev/null || true)
|
|
before_capacity=
|
|
before_allocatable=
|
|
if [ "$service_before" = active ]; then
|
|
for _ in $(seq 1 10); do
|
|
before_capacity=$(kubectl get node "$node" -o 'jsonpath={.status.capacity.pods}' 2>/dev/null || true)
|
|
before_allocatable=$(kubectl get node "$node" -o 'jsonpath={.status.allocatable.pods}' 2>/dev/null || true)
|
|
[ -n "$before_capacity" ] && [ -n "$before_allocatable" ] && break
|
|
sleep 2
|
|
done
|
|
fi
|
|
changed=false
|
|
restart_required=false
|
|
if ! cmp -s "$expected" "$dropin" 2>/dev/null; then changed=true; fi
|
|
if [ "$changed" = true ] || [ "$service_before" != active ]; then
|
|
restart_required=true
|
|
elif [ -n "$before_capacity" ] && [ -n "$before_allocatable" ] && \
|
|
{ [ "$before_capacity" != "$desired_max_pods" ] || [ "$before_allocatable" != "$desired_max_pods" ]; }; then
|
|
restart_required=true
|
|
fi
|
|
mutation=false
|
|
daemon_reload_rc=0
|
|
restart_rc=0
|
|
credential_backup_path=
|
|
credential_backup_rc=0
|
|
if [ "$apply" = true ] && [ "$restart_required" = true ]; then
|
|
mkdir -p "$(dirname "$dropin")"
|
|
if [ "$changed" = true ]; then
|
|
cp "$expected" "$dropin"
|
|
chown 0:0 "$dropin" 2>/dev/null || true
|
|
chmod 0644 "$dropin"
|
|
fi
|
|
systemctl daemon-reload || daemon_reload_rc=$?
|
|
if [ "$daemon_reload_rc" = 0 ]; then
|
|
derived_passwd=/var/lib/rancher/k3s/server/cred/passwd
|
|
systemctl stop "$service" || restart_rc=$?
|
|
if [ "$restart_rc" = 0 ] && [ -f "$derived_passwd" ]; then
|
|
credential_backup_path="$derived_passwd.unidesk-pre-restart-$(date -u +%Y%m%dT%H%M%SZ)-$$"
|
|
mv "$derived_passwd" "$credential_backup_path" || credential_backup_rc=$?
|
|
fi
|
|
if [ "$restart_rc" = 0 ]; then
|
|
systemctl start "$service" || restart_rc=$?
|
|
fi
|
|
fi
|
|
mutation=true
|
|
fi
|
|
rm -f "$expected"
|
|
ready=false
|
|
if [ "$apply" = true ] && [ "$restart_required" = true ] && [ "$daemon_reload_rc" = 0 ] && [ "$restart_rc" = 0 ]; then
|
|
for _ in $(seq 1 60); do
|
|
if [ "$(systemctl is-active "$service" 2>/dev/null || true)" = active ] && \
|
|
[ "$(kubectl get node "$node" -o 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" = True ]; then
|
|
ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
else
|
|
for _ in $(seq 1 5); do
|
|
if [ "$(systemctl is-active "$service" 2>/dev/null || true)" = active ] && \
|
|
[ "$(kubectl get node "$node" -o 'jsonpath={.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)" = True ]; then
|
|
ready=true
|
|
break
|
|
fi
|
|
sleep 2
|
|
done
|
|
fi
|
|
after_sha=
|
|
if [ -f "$dropin" ]; then after_sha=$(sha256sum "$dropin" | awk '{print "sha256:"$1}'); fi
|
|
after_capacity=$(kubectl get node "$node" -o 'jsonpath={.status.capacity.pods}' 2>/dev/null || true)
|
|
after_allocatable=$(kubectl get node "$node" -o 'jsonpath={.status.allocatable.pods}' 2>/dev/null || true)
|
|
service_active=$(systemctl is-active "$service" 2>/dev/null || true)
|
|
python3 - "$apply" "$mutation" "$changed" "$restart_required" "$daemon_reload_rc" "$restart_rc" "$credential_backup_path" "$credential_backup_rc" "$before_sha" "$after_sha" "$expected_sha" "$service_active" "$ready" "$before_capacity" "$before_allocatable" "$after_capacity" "$after_allocatable" "$desired_max_pods" <<'PY'
|
|
import json, sys
|
|
apply, mutation, changed, restart_required = [value == "true" for value in sys.argv[1:5]]
|
|
daemon_reload_rc, restart_rc = int(sys.argv[5]), int(sys.argv[6])
|
|
credential_backup_path, credential_backup_rc = sys.argv[7], int(sys.argv[8])
|
|
before_sha, after_sha, expected_sha = sys.argv[9:12]
|
|
service_active, ready = sys.argv[12], sys.argv[13] == "true"
|
|
before_capacity, before_allocatable, after_capacity, after_allocatable, desired = sys.argv[14:19]
|
|
def number(value):
|
|
return int(value) if value.isdigit() else None
|
|
warnings = []
|
|
if credential_backup_path:
|
|
warnings.append({
|
|
"code": "k3s-derived-credential-backed-up",
|
|
"blocking": False,
|
|
"message": "已在受控重启前备份 k3s 派生 passwd，避免 datastore freshness 门禁阻断变更",
|
|
"backupPath": credential_backup_path,
|
|
"backupExitCode": credential_backup_rc,
|
|
})
|
|
ok = (
|
|
after_sha == expected_sha
|
|
and service_active == "active"
|
|
and ready
|
|
and after_capacity == desired
|
|
and after_allocatable == desired
|
|
and daemon_reload_rc == 0
|
|
and restart_rc == 0
|
|
)
|
|
print(json.dumps({
|
|
"ok": ok,
|
|
"mode": "apply" if apply else "status",
|
|
"mutation": mutation,
|
|
"changed": changed,
|
|
"restartRequired": restart_required,
|
|
"daemonReloadExitCode": daemon_reload_rc,
|
|
"restartExitCode": restart_rc,
|
|
"warnings": warnings,
|
|
"beforeDropInSha256": before_sha or None,
|
|
"dropInSha256": after_sha or None,
|
|
"expectedDropInSha256": expected_sha,
|
|
"dropInMatches": after_sha == expected_sha,
|
|
"serviceActive": service_active == "active",
|
|
"nodeReady": ready,
|
|
"desiredMaxPods": int(desired),
|
|
"beforeCapacityPods": number(before_capacity),
|
|
"beforeAllocatablePods": number(before_allocatable),
|
|
"liveCapacityPods": number(after_capacity),
|
|
"liveAllocatablePods": number(after_allocatable),
|
|
"valuesPrinted": False,
|
|
}, ensure_ascii=False))
|
|
PY
|
|
rc=$?
|
|
if [ "$rc" != 0 ]; then exit "$rc"; fi
|
|
if [ "$apply" = true ]; then
|
|
[ "$after_sha" = "$expected_sha" ] && [ "$service_active" = active ] && [ "$ready" = true ] && \
|
|
[ "$after_capacity" = "$desired_max_pods" ] && [ "$after_allocatable" = "$desired_max_pods" ] && \
|
|
[ "$daemon_reload_rc" = 0 ] && [ "$restart_rc" = 0 ]
|
|
exit $?
|
|
fi
|
|
exit 0
|
|
`;
|
|
}
|
|
|
|
export function k3sServiceUnitContent(spec: ControlPlaneK3sNodeSpec, install: ControlPlaneK3sInstallSpec): string {
|
|
return [
|
|
"[Unit]",
|
|
"Description=Lightweight Kubernetes",
|
|
"Documentation=https://k3s.io",
|
|
"Wants=network-online.target",
|
|
"After=network-online.target",
|
|
"",
|
|
"[Install]",
|
|
"WantedBy=multi-user.target",
|
|
"",
|
|
"[Service]",
|
|
"Type=notify",
|
|
`EnvironmentFile=-${install.proxyEnvPath}`,
|
|
"KillMode=process",
|
|
"Delegate=yes",
|
|
"LimitNOFILE=1048576",
|
|
"LimitNPROC=infinity",
|
|
"LimitCORE=infinity",
|
|
"TasksMax=infinity",
|
|
"TimeoutStartSec=0",
|
|
"Restart=always",
|
|
"RestartSec=5s",
|
|
"ExecStartPre=-/sbin/modprobe br_netfilter",
|
|
"ExecStartPre=-/sbin/modprobe overlay",
|
|
`ExecStart=${["/usr/local/bin/k3s", ...spec.serverArgs].map(systemdExecArg).join(" ")}`,
|
|
"",
|
|
].join("\n");
|
|
}
|
|
|
|
export function k3sRegistriesYaml(install: ControlPlaneK3sInstallSpec): string {
|
|
const endpoint = install.localRegistry.bind.split(":").slice(0, 2).join(":");
|
|
return [
|
|
"mirrors:",
|
|
` "${endpoint}":`,
|
|
" endpoint:",
|
|
` - "http://${endpoint}"`,
|
|
"configs:",
|
|
` "${endpoint}":`,
|
|
" tls:",
|
|
" insecure_skip_verify: true",
|
|
"",
|
|
].join("\n");
|
|
}
|
|
|
|
export function controlPlaneEgressProxySummary(proxy: ControlPlaneEgressProxySpec | null): Record<string, unknown> | null {
|
|
if (proxy === null) return null;
|
|
if (proxy.mode === "host-route") {
|
|
return {
|
|
mode: proxy.mode,
|
|
clientName: proxy.clientName,
|
|
hostProxyConfigRef: proxy.hostProxyConfigRef,
|
|
proxyEnvPath: proxy.proxyEnvPath,
|
|
proxyUrl: proxy.proxyUrl,
|
|
noProxyCount: proxy.noProxy.length,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
return {
|
|
mode: proxy.mode,
|
|
clientName: proxy.clientName,
|
|
namespace: proxy.namespace,
|
|
serviceName: proxy.serviceName,
|
|
port: proxy.port,
|
|
sourceConfigRef: proxy.sourceConfigRef,
|
|
sourceType: proxy.sourceType,
|
|
sourceRef: proxy.sourceRef,
|
|
sourceKey: proxy.sourceKey,
|
|
sourceFingerprint: proxy.sourceFingerprint,
|
|
preferredOutbound: proxy.preferredOutbound,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function hostRouteNoProxy(proxy: ControlPlaneHostRouteEgressProxySpec): readonly string[] {
|
|
return [...new Set(["localhost", "127.0.0.1", "::1", "127.0.0.1:5000", "localhost:5000", ...proxy.noProxy])];
|
|
}
|
|
|
|
export function runtimeHostProxyConfig(node: ControlPlaneNodeSpec, spec: ControlPlaneRuntimeProxySpec): Record<string, unknown> {
|
|
if (!spec.enabled) {
|
|
return {
|
|
enabled: false,
|
|
mode: "host-route",
|
|
configRef: spec.configRef,
|
|
hostNetwork: false,
|
|
injectEnv: false,
|
|
deployments: [],
|
|
statefulSets: [],
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
if (node.egressProxy?.mode !== "host-route") {
|
|
throw new Error(`runtimeProxy enabled for ${node.id} requires nodes.${node.id}.egressProxy.mode=host-route`);
|
|
}
|
|
return {
|
|
enabled: true,
|
|
mode: spec.mode,
|
|
configRef: spec.configRef,
|
|
hostNetwork: spec.hostNetwork,
|
|
injectEnv: spec.injectEnv,
|
|
deployments: spec.deployments,
|
|
statefulSets: spec.statefulSets,
|
|
proxyUrl: node.egressProxy.proxyUrl,
|
|
noProxy: hostRouteNoProxy(node.egressProxy),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function runtimeHostProxyEnv(node: ControlPlaneNodeSpec, spec: ControlPlaneRuntimeProxySpec): readonly Record<string, string>[] {
|
|
if (!spec.enabled || !spec.injectEnv) return [];
|
|
if (node.egressProxy?.mode !== "host-route") {
|
|
throw new Error(`runtimeProxy env for ${node.id} requires nodes.${node.id}.egressProxy.mode=host-route`);
|
|
}
|
|
const proxyUrl = node.egressProxy.proxyUrl;
|
|
const noProxy = hostRouteNoProxy(node.egressProxy).join(",");
|
|
return [
|
|
{ name: "HTTP_PROXY", value: proxyUrl },
|
|
{ name: "HTTPS_PROXY", value: proxyUrl },
|
|
{ name: "ALL_PROXY", value: proxyUrl },
|
|
{ name: "NO_PROXY", value: noProxy },
|
|
{ name: "http_proxy", value: proxyUrl },
|
|
{ name: "https_proxy", value: proxyUrl },
|
|
{ name: "all_proxy", value: proxyUrl },
|
|
{ name: "no_proxy", value: noProxy },
|
|
];
|
|
}
|
|
|
|
export function runtimeProxyReady(status: Record<string, unknown>): boolean {
|
|
const proxy = record(status.runtimeProxy);
|
|
return proxy.enabled !== true || boolField(proxy, "ready");
|
|
}
|
|
|
|
export function gitMirrorRuntimeProxySpec(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): ControlPlaneRuntimeProxySpec {
|
|
const config = target.gitMirror.egressProxy;
|
|
if (config === null || config.mode !== "host-route") {
|
|
return { enabled: false, mode: "host-route", configRef: null, hostNetwork: false, injectEnv: false, deployments: [], statefulSets: [] };
|
|
}
|
|
return {
|
|
enabled: config.podHostNetwork || config.injectPodEnv,
|
|
mode: "host-route",
|
|
configRef: `nodes.${node.id}.egressProxy`,
|
|
hostNetwork: config.podHostNetwork,
|
|
injectEnv: config.injectPodEnv,
|
|
deployments: [],
|
|
statefulSets: [],
|
|
};
|
|
}
|
|
|
|
export function gitMirrorEffectiveEgressProxySummary(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): Record<string, unknown> {
|
|
const config = target.gitMirror.egressProxy;
|
|
if (config === null || config.mode === "direct") {
|
|
return {
|
|
mode: "direct",
|
|
required: false,
|
|
transport: target.gitMirror.githubTransport.mode,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
const proxy = node.egressProxy;
|
|
if (config.mode === "host-route") {
|
|
return {
|
|
mode: config.mode,
|
|
required: config.required,
|
|
transport: target.gitMirror.githubTransport.mode,
|
|
podHostNetwork: config.podHostNetwork,
|
|
injectPodEnv: config.injectPodEnv,
|
|
ready: proxy?.mode === "host-route",
|
|
nodeProxy: controlPlaneEgressProxySummary(proxy),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
return {
|
|
mode: config.mode,
|
|
required: config.required,
|
|
transport: target.gitMirror.githubTransport.mode,
|
|
podHostNetwork: config.podHostNetwork,
|
|
injectPodEnv: config.injectPodEnv,
|
|
ready: proxy !== null,
|
|
nodeProxy: controlPlaneEgressProxySummary(proxy),
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function gitMirrorGithubTransportSummary(transport: ControlPlaneGitMirrorGithubTransportSpec): Record<string, unknown> {
|
|
if (transport.mode === "ssh") {
|
|
return {
|
|
mode: "ssh",
|
|
privateKeySecretKey: transport.privateKeySecretKey,
|
|
privateKeySourceRef: transport.privateKeySourceRef,
|
|
privateKeySourceKey: transport.privateKeySourceKey,
|
|
privateKeySourceEncoding: transport.privateKeySourceEncoding,
|
|
knownHostsSecretKey: transport.knownHostsSecretKey,
|
|
knownHostsSourceRef: transport.knownHostsSourceRef,
|
|
knownHostsSourceKey: transport.knownHostsSourceKey,
|
|
knownHostsSourceEncoding: transport.knownHostsSourceEncoding,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
return {
|
|
mode: "https",
|
|
username: transport.username,
|
|
tokenSecretName: transport.tokenSecretName,
|
|
tokenSecretKey: transport.tokenSecretKey,
|
|
tokenSourceRef: transport.tokenSourceRef,
|
|
tokenSourceKey: transport.tokenSourceKey,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function tektonGitWorkspaceSecretSummary(target: ControlPlaneTargetSpec): Record<string, unknown> {
|
|
const transport = target.gitMirror.githubTransport;
|
|
const secret = target.tekton.gitWorkspaceSecret;
|
|
return {
|
|
name: secret.name,
|
|
namespace: secret.namespace,
|
|
sourceRefFrom: secret.sourceRefFrom,
|
|
privateKeySecretKey: secret.privateKeySecretKey,
|
|
privateKeySourceRef: transport.mode === "ssh" ? transport.privateKeySourceRef : null,
|
|
privateKeySourceKey: transport.mode === "ssh" ? transport.privateKeySourceKey : null,
|
|
knownHostsSecretKey: secret.knownHostsSecretKey,
|
|
knownHostsSourceRef: transport.mode === "ssh" ? transport.knownHostsSourceRef : null,
|
|
knownHostsSourceKey: transport.mode === "ssh" ? transport.knownHostsSourceKey : null,
|
|
valuesPrinted: false,
|
|
};
|
|
}
|
|
|
|
export function systemdExecArg(value: string): string {
|
|
if (/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)) return value;
|
|
return `"${value.replaceAll("\\", "\\\\").replaceAll("\"", "\\\"").replaceAll("$", "\\$").replaceAll("`", "\\`")}"`;
|
|
}
|
|
|
|
export function k3sInstallSubmitScript(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, install: ControlPlaneK3sInstallSpec): string {
|
|
if (node.k3s === null) throw new Error("k3sInstallSubmitScript requires node.k3s");
|
|
const runner = k3sInstallRunnerScript(node, target, install);
|
|
const encoded = Buffer.from(runner, "utf8").toString("base64");
|
|
return `
|
|
set -eu
|
|
state_dir=${shQuote(install.state.dir)}
|
|
runner="$state_dir/install-runner.sh"
|
|
pid_file="$state_dir/pid"
|
|
status_file=${shQuote(install.state.statusPath)}
|
|
mkdir -p "$state_dir"
|
|
if [ -s "$pid_file" ]; then
|
|
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
|
if [ -n "$pid" ] && kill -0 "$pid" >/dev/null 2>&1; then
|
|
python3 - "$pid" "$status_file" <<'PY'
|
|
import json, pathlib, sys
|
|
status = None
|
|
path = pathlib.Path(sys.argv[2])
|
|
if path.exists():
|
|
try:
|
|
status = json.loads(path.read_text())
|
|
except Exception as exc:
|
|
status = {"parseError": str(exc)}
|
|
print(json.dumps({"ok": True, "alreadyRunning": True, "pid": sys.argv[1], "status": status, "valuesPrinted": False}, ensure_ascii=False))
|
|
PY
|
|
exit 0
|
|
fi
|
|
fi
|
|
printf %s ${shQuote(encoded)} | base64 -d >"$runner"
|
|
chmod 0700 "$runner"
|
|
cat >"$status_file" <<JSON
|
|
{"ok":false,"phase":"submitted","node":"${node.id}","lane":"${target.lane}","updatedAt":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","valuesPrinted":false}
|
|
JSON
|
|
nohup sh "$runner" >${shQuote(install.state.logPath)} 2>&1 &
|
|
pid=$!
|
|
printf '%s\\n' "$pid" >"$pid_file"
|
|
python3 - "$pid" "$state_dir" "$status_file" <<'PY'
|
|
import json, sys
|
|
print(json.dumps({"ok": True, "submitted": True, "pid": sys.argv[1], "stateDir": sys.argv[2], "statusPath": sys.argv[3], "valuesPrinted": False}, ensure_ascii=False))
|
|
PY
|
|
`;
|
|
}
|
|
|
|
export function k3sInstallRunnerScript(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, install: ControlPlaneK3sInstallSpec): string {
|
|
if (node.k3s === null) throw new Error("k3sInstallRunnerScript requires node.k3s");
|
|
const unit = Buffer.from(k3sServiceUnitContent(node.k3s, install), "utf8").toString("base64");
|
|
const dropIn = Buffer.from(k3sDropInContent(node.k3s), "utf8").toString("base64");
|
|
const registries = Buffer.from(k3sRegistriesYaml(install), "utf8").toString("base64");
|
|
return `#!/bin/sh
|
|
set -eu
|
|
state_dir=${shQuote(install.state.dir)}
|
|
status_file=${shQuote(install.state.statusPath)}
|
|
proxy_env=${shQuote(install.proxyEnvPath)}
|
|
registries_yaml=${shQuote(install.registriesYamlPath)}
|
|
binary_url=${shQuote(install.binaryUrl)}
|
|
sha256_url=${shQuote(install.sha256Url)}
|
|
expected_sha=${shQuote(install.expectedSha256)}
|
|
version=${shQuote(install.version)}
|
|
node_name=${shQuote(node.k3s.nodeStatusName)}
|
|
service_name=${shQuote(node.k3s.serviceName)}
|
|
registry_name=${shQuote(install.localRegistry.containerName)}
|
|
registry_image=${shQuote(install.localRegistry.image)}
|
|
registry_bind=${shQuote(install.localRegistry.bind)}
|
|
log_status() {
|
|
phase="$1"; ok="$2"; message="$3"
|
|
python3 - "$status_file" "$phase" "$ok" "$message" <<'PY'
|
|
import json, pathlib, sys, datetime
|
|
path = pathlib.Path(sys.argv[1])
|
|
payload = {
|
|
"ok": sys.argv[3] == "true",
|
|
"phase": sys.argv[2],
|
|
"message": sys.argv[4],
|
|
"updatedAt": datetime.datetime.utcnow().replace(microsecond=0).isoformat() + "Z",
|
|
"node": "${node.id}",
|
|
"lane": "${target.lane}",
|
|
"valuesPrinted": False,
|
|
}
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.write_text(json.dumps(payload, ensure_ascii=False) + "\\n")
|
|
PY
|
|
}
|
|
log_status starting false "k3s install started"
|
|
mkdir -p "$state_dir" /usr/local/bin /etc/rancher/k3s /etc/systemd/system/k3s.service.d
|
|
if [ ! -s "$proxy_env" ]; then
|
|
log_status failed false "proxy env missing"
|
|
exit 41
|
|
fi
|
|
. "$proxy_env"
|
|
export HTTP_PROXY HTTPS_PROXY ALL_PROXY NO_PROXY http_proxy https_proxy all_proxy no_proxy
|
|
curl -fsS --max-time 20 --proxy "$HTTPS_PROXY" -o /dev/null https://www.gstatic.com/generate_204
|
|
log_status proxy-ready false "host proxy probe passed"
|
|
if command -v docker >/dev/null 2>&1; then
|
|
if docker ps --format '{{.Names}}' | grep -Fx "$registry_name" >/dev/null 2>&1; then
|
|
:
|
|
elif docker ps -a --format '{{.Names}}' | grep -Fx "$registry_name" >/dev/null 2>&1; then
|
|
docker start "$registry_name" >/dev/null 2>&1 || true
|
|
else
|
|
docker run -d --restart unless-stopped --name "$registry_name" -p "$registry_bind" "$registry_image" >/dev/null 2>&1 || true
|
|
fi
|
|
fi
|
|
log_status downloading false "downloading k3s binary through host proxy"
|
|
current_sha="$(sha256sum /usr/local/bin/k3s 2>/dev/null | cut -d' ' -f1 || true)"
|
|
if [ "$current_sha" != "$expected_sha" ]; then
|
|
sha_file="$state_dir/sha256sum-amd64.txt"
|
|
curl -fL --connect-timeout ${install.downloads.connectTimeoutSeconds} --max-time ${install.downloads.maxTimeSeconds} --retry ${install.downloads.retry} --retry-delay ${install.downloads.retryDelaySeconds} -o "$sha_file" "$sha256_url"
|
|
if ! grep -F "$expected_sha" "$sha_file" >/dev/null 2>&1; then
|
|
log_status failed false "expected k3s sha256 missing from upstream sha256 file"
|
|
exit 44
|
|
fi
|
|
tmp_bin=/usr/local/bin/k3s.tmp.$$
|
|
curl -fL --connect-timeout ${install.downloads.connectTimeoutSeconds} --max-time ${install.downloads.maxTimeSeconds} --retry ${install.downloads.retry} --retry-delay ${install.downloads.retryDelaySeconds} -o "$tmp_bin" "$binary_url"
|
|
actual_sha=$(sha256sum "$tmp_bin" | cut -d' ' -f1)
|
|
if [ "$actual_sha" != "$expected_sha" ]; then
|
|
rm -f "$tmp_bin"
|
|
log_status failed false "k3s binary sha256 mismatch"
|
|
exit 42
|
|
fi
|
|
chmod 0755 "$tmp_bin"
|
|
mv -f "$tmp_bin" /usr/local/bin/k3s
|
|
fi
|
|
ln -sf /usr/local/bin/k3s /usr/local/bin/kubectl
|
|
ln -sf /usr/local/bin/k3s /usr/local/bin/crictl
|
|
ln -sf /usr/local/bin/k3s /usr/local/bin/ctr
|
|
printf %s ${shQuote(registries)} | base64 -d >"$registries_yaml"
|
|
printf %s ${shQuote(unit)} | base64 -d >/etc/systemd/system/k3s.service
|
|
printf %s ${shQuote(dropIn)} | base64 -d >${shQuote(node.k3s.dropInPath)}
|
|
log_status systemd false "starting k3s service"
|
|
systemctl daemon-reload
|
|
systemctl enable --now "$service_name"
|
|
for _ in $(seq 1 120); do
|
|
if /usr/local/bin/kubectl get node "$node_name" >/tmp/unidesk-k3s-node.out 2>/tmp/unidesk-k3s-node.err; then
|
|
if /usr/local/bin/kubectl get node "$node_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null | grep -Fx True >/dev/null 2>&1; then
|
|
log_status succeeded true "k3s node ready"
|
|
exit 0
|
|
fi
|
|
fi
|
|
sleep 5
|
|
done
|
|
log_status failed false "timed out waiting for k3s node ready"
|
|
exit 43
|
|
`;
|
|
}
|
|
|
|
export function k3sInstallStatusScript(node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec, install: ControlPlaneK3sInstallSpec, tailLines: number): string {
|
|
if (node.k3s === null) throw new Error("k3sInstallStatusScript requires node.k3s");
|
|
return `
|
|
set +e
|
|
status_path=${shQuote(install.state.statusPath)}
|
|
log_path=${shQuote(install.state.logPath)}
|
|
pid_file=${shQuote(`${install.state.dir}/pid`)}
|
|
service_name=${shQuote(node.k3s.serviceName)}
|
|
node_name=${shQuote(node.k3s.nodeStatusName)}
|
|
expected_sha=${shQuote(install.expectedSha256)}
|
|
proxy_env=${shQuote(install.proxyEnvPath)}
|
|
registry_name=${shQuote(install.localRegistry.containerName)}
|
|
running=false
|
|
pid=
|
|
if [ -s "$pid_file" ]; then
|
|
pid="$(cat "$pid_file" 2>/dev/null || true)"
|
|
if [ -n "$pid" ] && kill -0 "$pid" >/dev/null 2>&1; then running=true; fi
|
|
fi
|
|
service_active="$(systemctl is-active "$service_name" 2>/dev/null || true)"
|
|
binary_sha="$(sha256sum /usr/local/bin/k3s 2>/dev/null | cut -d' ' -f1 || true)"
|
|
proxy_probe=false
|
|
if [ -s "$proxy_env" ]; then
|
|
. "$proxy_env"
|
|
curl -fsS --max-time 20 --proxy "$HTTPS_PROXY" -o /dev/null https://www.gstatic.com/generate_204 >/dev/null 2>&1 && proxy_probe=true
|
|
fi
|
|
registry_running=false
|
|
if command -v docker >/dev/null 2>&1; then
|
|
docker ps --format '{{.Names}}' 2>/dev/null | grep -Fx "$registry_name" >/dev/null 2>&1 && registry_running=true
|
|
fi
|
|
node_ready=false
|
|
node_capacity=
|
|
node_allocatable=
|
|
if command -v kubectl >/dev/null 2>&1; then
|
|
ready="$(kubectl get node "$node_name" -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}' 2>/dev/null || true)"
|
|
[ "$ready" = "True" ] && node_ready=true
|
|
node_capacity="$(kubectl get node "$node_name" -o jsonpath='{.status.capacity.pods}' 2>/dev/null || true)"
|
|
node_allocatable="$(kubectl get node "$node_name" -o jsonpath='{.status.allocatable.pods}' 2>/dev/null || true)"
|
|
fi
|
|
python3 - "$status_path" "$log_path" "$running" "$pid" "$service_active" "$binary_sha" "$expected_sha" "$proxy_probe" "$registry_running" "$node_ready" "$node_capacity" "$node_allocatable" ${shQuote(String(tailLines))} <<'PY'
|
|
import json, pathlib, sys
|
|
status_path = pathlib.Path(sys.argv[1])
|
|
log_path = pathlib.Path(sys.argv[2])
|
|
status = None
|
|
if status_path.exists():
|
|
try:
|
|
status = json.loads(status_path.read_text())
|
|
except Exception as exc:
|
|
status = {"parseError": str(exc), "rawTail": status_path.read_text(errors="replace")[-1000:]}
|
|
tail_lines = int(sys.argv[13])
|
|
log_tail = ""
|
|
if log_path.exists():
|
|
log_tail = "\\n".join(log_path.read_text(errors="replace").splitlines()[-tail_lines:])
|
|
binary_sha = sys.argv[6]
|
|
payload = {
|
|
"node": "${node.id}",
|
|
"lane": "${target.lane}",
|
|
"running": sys.argv[3] == "true",
|
|
"pid": sys.argv[4] or None,
|
|
"status": status,
|
|
"checks": {
|
|
"serviceActive": sys.argv[5] == "active",
|
|
"binarySha256Ok": binary_sha == sys.argv[7],
|
|
"binarySha256Prefix": binary_sha[:12] if binary_sha else "",
|
|
"proxyProbe": sys.argv[8] == "true",
|
|
"registryRunning": sys.argv[9] == "true",
|
|
"nodeReady": sys.argv[10] == "true",
|
|
"capacityPods": int(sys.argv[11]) if sys.argv[11].isdigit() else None,
|
|
"allocatablePods": int(sys.argv[12]) if sys.argv[12].isdigit() else None,
|
|
},
|
|
"logBytes": log_path.stat().st_size if log_path.exists() else 0,
|
|
"logTail": log_tail,
|
|
"valuesPrinted": False,
|
|
}
|
|
print(json.dumps(payload, ensure_ascii=False))
|
|
PY
|
|
`;
|
|
}
|
|
|
|
export function statusScript(nodeSpec: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
|
|
const tektonRequiredCrds = shellJsonArray(target.tekton.install.requiredCrds);
|
|
const tektonDeploymentNamespaces = shellJsonArray(target.tekton.install.expectedDeploymentNamespaces);
|
|
const requiredCrds = shellJsonArray(target.argo.install.requiredCrds);
|
|
const argoDeployments = shellJsonArray(target.argo.install.expectedDeployments);
|
|
const argoStatefulSets = shellJsonArray(target.argo.install.expectedStatefulSets);
|
|
const tektonRuntimeProxy = JSON.stringify(runtimeHostProxyConfig(nodeSpec, target.tekton.install.runtimeProxy));
|
|
const argoRuntimeProxy = JSON.stringify(runtimeHostProxyConfig(nodeSpec, target.argo.install.runtimeProxy));
|
|
const argoInternalHttp = JSON.stringify(target.argo.install.internalHttp === null ? null : {
|
|
...target.argo.install.internalHttp,
|
|
rolloutIdentity: argoInternalHttpIdentity(target),
|
|
});
|
|
const argoReconciliation = JSON.stringify(target.argo.install.reconciliation);
|
|
const k3s = nodeSpec.k3s;
|
|
const k3sDropIn = k3s === null ? "" : k3sDropInContent(k3s);
|
|
const gitMirrorEgressProxyJson = JSON.stringify(gitMirrorEffectiveEgressProxySummary(nodeSpec, target));
|
|
return `
|
|
set +e
|
|
node=${shQuote(target.node)}
|
|
lane=${shQuote(target.lane)}
|
|
ci_ns=${shQuote(target.ciNamespace)}
|
|
runtime_ns=${shQuote(target.runtimeNamespace)}
|
|
gitmirror_ns=${shQuote(target.gitMirror.namespace)}
|
|
read_deploy=${shQuote(target.gitMirror.serviceReadName)}
|
|
write_deploy=${shQuote(target.gitMirror.serviceWriteName)}
|
|
read_svc=${shQuote(target.gitMirror.serviceReadName)}
|
|
write_svc=${shQuote(target.gitMirror.serviceWriteName)}
|
|
cache_pvc=${shQuote(target.gitMirror.cachePvcName)}
|
|
cache_host_path=${shQuote(target.gitMirror.cacheHostPath ?? "")}
|
|
github_transport_mode=${shQuote(target.gitMirror.githubTransport.mode)}
|
|
github_ssh_secret=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.secretName : "")}
|
|
github_ssh_private_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySecretKey : "")}
|
|
github_ssh_private_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceRef : "")}
|
|
github_ssh_private_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceKey : "")}
|
|
github_ssh_known_hosts_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSecretKey ?? "" : "")}
|
|
github_ssh_known_hosts_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceRef ?? "" : "")}
|
|
github_ssh_known_hosts_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceKey ?? "" : "")}
|
|
github_token_secret=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSecretName : "")}
|
|
github_token_key=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSecretKey : "")}
|
|
github_token_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSourceRef : "")}
|
|
github_token_source_key=${shQuote(target.gitMirror.githubTransport.mode === "https" ? target.gitMirror.githubTransport.tokenSourceKey : "")}
|
|
gitmirror_egress_proxy_json=${shQuote(gitMirrorEgressProxyJson)}
|
|
pipeline=${shQuote(target.tekton.pipelineName)}
|
|
service_account=${shQuote(target.tekton.serviceAccountName)}
|
|
runtime_observer_role=${shQuote(target.tekton.runtimeObserverRbac.roleName)}
|
|
runtime_observer_rolebinding=${shQuote(target.tekton.runtimeObserverRbac.roleBindingName)}
|
|
ci_git_secret=${shQuote(target.tekton.gitWorkspaceSecret.name)}
|
|
ci_git_private_key=${shQuote(target.tekton.gitWorkspaceSecret.privateKeySecretKey)}
|
|
ci_git_known_hosts_key=${shQuote(target.tekton.gitWorkspaceSecret.knownHostsSecretKey)}
|
|
ci_git_source_ref_from=${shQuote(target.tekton.gitWorkspaceSecret.sourceRefFrom)}
|
|
ci_git_private_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceRef : "")}
|
|
ci_git_private_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.privateKeySourceKey : "")}
|
|
ci_git_known_hosts_source_ref=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceRef ?? "" : "")}
|
|
ci_git_known_hosts_source_key=${shQuote(target.gitMirror.githubTransport.mode === "ssh" ? target.gitMirror.githubTransport.knownHostsSourceKey ?? "" : "")}
|
|
argo_ns=${shQuote(target.argo.namespace)}
|
|
argo_project=${shQuote(target.argo.projectName)}
|
|
argo_app=${shQuote(target.argo.applicationName)}
|
|
argo_observer_role=${shQuote(target.tekton.argoObserverRbac.roleName)}
|
|
argo_observer_rolebinding=${shQuote(target.tekton.argoObserverRbac.roleBindingName)}
|
|
registry=${shQuote(nodeSpec.registry.endpoint)}
|
|
registry_spec_json=${shQuote(JSON.stringify(controlPlaneRegistrySummary(nodeSpec.registry)))}
|
|
tools_image=${shQuote(target.tekton.toolsImage.output)}
|
|
tekton_required_crds_json=${shQuote(tektonRequiredCrds)}
|
|
tekton_deployment_namespaces_json=${shQuote(tektonDeploymentNamespaces)}
|
|
required_crds_json=${shQuote(requiredCrds)}
|
|
argo_deployments_json=${shQuote(argoDeployments)}
|
|
argo_statefulsets_json=${shQuote(argoStatefulSets)}
|
|
tekton_runtime_proxy_json=${shQuote(tektonRuntimeProxy)}
|
|
argo_runtime_proxy_json=${shQuote(argoRuntimeProxy)}
|
|
argo_internal_http_json=${shQuote(argoInternalHttp)}
|
|
argo_reconciliation_json=${shQuote(argoReconciliation)}
|
|
k3s_managed=${k3s === null ? "false" : "true"}
|
|
k3s_service=${shQuote(k3s?.serviceName ?? "")}
|
|
k3s_dropin=${shQuote(k3s?.dropInPath ?? "")}
|
|
k3s_node=${shQuote(k3s?.nodeStatusName ?? "")}
|
|
k3s_desired_max_pods=${shQuote(String(k3s?.kubelet.maxPods ?? ""))}
|
|
k3s_expected_sha=${shQuote(k3s === null ? "" : sha256Short(k3sDropIn))}
|
|
exists_ns() { kubectl get ns "$1" >/dev/null 2>&1 && printf true || printf false; }
|
|
exists_res() { kubectl -n "$1" get "$2" "$3" >/dev/null 2>&1 && printf true || printf false; }
|
|
deploy_ready() { desired=$(kubectl -n "$1" get deploy "$2" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n "$1" get deploy "$2" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n "$desired" ] && [ "$desired" -gt 0 ] 2>/dev/null && [ "\${ready:-0}" = "$desired" ] && printf true || printf false; }
|
|
sts_ready() { desired=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.spec.replicas}' 2>/dev/null || true); ready=$(kubectl -n "$1" get statefulset "$2" -o 'jsonpath={.status.readyReplicas}' 2>/dev/null || true); [ -n "$desired" ] && [ "$desired" -gt 0 ] 2>/dev/null && [ "\${ready:-0}" = "$desired" ] && printf true || printf false; }
|
|
endpoint_ready() { endpoints=$(kubectl -n "$1" get endpoints "$2" -o 'jsonpath={.subsets[*].addresses[*].ip}' 2>/dev/null || true); [ -n "$endpoints" ] && printf true || printf false; }
|
|
can_i_runtime() { kubectl auth can-i "$1" "$2" --as="system:serviceaccount:$ci_ns:$service_account" -n "$runtime_ns" >/dev/null 2>&1 && printf true || printf false; }
|
|
can_i_argo() { kubectl auth can-i "$1" "$2" --as="system:serviceaccount:$ci_ns:$service_account" -n "$argo_ns" >/dev/null 2>&1 && printf true || printf false; }
|
|
runtime_observer_role_exists=$(exists_res "$runtime_ns" role "$runtime_observer_role")
|
|
runtime_observer_rolebinding_exists=$(exists_res "$runtime_ns" rolebinding "$runtime_observer_rolebinding")
|
|
runtime_observer_can_list_deployments=$(can_i_runtime list deployments.apps)
|
|
runtime_observer_can_list_statefulsets=$(can_i_runtime list statefulsets.apps)
|
|
runtime_observer_ready=false
|
|
if [ "$runtime_observer_role_exists" = true ] && [ "$runtime_observer_rolebinding_exists" = true ] && [ "$runtime_observer_can_list_deployments" = true ] && [ "$runtime_observer_can_list_statefulsets" = true ]; then runtime_observer_ready=true; fi
|
|
argo_observer_role_exists=$(exists_res "$argo_ns" role "$argo_observer_role")
|
|
argo_observer_rolebinding_exists=$(exists_res "$argo_ns" rolebinding "$argo_observer_rolebinding")
|
|
argo_observer_can_get_application=$(can_i_argo get applications.argoproj.io)
|
|
argo_observer_ready=false
|
|
if [ "$argo_observer_role_exists" = true ] && [ "$argo_observer_rolebinding_exists" = true ] && [ "$argo_observer_can_get_application" = true ]; then argo_observer_ready=true; fi
|
|
github_transport_json=$(python3 - "$github_transport_mode" "$gitmirror_ns" "$github_ssh_secret" "$github_ssh_private_key" "$github_ssh_private_source_ref" "$github_ssh_private_source_key" "$github_ssh_known_hosts_key" "$github_ssh_known_hosts_source_ref" "$github_ssh_known_hosts_source_key" "$github_token_secret" "$github_token_key" "$github_token_source_ref" "$github_token_source_key" <<'PY'
|
|
import hashlib, json, subprocess, sys
|
|
mode, namespace, ssh_secret, ssh_private_key, ssh_private_source_ref, ssh_private_source_key, ssh_known_hosts_key, ssh_known_hosts_source_ref, ssh_known_hosts_source_key, token_secret, token_key, token_source_ref, token_source_key = sys.argv[1:14]
|
|
def read_secret(name):
|
|
proc = subprocess.run(["kubectl", "-n", namespace, "get", "secret", name, "-o", "json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
if proc.returncode != 0:
|
|
return False, {}, {}
|
|
try:
|
|
obj = json.loads(proc.stdout)
|
|
except Exception:
|
|
obj = {}
|
|
return True, obj.get("data") or {}, obj.get("metadata", {}).get("annotations") or {}
|
|
def fingerprint(value):
|
|
return "sha256:" + hashlib.sha256(value.encode()).hexdigest()[:16] if value else None
|
|
if mode == "ssh":
|
|
exists, data, annotations = read_secret(ssh_secret)
|
|
private_encoded = data.get(ssh_private_key) if isinstance(data, dict) else None
|
|
known_hosts_encoded = data.get(ssh_known_hosts_key) if ssh_known_hosts_key and isinstance(data, dict) else None
|
|
private_present = isinstance(private_encoded, str) and len(private_encoded) > 0
|
|
known_hosts_expected = bool(ssh_known_hosts_key)
|
|
known_hosts_present = isinstance(known_hosts_encoded, str) and len(known_hosts_encoded) > 0
|
|
print(json.dumps({
|
|
"mode": mode,
|
|
"required": True,
|
|
"ready": exists and private_present and (not known_hosts_expected or known_hosts_present),
|
|
"secretName": ssh_secret,
|
|
"privateKeySecretKey": ssh_private_key,
|
|
"privateKeySourceRef": ssh_private_source_ref,
|
|
"privateKeySourceKey": ssh_private_source_key,
|
|
"privateKeySecretExists": exists,
|
|
"privateKeyPresent": private_present,
|
|
"privateKeyBytes": len(private_encoded) if private_present else 0,
|
|
"privateKeyFingerprint": annotations.get("unidesk.ai/private-key-fingerprint") or fingerprint(private_encoded),
|
|
"knownHostsSecretKey": ssh_known_hosts_key or None,
|
|
"knownHostsSourceRef": ssh_known_hosts_source_ref or None,
|
|
"knownHostsSourceKey": ssh_known_hosts_source_key or None,
|
|
"knownHostsPresent": (known_hosts_present if known_hosts_expected else None),
|
|
"knownHostsBytes": (len(known_hosts_encoded) if known_hosts_present else 0) if known_hosts_expected else None,
|
|
"knownHostsFingerprint": annotations.get("unidesk.ai/known-hosts-fingerprint") or fingerprint(known_hosts_encoded),
|
|
"valuesPrinted": False,
|
|
}))
|
|
raise SystemExit(0)
|
|
if mode != "https":
|
|
print(json.dumps({"mode": mode, "required": True, "ready": False, "valuesPrinted": False}))
|
|
raise SystemExit(0)
|
|
exists, data, _ = read_secret(token_secret)
|
|
encoded = data.get(token_key) if isinstance(data, dict) else None
|
|
present = isinstance(encoded, str) and len(encoded) > 0
|
|
print(json.dumps({"mode": mode, "required": True, "ready": exists and present, "tokenSecretName": token_secret, "tokenSecretKey": token_key, "tokenSourceRef": token_source_ref, "tokenSourceKey": token_source_key, "tokenSecretExists": exists, "tokenKeyPresent": present, "tokenKeyBytes": len(encoded) if present else 0, "tokenFingerprint": fingerprint(encoded), "valuesPrinted": False}))
|
|
PY
|
|
)
|
|
ci_git_workspace_json=$(python3 - "$ci_ns" "$ci_git_secret" "$ci_git_private_key" "$ci_git_known_hosts_key" "$ci_git_source_ref_from" "$ci_git_private_source_ref" "$ci_git_private_source_key" "$ci_git_known_hosts_source_ref" "$ci_git_known_hosts_source_key" <<'PY'
|
|
import hashlib, json, subprocess, sys
|
|
namespace, secret, private_key, known_hosts_key, source_ref_from, private_source_ref, private_source_key, known_hosts_source_ref, known_hosts_source_key = sys.argv[1:10]
|
|
proc = subprocess.run(["kubectl", "-n", namespace, "get", "secret", secret, "-o", "json"], stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
exists = proc.returncode == 0
|
|
obj = {}
|
|
if exists:
|
|
try:
|
|
obj = json.loads(proc.stdout)
|
|
except Exception:
|
|
obj = {}
|
|
data = obj.get("data") or {}
|
|
annotations = obj.get("metadata", {}).get("annotations") or {}
|
|
def fingerprint(value):
|
|
return "sha256:" + hashlib.sha256(value.encode()).hexdigest()[:16] if value else None
|
|
private_encoded = data.get(private_key) if isinstance(data, dict) else None
|
|
known_hosts_encoded = data.get(known_hosts_key) if isinstance(data, dict) else None
|
|
private_present = isinstance(private_encoded, str) and len(private_encoded) > 0
|
|
known_hosts_present = isinstance(known_hosts_encoded, str) and len(known_hosts_encoded) > 0
|
|
print(json.dumps({
|
|
"required": True,
|
|
"ready": exists and private_present and known_hosts_present,
|
|
"namespace": namespace,
|
|
"secretName": secret,
|
|
"sourceRefFrom": source_ref_from,
|
|
"privateKeySecretKey": private_key,
|
|
"privateKeySourceRef": private_source_ref,
|
|
"privateKeySourceKey": private_source_key,
|
|
"privateKeySecretExists": exists,
|
|
"privateKeyPresent": private_present,
|
|
"privateKeyBytes": len(private_encoded) if private_present else 0,
|
|
"privateKeyFingerprint": annotations.get("unidesk.ai/private-key-fingerprint") or fingerprint(private_encoded),
|
|
"knownHostsSecretKey": known_hosts_key,
|
|
"knownHostsSourceRef": known_hosts_source_ref,
|
|
"knownHostsSourceKey": known_hosts_source_key,
|
|
"knownHostsPresent": known_hosts_present,
|
|
"knownHostsBytes": len(known_hosts_encoded) if known_hosts_present else 0,
|
|
"knownHostsFingerprint": annotations.get("unidesk.ai/known-hosts-fingerprint") or fingerprint(known_hosts_encoded),
|
|
"valuesPrinted": False,
|
|
}))
|
|
PY
|
|
)
|
|
registry_endpoint_ready=false
|
|
if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 3 "http://$registry/v2/" >/tmp/hwlab-registry.out 2>/tmp/hwlab-registry.err && registry_endpoint_ready=true; fi
|
|
registry_workload_json=$(python3 - "$registry_spec_json" "$registry_endpoint_ready" <<'PY'
|
|
import json, subprocess, sys
|
|
spec=json.loads(sys.argv[1])
|
|
endpoint_ready=sys.argv[2] == "true"
|
|
def run(args):
|
|
return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
def exists(kind, namespace, name):
|
|
return run(["kubectl", "-n", namespace, "get", kind, name]).returncode == 0
|
|
def deploy_ready(namespace, name):
|
|
proc=run(["kubectl", "-n", namespace, "get", "deploy", name, "-o", "json"])
|
|
if proc.returncode != 0:
|
|
return {"exists": False, "ready": False, "desired": None, "readyReplicas": None}
|
|
try:
|
|
obj=json.loads(proc.stdout or "{}")
|
|
except Exception:
|
|
obj={}
|
|
desired=int(obj.get("spec", {}).get("replicas") or 0)
|
|
ready=int(obj.get("status", {}).get("readyReplicas") or 0)
|
|
return {"exists": True, "ready": desired > 0 and ready == desired, "desired": desired, "readyReplicas": ready}
|
|
if spec.get("mode") != "k8s-workload":
|
|
print(json.dumps({"mode": spec.get("mode") or "host-docker", "managed": False, "ready": endpoint_ready, "endpointReady": endpoint_ready, "valuesPrinted": False}))
|
|
raise SystemExit(0)
|
|
ns=str(spec.get("namespace") or "")
|
|
deploy=str(spec.get("deploymentName") or "")
|
|
svc=str(spec.get("serviceName") or "")
|
|
pvc=str(spec.get("pvcName") or "")
|
|
deployment=deploy_ready(ns, deploy)
|
|
namespace_exists=run(["kubectl", "get", "ns", ns]).returncode == 0
|
|
pvc_exists=exists("pvc", ns, pvc)
|
|
service_exists=exists("service", ns, svc)
|
|
ready=endpoint_ready and namespace_exists and pvc_exists and service_exists and deployment.get("ready") is True
|
|
print(json.dumps({
|
|
"mode": "k8s-workload",
|
|
"managed": True,
|
|
"ready": ready,
|
|
"endpointReady": endpoint_ready,
|
|
"namespace": ns,
|
|
"namespaceExists": namespace_exists,
|
|
"deploymentName": deploy,
|
|
"deployment": deployment,
|
|
"serviceName": svc,
|
|
"serviceExists": service_exists,
|
|
"pvcName": pvc,
|
|
"pvcExists": pvc_exists,
|
|
"seededFromOldRegistry": False,
|
|
"zeroSeeded": True,
|
|
"valuesPrinted": False,
|
|
}))
|
|
PY
|
|
)
|
|
registry_ready=$(python3 - "$registry_workload_json" <<'PY'
|
|
import json, sys
|
|
try:
|
|
data=json.loads(sys.argv[1] or "{}")
|
|
except Exception:
|
|
data={}
|
|
print("true" if data.get("ready") is True else "false")
|
|
PY
|
|
)
|
|
tools_repo_tag=\${tools_image#\${registry}/}
|
|
tools_repo=\${tools_repo_tag%:*}
|
|
tools_tag=\${tools_repo_tag##*:}
|
|
tools_image_ready=false
|
|
manifest_accept='application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.list.v2+json'
|
|
if [ "$tools_repo" != "$tools_repo_tag" ] && command -v curl >/dev/null 2>&1; then curl -fsS --max-time 5 -H "Accept: $manifest_accept" "http://$registry/v2/$tools_repo/manifests/$tools_tag" >/tmp/hwlab-tools-image.out 2>/tmp/hwlab-tools-image.err && tools_image_ready=true; fi
|
|
cache_host_path_ready=false
|
|
if [ -n "$cache_host_path" ] && kubectl -n "$gitmirror_ns" exec deploy/"$read_deploy" -- sh -lc 'test -d /cache' >/dev/null 2>&1; then cache_host_path_ready=true; fi
|
|
k3s_fragment=$(python3 - "$k3s_managed" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_expected_sha" <<'PY'
|
|
import hashlib, json, re, subprocess, sys
|
|
managed = sys.argv[1] == "true"
|
|
service, dropin, node_name, desired_raw, expected_sha = sys.argv[2:7]
|
|
def run(args):
|
|
return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
def to_int(value):
|
|
try:
|
|
return int(value)
|
|
except Exception:
|
|
return None
|
|
if not managed:
|
|
print(json.dumps({"managed": False, "ready": True}))
|
|
raise SystemExit(0)
|
|
desired = to_int(desired_raw)
|
|
node_json = run(["kubectl", "get", "node", node_name, "-o", "json"])
|
|
capacity = None
|
|
allocatable = None
|
|
node_ready = False
|
|
if node_json.returncode == 0:
|
|
data = json.loads(node_json.stdout)
|
|
capacity = to_int(data.get("status", {}).get("capacity", {}).get("pods"))
|
|
allocatable = to_int(data.get("status", {}).get("allocatable", {}).get("pods"))
|
|
for condition in data.get("status", {}).get("conditions", []):
|
|
if condition.get("type") == "Ready":
|
|
node_ready = condition.get("status") == "True"
|
|
unit = run(["systemctl", "cat", service])
|
|
unit_text = unit.stdout if unit.returncode == 0 else ""
|
|
dropin_read = run(["cat", dropin])
|
|
dropin_exists = dropin_read.returncode == 0
|
|
dropin_text = dropin_read.stdout if dropin_exists else ""
|
|
dropin_sha = "sha256:" + hashlib.sha256(dropin_text.encode()).hexdigest() if dropin_exists else None
|
|
matches = re.findall(r"max-pods=([0-9]+)", unit_text + "\\n" + dropin_text)
|
|
configured = to_int(matches[-1]) if matches else None
|
|
dropin_matches = dropin_sha == expected_sha
|
|
ready = dropin_matches and capacity == desired and allocatable == desired
|
|
source = "managed-dropin" if dropin_matches else ("systemd-or-config" if configured is not None else "kubelet-default")
|
|
print(json.dumps({
|
|
"managed": True,
|
|
"ready": ready,
|
|
"serviceName": service,
|
|
"dropInPath": dropin,
|
|
"dropInExists": dropin_exists,
|
|
"dropInSha256": dropin_sha,
|
|
"expectedDropInSha256": expected_sha,
|
|
"dropInMatches": dropin_matches,
|
|
"configuredMaxPods": configured,
|
|
"desiredMaxPods": desired,
|
|
"liveNodeName": node_name,
|
|
"liveCapacityPods": capacity,
|
|
"liveAllocatablePods": allocatable,
|
|
"nodeReady": node_ready,
|
|
"restartRequired": not ready,
|
|
"source": source,
|
|
"unitReadable": unit.returncode == 0,
|
|
}))
|
|
PY
|
|
)
|
|
python3 - "$tekton_required_crds_json" "$tekton_deployment_namespaces_json" "$tekton_runtime_proxy_json" <<'PY' >/tmp/hwlab-node-tekton-status-fragments.json
|
|
import json, subprocess, sys
|
|
required_crds=json.loads(sys.argv[1])
|
|
namespaces=json.loads(sys.argv[2])
|
|
runtime_proxy=json.loads(sys.argv[3])
|
|
def run(args):
|
|
return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
def exists(args):
|
|
return run(args).returncode == 0
|
|
def env_map(container):
|
|
result={}
|
|
for item in container.get("env") or []:
|
|
name=item.get("name")
|
|
if isinstance(name, str) and "value" in item:
|
|
result[name]=str(item.get("value") or "")
|
|
return result
|
|
def runtime_proxy_workload(kind, namespace, name, cfg):
|
|
proc=run(["kubectl", "-n", namespace, "get", kind, name, "-o", "json"])
|
|
if proc.returncode != 0:
|
|
return {"kind": kind, "namespace": namespace, "name": name, "exists": False, "ready": False}
|
|
obj=json.loads(proc.stdout or "{}")
|
|
template=obj.get("spec", {}).get("template", {})
|
|
spec=template.get("spec", {})
|
|
annotations=template.get("metadata", {}).get("annotations") or {}
|
|
containers=spec.get("containers") or []
|
|
expected_env={
|
|
"HTTP_PROXY": str(cfg.get("proxyUrl") or ""),
|
|
"HTTPS_PROXY": str(cfg.get("proxyUrl") or ""),
|
|
"ALL_PROXY": str(cfg.get("proxyUrl") or ""),
|
|
"NO_PROXY": ",".join([str(item) for item in cfg.get("noProxy") or []]),
|
|
"http_proxy": str(cfg.get("proxyUrl") or ""),
|
|
"https_proxy": str(cfg.get("proxyUrl") or ""),
|
|
"all_proxy": str(cfg.get("proxyUrl") or ""),
|
|
"no_proxy": ",".join([str(item) for item in cfg.get("noProxy") or []]),
|
|
}
|
|
env_matches=True
|
|
if cfg.get("injectEnv"):
|
|
env_matches=bool(containers) and all(all(env_map(container).get(key) == value for key, value in expected_env.items()) for container in containers)
|
|
host_network_matches=(not cfg.get("hostNetwork")) or spec.get("hostNetwork") is True
|
|
dns_policy_matches=(not cfg.get("hostNetwork")) or spec.get("dnsPolicy") == "ClusterFirstWithHostNet"
|
|
annotation_matches=annotations.get("unidesk.ai/runtime-proxy") == "host-route" and annotations.get("unidesk.ai/runtime-proxy-config-ref") == str(cfg.get("configRef") or "")
|
|
return {
|
|
"kind": kind,
|
|
"namespace": namespace,
|
|
"name": name,
|
|
"exists": True,
|
|
"ready": host_network_matches and dns_policy_matches and env_matches and annotation_matches,
|
|
"hostNetwork": spec.get("hostNetwork") is True,
|
|
"hostNetworkMatches": host_network_matches,
|
|
"dnsPolicy": spec.get("dnsPolicy") or None,
|
|
"dnsPolicyMatches": dns_policy_matches,
|
|
"injectEnv": bool(cfg.get("injectEnv")),
|
|
"envMatches": env_matches,
|
|
"annotationMatches": annotation_matches,
|
|
}
|
|
def runtime_proxy_status(cfg, namespace_values):
|
|
if not cfg.get("enabled"):
|
|
return {"enabled": False, "ready": True, "mode": cfg.get("mode") or "host-route", "workloads": [], "valuesPrinted": False}
|
|
workloads=[]
|
|
missing=[]
|
|
for kind, names in (("deployment", cfg.get("deployments") or []), ("statefulset", cfg.get("statefulSets") or [])):
|
|
for name in [str(item) for item in names]:
|
|
found=[]
|
|
for namespace in namespace_values:
|
|
proc=run(["kubectl", "-n", namespace, "get", kind, name, "-o", "name"])
|
|
if proc.returncode == 0:
|
|
found.append(namespace)
|
|
if not found:
|
|
missing.append({"kind": kind, "name": name})
|
|
workloads.append({"kind": kind, "name": name, "exists": False, "ready": False})
|
|
else:
|
|
workloads.extend(runtime_proxy_workload(kind, namespace, name, cfg) for namespace in found)
|
|
return {
|
|
"enabled": True,
|
|
"mode": cfg.get("mode") or "host-route",
|
|
"configRef": cfg.get("configRef"),
|
|
"hostNetwork": bool(cfg.get("hostNetwork")),
|
|
"injectEnv": bool(cfg.get("injectEnv")),
|
|
"selectedDeployments": cfg.get("deployments") or [],
|
|
"selectedStatefulSets": cfg.get("statefulSets") or [],
|
|
"missing": missing,
|
|
"workloads": workloads,
|
|
"ready": len(missing) == 0 and all(item.get("ready") for item in workloads),
|
|
"valuesPrinted": False,
|
|
}
|
|
def namespace_deployments(namespace):
|
|
proc = run(["kubectl", "-n", namespace, "get", "deploy", "-o", "json"])
|
|
if proc.returncode != 0:
|
|
return {"namespace": namespace, "namespaceExists": False, "deployments": [], "ready": False}
|
|
try:
|
|
data=json.loads(proc.stdout)
|
|
except Exception:
|
|
return {"namespace": namespace, "namespaceExists": True, "deployments": [], "ready": False}
|
|
deployments=[]
|
|
for item in data.get("items", []):
|
|
desired=int(item.get("spec", {}).get("replicas") or 0)
|
|
ready=int(item.get("status", {}).get("readyReplicas") or 0)
|
|
deployments.append({"name": item.get("metadata", {}).get("name"), "desired": desired, "readyReplicas": ready, "ready": desired > 0 and ready == desired})
|
|
return {"namespace": namespace, "namespaceExists": True, "deployments": deployments, "ready": len(deployments) > 0 and all(item["ready"] for item in deployments)}
|
|
crds=[{"name": name, "exists": exists(["kubectl", "get", "crd", name])} for name in required_crds]
|
|
deployment_namespaces=[namespace_deployments(ns) for ns in namespaces]
|
|
print(json.dumps({"crds": crds, "deploymentNamespaces": deployment_namespaces, "crdsReady": all(item["exists"] for item in crds), "deploymentsReady": all(item["ready"] for item in deployment_namespaces) if deployment_namespaces else True, "runtimeProxy": runtime_proxy_status(runtime_proxy, namespaces)}))
|
|
PY
|
|
tekton_fragment=$(cat /tmp/hwlab-node-tekton-status-fragments.json 2>/dev/null || printf '{}')
|
|
tekton_installed=$(python3 - "$tekton_fragment" <<'PY'
|
|
import json, sys
|
|
data=json.loads(sys.argv[1] or '{}')
|
|
print('true' if data.get('crdsReady') and data.get('deploymentsReady') else 'false')
|
|
PY
|
|
)
|
|
python3 - "$required_crds_json" "$argo_deployments_json" "$argo_statefulsets_json" "$argo_runtime_proxy_json" "$argo_internal_http_json" "$argo_reconciliation_json" <<'PY' >/tmp/hwlab-node-status-fragments.json
|
|
import json, subprocess, sys
|
|
required_crds=json.loads(sys.argv[1])
|
|
deployments=json.loads(sys.argv[2])
|
|
statefulsets=json.loads(sys.argv[3])
|
|
runtime_proxy=json.loads(sys.argv[4])
|
|
internal_http=json.loads(sys.argv[5])
|
|
reconciliation=json.loads(sys.argv[6])
|
|
ns="${target.argo.namespace}"
|
|
def run(args):
|
|
return subprocess.run(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True)
|
|
def exists(args):
|
|
return run(args).returncode == 0
|
|
def env_map(container):
|
|
result={}
|
|
for item in container.get("env") or []:
|
|
name=item.get("name")
|
|
if isinstance(name, str) and "value" in item:
|
|
result[name]=str(item.get("value") or "")
|
|
return result
|
|
def runtime_proxy_workload(kind, namespace, name, cfg):
|
|
proc=run(["kubectl", "-n", namespace, "get", kind, name, "-o", "json"])
|
|
if proc.returncode != 0:
|
|
return {"kind": kind, "namespace": namespace, "name": name, "exists": False, "ready": False}
|
|
obj=json.loads(proc.stdout or "{}")
|
|
template=obj.get("spec", {}).get("template", {})
|
|
spec=template.get("spec", {})
|
|
annotations=template.get("metadata", {}).get("annotations") or {}
|
|
containers=spec.get("containers") or []
|
|
expected_env={
|
|
"HTTP_PROXY": str(cfg.get("proxyUrl") or ""),
|
|
"HTTPS_PROXY": str(cfg.get("proxyUrl") or ""),
|
|
"ALL_PROXY": str(cfg.get("proxyUrl") or ""),
|
|
"NO_PROXY": ",".join([str(item) for item in cfg.get("noProxy") or []]),
|
|
"http_proxy": str(cfg.get("proxyUrl") or ""),
|
|
"https_proxy": str(cfg.get("proxyUrl") or ""),
|
|
"all_proxy": str(cfg.get("proxyUrl") or ""),
|
|
"no_proxy": ",".join([str(item) for item in cfg.get("noProxy") or []]),
|
|
}
|
|
env_matches=True
|
|
if cfg.get("injectEnv"):
|
|
env_matches=bool(containers) and all(all(env_map(container).get(key) == value for key, value in expected_env.items()) for container in containers)
|
|
host_network_matches=(not cfg.get("hostNetwork")) or spec.get("hostNetwork") is True
|
|
dns_policy_matches=(not cfg.get("hostNetwork")) or spec.get("dnsPolicy") == "ClusterFirstWithHostNet"
|
|
annotation_matches=annotations.get("unidesk.ai/runtime-proxy") == "host-route" and annotations.get("unidesk.ai/runtime-proxy-config-ref") == str(cfg.get("configRef") or "")
|
|
return {
|
|
"kind": kind,
|
|
"namespace": namespace,
|
|
"name": name,
|
|
"exists": True,
|
|
"ready": host_network_matches and dns_policy_matches and env_matches and annotation_matches,
|
|
"hostNetwork": spec.get("hostNetwork") is True,
|
|
"hostNetworkMatches": host_network_matches,
|
|
"dnsPolicy": spec.get("dnsPolicy") or None,
|
|
"dnsPolicyMatches": dns_policy_matches,
|
|
"injectEnv": bool(cfg.get("injectEnv")),
|
|
"envMatches": env_matches,
|
|
"annotationMatches": annotation_matches,
|
|
}
|
|
def runtime_proxy_status(cfg):
|
|
if not cfg.get("enabled"):
|
|
return {"enabled": False, "ready": True, "mode": cfg.get("mode") or "host-route", "workloads": [], "valuesPrinted": False}
|
|
workloads=[]
|
|
missing=[]
|
|
for kind, names in (("deployment", cfg.get("deployments") or []), ("statefulset", cfg.get("statefulSets") or [])):
|
|
for name in [str(item) for item in names]:
|
|
proc=run(["kubectl", "-n", ns, "get", kind, name, "-o", "name"])
|
|
if proc.returncode != 0:
|
|
missing.append({"kind": kind, "name": name})
|
|
workloads.append({"kind": kind, "name": name, "exists": False, "ready": False})
|
|
else:
|
|
workloads.append(runtime_proxy_workload(kind, ns, name, cfg))
|
|
return {
|
|
"enabled": True,
|
|
"mode": cfg.get("mode") or "host-route",
|
|
"configRef": cfg.get("configRef"),
|
|
"hostNetwork": bool(cfg.get("hostNetwork")),
|
|
"injectEnv": bool(cfg.get("injectEnv")),
|
|
"selectedDeployments": cfg.get("deployments") or [],
|
|
"selectedStatefulSets": cfg.get("statefulSets") or [],
|
|
"missing": missing,
|
|
"workloads": workloads,
|
|
"ready": len(missing) == 0 and all(item.get("ready") for item in workloads),
|
|
"valuesPrinted": False,
|
|
}
|
|
def ready(kind, name):
|
|
data = run(["kubectl", "-n", ns, "get", kind, name, "-o", "json"])
|
|
if data.returncode != 0:
|
|
return {"name": name, "exists": False, "ready": False, "desired": None, "readyReplicas": None}
|
|
obj=json.loads(data.stdout)
|
|
desired=int(obj.get("spec", {}).get("replicas") or 0)
|
|
ready_replicas=int(obj.get("status", {}).get("readyReplicas") or 0)
|
|
return {"name": name, "exists": True, "ready": desired > 0 and ready_replicas == desired, "desired": desired, "readyReplicas": ready_replicas}
|
|
def internal_http_status(cfg):
|
|
if not isinstance(cfg, dict):
|
|
return {"enabled": False, "ready": True, "valuesPrinted": False}
|
|
config_map=run(["kubectl", "-n", ns, "get", "configmap", str(cfg.get("configMapName") or ""), "-o", "json"])
|
|
deployment=run(["kubectl", "-n", ns, "get", "deployment", str(cfg.get("deploymentName") or ""), "-o", "json"])
|
|
config={}
|
|
workload={}
|
|
if config_map.returncode == 0:
|
|
try:
|
|
config=json.loads(config_map.stdout or "{}")
|
|
except Exception:
|
|
config={}
|
|
if deployment.returncode == 0:
|
|
try:
|
|
workload=json.loads(deployment.stdout or "{}")
|
|
except Exception:
|
|
workload={}
|
|
config_value=str((config.get("data") or {}).get(str(cfg.get("configKey") or "")) or "")
|
|
annotation_value=str((((workload.get("spec") or {}).get("template") or {}).get("metadata") or {}).get("annotations", {}).get(str(cfg.get("rolloutAnnotationKey") or "")) or "")
|
|
config_matches=config_value == "true"
|
|
rollout_matches=annotation_value == str(cfg.get("rolloutIdentity") or "")
|
|
return {
|
|
"enabled": True,
|
|
"ready": config_map.returncode == 0 and deployment.returncode == 0 and config_matches and rollout_matches,
|
|
"configMapName": cfg.get("configMapName"),
|
|
"configKey": cfg.get("configKey"),
|
|
"configMatches": config_matches,
|
|
"deploymentName": cfg.get("deploymentName"),
|
|
"rolloutAnnotationKey": cfg.get("rolloutAnnotationKey"),
|
|
"rolloutMatches": rollout_matches,
|
|
"valuesPrinted": False,
|
|
}
|
|
${argoReconciliationStatusPython()}
|
|
crds=[{"name": name, "exists": exists(["kubectl", "get", "crd", name])} for name in required_crds]
|
|
deploy=[ready("deployment", name) for name in deployments]
|
|
sts=[ready("statefulset", name) for name in statefulsets]
|
|
print(json.dumps({"crds": crds, "deployments": deploy, "statefulSets": sts, "crdsReady": all(item["exists"] for item in crds), "deploymentsReady": all(item["ready"] for item in deploy) if deploy else True, "statefulSetsReady": all(item["ready"] for item in sts) if sts else True, "runtimeProxy": runtime_proxy_status(runtime_proxy), "internalHttp": internal_http_status(internal_http), "reconciliation": reconciliation_status(reconciliation)}))
|
|
PY
|
|
argo_fragment=$(cat /tmp/hwlab-node-status-fragments.json 2>/dev/null || printf '{}')
|
|
cat <<JSON
|
|
{"observedAt":"$(date -u +%Y-%m-%dT%H:%M:%SZ)","node":"$node","lane":"$lane","components":{"k3sNodeConfig":$k3s_fragment,"tekton":{"installed":$tekton_installed,"controllerReady":$(deploy_ready tekton-pipelines tekton-pipelines-controller),"webhookReady":$(deploy_ready tekton-pipelines tekton-pipelines-webhook),"install":$tekton_fragment},"ciNamespace":{"name":"$ci_ns","exists":$(exists_ns "$ci_ns"),"serviceAccountExists":$(exists_res "$ci_ns" serviceaccount "$service_account"),"pipelineExists":$(exists_res "$ci_ns" pipeline "$pipeline"),"gitWorkspaceSecret":$ci_git_workspace_json},"gitMirror":{"namespace":"$gitmirror_ns","namespaceExists":$(exists_ns "$gitmirror_ns"),"readDeploymentReady":$(deploy_ready "$gitmirror_ns" "$read_deploy"),"writeDeploymentReady":$(deploy_ready "$gitmirror_ns" "$write_deploy"),"readServiceExists":$(exists_res "$gitmirror_ns" service "$read_svc"),"writeServiceExists":$(exists_res "$gitmirror_ns" service "$write_svc"),"readEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$read_svc"),"writeEndpointsReady":$(endpoint_ready "$gitmirror_ns" "$write_svc"),"cachePvcExists":$(exists_res "$gitmirror_ns" pvc "$cache_pvc"),"cacheHostPath":"$cache_host_path","cacheHostPathReady":$cache_host_path_ready,"egressProxy":$gitmirror_egress_proxy_json,"githubTransport":$github_transport_json,"summary":{"localSource":null,"githubSource":null,"localGitops":null,"githubGitops":null,"pendingFlush":null,"flushNeeded":null,"githubInSync":null}},"argo":{"namespace":"$argo_ns","namespaceExists":$(exists_ns "$argo_ns"),"installed":$(kubectl get crd applications.argoproj.io appprojects.argoproj.io >/dev/null 2>&1 && printf true || printf false),"projectExists":$(kubectl -n "$argo_ns" get appproject "$argo_project" >/dev/null 2>&1 && printf true || printf false),"applicationExists":$(kubectl -n "$argo_ns" get application "$argo_app" >/dev/null 2>&1 && printf true || printf false),"argoObserverRbac":{"roleName":"$argo_observer_role","roleExists":$argo_observer_role_exists,"roleBindingName":"$argo_observer_rolebinding","roleBindingExists":$argo_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canGetApplication":$argo_observer_can_get_application,"ready":$argo_observer_ready},"install":$argo_fragment},"registry":{"endpoint":"$registry","ready":$registry_ready,"endpointReady":$registry_endpoint_ready,"workload":$registry_workload_json,"toolsImage":"$tools_image","toolsImageReady":$tools_image_ready},"runtimeNamespace":{"name":"$runtime_ns","exists":$(exists_ns "$runtime_ns"),"runtimeObserverRbac":{"roleName":"$runtime_observer_role","roleExists":$runtime_observer_role_exists,"roleBindingName":"$runtime_observer_rolebinding","roleBindingExists":$runtime_observer_rolebinding_exists,"serviceAccountNamespace":"$ci_ns","serviceAccountName":"$service_account","canListDeployments":$runtime_observer_can_list_deployments,"canListStatefulSets":$runtime_observer_can_list_statefulsets,"ready":$runtime_observer_ready}}}}
|
|
JSON
|
|
`;
|
|
}
|
|
|
|
export function applyScript(yaml: string, node: ControlPlaneNodeSpec, target: ControlPlaneTargetSpec): string {
|
|
const encoded = Buffer.from(yaml, "utf8").toString("base64");
|
|
return `
|
|
set +e
|
|
manifest=$(mktemp /tmp/hwlab-node-infra.XXXXXX.yaml)
|
|
printf %s ${shQuote(encoded)} | base64 -d >"$manifest"
|
|
field_manager=${shQuote(controlPlaneFieldManager(target))}
|
|
${registryPreApplyScript(node)}
|
|
kubectl apply --server-side --force-conflicts --field-manager="$field_manager" -f "$manifest" >/tmp/hwlab-node-infra-apply.out 2>/tmp/hwlab-node-infra-apply.err
|
|
kubectl_rc=$?
|
|
${k3sApplyScriptFragment(node.k3s, target)}
|
|
${registryPostApplyScript(node)}
|
|
python3 - "$kubectl_rc" "$k3s_report_file" "$registry_pre_report_file" "$registry_report_file" <<'PY'
|
|
import json, pathlib, sys
|
|
k3s_report = {}
|
|
try:
|
|
k3s_report = json.loads(pathlib.Path(sys.argv[2]).read_text(errors='replace'))
|
|
except Exception as exc:
|
|
k3s_report = {"managed": None, "ok": False, "parseError": str(exc)}
|
|
registry_pre = {}
|
|
try:
|
|
registry_pre = json.loads(pathlib.Path(sys.argv[3]).read_text(errors='replace'))
|
|
except Exception as exc:
|
|
registry_pre = {"managed": None, "ok": False, "parseError": str(exc)}
|
|
registry_report = {}
|
|
try:
|
|
registry_report = json.loads(pathlib.Path(sys.argv[4]).read_text(errors='replace'))
|
|
except Exception as exc:
|
|
registry_report = {"managed": None, "ok": False, "parseError": str(exc)}
|
|
out=pathlib.Path('/tmp/hwlab-node-infra-apply.out').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.out').exists() else ''
|
|
err=pathlib.Path('/tmp/hwlab-node-infra-apply.err').read_text(errors='replace') if pathlib.Path('/tmp/hwlab-node-infra-apply.err').exists() else ''
|
|
print(json.dumps({'k3sNodeConfig': k3s_report, 'registryMigration': {'preApply': registry_pre, 'postApply': registry_report}, 'kubernetesApply': {'applyExitCode': int(sys.argv[1]), 'stdoutPreview': out[-2000:], 'stderrPreview': err[-2000:], 'runtimeRolloutTriggered': False, 'pk01Touched': False}}, ensure_ascii=False))
|
|
PY
|
|
rm -f "$manifest"
|
|
if [ "$registry_pre_rc" != 0 ]; then exit "$registry_pre_rc"; fi
|
|
if [ "$kubectl_rc" != 0 ]; then exit "$kubectl_rc"; fi
|
|
if [ "$registry_rc" != 0 ]; then exit "$registry_rc"; fi
|
|
exit "$k3s_rc"
|
|
`;
|
|
}
|
|
|
|
export function registryPreApplyScript(node: ControlPlaneNodeSpec): string {
|
|
if (node.registry.mode !== "k8s-workload") {
|
|
return `
|
|
registry_pre_report_file=$(mktemp /tmp/hwlab-node-registry-pre.XXXXXX.json)
|
|
printf '{"managed":false,"ok":true,"mutation":false}\\n' >"$registry_pre_report_file"
|
|
registry_pre_rc=0
|
|
`;
|
|
}
|
|
const installRegistry = node.k3s?.install?.localRegistry ?? null;
|
|
return `
|
|
registry_pre_report_file=$(mktemp /tmp/hwlab-node-registry-pre.XXXXXX.json)
|
|
host_registry_name=${shQuote(installRegistry?.containerName ?? "")}
|
|
registry_pre_rc=0
|
|
host_registry_exists=false
|
|
host_registry_running=false
|
|
docker_stop_rc=0
|
|
if [ -n "$host_registry_name" ] && command -v docker >/dev/null 2>&1; then
|
|
if docker ps -a --format '{{.Names}}' 2>/dev/null | grep -Fx "$host_registry_name" >/dev/null 2>&1; then
|
|
host_registry_exists=true
|
|
if docker ps --format '{{.Names}}' 2>/dev/null | grep -Fx "$host_registry_name" >/dev/null 2>&1; then host_registry_running=true; fi
|
|
docker stop "$host_registry_name" >/tmp/hwlab-registry-docker-stop.out 2>/tmp/hwlab-registry-docker-stop.err || docker_stop_rc=$?
|
|
if [ "$docker_stop_rc" != 0 ]; then registry_pre_rc=$docker_stop_rc; fi
|
|
fi
|
|
fi
|
|
python3 - "$host_registry_exists" "$host_registry_running" "$docker_stop_rc" "$host_registry_name" <<'PY' >"$registry_pre_report_file"
|
|
import json, pathlib, sys
|
|
def tail(path):
|
|
p = pathlib.Path(path)
|
|
return p.read_text(errors='replace')[-1000:] if p.exists() else ''
|
|
print(json.dumps({
|
|
"managed": True,
|
|
"ok": int(sys.argv[3] or "0") == 0,
|
|
"mutation": sys.argv[1] == "true",
|
|
"mode": "k8s-workload",
|
|
"hostDockerRegistryName": sys.argv[4] or None,
|
|
"hostDockerRegistryExists": sys.argv[1] == "true",
|
|
"hostDockerRegistryWasRunning": sys.argv[2] == "true",
|
|
"stoppedBeforeApply": sys.argv[1] == "true" and int(sys.argv[3] or "0") == 0,
|
|
"seededFromOldRegistry": False,
|
|
"zeroSeeded": True,
|
|
"dockerStopExitCode": int(sys.argv[3] or "0"),
|
|
"dockerStopStderrTail": tail('/tmp/hwlab-registry-docker-stop.err'),
|
|
"valuesPrinted": False,
|
|
}, ensure_ascii=False))
|
|
PY
|
|
`;
|
|
}
|
|
|
|
export function registryPostApplyScript(node: ControlPlaneNodeSpec): string {
|
|
if (node.registry.mode !== "k8s-workload") {
|
|
return `
|
|
registry_report_file=$(mktemp /tmp/hwlab-node-registry.XXXXXX.json)
|
|
printf '{"managed":false,"ok":true,"mutation":false}\\n' >"$registry_report_file"
|
|
registry_rc=0
|
|
`;
|
|
}
|
|
return `
|
|
registry_report_file=$(mktemp /tmp/hwlab-node-registry.XXXXXX.json)
|
|
registry_ns=${shQuote(node.registry.namespace)}
|
|
registry_deploy=${shQuote(node.registry.deploymentName)}
|
|
registry_pvc=${shQuote(node.registry.pvcName)}
|
|
registry_endpoint=${shQuote(node.registry.endpoint)}
|
|
registry_rc=0
|
|
registry_pv=
|
|
registry_pv_path=
|
|
registry_pvc_bound=false
|
|
for _ in $(seq 1 60); do
|
|
registry_phase=$(kubectl -n "$registry_ns" get pvc "$registry_pvc" -o 'jsonpath={.status.phase}' 2>/dev/null || true)
|
|
if [ "$registry_phase" = Bound ]; then registry_pvc_bound=true; break; fi
|
|
sleep 2
|
|
done
|
|
if [ "$registry_pvc_bound" = true ]; then
|
|
registry_pv=$(kubectl -n "$registry_ns" get pvc "$registry_pvc" -o 'jsonpath={.spec.volumeName}' 2>/dev/null || true)
|
|
if [ -n "$registry_pv" ]; then
|
|
registry_pv_path=$(kubectl get pv "$registry_pv" -o 'jsonpath={.spec.hostPath.path}' 2>/dev/null || true)
|
|
if [ -z "$registry_pv_path" ]; then registry_pv_path=$(kubectl get pv "$registry_pv" -o 'jsonpath={.spec.local.path}' 2>/dev/null || true); fi
|
|
fi
|
|
fi
|
|
kubectl -n "$registry_ns" delete pod -l "app.kubernetes.io/name=$registry_deploy" --ignore-not-found >/tmp/hwlab-registry-pod-delete.out 2>/tmp/hwlab-registry-pod-delete.err || true
|
|
kubectl -n "$registry_ns" rollout status deploy/"$registry_deploy" --timeout=180s >/tmp/hwlab-registry-rollout.out 2>/tmp/hwlab-registry-rollout.err
|
|
registry_rollout_rc=$?
|
|
registry_endpoint_ready=false
|
|
if command -v curl >/dev/null 2>&1; then curl -fsS --max-time 5 "http://$registry_endpoint/v2/" >/tmp/hwlab-registry-endpoint.out 2>/tmp/hwlab-registry-endpoint.err && registry_endpoint_ready=true; fi
|
|
if [ "$registry_rollout_rc" != 0 ] || [ "$registry_endpoint_ready" != true ]; then registry_rc=1; fi
|
|
python3 - "$registry_rc" "$registry_pvc_bound" "$registry_pv" "$registry_pv_path" "$registry_rollout_rc" "$registry_endpoint_ready" "$registry_ns" "$registry_deploy" "$registry_pvc" "$registry_endpoint" <<'PY' >"$registry_report_file"
|
|
import json, pathlib, sys
|
|
def tail(path):
|
|
p = pathlib.Path(path)
|
|
return p.read_text(errors='replace')[-1000:] if p.exists() else ''
|
|
payload = {
|
|
"managed": True,
|
|
"ok": int(sys.argv[1]) == 0,
|
|
"mutation": True,
|
|
"mode": "k8s-workload",
|
|
"namespace": sys.argv[7],
|
|
"deploymentName": sys.argv[8],
|
|
"pvcName": sys.argv[9],
|
|
"endpoint": sys.argv[10],
|
|
"pvcBound": sys.argv[2] == "true",
|
|
"pvName": sys.argv[3] or None,
|
|
"pvPathPresent": bool(sys.argv[4]),
|
|
"seededFromOldRegistry": False,
|
|
"zeroSeeded": True,
|
|
"rolloutExitCode": int(sys.argv[5] or "1"),
|
|
"endpointReady": sys.argv[6] == "true",
|
|
"rolloutStdoutTail": tail('/tmp/hwlab-registry-rollout.out'),
|
|
"rolloutStderrTail": tail('/tmp/hwlab-registry-rollout.err'),
|
|
"valuesPrinted": False,
|
|
}
|
|
print(json.dumps(payload, ensure_ascii=False))
|
|
PY
|
|
`;
|
|
}
|
|
|
|
export function controlPlaneFieldManager(target: ControlPlaneTargetSpec): string {
|
|
return `unidesk-hwlab-${target.node.toLowerCase()}-${target.lane}-control-plane`;
|
|
}
|
|
|
|
export function k3sApplyScriptFragment(spec: ControlPlaneK3sNodeSpec | null, target: ControlPlaneTargetSpec): string {
|
|
if (spec === null) {
|
|
return `
|
|
k3s_report_file=$(mktemp /tmp/hwlab-node-k3s.XXXXXX.json)
|
|
printf '{"managed":false,"ok":true,"mutation":false}\\n' >"$k3s_report_file"
|
|
k3s_rc=0
|
|
`;
|
|
}
|
|
const content = k3sDropInContent(spec);
|
|
const encoded = Buffer.from(content, "utf8").toString("base64");
|
|
return `
|
|
k3s_report_file=$(mktemp /tmp/hwlab-node-k3s.XXXXXX.json)
|
|
k3s_service=${shQuote(spec.serviceName)}
|
|
k3s_dropin=${shQuote(spec.dropInPath)}
|
|
k3s_node=${shQuote(spec.nodeStatusName)}
|
|
k3s_namespace=${shQuote(target.ciNamespace)}
|
|
k3s_image=${shQuote(target.tekton.toolsImage.output)}
|
|
k3s_desired_max_pods=${shQuote(String(spec.kubelet.maxPods))}
|
|
k3s_expected_sha=${shQuote(sha256Short(content))}
|
|
k3s_before_capacity=$(kubectl get node "$k3s_node" -o 'jsonpath={.status.capacity.pods}' 2>/dev/null || true)
|
|
k3s_before_allocatable=$(kubectl get node "$k3s_node" -o 'jsonpath={.status.allocatable.pods}' 2>/dev/null || true)
|
|
capacity_restart=false
|
|
if [ "$k3s_before_capacity" != "$k3s_desired_max_pods" ] || [ "$k3s_before_allocatable" != "$k3s_desired_max_pods" ]; then capacity_restart=true; fi
|
|
k3s_current_dropin_sha=
|
|
if [ -f "$k3s_dropin" ]; then k3s_current_dropin_sha=$(sha256sum "$k3s_dropin" | awk '{print "sha256:"$1}'); fi
|
|
if [ "$k3s_current_dropin_sha" = "$k3s_expected_sha" ] && [ "$capacity_restart" = false ]; then
|
|
python3 - "$k3s_current_dropin_sha" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_before_capacity" "$k3s_before_allocatable" <<'PY' >"$k3s_report_file"
|
|
import json, sys
|
|
dropin_sha, expected_sha, service, dropin, node_name, desired, before_capacity, before_allocatable = sys.argv[1:9]
|
|
print(json.dumps({
|
|
"managed": True,
|
|
"ok": True,
|
|
"mutation": False,
|
|
"applyMode": "noop",
|
|
"completionPending": False,
|
|
"serviceName": service,
|
|
"dropInPath": dropin,
|
|
"dropInSha256": dropin_sha,
|
|
"expectedDropInSha256": expected_sha,
|
|
"dropInMatches": dropin_sha == expected_sha,
|
|
"nodeName": node_name,
|
|
"desiredMaxPods": int(desired),
|
|
"beforeCapacityPods": int(before_capacity) if before_capacity.isdigit() else None,
|
|
"beforeAllocatablePods": int(before_allocatable) if before_allocatable.isdigit() else None,
|
|
}, ensure_ascii=False))
|
|
PY
|
|
k3s_rc=0
|
|
else
|
|
k3s_job="hwlab-node-k3s-config-$(date +%s)"
|
|
k3s_job_manifest=$(mktemp /tmp/hwlab-node-k3s-job.XXXXXX.json)
|
|
k3s_host_script=$(mktemp /tmp/hwlab-node-k3s-host.XXXXXX.sh)
|
|
k3s_job_apply_stdout=/tmp/hwlab-node-k3s-job-apply.out
|
|
k3s_job_apply_stderr=/tmp/hwlab-node-k3s-job-apply.err
|
|
k3s_docker_stdout=/tmp/hwlab-node-k3s-docker.out
|
|
k3s_docker_stderr=/tmp/hwlab-node-k3s-docker.err
|
|
k3s_host_report="/tmp/$k3s_job-report.json"
|
|
rm -f "$k3s_host_report"
|
|
python3 - "$k3s_job_manifest" "$k3s_host_script" "$k3s_job" "$k3s_namespace" "$k3s_image" "$k3s_dropin" ${shQuote(encoded)} "$k3s_service" "$k3s_desired_max_pods" "$k3s_expected_sha" "$capacity_restart" "$k3s_host_report" <<'PY'
|
|
import json, os, shlex, sys
|
|
manifest_path, host_script_path, job, namespace, image, dropin, encoded, service, desired, expected_sha, capacity_restart, report_path = sys.argv[1:13]
|
|
script = f"""#!/bin/sh
|
|
set -eu
|
|
expected=/tmp/unidesk-k3s-dropin.conf
|
|
printf %s {shlex.quote(encoded)} | base64 -d > "$expected"
|
|
host_dropin=/host{shlex.quote(dropin)}
|
|
host_report=/host{shlex.quote(report_path)}
|
|
mkdir -p "$(dirname "$host_dropin")"
|
|
before_sha=
|
|
if [ -f "$host_dropin" ]; then before_sha=$(sha256sum "$host_dropin" | awk '{{print "sha256:"$1}}'); fi
|
|
changed=false
|
|
if ! cmp -s "$expected" "$host_dropin" 2>/dev/null; then
|
|
cp "$expected" "$host_dropin"
|
|
chown 0:0 "$host_dropin" 2>/dev/null || true
|
|
chmod 0644 "$host_dropin"
|
|
changed=true
|
|
fi
|
|
nsenter_path=$(command -v nsenter || true)
|
|
host_systemctl() {{
|
|
if command -v chroot >/dev/null 2>&1 && [ -x /host/usr/bin/systemctl ]; then
|
|
chroot /host /usr/bin/systemctl "$@"
|
|
return $?
|
|
fi
|
|
if [ -n "$nsenter_path" ]; then
|
|
"$nsenter_path" -t 1 -m -u -i -n -p -- /usr/bin/systemctl "$@"
|
|
return $?
|
|
fi
|
|
return 127
|
|
}}
|
|
daemon_reload_rc=0
|
|
restart_rc=0
|
|
restarted=false
|
|
if command -v chroot >/dev/null 2>&1 || [ -n "$nsenter_path" ]; then
|
|
host_systemctl daemon-reload || daemon_reload_rc=$?
|
|
if [ "$changed" = true ] || [ {shlex.quote(capacity_restart)} = true ]; then
|
|
restarted=true
|
|
host_systemctl restart {shlex.quote(service)} || restart_rc=$?
|
|
fi
|
|
else
|
|
daemon_reload_rc=127
|
|
restart_rc=127
|
|
fi
|
|
after_sha=
|
|
if [ -f "$host_dropin" ]; then after_sha=$(sha256sum "$host_dropin" | awk '{{print "sha256:"$1}}'); fi
|
|
service_active=unknown
|
|
if command -v chroot >/dev/null 2>&1 || [ -n "$nsenter_path" ]; then service_active=$(host_systemctl is-active {shlex.quote(service)} 2>/dev/null || true); fi
|
|
python3 - "$changed" "$restarted" "$daemon_reload_rc" "$restart_rc" "$before_sha" "$after_sha" "$service_active" "$nsenter_path" <<'REPORT' >"$host_report"
|
|
import json, sys
|
|
changed, restarted = sys.argv[1] == "true", sys.argv[2] == "true"
|
|
daemon_reload_rc, restart_rc = int(sys.argv[3] or "0"), int(sys.argv[4] or "0")
|
|
print(json.dumps({{
|
|
"jobChanged": changed,
|
|
"jobRestarted": restarted,
|
|
"daemonReloadExitCode": daemon_reload_rc,
|
|
"restartExitCode": restart_rc,
|
|
"beforeDropInSha256": sys.argv[5] or None,
|
|
"dropInSha256": sys.argv[6] or None,
|
|
"expectedDropInSha256": {json.dumps(expected_sha)},
|
|
"dropInMatches": sys.argv[6] == {json.dumps(expected_sha)},
|
|
"serviceActiveText": sys.argv[7] or None,
|
|
"nsenterPresent": bool(sys.argv[8]),
|
|
}}))
|
|
REPORT
|
|
chmod 0644 "$host_report" 2>/dev/null || true
|
|
cat "$host_report"
|
|
"""
|
|
with open(host_script_path, "w", encoding="utf-8") as handle:
|
|
handle.write(script)
|
|
os.chmod(host_script_path, 0o755)
|
|
manifest = {
|
|
"apiVersion": "batch/v1",
|
|
"kind": "Job",
|
|
"metadata": {"name": job, "namespace": namespace, "labels": {"app.kubernetes.io/part-of": "hwlab-node-control-plane", "unidesk.ai/operation": "k3s-node-config"}},
|
|
"spec": {
|
|
"backoffLimit": 0,
|
|
"ttlSecondsAfterFinished": 300,
|
|
"template": {
|
|
"metadata": {"labels": {"app.kubernetes.io/part-of": "hwlab-node-control-plane", "unidesk.ai/operation": "k3s-node-config"}},
|
|
"spec": {
|
|
"restartPolicy": "Never",
|
|
"hostPID": True,
|
|
"hostNetwork": True,
|
|
"containers": [{
|
|
"name": "apply-k3s-node-config",
|
|
"image": image,
|
|
"imagePullPolicy": "IfNotPresent",
|
|
"securityContext": {"privileged": True},
|
|
"command": ["/bin/sh", "-lc", script],
|
|
"volumeMounts": [{"name": "host-root", "mountPath": "/host"}],
|
|
}],
|
|
"volumes": [{"name": "host-root", "hostPath": {"path": "/", "type": "Directory"}}],
|
|
},
|
|
},
|
|
},
|
|
}
|
|
with open(manifest_path, "w", encoding="utf-8") as handle:
|
|
json.dump(manifest, handle)
|
|
PY
|
|
k3s_render_rc=$?
|
|
if [ "$k3s_render_rc" != 0 ]; then
|
|
python3 - "$k3s_render_rc" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" <<'PY' >"$k3s_report_file"
|
|
import json, sys
|
|
render_rc = int(sys.argv[1] or "1")
|
|
expected_sha, service, dropin, node_name, desired = sys.argv[2:7]
|
|
print(json.dumps({
|
|
"managed": True,
|
|
"ok": False,
|
|
"mutation": False,
|
|
"renderExitCode": render_rc,
|
|
"serviceName": service,
|
|
"dropInPath": dropin,
|
|
"expectedDropInSha256": expected_sha,
|
|
"nodeName": node_name,
|
|
"desiredMaxPods": int(desired),
|
|
}, ensure_ascii=False))
|
|
PY
|
|
k3s_rc=$k3s_render_rc
|
|
else
|
|
kubectl apply -f "$k3s_job_manifest" >"$k3s_job_apply_stdout" 2>"$k3s_job_apply_stderr"
|
|
k3s_job_apply_rc=$?
|
|
k3s_apply_mode=kubernetes-job
|
|
k3s_docker_rc=127
|
|
if [ "$k3s_job_apply_rc" != 0 ] && command -v docker >/dev/null 2>&1; then
|
|
k3s_apply_mode=docker-host-fallback
|
|
docker run --rm --privileged --pid=host --network=host -v /:/host --entrypoint /bin/sh "$k3s_image" "/host$k3s_host_script" >"$k3s_docker_stdout" 2>"$k3s_docker_stderr"
|
|
k3s_docker_rc=$?
|
|
fi
|
|
k3s_submit_rc=$k3s_job_apply_rc
|
|
if [ "$k3s_job_apply_rc" != 0 ] && [ "$k3s_docker_rc" = 0 ]; then k3s_submit_rc=0; fi
|
|
python3 - "$k3s_submit_rc" "$k3s_job_apply_rc" "$k3s_docker_rc" "$k3s_apply_mode" "$k3s_before_capacity" "$k3s_before_allocatable" "$k3s_expected_sha" "$k3s_service" "$k3s_dropin" "$k3s_node" "$k3s_desired_max_pods" "$k3s_job" "$k3s_namespace" "$k3s_host_report" "$k3s_job_apply_stdout" "$k3s_job_apply_stderr" "$k3s_docker_stdout" "$k3s_docker_stderr" <<'PY' >"$k3s_report_file"
|
|
import json, pathlib, sys
|
|
submit_rc, job_apply_rc, docker_rc = [int(value or "0") for value in sys.argv[1:4]]
|
|
apply_mode = sys.argv[4]
|
|
before_capacity, before_allocatable = sys.argv[5:7]
|
|
expected_sha, service, dropin, node_name, desired, job_name, namespace, host_report = sys.argv[7:15]
|
|
def read(path):
|
|
return pathlib.Path(path).read_text(errors='replace') if pathlib.Path(path).exists() else ''
|
|
try:
|
|
host_report_data = json.loads(read(host_report) or "{}")
|
|
except Exception:
|
|
host_report_data = {}
|
|
apply_ok = submit_rc == 0
|
|
print(json.dumps({
|
|
"managed": True,
|
|
"ok": apply_ok,
|
|
"mutation": apply_ok,
|
|
"completionPending": apply_ok and apply_mode == "kubernetes-job",
|
|
"applyMode": apply_mode,
|
|
"jobName": job_name,
|
|
"namespace": namespace,
|
|
"jobApplyExitCode": job_apply_rc,
|
|
"dockerFallbackExitCode": docker_rc,
|
|
"serviceName": service,
|
|
"dropInPath": dropin,
|
|
"dropInSha256": host_report_data.get("dropInSha256"),
|
|
"expectedDropInSha256": expected_sha,
|
|
"dropInMatches": host_report_data.get("dropInSha256") == expected_sha if host_report_data else None,
|
|
"daemonReloadExitCode": host_report_data.get("daemonReloadExitCode"),
|
|
"restartExitCode": host_report_data.get("restartExitCode"),
|
|
"serviceActive": host_report_data.get("serviceActiveText") == "active" if host_report_data else None,
|
|
"nodeName": node_name,
|
|
"desiredMaxPods": int(desired),
|
|
"beforeCapacityPods": int(before_capacity) if before_capacity.isdigit() else None,
|
|
"beforeAllocatablePods": int(before_allocatable) if before_allocatable.isdigit() else None,
|
|
"hostReportPath": host_report,
|
|
"statusCommand": f"bun scripts/cli.ts hwlab nodes control-plane infra status --node {node_name.upper()} --lane ${target.lane}",
|
|
"jobCompletionCommand": f"kubectl -n {namespace} wait --for=condition=complete job/{job_name} --timeout=120s",
|
|
"jobLogsCommand": f"kubectl -n {namespace} logs job/{job_name} --tail=120",
|
|
"jobApplyStdoutPreview": read(sys.argv[15])[-1000:],
|
|
"jobApplyStderrPreview": read(sys.argv[16])[-1000:],
|
|
"dockerStdoutPreview": read(sys.argv[17])[-1000:],
|
|
"dockerStderrPreview": read(sys.argv[18])[-1000:],
|
|
}, ensure_ascii=False))
|
|
PY
|
|
k3s_rc=$k3s_submit_rc
|
|
fi
|
|
rm -f "$k3s_job_manifest" "$k3s_host_script"
|
|
fi
|
|
`;
|
|
}
|
|
|
|
export function asRecord(value: unknown, path: string): Record<string, unknown> {
|
|
if (typeof value !== "object" || value === null || Array.isArray(value)) throw new Error(`${path} must be an object`);
|
|
return value as Record<string, unknown>;
|
|
}
|
|
|
|
export function record(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
export function renderTable(headers: string[], rows: string[][]): string[] {
|
|
const widths = headers.map((header, index) => Math.max(header.length, ...rows.map((row) => row[index]?.length ?? 0)));
|
|
const render = (row: string[]) => row.map((cell, index) => cell.padEnd(widths[index] ?? cell.length)).join(" ").trimEnd();
|
|
return [render(headers), ...rows.map(render)];
|
|
}
|
|
|
|
export function renderCell(value: unknown, fallback = "-"): string {
|
|
if (value === undefined || value === null || value === "") return fallback;
|
|
return String(value);
|
|
}
|
|
|
|
export function optionsModeFromCommand(command: unknown): string {
|
|
const value = String(command ?? "");
|
|
if (value.endsWith(" status") || value.endsWith(" logs")) return "status";
|
|
return "benchmark";
|
|
}
|
|
|
|
export function stringField(obj: Record<string, unknown>, key: string, path: string): string {
|
|
const value = obj[key];
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`);
|
|
return value;
|
|
}
|
|
|
|
export function optionalStringField(obj: Record<string, unknown>, key: string, path: string): string | undefined {
|
|
const value = obj[key];
|
|
if (value === undefined) return undefined;
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`);
|
|
return value;
|
|
}
|
|
|
|
export function absoluteConfigPathField(obj: Record<string, unknown>, key: string, path: string): string {
|
|
const value = stringField(obj, key, path);
|
|
if (!value.startsWith("/") || value.includes("\0") || value.includes("..")) throw new Error(`${path}.${key} must be an absolute path without '..'`);
|
|
return value;
|
|
}
|
|
|
|
export function validateKubernetesName(value: string, path: string): void {
|
|
if (!/^[a-z0-9]([-a-z0-9]*[a-z0-9])?$/u.test(value) || value.length > 253) throw new Error(`${path} must be a Kubernetes resource name`);
|
|
}
|
|
|
|
export function validateSecretKey(value: string, path: string): void {
|
|
if (!/^[A-Za-z0-9._-]+$/u.test(value)) throw new Error(`${path} must be a Kubernetes Secret key`);
|
|
}
|
|
|
|
export function validateEnvKey(value: string, path: string): void {
|
|
if (!/^[A-Z0-9_]+$/u.test(value)) throw new Error(`${path} must be an env key`);
|
|
}
|
|
|
|
export function validateSourceRef(value: string, path: string): void {
|
|
if (!/^[A-Za-z0-9_./-]+$/u.test(value) || value.includes("..")) throw new Error(`${path} has an unsupported sourceRef format`);
|
|
}
|
|
|
|
export function numberField(obj: Record<string, unknown>, key: string, path: string): number {
|
|
const value = obj[key];
|
|
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path}.${key} must be a positive integer`);
|
|
return value;
|
|
}
|
|
|
|
export function positiveConfigIntegerField(obj: Record<string, unknown>, key: string, path: string): number {
|
|
const value = obj[key];
|
|
if (typeof value !== "number" || !Number.isInteger(value) || value <= 0) throw new Error(`${path}.${key} must be a positive integer`);
|
|
return value;
|
|
}
|
|
|
|
export function nonNegativeIntegerField(obj: Record<string, unknown>, key: string, path: string): number {
|
|
const value = obj[key];
|
|
if (typeof value !== "number" || !Number.isInteger(value) || value < 0) throw new Error(`${path}.${key} must be a non-negative integer`);
|
|
return value;
|
|
}
|
|
|
|
export function numberArrayField(obj: Record<string, unknown>, key: string, path: string): number[] {
|
|
const value = obj[key];
|
|
if (!Array.isArray(value) || value.some((item) => typeof item !== "number" || !Number.isInteger(item))) throw new Error(`${path}.${key} must be an array of integers`);
|
|
return [...value] as number[];
|
|
}
|
|
|
|
export function stringArrayField(obj: Record<string, unknown>, key: string, path: string): string[] {
|
|
const value = obj[key];
|
|
if (!Array.isArray(value) || value.some((item) => typeof item !== "string" || item.length === 0)) throw new Error(`${path}.${key} must be an array of non-empty strings`);
|
|
return [...value] as string[];
|
|
}
|
|
|
|
export function stringRecordField(obj: Record<string, unknown>, path: string): Record<string, string> {
|
|
const result: Record<string, string> = {};
|
|
for (const [key, value] of Object.entries(obj)) {
|
|
if (!/^[A-Za-z_][A-Za-z0-9_]*$/u.test(key)) throw new Error(`${path}.${key} has an unsupported key format`);
|
|
if (typeof value !== "string" || value.length === 0) throw new Error(`${path}.${key} must be a non-empty string`);
|
|
result[key] = value;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
export function booleanField(obj: Record<string, unknown>, key: string, path: string): boolean {
|
|
const value = obj[key];
|
|
if (typeof value !== "boolean") throw new Error(`${path}.${key} must be a boolean`);
|
|
return value;
|
|
}
|
|
|
|
export function boolField(obj: Record<string, unknown>, key: string): boolean {
|
|
return obj[key] === true;
|
|
}
|
|
|
|
export function numberValue(value: unknown): number | null {
|
|
return typeof value === "number" && Number.isFinite(value) ? value : null;
|
|
}
|
|
|
|
export function requiredOption(args: string[], name: string): string {
|
|
const index = args.indexOf(name);
|
|
if (index === -1) throw new Error(`${name} is required`);
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--") || value.length === 0) throw new Error(`${name} requires a value`);
|
|
return value;
|
|
}
|
|
|
|
export function optionValue(args: string[], name: string): string | null {
|
|
const index = args.indexOf(name);
|
|
if (index === -1) return null;
|
|
const value = args[index + 1];
|
|
if (value === undefined || value.startsWith("--") || value.length === 0) throw new Error(`${name} requires a value`);
|
|
return value;
|
|
}
|
|
|
|
export function positiveIntegerOption(args: string[], name: string, defaultValue: number, maxValue: number): number {
|
|
const raw = optionValue(args, name);
|
|
if (raw === null) return defaultValue;
|
|
const value = Number(raw);
|
|
if (!Number.isInteger(value) || value <= 0) throw new Error(`${name} must be a positive integer`);
|
|
return Math.min(value, maxValue);
|
|
}
|
|
|
|
export function compactCommandResult(result: CommandResult): Record<string, unknown> {
|
|
return {
|
|
exitCode: result.exitCode,
|
|
timedOut: result.timedOut,
|
|
stdoutBytes: Buffer.byteLength(result.stdout),
|
|
stderrBytes: Buffer.byteLength(result.stderr),
|
|
stdoutTail: result.stdout.slice(-2000),
|
|
stderrTail: result.stderr.slice(-2000),
|
|
};
|
|
}
|
|
|
|
export function shellJsonArray(items: readonly string[]): string {
|
|
return JSON.stringify([...items]);
|
|
}
|
|
|
|
export function shQuote(value: string): string {
|
|
return `'${value.replace(/'/gu, `'"'"'`)}'`;
|
|
}
|
|
|
|
export function validateHttpsUrl(value: string, path: string): void {
|
|
let parsed: URL;
|
|
try {
|
|
parsed = new URL(value);
|
|
} catch {
|
|
throw new Error(`${path} must be a valid URL`);
|
|
}
|
|
if (parsed.protocol !== "https:") throw new Error(`${path} must use https://`);
|
|
}
|
|
|
|
export function hostRouteProxyHostAllowed(value: string): boolean {
|
|
if (value === "127.0.0.1" || value === "localhost") return true;
|
|
const parts = value.split(".").map((part) => Number(part));
|
|
if (parts.length !== 4 || parts.some((part) => !Number.isInteger(part) || part < 0 || part > 255)) return false;
|
|
return parts[0] === 10
|
|
|| (parts[0] === 172 && parts[1] >= 16 && parts[1] <= 31)
|
|
|| (parts[0] === 192 && parts[1] === 168);
|
|
}
|
|
|
|
export function sha256Short(text: string): string {
|
|
return `sha256:${createHash("sha256").update(text).digest("hex")}`;
|
|
}
|