mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: 优化异地 artifact registry 拉取

Browse Source
This commit is contained in:
pikastech
2026-07-20 08:40:01 +02:00
parent b89e8c3861
commit 84fcdd02b9
9 changed files with 475 additions and 20 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/platform-infra-k3s-cluster-config.ts
+67 -2
View File
@@ -51,13 +51,32 @@ export interface K3sClusterTarget {
workerPrivateKey: string;
workerPublicKey: string;
agentToken: string;
registryUsername: string;
registryPassword: string;
registryPasswordHash: string;
};
k3s: {
serverUrl: string;
agentTokenFile: string;
flannelInterface: string;
maxPods: number;
registryMirror: { source: string; endpoint: string };
registryMirror: {
source: string;
endpoint: string;
auth: {
sourceRef: string;
usernameKey: string;
passwordKey: string;
passwordHashKey: string;
};
};
registryPullSmoke: {
image: string;
expectedDigest: string;
expectedBytes: number;
concurrentPulls: number;
timeoutSeconds: number;
};
serviceLb: { enabledNodeLabel: string; enabledNode: string };
workerDns: {
namespace: string;
@@ -93,6 +112,8 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri
const secrets = record(raw.secrets, `targets.${id}.secrets`);
const k3s = record(raw.k3s, `targets.${id}.k3s`);
const mirror = record(k3s.registryMirror, `targets.${id}.k3s.registryMirror`);
const mirrorAuth = record(mirror.auth, `targets.${id}.k3s.registryMirror.auth`);
const registryPullSmoke = record(k3s.registryPullSmoke, `targets.${id}.k3s.registryPullSmoke`);
const serviceLb = record(k3s.serviceLb, `targets.${id}.k3s.serviceLb`);
const workerDns = record(k3s.workerDns, `targets.${id}.k3s.workerDns`);
const smoke = record(raw.smoke, `targets.${id}.smoke`);
@@ -151,13 +172,32 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri
workerPrivateKey: text(secrets, "workerPrivateKey", `targets.${id}.secrets`),
workerPublicKey: text(secrets, "workerPublicKey", `targets.${id}.secrets`),
agentToken: text(secrets, "agentToken", `targets.${id}.secrets`),
registryUsername: text(secrets, "registryUsername", `targets.${id}.secrets`),
registryPassword: text(secrets, "registryPassword", `targets.${id}.secrets`),
registryPasswordHash: text(secrets, "registryPasswordHash", `targets.${id}.secrets`),
},
k3s: {
serverUrl: text(k3s, "serverUrl", `targets.${id}.k3s`),
agentTokenFile: text(k3s, "agentTokenFile", `targets.${id}.k3s`),
flannelInterface: text(k3s, "flannelInterface", `targets.${id}.k3s`),
maxPods: integer(k3s, "maxPods", `targets.${id}.k3s`),
registryMirror: { source: text(mirror, "source", "registryMirror"), endpoint: text(mirror, "endpoint", "registryMirror") },
registryMirror: {
source: text(mirror, "source", "registryMirror"),
endpoint: httpsUrl(mirror, "endpoint", "registryMirror"),
auth: {
sourceRef: text(mirrorAuth, "sourceRef", "registryMirror.auth"),
usernameKey: text(mirrorAuth, "usernameKey", "registryMirror.auth"),
passwordKey: text(mirrorAuth, "passwordKey", "registryMirror.auth"),
passwordHashKey: text(mirrorAuth, "passwordHashKey", "registryMirror.auth"),
},
},
registryPullSmoke: {
image: text(registryPullSmoke, "image", "registryPullSmoke"),
expectedDigest: sha256Digest(registryPullSmoke, "expectedDigest", "registryPullSmoke"),
expectedBytes: integer(registryPullSmoke, "expectedBytes", "registryPullSmoke"),
concurrentPulls: integer(registryPullSmoke, "concurrentPulls", "registryPullSmoke"),
timeoutSeconds: integer(registryPullSmoke, "timeoutSeconds", "registryPullSmoke"),
},
serviceLb: { enabledNodeLabel: text(serviceLb, "enabledNodeLabel", "serviceLb"), enabledNode: text(serviceLb, "enabledNode", "serviceLb") },
workerDns: {
namespace: kubernetesName(workerDns, "namespace", "workerDns"),
@@ -181,6 +221,16 @@ export function readK3sClusterConfig(targetId?: string): { defaultTargetId: stri
if (target.k3s.flannelInterface !== target.wireguard.interfaceName) throw new Error(`targets.${id}.k3s.flannelInterface must match wireguard.interfaceName`);
if (target.k3s.serviceLb.enabledNode !== target.controlPlane.nodeName) throw new Error(`targets.${id}.k3s.serviceLb.enabledNode must match controlPlane.nodeName`);
if (!target.tcpTuning.sysctlPath.startsWith("/etc/sysctl.d/")) throw new Error(`targets.${id}.tcpTuning.sysctlPath must stay under /etc/sysctl.d`);
if (target.k3s.registryMirror.auth.sourceRef !== target.secrets.sourceRef) {
throw new Error(`targets.${id}.k3s.registryMirror.auth.sourceRef must match targets.${id}.secrets.sourceRef`);
}
if (
target.k3s.registryMirror.auth.usernameKey !== target.secrets.registryUsername
|| target.k3s.registryMirror.auth.passwordKey !== target.secrets.registryPassword
|| target.k3s.registryMirror.auth.passwordHashKey !== target.secrets.registryPasswordHash
) {
throw new Error(`targets.${id}.k3s.registryMirror.auth keys must match targets.${id}.secrets registry keys`);
}
return { defaultTargetId, target };
}
@@ -227,6 +277,21 @@ function integer(value: Record<string, unknown>, key: string, label: string): nu
return Number(result);
}
function httpsUrl(value: Record<string, unknown>, key: string, label: string): string {
const result = text(value, key, label);
const parsed = new URL(result);
if (parsed.protocol !== "https:" || parsed.username.length > 0 || parsed.password.length > 0) {
throw new Error(`${label}.${key} must be an HTTPS URL without embedded credentials`);
}
return result;
}
function sha256Digest(value: Record<string, unknown>, key: string, label: string): string {
const result = text(value, key, label);
if (!/^sha256:[0-9a-f]{64}$/u.test(result)) throw new Error(`${label}.${key} must be a sha256 digest`);
return result;
}
function enumText<const T extends string>(value: Record<string, unknown>, key: string, label: string, allowed: readonly T[]): T {
const result = text(value, key, label);
if (!allowed.includes(result as T)) throw new Error(`${label}.${key} must be one of ${allowed.join(", ")}`);
scripts/src/platform-infra-k3s-cluster.ts
+239 -8
View File
@@ -1,5 +1,5 @@
import { createHash, generateKeyPairSync, randomBytes } from "node:crypto";
import { chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { appendFileSync, chmodSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
import { dirname } from "node:path";
import type { UniDeskConfig } from "./config";
import { rootPath } from "./config";
@@ -13,7 +13,7 @@ import {
} from "./platform-infra-host-proxy";
import { address, K3S_CLUSTER_CONFIG_LABEL, readK3sClusterConfig, type K3sClusterTarget } from "./platform-infra-k3s-cluster-config";
type Action = "plan" | "secret-init" | "apply" | "status" | "smoke";
type Action = "plan" | "secret-init" | "registry-credential-init" | "apply" | "status" | "smoke" | "registry-smoke";
interface Options {
action: Action;
@@ -27,6 +27,9 @@ interface SecretMaterial {
workerPrivate: string;
workerPublic: string;
agentToken: string;
registryUsername: string;
registryPassword: string;
registryPasswordHash: string;
fingerprint: string;
}
@@ -35,16 +38,18 @@ export async function runPlatformInfraK3sClusterCommand(_config: UniDeskConfig,
const { target } = readK3sClusterConfig(options.targetId);
if (options.action === "plan") return plan(target);
if (options.action === "secret-init") return secretInit(target, options.confirm);
if (options.action === "registry-credential-init") return registryCredentialInit(target, options.confirm);
if (options.action === "status") return status(target);
if (options.action === "smoke") return smoke(target, options.confirm);
if (options.action === "registry-smoke") return registrySmoke(target, options.confirm);
if (!options.confirm) return { ...plan(target), mode: "dry-run", mutation: false };
return apply(target);
}
function parseOptions(args: string[]): Options {
const action = args[0] ?? "plan";
if (action !== "plan" && action !== "secret-init" && action !== "apply" && action !== "status" && action !== "smoke") {
throw new Error("platform-infra k3s-cluster usage: plan|secret-init|apply|status|smoke [--target <id>] [--confirm]");
if (action !== "plan" && action !== "secret-init" && action !== "registry-credential-init" && action !== "apply" && action !== "status" && action !== "smoke" && action !== "registry-smoke") {
throw new Error("platform-infra k3s-cluster usage: plan|secret-init|registry-credential-init|apply|status|smoke|registry-smoke [--target <id>] [--confirm]");
}
return { action, targetId: option(args, "--target") ?? undefined, confirm: args.includes("--confirm") };
}
@@ -72,15 +77,18 @@ function plan(target: K3sClusterTarget): Record<string, unknown> {
serverUrl: target.k3s.serverUrl,
agentTokenFile: target.k3s.agentTokenFile,
registryMirror: target.k3s.registryMirror,
registryPullSmoke: target.k3s.registryPullSmoke,
serviceLb: target.k3s.serviceLb,
workerDns: target.k3s.workerDns,
},
secret: secretSummary(target),
next: {
secretInit: `bun scripts/cli.ts platform-infra k3s-cluster secret-init --target ${target.id} --confirm`,
registryCredentialInit: `bun scripts/cli.ts platform-infra k3s-cluster registry-credential-init --target ${target.id} --confirm`,
apply: `bun scripts/cli.ts platform-infra k3s-cluster apply --target ${target.id} --confirm`,
status: `bun scripts/cli.ts platform-infra k3s-cluster status --target ${target.id}`,
smoke: `bun scripts/cli.ts platform-infra k3s-cluster smoke --target ${target.id} --confirm`,
registrySmoke: `bun scripts/cli.ts platform-infra k3s-cluster registry-smoke --target ${target.id} --confirm`,
},
};
}
@@ -91,12 +99,16 @@ function secretInit(target: K3sClusterTarget, confirm: boolean): Record<string,
if (existsSync(path)) throw new Error(`${path} already exists; secret-init never overwrites cluster credentials`);
const control = wireguardKeyPair();
const worker = wireguardKeyPair();
const registry = createRegistryCredential();
const values: Record<string, string> = {
[target.secrets.controlPlanePrivateKey]: control.privateKey,
[target.secrets.controlPlanePublicKey]: control.publicKey,
[target.secrets.workerPrivateKey]: worker.privateKey,
[target.secrets.workerPublicKey]: worker.publicKey,
[target.secrets.agentToken]: randomBytes(32).toString("base64url"),
[target.secrets.registryUsername]: registry.username,
[target.secrets.registryPassword]: registry.password,
[target.secrets.registryPasswordHash]: registry.passwordHash,
};
mkdirSync(dirname(path), { recursive: true, mode: 0o700 });
writeFileSync(path, Object.entries(values).map(([key, value]) => `${key}=${value}`).join("\n") + "\n", { encoding: "utf8", mode: 0o600 });
@@ -104,6 +116,44 @@ function secretInit(target: K3sClusterTarget, confirm: boolean): Record<string,
return { ok: true, mutation: true, targetId: target.id, secret: secretSummary(target), valuesPrinted: false };
}
function registryCredentialInit(target: K3sClusterTarget, confirm: boolean): Record<string, unknown> {
const path = target.secrets.sourceRef;
const keys = [target.secrets.registryUsername, target.secrets.registryPassword, target.secrets.registryPasswordHash];
if (!existsSync(path)) throw new Error(`${path} is missing; run platform-infra k3s-cluster secret-init --confirm`);
const values = parseEnvFile(readFileSync(path, "utf8"));
const present = keys.filter((key) => typeof values[key] === "string" && values[key]!.length > 0);
if (present.length === keys.length) {
return { ok: true, mutation: false, mode: "already-present", targetId: target.id, secret: secretSummary(target), valuesPrinted: false };
}
if (present.length > 0) throw new Error(`${path} contains a partial artifact registry credential; refusing to overwrite`);
if (!confirm) {
return { ok: true, mutation: false, mode: "dry-run", targetId: target.id, sourceRef: path, keys, valuesPrinted: false };
}
const credential = createRegistryCredential();
appendFileSync(path, [
`${target.secrets.registryUsername}=${credential.username}`,
`${target.secrets.registryPassword}=${credential.password}`,
`${target.secrets.registryPasswordHash}=${credential.passwordHash}`,
"",
].join("\n"), { encoding: "utf8", mode: 0o600 });
chmodSync(path, 0o600);
return { ok: true, mutation: true, targetId: target.id, secret: secretSummary(target), valuesPrinted: false };
}
function createRegistryCredential(): { username: string; password: string; passwordHash: string } {
const username = "unidesk-pull";
const password = randomBytes(32).toString("base64url");
const result = runCommand(
["docker", "run", "--rm", "-i", "caddy:2.10.2-alpine", "sh", "-c", 'IFS= read -r password; caddy hash-password --algorithm bcrypt --plaintext "$password"'],
rootPath(),
{ input: `${password}\n`, timeoutMs: 60_000 },
);
if (result.exitCode !== 0 || !/^\$2[aby]\$\d{2}\$[./A-Za-z0-9]{53}$/u.test(result.stdout.trim())) {
throw new Error("failed to generate artifact registry bcrypt credential");
}
return { username, password, passwordHash: result.stdout.trim() };
}
function apply(target: K3sClusterTarget): Record<string, unknown> {
const secret = readSecrets(target);
const workerNetwork = runWorker(target, workerWireguardApplyScript(target, secret), false, 120_000);
@@ -237,6 +287,175 @@ function smoke(target: K3sClusterTarget, confirm: boolean): Record<string, unkno
};
}
function registrySmoke(target: K3sClusterTarget, confirm: boolean): Record<string, unknown> {
const smoke = target.k3s.registryPullSmoke;
if (!confirm) {
return {
ok: true,
mutation: false,
mode: "dry-run",
targetId: target.id,
workerNode: target.worker.nodeName,
registry: { ...target.k3s.registryMirror, auth: { ...target.k3s.registryMirror.auth, valuesPrinted: false } },
smoke,
valuesPrinted: false,
};
}
const secret = readSecrets(target);
const before = runControlPlane(target, nodeReadinessScript(target), 30_000);
requireSuccess(before, "registry-smoke-node-readiness-before");
const worker = runWorker(target, registrySmokeScript(target, secret), true, (smoke.timeoutSeconds + 120) * 1000);
const after = runControlPlane(target, nodeReadinessScript(target), 30_000);
const checks = parseFirstJson(worker.stdout);
const beforeChecks = parseFirstJson(before.stdout);
const afterChecks = parseFirstJson(after.stdout);
const ok = worker.exitCode === 0
&& beforeChecks.ok === true
&& after.exitCode === 0
&& afterChecks.ok === true
&& checks.ok === true;
return {
ok,
mutation: true,
targetId: target.id,
workerNode: target.worker.nodeName,
registry: {
source: target.k3s.registryMirror.source,
endpoint: target.k3s.registryMirror.endpoint,
authSourceRef: target.k3s.registryMirror.auth.sourceRef,
valuesPrinted: false,
},
smoke,
checks,
clusterBefore: beforeChecks,
clusterAfter: afterChecks,
providerControlChannelReturned: worker.exitCode === 0,
result: { worker: compact(worker), before: compact(before), after: compact(after) },
valuesPrinted: false,
};
}
function nodeReadinessScript(target: K3sClusterTarget): string {
return `set -eu
cp_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.controlPlane.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')
worker_ready=$(/usr/local/bin/kubectl get node ${shQuote(target.worker.nodeName)} -o jsonpath='{.status.conditions[?(@.type=="Ready")].status}')
python3 - "$cp_ready" "$worker_ready" <<'PY'
import json, sys
cp, worker = sys.argv[1:]
print(json.dumps({"ok": cp == "True" and worker == "True", "controlPlaneReady": cp == "True", "workerReady": worker == "True"}, separators=(",", ":")))
PY
`;
}
function registrySmokeScript(target: K3sClusterTarget, secret: SecretMaterial): string {
const smoke = target.k3s.registryPullSmoke;
const endpoint = new URL(target.k3s.registryMirror.endpoint);
const imagePath = smoke.image.slice(target.k3s.registryMirror.source.length + 1);
const separator = imagePath.lastIndexOf(":");
if (separator < 1) throw new Error(`registry smoke image must use a tag: ${smoke.image}`);
const repository = imagePath.slice(0, separator);
const tag = imagePath.slice(separator + 1);
return `set -eu
tmp="$(mktemp -d)"
monitor_pid=""
cleanup() {
touch "$tmp/stop"
if [ -n "$monitor_pid" ]; then wait "$monitor_pid" 2>/dev/null || true; fi
rm -rf "$tmp"
}
trap cleanup EXIT
cat >"$tmp/netrc" <<'EOF'
machine ${endpoint.hostname}
login ${secret.registryUsername}
password ${secret.registryPassword}
EOF
chmod 0600 "$tmp/netrc"
base=${shQuote(target.k3s.registryMirror.endpoint)}
unauth_status="$(curl -sS -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 20 "$base/v2/" || true)"
auth_status="$(curl -sS --netrc-file "$tmp/netrc" -o /dev/null -w '%{http_code}' --connect-timeout 5 --max-time 20 "$base/v2/" || true)"
manifest_headers="$tmp/manifest.headers"
curl -fsS --netrc-file "$tmp/netrc" -D "$manifest_headers" -o "$tmp/manifest.json" -H 'Accept: application/vnd.docker.distribution.manifest.v2+json' "$base/v2/${repository}/manifests/${tag}"
manifest_digest="$(tr -d '\\r' <"$manifest_headers" | awk 'tolower($1)=="docker-content-digest:" {print $2}' | tail -1)"
layer_digest="$(python3 - "$tmp/manifest.json" <<'PY'
import json, sys
manifest = json.load(open(sys.argv[1], encoding="utf-8"))
print(manifest["layers"][0]["digest"])
PY
)"
range_status="$(curl -sS --netrc-file "$tmp/netrc" -H 'Range: bytes=0-1023' -o "$tmp/range.bin" -w '%{http_code}' "$base/v2/${repository}/blobs/$layer_digest" || true)"
range_bytes="$(wc -c <"$tmp/range.bin" | tr -d ' ')"
write_status="$(curl -sS --netrc-file "$tmp/netrc" -X POST -o /dev/null -w '%{http_code}' "$base/v2/unidesk-registry-smoke/blobs/uploads/" || true)"
(
: >"$tmp/api.failures"
while [ ! -f "$tmp/stop" ]; do
nc -z -w 3 ${address(target.wireguard.controlPlaneAddress)} 6443 || printf x >>"$tmp/api.failures"
sleep 2
done
) &
monitor_pid=$!
k3s crictl rmi ${shQuote(smoke.image)} >/dev/null 2>&1 || true
cold_start="$(date +%s%3N)"
k3s crictl pull ${shQuote(smoke.image)} >"$tmp/cold.log" 2>&1
cold_end="$(date +%s%3N)"
k3s crictl inspecti ${shQuote(smoke.image)} >"$tmp/inspect.json"
grep -F ${shQuote(smoke.expectedDigest)} "$tmp/inspect.json" >/dev/null
k3s crictl rmi ${shQuote(smoke.image)} >/dev/null 2>&1 || true
concurrent_start="$(date +%s%3N)"
pids=""
for index in $(seq 1 ${smoke.concurrentPulls}); do
k3s crictl pull ${shQuote(smoke.image)} >"$tmp/concurrent-$index.log" 2>&1 &
pids="$pids $!"
done
concurrent_ok=true
for pid in $pids; do wait "$pid" || concurrent_ok=false; done
concurrent_end="$(date +%s%3N)"
hot_start="$(date +%s%3N)"
k3s crictl pull ${shQuote(smoke.image)} >"$tmp/hot.log" 2>&1
hot_end="$(date +%s%3N)"
touch "$tmp/stop"
wait "$monitor_pid" 2>/dev/null || true
monitor_pid=""
api_failures="$(wc -c <"$tmp/api.failures" | tr -d ' ')"
python3 - "$unauth_status" "$auth_status" "$manifest_digest" "$range_status" "$range_bytes" "$write_status" "$cold_start" "$cold_end" "$concurrent_start" "$concurrent_end" "$hot_start" "$hot_end" "$concurrent_ok" "$api_failures" <<'PY'
import json, sys
(unauth, auth, digest, range_status, range_bytes, write_status, cold_start, cold_end,
concurrent_start, concurrent_end, hot_start, hot_end, concurrent_ok, api_failures) = sys.argv[1:]
cold_ms = int(cold_end) - int(cold_start)
concurrent_ms = int(concurrent_end) - int(concurrent_start)
hot_ms = int(hot_end) - int(hot_start)
expected_bytes = ${smoke.expectedBytes}
throughput = None if cold_ms <= 0 else round(expected_bytes / (cold_ms / 1000), 2)
checks = {
"unauthenticatedRejected": unauth == "401",
"authenticatedV2": auth == "200",
"manifestDigest": digest == ${JSON.stringify(smoke.expectedDigest)},
"blobRange": range_status == "206" and int(range_bytes or 0) == 1024,
"writesRejected": write_status in {"403", "405"},
"coldPull": cold_ms > 0,
"concurrentPulls": concurrent_ok == "true",
"repeatPull": hot_ms >= 0,
"apiStable": int(api_failures) == 0,
}
print(json.dumps({
"ok": all(checks.values()),
"checks": checks,
"unauthenticatedStatus": unauth,
"authenticatedStatus": auth,
"manifestDigest": digest,
"rangeStatus": range_status,
"rangeBytes": int(range_bytes or 0),
"writeStatus": write_status,
"coldPullDurationMs": cold_ms,
"coldPullBytesPerSecond": throughput,
"concurrentPullDurationMs": concurrent_ms,
"hotPullDurationMs": hot_ms,
"apiProbeFailures": int(api_failures),
"valuesPrinted": False,
}, separators=(",", ":")))
PY
`;
}
function reconcileWorkerDns(target: K3sClusterTarget): Record<string, unknown> {
const before = workerDnsStatus(target);
const mutation = before.data.configurationCurrent !== true;
@@ -556,7 +775,6 @@ function workerApplyScript(target: K3sClusterTarget, secret: SecretMaterial): st
const desiredHash = createHash("sha256").update(JSON.stringify({
version: target.worker.k3sVersion,
exec,
registryMirror: target.k3s.registryMirror,
agentToken: secret.agentToken,
})).digest("hex");
return `set -eu
@@ -586,8 +804,14 @@ mirrors:
"${target.k3s.registryMirror.source}":
endpoint:
- "${target.k3s.registryMirror.endpoint}"
configs:
"${new URL(target.k3s.registryMirror.endpoint).host}":
auth:
username: ${JSON.stringify(secret.registryUsername)}
password: ${JSON.stringify(secret.registryPassword)}
EOF
wg_changed=false
registry_changed=false
if ! cmp -s "$wg_candidate" /etc/wireguard/${wg.interfaceName}.conf 2>/dev/null; then
install -m 0600 "$wg_candidate" /etc/wireguard/${wg.interfaceName}.conf
wg_changed=true
@@ -597,6 +821,7 @@ if ! cmp -s "$token_candidate" ${shQuote(target.k3s.agentTokenFile)} 2>/dev/null
fi
if ! cmp -s "$registry_candidate" /etc/rancher/k3s/registries.yaml 2>/dev/null; then
install -m 0600 "$registry_candidate" /etc/rancher/k3s/registries.yaml
registry_changed=true
fi
systemctl enable wg-quick@${wg.interfaceName} >/dev/null 2>&1 || true
if [ "$wg_changed" = true ]; then
@@ -610,7 +835,10 @@ desired_hash_file=/etc/unidesk/k3s-agent-desired.sha256
current_hash=$(cat "$desired_hash_file" 2>/dev/null || true)
agent_state="$(systemctl show k3s-agent -p ActiveState --value 2>/dev/null || true)"
agent_pid="$(systemctl show k3s-agent -p MainPID --value 2>/dev/null || true)"
if [ "$current_hash" = "$desired_hash" ] && { [ "$agent_state" = active ] || [ "$agent_state" = activating ]; } && [ "\${agent_pid:-0}" -gt 0 ]; then exit 0; fi
if [ "$current_hash" = "$desired_hash" ] && { [ "$agent_state" = active ] || [ "$agent_state" = activating ]; } && [ "\${agent_pid:-0}" -gt 0 ]; then
if [ "$registry_changed" = true ]; then systemctl restart --no-block k3s-agent; fi
exit 0
fi
if [ -x /usr/local/bin/k3s-uninstall.sh ]; then /usr/local/bin/k3s-uninstall.sh; fi
if [ -x /usr/local/bin/k3s-agent-uninstall.sh ]; then /usr/local/bin/k3s-agent-uninstall.sh; fi
install -d -m 700 /etc/rancher/k3s
@@ -848,9 +1076,9 @@ ${bundle.applyScript}
function readSecrets(target: K3sClusterTarget): SecretMaterial {
if (!existsSync(target.secrets.sourceRef)) throw new Error(`${target.secrets.sourceRef} is missing; run platform-infra k3s-cluster secret-init --confirm`);
const values = parseEnvFile(readFileSync(target.secrets.sourceRef, "utf8"));
const required = (key: string) => {
const required = (key: string, minimumLength = 32) => {
const value = values[key];
if (typeof value !== "string" || value.length < 32) throw new Error(`${target.secrets.sourceRef}.${key} is missing or invalid`);
if (typeof value !== "string" || value.length < minimumLength) throw new Error(`${target.secrets.sourceRef}.${key} is missing or invalid`);
return value;
};
const material = {
@@ -859,6 +1087,9 @@ function readSecrets(target: K3sClusterTarget): SecretMaterial {
workerPrivate: required(target.secrets.workerPrivateKey),
workerPublic: required(target.secrets.workerPublicKey),
agentToken: required(target.secrets.agentToken),
registryUsername: required(target.secrets.registryUsername, 1),
registryPassword: required(target.secrets.registryPassword),
registryPasswordHash: required(target.secrets.registryPasswordHash),
};
return { ...material, fingerprint: hash(Object.entries(material).sort().map(([key, value]) => `${key}=${value}`).join("\n")) };
}
scripts/src/platform-infra-public-edge.ts
+89 -9
View File
@@ -67,6 +67,15 @@ export interface ResolvedSite extends SiteReference {
upstreamTlsServerName?: string;
healthPath: string;
expectedA: string;
healthExpectedStatuses?: number[];
accessPolicy?: {
allowedMethods: string[];
basicAuth: {
sourceRef: string;
usernameKey: string;
passwordHashKey: string;
};
};
routes?: ResolvedSiteRoute[];
}
@@ -334,11 +343,23 @@ export function resolveSites(target: PublicEdgeTarget): ResolutionResult {
throw new Error(`${label}.upstreamTlsServerName 仅允许 HTTPS upstream`);
}
const healthPath = resolveHealthPath(owner, exposure, label);
const healthExpectedStatuses = resolveHealthExpectedStatuses(exposure, label);
const accessPolicy = resolveAccessPolicy(owner, exposure, label);
const routes = resolveSiteRoutes(exposure, label);
const previous = hostnames.get(hostname);
if (previous !== undefined) throw new Error(`hostname 与 site ${previous} 重复：${hostname}`);
hostnames.set(hostname, reference.id);
sites.push({ ...reference, hostname, upstream, upstreamTlsServerName, healthPath, expectedA, ...(routes === undefined ? {} : { routes }) });
sites.push({
...reference,
hostname,
upstream,
upstreamTlsServerName,
healthPath,
expectedA,
...(healthExpectedStatuses === undefined ? {} : { healthExpectedStatuses }),
...(accessPolicy === undefined ? {} : { accessPolicy }),
...(routes === undefined ? {} : { routes }),
});
} catch (error) {
unresolved.push({ ...reference, code: "config-ref-unresolved", detail: error instanceof Error ? error.message : String(error) });
}
@@ -346,6 +367,40 @@ export function resolveSites(target: PublicEdgeTarget): ResolutionResult {
return { sites, unresolved };
}
function resolveHealthExpectedStatuses(exposure: Record<string, unknown>, label: string): number[] | undefined {
if (exposure.healthExpectedStatuses === undefined) return undefined;
const statuses = array(exposure.healthExpectedStatuses, `${label}.healthExpectedStatuses`).map((value, index) => {
if (!Number.isInteger(value) || Number(value) < 100 || Number(value) > 599) {
throw new Error(`${label}.healthExpectedStatuses[${index}] 必须是 HTTP status`);
}
return Number(value);
});
if (statuses.length === 0) throw new Error(`${label}.healthExpectedStatuses 不能为空`);
return [...new Set(statuses)];
}
function resolveAccessPolicy(
owner: Record<string, unknown>,
exposure: Record<string, unknown>,
label: string,
): ResolvedSite["accessPolicy"] {
if (exposure.accessPolicy === undefined) return undefined;
const policy = record(exposure.accessPolicy, `${label}.accessPolicy`);
const allowedMethods = stringArray(policy.allowedMethods, `${label}.accessPolicy.allowedMethods`);
if (allowedMethods.length === 0 || allowedMethods.some((method) => !/^[A-Z]+$/u.test(method))) {
throw new Error(`${label}.accessPolicy.allowedMethods 必须是非空大写 HTTP method 列表`);
}
const auth = record(owner.auth, `${label}.auth`);
return {
allowedMethods: [...new Set(allowedMethods)],
basicAuth: {
sourceRef: absolutePath(auth.sourceRef, `${label}.auth.sourceRef`),
usernameKey: envName(auth.usernameKey, `${label}.auth.usernameKey`),
passwordHashKey: envName(auth.passwordHashKey, `${label}.auth.passwordHashKey`),
},
};
}
function readSiteOwner(reference: SiteReference, ownerPath: string): Record<string, unknown> {
if (!isAbsolute(reference.configRef)) {
if (reference.sourceCommit !== undefined) throw new Error(`仓库内 configRef 不接受 sourceCommit：${reference.configRef}`);
@@ -650,7 +705,9 @@ export function renderPublicEdgeArtifacts(target: PublicEdgeTarget, sites: Resol
.map((site) => renderManagedBlock(`public-edge-site:${site.id}`, renderSiteBlock(site, target.runtime.responseHeaderTimeoutSeconds)))
.join("\n");
const dnsServers = target.runtime.dnsServers.map((server) => ` - ${server}`).join("\n");
const compose = `services:\n caddy:\n image: ${target.runtime.image}\n container_name: ${target.runtime.containerName}\n network_mode: host\n restart: unless-stopped\n dns:\n${dnsServers}\n volumes:\n - ${target.runtime.caddyfilePath}:/etc/caddy/Caddyfile:ro\n - ${target.runtime.dataDir}:/data\n - ${target.runtime.configDir}:/config\n`;
const envFiles = [...new Set(sites.flatMap((site) => site.accessPolicy === undefined ? [] : [site.accessPolicy.basicAuth.sourceRef]))];
const envFileBlock = envFiles.length === 0 ? "" : ` env_file:\n${envFiles.map((path) => ` - ${path}`).join("\n")}\n`;
const compose = `services:\n caddy:\n image: ${target.runtime.image}\n container_name: ${target.runtime.containerName}\n network_mode: host\n restart: unless-stopped\n${envFileBlock} dns:\n${dnsServers}\n volumes:\n - ${target.runtime.caddyfilePath}:/etc/caddy/Caddyfile:ro\n - ${target.runtime.dataDir}:/data\n - ${target.runtime.configDir}:/config\n`;
return { caddyfile: `${global}\n${blocks}`, compose };
}
@@ -660,6 +717,11 @@ function renderManagedBlock(owner: string, body: string): string {
function renderSiteBlock(site: ResolvedSite, defaultTimeoutSeconds: number): string {
if (site.routes === undefined) {
if (site.accessPolicy !== undefined) {
const deniedMethods = site.accessPolicy.allowedMethods.join(" ");
const auth = site.accessPolicy.basicAuth;
return `${site.hostname} \{\n\troute \{\n\t\t@denied not method ${deniedMethods}\n\t\trespond @denied 405\n\t\tbasic_auth \{\n\t\t\t{$${auth.usernameKey}} {$${auth.passwordHashKey}}\n\t\t\}\n${renderReverseProxy(site.upstream, defaultTimeoutSeconds, site.upstreamTlsServerName, 2)}\n\t\}\n\}\n`;
}
return `${site.hostname} \{\n${renderReverseProxy(site.upstream, defaultTimeoutSeconds, site.upstreamTlsServerName, 1)}\n\}\n`;
}
const routes = site.routes.map((route) => {
@@ -700,7 +762,9 @@ export function applyScript(
const desiredCaddyFingerprint = sha256Fingerprint(artifacts.caddyfile);
const desiredComposeFingerprint = sha256Fingerprint(artifacts.compose);
const dnsItems = sites.map((site) => shQuote(`${site.hostname}|${site.expectedA}`)).join(" ");
const probeItems = sites.map((site) => shQuote(`${site.hostname}|${site.healthPath}`)).join(" ");
const probeItems = sites.map((site) => shQuote(`${site.hostname}|${site.healthPath}|${site.healthExpectedStatuses?.join(",") ?? "2xx3xx"}`)).join(" ");
const authFiles = [...new Set(sites.flatMap((site) => site.accessPolicy === undefined ? [] : [site.accessPolicy.basicAuth.sourceRef]))];
const caddyEnvArgs = authFiles.map((path) => `--env-file ${shQuote(path)}`).join(" ");
return `set -u
mkdir -p "$(dirname ${shQuote(target.runtime.lockPath)})"
exec 9>${shQuote(target.runtime.lockPath)}
@@ -717,6 +781,7 @@ desired_caddy="$tmp/Caddyfile.desired"
current_caddy="$tmp/Caddyfile.current"
current_caddy_source="empty"
mkdir -p ${shQuote(target.runtime.workDir)} ${shQuote(target.runtime.dataDir)} ${shQuote(target.runtime.configDir)} "$(dirname ${shQuote(target.runtime.statePath)})"
${authFiles.map((path) => `[ -s ${shQuote(path)} ] || { printf '%s\\n' '{"ok":false,"mutation":false,"error":"public-edge-auth-source-missing","sourceRef":${JSON.stringify(path)},"valuesPrinted":false}'; exit 1; }`).join("\n")}
printf '%s' ${shQuote(ancestry)} | base64 -d >"$ancestry_file"
printf '%s' ${shQuote(caddyfile)} | base64 -d >"$desired_caddy"
printf '%s' ${shQuote(compose)} | base64 -d >"$compose_candidate"
@@ -815,8 +880,8 @@ for item in ${dnsItems}; do
fi
done
compose_rc=0; docker compose -f "$compose_candidate" config >/dev/null 2>"$tmp/compose.err" || compose_rc=$?
adapt_rc=0; docker run --rm -v "$caddy_candidate:/etc/caddy/Caddyfile:ro" ${shQuote(target.runtime.image)} caddy adapt --validate --config /etc/caddy/Caddyfile >/dev/null 2>"$tmp/adapt.err" || adapt_rc=$?
validate_rc=0; docker run --rm -v "$caddy_candidate:/etc/caddy/Caddyfile:ro" ${shQuote(target.runtime.image)} caddy validate --config /etc/caddy/Caddyfile >/dev/null 2>"$tmp/validate.err" || validate_rc=$?
adapt_rc=0; docker run --rm ${caddyEnvArgs} -v "$caddy_candidate:/etc/caddy/Caddyfile:ro" ${shQuote(target.runtime.image)} caddy adapt --validate --config /etc/caddy/Caddyfile >/dev/null 2>"$tmp/adapt.err" || adapt_rc=$?
validate_rc=0; docker run --rm ${caddyEnvArgs} -v "$caddy_candidate:/etc/caddy/Caddyfile:ro" ${shQuote(target.runtime.image)} caddy validate --config /etc/caddy/Caddyfile >/dev/null 2>"$tmp/validate.err" || validate_rc=$?
if [ "$listeners_rc" -ne 0 ] || [ "$udp_rc" -ne 0 ] || [ "$compose_rc" -ne 0 ] || [ "$adapt_rc" -ne 0 ] || [ "$validate_rc" -ne 0 ]; then
python3 - "$listeners_rc" "$udp_rc" "$dns_rc" "$compose_rc" "$adapt_rc" "$validate_rc" <<'PY'
import json, sys
@@ -918,10 +983,14 @@ probe_index=0
for item in ${probeItems}; do
probe_index=$((probe_index + 1))
(
host="${"${item%%|*}"}"; path="${"${item#*|}"}"; path="/${"${path#/}"}"
if ! curl --noproxy '*' -fsS --connect-timeout 3 --max-time 10 --resolve "$host:${target.listener.httpsPort}:127.0.0.1" "https://$host$path" >/dev/null 2>"$tmp/probe-$probe_index.err"; then
host="${"${item%%|*}"}"; rest="${"${item#*|}"}"; path="${"${rest%%|*}"}"; expected="${"${rest#*|}"}"; path="/${"${path#/}"}"
status="$(curl --noproxy '*' -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 --resolve "$host:${target.listener.httpsPort}:127.0.0.1" "https://$host$path" 2>"$tmp/probe-$probe_index.err" || true)"
case "$expected" in
2xx3xx) printf '%s' "$status" | grep -Eq '^[23][0-9][0-9]$' ;;
*) printf ',%s,' "$expected" | grep -Fq ",$status," ;;
esac || {
printf '%s\t%s\\n' "$host" "$path" >"$tmp/probe-failure-$probe_index.tsv"
fi
}
) &
done
wait || true
@@ -977,11 +1046,16 @@ export function renderPublicEdgeStatusScript(
hostname: site.hostname,
path: site.healthPath,
}), "utf8").toString("base64");
const expected = site.healthExpectedStatuses;
const statusCheck = expected === undefined
? `printf '%s' "$status" | grep -Eq '^[23][0-9][0-9]$'`
: `printf ',${expected.join(",")},' | grep -Fq ",$status,"`;
return `(
if ! getent ahostsv4 ${shQuote(site.hostname)} | awk '{print $1}' | grep -Fxq ${shQuote(site.expectedA)}; then
printf '%s' ${shQuote(dnsFailure)} | base64 -d >"$dns_failures_dir/${failureFile}"
fi
if ! curl --noproxy '*' -fsS --connect-timeout 3 --max-time 10 --resolve ${shQuote(`${site.hostname}:${target.listener.httpsPort}:127.0.0.1`)} ${shQuote(`https://${site.hostname}${site.healthPath}`)} >/dev/null; then
status="$(curl --noproxy '*' -sS -o /dev/null -w '%{http_code}' --connect-timeout 3 --max-time 10 --resolve ${shQuote(`${site.hostname}:${target.listener.httpsPort}:127.0.0.1`)} ${shQuote(`https://${site.hostname}${site.healthPath}`)} || true)"
if ! ${statusCheck}; then
printf '%s' ${shQuote(siteFailure)} | base64 -d >"$site_failures_dir/${failureFile}"
fi
) &`;
@@ -1200,6 +1274,12 @@ function absolutePath(value: unknown, path: string): string {
return result;
}
function envName(value: unknown, path: string): string {
const parsed = string(value, path);
if (!/^[A-Z_][A-Z0-9_]*$/u.test(parsed)) throw inputError(`${path} 必须是大写环境变量名`, "invalid-env-name", path);
return parsed;
}
function yamlConfigPath(value: unknown, path: string): string {
const result = string(value, path);
if (result.includes("..") || !result.endsWith(".yaml")) throw new Error(`${path} 必须是 YAML 路径`);