pikastech
25598b46d0
# Conflicts: # scripts/src/gc-remote-policy-runner.py # scripts/src/gc-remote-runner.py
700 lines
33 KiB
TypeScript
700 lines
33 KiB
TypeScript
import { Buffer } from "node:buffer";
|
|
import { existsSync, readFileSync } from "node:fs";
|
|
|
|
import { type UniDeskConfig, rootPath } from "./config";
|
|
import { remoteGcDegradedFailure } from "./gc-remote-degraded";
|
|
import { runSshCommandCapture } from "./ssh";
|
|
|
|
type RemoteGcAction = "plan" | "memory" | "memory-distribution" | "retention-plan" | "retention-run" | "merged-worktrees-plan" | "merged-worktrees-run" | "snapshot" | "trend" | "run" | "status" | "policy-plan" | "policy-install" | "policy-status";
|
|
|
|
interface RemoteGcOptions {
|
|
confirm: boolean;
|
|
journal: boolean;
|
|
journalTargetBytes: number;
|
|
dockerLogs: boolean;
|
|
dockerLogMaxBytes: number;
|
|
buildCache: boolean;
|
|
buildCacheUntil: string;
|
|
tmp: boolean;
|
|
tmpMinAgeHours: number;
|
|
toolCaches: boolean;
|
|
webObserveArtifacts: boolean;
|
|
k3sImageCache: boolean;
|
|
hostContainerdCache: boolean;
|
|
localPathOrphans: boolean;
|
|
aptCache: boolean;
|
|
coreDumps: boolean;
|
|
coreDumpMinAgeHours: number;
|
|
registry: boolean;
|
|
hwlabRegistry: boolean;
|
|
registryGcOnly: boolean;
|
|
registryKeepPerRepo: number;
|
|
registryMinAgeHours: number;
|
|
targetUsePercent?: number;
|
|
jobId?: string;
|
|
limit: number;
|
|
resultLimit: number;
|
|
full: boolean;
|
|
historyLimit: number;
|
|
saveSnapshot: boolean;
|
|
memoryPressureOnly: boolean;
|
|
buildCacheOnly: boolean;
|
|
hostDockerOnly: boolean;
|
|
}
|
|
|
|
const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = {
|
|
confirm: false,
|
|
journal: true,
|
|
journalTargetBytes: 512 * 1024 * 1024,
|
|
dockerLogs: true,
|
|
dockerLogMaxBytes: 50 * 1024 * 1024,
|
|
buildCache: true,
|
|
buildCacheUntil: "",
|
|
tmp: true,
|
|
tmpMinAgeHours: 24,
|
|
toolCaches: false,
|
|
webObserveArtifacts: false,
|
|
k3sImageCache: false,
|
|
hostContainerdCache: false,
|
|
localPathOrphans: false,
|
|
aptCache: true,
|
|
coreDumps: true,
|
|
coreDumpMinAgeHours: 1,
|
|
registry: false,
|
|
hwlabRegistry: false,
|
|
registryGcOnly: false,
|
|
registryKeepPerRepo: 20,
|
|
registryMinAgeHours: 48,
|
|
limit: 50,
|
|
resultLimit: 50,
|
|
full: false,
|
|
historyLimit: 12,
|
|
saveSnapshot: true,
|
|
memoryPressureOnly: false,
|
|
buildCacheOnly: false,
|
|
hostDockerOnly: false,
|
|
};
|
|
|
|
const GC_CONFIG_RELATIVE_PATH = "config/unidesk-cli.yaml";
|
|
const GC_REMOTE_CONFIG_REF = `${GC_CONFIG_RELATIVE_PATH}#gc.remote.targets`;
|
|
const GC_REMOTE_RUNNER_RELATIVE_PATH = "scripts/src/gc-remote-runner.py";
|
|
const GC_REMOTE_RUNNER_CONFIG_PLACEHOLDER = "__UNIDESK_GC_REMOTE_CONFIG_BASE64__";
|
|
const GC_REMOTE_WEB_OBSERVE_RELATIVE_PATH = "scripts/src/gc-remote-web-observe.py";
|
|
const GC_REMOTE_WEB_OBSERVE_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_WEB_OBSERVE_HELPERS__";
|
|
const GC_REMOTE_MEMORY_DISTRIBUTION_RELATIVE_PATH = "scripts/src/gc-remote-memory-distribution.py";
|
|
const GC_REMOTE_MEMORY_DISTRIBUTION_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_MEMORY_DISTRIBUTION_HELPERS__";
|
|
const GC_REMOTE_KUBERNETES_RETENTION_RELATIVE_PATH = "scripts/src/gc-remote-kubernetes-retention.py";
|
|
const GC_REMOTE_KUBERNETES_RETENTION_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_KUBERNETES_RETENTION_HELPERS__";
|
|
const GC_REMOTE_POLICY_KUBERNETES_RETENTION_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_POLICY_KUBERNETES_RETENTION_HELPERS__";
|
|
const GC_REMOTE_MERGED_WORKTREE_RELATIVE_PATH = "scripts/src/gc-remote-merged-worktrees.py";
|
|
const GC_REMOTE_MERGED_WORKTREE_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_MERGED_WORKTREE_HELPERS__";
|
|
const GC_REMOTE_POLICY_MERGED_WORKTREE_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_POLICY_MERGED_WORKTREE_HELPERS__";
|
|
const GC_REMOTE_CONTAINERD_RELATIVE_PATH = "scripts/src/gc-remote-containerd.py";
|
|
const GC_REMOTE_CONTAINERD_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_CONTAINERD_HELPERS__";
|
|
const GC_REMOTE_PVC_RELATIVE_PATH = "scripts/src/gc-remote-pvc.py";
|
|
const GC_REMOTE_PVC_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_PVC_HELPERS__";
|
|
const GC_REMOTE_GROWTH_RELATIVE_PATH = "scripts/src/gc-remote-growth.py";
|
|
const GC_REMOTE_GROWTH_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_GROWTH_HELPERS__";
|
|
const GC_REMOTE_REGISTRY_RELATIVE_PATH = "scripts/src/gc-remote-registry.py";
|
|
const GC_REMOTE_REGISTRY_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_REGISTRY_HELPERS__";
|
|
const GC_REMOTE_HOST_DOCKER_RELATIVE_PATH = "scripts/src/gc-remote-host-docker.py";
|
|
const GC_REMOTE_HOST_DOCKER_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_HOST_DOCKER_HELPERS__";
|
|
const GC_REMOTE_POLICY_RUNNER_RELATIVE_PATH = "scripts/src/gc-remote-policy-runner.py";
|
|
const GC_REMOTE_POLICY_RUNNER_PLACEHOLDER = "__UNIDESK_GC_REMOTE_POLICY_RUNNER_BASE64__";
|
|
|
|
export async function runRemoteGcCommand(config: UniDeskConfig, providerId: string | undefined, action: string | undefined, args: string[]): Promise<unknown> {
|
|
if (providerId === undefined || providerId.length === 0) {
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-provider-required",
|
|
usage: "bun scripts/cli.ts gc remote <providerId> memory|memory-distribution|retention|plan|snapshot|trend|run|status|policy [--include-registry] [--confirm]",
|
|
};
|
|
}
|
|
const subaction = action ?? "plan";
|
|
if (subaction === "retention") {
|
|
const [retentionAction = "plan", ...retentionArgs] = args;
|
|
const options = parseRemoteGcOptions(retentionArgs);
|
|
if (retentionAction === "plan" || retentionAction === "dry-run") {
|
|
return await runRemoteGc(config, providerId, "retention-plan", options);
|
|
}
|
|
if (retentionAction === "run") {
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-retention-run-requires-confirm",
|
|
dryRun: true,
|
|
mutation: false,
|
|
requiredFlag: "--confirm",
|
|
planCommand: `bun scripts/cli.ts gc remote ${providerId} retention plan --limit ${options.limit}`,
|
|
runCommand: `bun scripts/cli.ts gc remote ${providerId} retention run --confirm --limit ${options.limit}`,
|
|
};
|
|
}
|
|
return await runRemoteGc(config, providerId, "retention-run", options);
|
|
}
|
|
if (retentionAction === "status") {
|
|
return await runRemoteGc(config, providerId, "status", options);
|
|
}
|
|
return {
|
|
ok: false,
|
|
error: "unsupported-gc-remote-retention-action",
|
|
action: retentionAction,
|
|
supportedActions: ["plan", "run", "status"],
|
|
};
|
|
}
|
|
if (subaction === "merged-worktrees") {
|
|
const [worktreeAction = "plan", ...worktreeArgs] = args;
|
|
const options = parseRemoteGcOptions(worktreeArgs);
|
|
if (worktreeAction === "plan" || worktreeAction === "dry-run") return await runRemoteGc(config, providerId, "merged-worktrees-plan", options);
|
|
if (worktreeAction === "run") {
|
|
if (!options.confirm) return { ok: false, error: "gc-remote-merged-worktrees-run-requires-confirm", dryRun: true, mutation: false, requiredFlag: "--confirm", planCommand: `bun scripts/cli.ts gc remote ${providerId} merged-worktrees plan`, runCommand: `bun scripts/cli.ts gc remote ${providerId} merged-worktrees run --confirm` };
|
|
return await runRemoteGc(config, providerId, "merged-worktrees-run", options);
|
|
}
|
|
return { ok: false, error: "unsupported-gc-remote-merged-worktrees-action", action: worktreeAction, supportedActions: ["plan", "run"] };
|
|
}
|
|
if (subaction === "policy") {
|
|
const [policyAction = "plan", ...policyArgs] = args;
|
|
const options = parseRemoteGcOptions(policyArgs);
|
|
if (policyAction === "plan" || policyAction === "render" || policyAction === "dry-run") {
|
|
return await runRemoteGc(config, providerId, "policy-plan", options);
|
|
}
|
|
if (policyAction === "install") {
|
|
if (!options.confirm) {
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-policy-install-requires-confirm",
|
|
dryRun: true,
|
|
mutation: false,
|
|
requiredFlag: "--confirm",
|
|
planCommand: `bun scripts/cli.ts gc remote ${providerId} policy plan`,
|
|
installCommand: `bun scripts/cli.ts gc remote ${providerId} policy install --confirm`,
|
|
};
|
|
}
|
|
return await runRemoteGc(config, providerId, "policy-install", options);
|
|
}
|
|
if (policyAction === "status") {
|
|
return await runRemoteGc(config, providerId, "policy-status", options);
|
|
}
|
|
return {
|
|
ok: false,
|
|
error: "unsupported-gc-remote-policy-action",
|
|
action: policyAction,
|
|
supportedActions: ["plan", "render", "dry-run", "install", "status"],
|
|
};
|
|
}
|
|
const options = parseRemoteGcOptions(args);
|
|
if (options.memoryPressureOnly && subaction !== "plan" && subaction !== "run") {
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-memory-pressure-only-action-invalid",
|
|
action: subaction,
|
|
supportedActions: ["plan", "run"],
|
|
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan --memory-pressure-only`,
|
|
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm --memory-pressure-only`,
|
|
};
|
|
}
|
|
if (subaction === "memory") return await runRemoteGc(config, providerId, "memory", options);
|
|
if (subaction === "memory-distribution" || subaction === "memory-overview") {
|
|
return await runRemoteGc(config, providerId, "memory-distribution", options);
|
|
}
|
|
if (subaction === "plan" || subaction === "dry-run") return await runRemoteGc(config, providerId, "plan", options);
|
|
if (subaction === "snapshot" || subaction === "growth") return await runRemoteGc(config, providerId, "snapshot", options);
|
|
if (subaction === "trend") return await runRemoteGc(config, providerId, "trend", options);
|
|
if (subaction === "status") return await runRemoteGc(config, providerId, "status", options);
|
|
if (subaction === "run") {
|
|
if (!options.confirm) {
|
|
const registryFlag = options.registry ? " --include-registry" : "";
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-run-requires-confirm",
|
|
dryRun: true,
|
|
mutation: false,
|
|
requiredFlag: "--confirm",
|
|
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan${options.memoryPressureOnly ? " --memory-pressure-only" : registryFlag}`,
|
|
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm${options.memoryPressureOnly ? " --memory-pressure-only" : registryFlag}`,
|
|
};
|
|
}
|
|
return await runRemoteGc(config, providerId, "run", options);
|
|
}
|
|
return {
|
|
ok: false,
|
|
error: "unsupported-gc-remote-action",
|
|
action: subaction,
|
|
supportedActions: ["memory", "memory-distribution", "retention", "merged-worktrees", "plan", "snapshot", "trend", "run", "status", "policy"],
|
|
};
|
|
}
|
|
|
|
function parseRemoteGcOptions(args: string[]): RemoteGcOptions {
|
|
const options = { ...DEFAULT_REMOTE_OPTIONS };
|
|
for (let index = 0; index < args.length; index += 1) {
|
|
const arg = args[index] ?? "";
|
|
if (arg === "--confirm") {
|
|
options.confirm = true;
|
|
} else if (arg === "--dry-run") {
|
|
// accepted for plan compatibility
|
|
} else if (arg === "--journal-target-size") {
|
|
options.journalTargetBytes = parseSizeOption(arg, args[++index]);
|
|
} else if (arg === "--docker-log-max-bytes") {
|
|
options.dockerLogMaxBytes = parseSizeOption(arg, args[++index]);
|
|
} else if (arg === "--build-cache-until") {
|
|
const value = args[++index];
|
|
if (!value || !/^\d+(s|m|h|d)$/u.test(value)) throw new Error(`${arg} must look like 24h, 7d, 30m or 60s`);
|
|
options.buildCacheUntil = value;
|
|
} else if (arg === "--tmp-min-age-hours") {
|
|
options.tmpMinAgeHours = parseNonNegativeNumber(arg, args[++index]);
|
|
} else if (arg === "--core-dump-min-age-hours") {
|
|
options.coreDumpMinAgeHours = parseNonNegativeNumber(arg, args[++index]);
|
|
} else if (arg === "--registry-keep-per-repo") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
if (!Number.isInteger(value) || value < 1) throw new Error("--registry-keep-per-repo must be an integer >= 1");
|
|
options.registryKeepPerRepo = Math.min(value, 50);
|
|
} else if (arg === "--registry-min-age-hours") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
options.registryMinAgeHours = value;
|
|
} else if (arg === "--target-use-percent") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
if (!Number.isInteger(value) || value < 1 || value > 99) throw new Error("--target-use-percent must be an integer from 1 to 99");
|
|
options.targetUsePercent = value;
|
|
} else if (arg === "--job-id") {
|
|
const value = args[++index];
|
|
if (!value || !/^[A-Za-z0-9._-]{1,128}$/u.test(value)) throw new Error("--job-id must be a safe job id");
|
|
options.jobId = value;
|
|
} else if (arg === "--limit") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
if (!Number.isInteger(value) || value <= 0) throw new Error("--limit must be a positive integer");
|
|
options.limit = Math.min(value, 5000);
|
|
} else if (arg === "--result-limit") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
if (!Number.isInteger(value) || value <= 0) throw new Error("--result-limit must be a positive integer");
|
|
options.resultLimit = Math.min(value, 5000);
|
|
} else if (arg === "--history-limit") {
|
|
const value = parseNonNegativeNumber(arg, args[++index]);
|
|
if (!Number.isInteger(value) || value <= 1) throw new Error("--history-limit must be an integer greater than 1");
|
|
options.historyLimit = Math.min(value, 200);
|
|
} else if (arg === "--no-journal") {
|
|
options.journal = false;
|
|
} else if (arg === "--no-docker-logs") {
|
|
options.dockerLogs = false;
|
|
} else if (arg === "--no-build-cache") {
|
|
options.buildCache = false;
|
|
} else if (arg === "--no-tmp") {
|
|
options.tmp = false;
|
|
} else if (arg === "--include-tool-caches") {
|
|
options.toolCaches = true;
|
|
} else if (arg === "--no-tool-caches") {
|
|
options.toolCaches = false;
|
|
} else if (arg === "--include-web-observe-artifacts") {
|
|
options.webObserveArtifacts = true;
|
|
} else if (arg === "--no-web-observe-artifacts") {
|
|
options.webObserveArtifacts = false;
|
|
} else if (arg === "--include-k3s-image-cache") {
|
|
options.k3sImageCache = true;
|
|
} else if (arg === "--no-k3s-image-cache") {
|
|
options.k3sImageCache = false;
|
|
} else if (arg === "--include-host-containerd-cache") {
|
|
options.hostContainerdCache = true;
|
|
} else if (arg === "--no-host-containerd-cache") {
|
|
options.hostContainerdCache = false;
|
|
} else if (arg === "--include-local-path-orphans") {
|
|
options.localPathOrphans = true;
|
|
} else if (arg === "--no-local-path-orphans") {
|
|
options.localPathOrphans = false;
|
|
} else if (arg === "--no-apt-cache") {
|
|
options.aptCache = false;
|
|
} else if (arg === "--no-core-dumps") {
|
|
options.coreDumps = false;
|
|
} else if (arg === "--include-registry") {
|
|
options.registry = true;
|
|
} else if (arg === "--include-hwlab-registry") {
|
|
options.registry = true;
|
|
options.hwlabRegistry = true;
|
|
} else if (arg === "--registry-gc-only") {
|
|
options.registry = true;
|
|
options.hwlabRegistry = true;
|
|
options.registryGcOnly = true;
|
|
} else if (arg === "--full" || arg === "--raw") {
|
|
options.full = true;
|
|
} else if (arg === "--no-save" || arg === "--no-snapshot-save") {
|
|
options.saveSnapshot = false;
|
|
} else if (arg === "--memory-pressure-only") {
|
|
options.memoryPressureOnly = true;
|
|
} else if (arg === "--build-cache-only") {
|
|
options.buildCacheOnly = true;
|
|
} else if (arg === "--host-docker-only") {
|
|
options.hostDockerOnly = true;
|
|
} else {
|
|
throw new Error(`unknown gc remote option: ${arg}`);
|
|
}
|
|
}
|
|
return options;
|
|
}
|
|
|
|
function parseNonNegativeNumber(name: string, raw: string | undefined): number {
|
|
const value = Number(raw);
|
|
if (!Number.isFinite(value) || value < 0) throw new Error(`${name} must be a non-negative number`);
|
|
return value;
|
|
}
|
|
|
|
function parseSizeOption(name: string, raw: string | undefined): number {
|
|
const value = parseSize(raw ?? "");
|
|
if (value === null || value <= 0) throw new Error(`${name} must be a positive size such as 512M, 1GiB or 50000000`);
|
|
return value;
|
|
}
|
|
|
|
function parseSize(raw: string): number | null {
|
|
const match = raw.trim().match(/^(\d+(?:\.\d+)?)\s*(b|k|kb|kib|m|mb|mib|g|gb|gib)?$/iu);
|
|
if (!match) return null;
|
|
const value = Number(match[1]);
|
|
const unit = (match[2] ?? "b").toLowerCase();
|
|
const multiplier =
|
|
unit === "g" || unit === "gb" || unit === "gib" ? 1024 ** 3
|
|
: unit === "m" || unit === "mb" || unit === "mib" ? 1024 ** 2
|
|
: unit === "k" || unit === "kb" || unit === "kib" ? 1024
|
|
: 1;
|
|
const bytes = Math.floor(value * multiplier);
|
|
return Number.isFinite(bytes) ? bytes : null;
|
|
}
|
|
|
|
function yamlRecordOrEmpty(value: unknown, label: string): Record<string, unknown> {
|
|
if (value === undefined || value === null) return {};
|
|
if (typeof value !== "object" || Array.isArray(value)) throw new Error(`${label} must be a YAML object`);
|
|
return value as Record<string, unknown>;
|
|
}
|
|
|
|
function loadRemoteGcTargetConfig(providerId: string): Record<string, unknown> {
|
|
const configPath = rootPath(GC_CONFIG_RELATIVE_PATH);
|
|
if (!existsSync(configPath)) return {};
|
|
const parsed = yamlRecordOrEmpty(Bun.YAML.parse(readFileSync(configPath, "utf8")) as unknown, GC_CONFIG_RELATIVE_PATH);
|
|
const gc = yamlRecordOrEmpty(parsed.gc, `${GC_CONFIG_RELATIVE_PATH}#gc`);
|
|
const remote = yamlRecordOrEmpty(gc.remote, `${GC_CONFIG_RELATIVE_PATH}#gc.remote`);
|
|
const targets = yamlRecordOrEmpty(remote.targets, GC_REMOTE_CONFIG_REF);
|
|
const candidates = [providerId, providerId.toUpperCase(), providerId.toLowerCase()];
|
|
for (const key of candidates) {
|
|
if (Object.prototype.hasOwnProperty.call(targets, key)) {
|
|
const target = yamlRecordOrEmpty(targets[key], `${GC_REMOTE_CONFIG_REF}.${key}`);
|
|
const registryRetention = yamlRecordOrEmpty(target.registryRetention, `${GC_REMOTE_CONFIG_REF}.${key}.registryRetention`);
|
|
if (Object.keys(registryRetention).length === 0) return target;
|
|
const configRef = typeof registryRetention.configRef === "string" ? registryRetention.configRef : "";
|
|
if (configRef.length === 0) throw new Error(`${GC_REMOTE_CONFIG_REF}.${key}.registryRetention.configRef must be a file#path reference`);
|
|
return {
|
|
...target,
|
|
registryRetention: {
|
|
...registryRetention,
|
|
resolvedRegistry: resolveYamlConfigRef(configRef),
|
|
},
|
|
};
|
|
}
|
|
}
|
|
return {};
|
|
}
|
|
|
|
function resolveYamlConfigRef(configRef: string): Record<string, unknown> {
|
|
const separator = configRef.indexOf("#");
|
|
if (separator <= 0 || separator === configRef.length - 1) throw new Error(`configRef must contain file#path: ${configRef}`);
|
|
const relativePath = configRef.slice(0, separator);
|
|
const selector = configRef.slice(separator + 1);
|
|
const absolutePath = rootPath(relativePath);
|
|
if (!existsSync(absolutePath)) throw new Error(`configRef file does not exist: ${relativePath}`);
|
|
let value: unknown = Bun.YAML.parse(readFileSync(absolutePath, "utf8")) as unknown;
|
|
for (const segment of selector.split(".").filter(Boolean)) {
|
|
const record = yamlRecordOrEmpty(value, `${relativePath}#${selector}`);
|
|
if (!Object.prototype.hasOwnProperty.call(record, segment)) throw new Error(`configRef path not found: ${configRef}`);
|
|
value = record[segment];
|
|
}
|
|
return yamlRecordOrEmpty(value, configRef);
|
|
}
|
|
|
|
function recordOrEmpty(value: unknown): Record<string, unknown> {
|
|
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
|
|
}
|
|
|
|
function positiveInteger(value: unknown): number | null {
|
|
return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null;
|
|
}
|
|
|
|
function compactMemoryPressureCandidate(value: unknown): Record<string, unknown> {
|
|
const candidate = recordOrEmpty(value);
|
|
const action = recordOrEmpty(candidate.action);
|
|
const base = {
|
|
id: candidate.id,
|
|
kind: candidate.kind,
|
|
reason: candidate.reason,
|
|
risk: candidate.risk,
|
|
action: { op: action.op, allowlist: action.allowlist },
|
|
};
|
|
if (candidate.kind === "cgroup-memory-reclaim") {
|
|
return {
|
|
...base,
|
|
configId: candidate.configId,
|
|
path: candidate.path,
|
|
reclaimBytes: candidate.reclaimBytes,
|
|
requestedMemoryReclaimBytes: candidate.requestedMemoryReclaimBytes,
|
|
requestedMemoryReclaim: candidate.requestedMemoryReclaim,
|
|
estimateKind: candidate.estimateKind,
|
|
swappiness: candidate.swappiness,
|
|
minimumInactiveAnonBytes: candidate.minimumInactiveAnonBytes,
|
|
targetMemAvailableBytes: candidate.targetMemAvailableBytes,
|
|
minimumSwapFreeBytesAfterReclaim: candidate.minimumSwapFreeBytesAfterReclaim,
|
|
memoryCurrentBytes: candidate.memoryCurrentBytes,
|
|
memorySwapCurrentBytes: candidate.memorySwapCurrentBytes,
|
|
};
|
|
}
|
|
const identities = Array.isArray(candidate.processIdentities) ? candidate.processIdentities.map(recordOrEmpty) : [];
|
|
const processCount = positiveInteger(candidate.processCount);
|
|
const identitiesComplete = processCount !== null
|
|
&& identities.length === processCount
|
|
&& identities.every((identity) => positiveInteger(identity.pid) !== null
|
|
&& typeof identity.ppid === "number" && Number.isInteger(identity.ppid) && identity.ppid >= 0
|
|
&& positiveInteger(identity.sid) !== null
|
|
&& positiveInteger(identity.startTicks) !== null);
|
|
return {
|
|
...base,
|
|
expectedMemoryRssBytes: candidate.expectedMemoryRssBytes,
|
|
expectedMemoryRss: candidate.expectedMemoryRss,
|
|
rootPid: candidate.rootPid,
|
|
sid: candidate.sid,
|
|
rootStartTicks: candidate.rootStartTicks,
|
|
processCount: candidate.processCount,
|
|
activeObserverIntersection: candidate.activeObserverIntersection,
|
|
treeClosure: candidate.treeClosure,
|
|
identityClosure: {
|
|
identityCount: identities.length,
|
|
identitiesComplete,
|
|
basis: "pre-term:pid-ppid-sid-startTicks;post-term:pid-startTicks",
|
|
},
|
|
};
|
|
}
|
|
|
|
function memoryPressureCandidateAllowed(value: unknown): boolean {
|
|
const candidate = recordOrEmpty(value);
|
|
const action = recordOrEmpty(candidate.action);
|
|
if (typeof candidate.id !== "string" || candidate.id.length === 0 || positiveInteger(candidate.expectedMemoryRssBytes) === null) return false;
|
|
if (candidate.kind === "cgroup-memory-reclaim") {
|
|
const swappiness = typeof candidate.swappiness === "number" && Number.isInteger(candidate.swappiness)
|
|
? candidate.swappiness
|
|
: null;
|
|
return typeof candidate.configId === "string" && candidate.configId.length > 0
|
|
&& typeof candidate.path === "string" && candidate.path.startsWith("/")
|
|
&& positiveInteger(candidate.reclaimBytes) !== null
|
|
&& swappiness !== null && swappiness >= 0 && swappiness <= 200
|
|
&& (swappiness === 0 || positiveInteger(candidate.minimumInactiveAnonBytes) !== null)
|
|
&& action.op === "cgroup-v2-memory-reclaim"
|
|
&& action.allowlist === "yaml-exact-cgroup-path";
|
|
}
|
|
if (candidate.kind !== "browser-orphan-process-tree" && candidate.kind !== "stale-web-observer-process-tree") return false;
|
|
const compact = compactMemoryPressureCandidate(candidate);
|
|
const identityClosure = recordOrEmpty(compact.identityClosure);
|
|
return positiveInteger(candidate.rootPid) !== null
|
|
&& positiveInteger(candidate.sid) !== null
|
|
&& positiveInteger(candidate.rootStartTicks) !== null
|
|
&& candidate.activeObserverIntersection === false
|
|
&& candidate.treeClosure === "unique-root-ppid-sid-closed"
|
|
&& identityClosure.identitiesComplete === true
|
|
&& action.op === "term-wait-kill-exact-process-tree"
|
|
&& action.allowlist === "yaml-web-probe-memory-pressure";
|
|
}
|
|
|
|
function compactMemoryPressureCandidateScan(value: unknown): Record<string, unknown> {
|
|
const scan = recordOrEmpty(value);
|
|
const processPressure = recordOrEmpty(scan.processPressure);
|
|
const observeRoots = recordOrEmpty(scan.observeRoots);
|
|
const cgroupReclaim = recordOrEmpty(scan.cgroupReclaim);
|
|
const cgroupProtected = Array.isArray(cgroupReclaim.protected) ? cgroupReclaim.protected : [];
|
|
return {
|
|
ok: scan.ok,
|
|
blocked: scan.blocked,
|
|
configError: scan.configError,
|
|
observeScanTruncated: scan.observeScanTruncated,
|
|
processScanTruncated: scan.processScanTruncated,
|
|
observeScanError: scan.observeScanError,
|
|
processScanError: scan.processScanError,
|
|
processPressure: {
|
|
processCount: processPressure.processCount,
|
|
zombieCount: processPressure.zombieCount,
|
|
zombieKindCount: Array.isArray(processPressure.zombiesByComm) ? processPressure.zombiesByComm.length : 0,
|
|
zombieParentCount: Array.isArray(processPressure.zombiesByParent) ? processPressure.zombiesByParent.length : 0,
|
|
topRssSampleCount: Array.isArray(processPressure.topRss) ? processPressure.topRss.length : 0,
|
|
},
|
|
observeRoots: {
|
|
configuredCount: observeRoots.configuredCount,
|
|
readableCount: observeRoots.readableCount,
|
|
missingCount: observeRoots.missingCount,
|
|
minimumReadable: observeRoots.minimumReadable,
|
|
},
|
|
cgroupReclaim: {
|
|
configuredCount: cgroupReclaim.configuredCount,
|
|
eligibleCount: cgroupReclaim.eligibleCount,
|
|
protectedCount: cgroupProtected.length,
|
|
protected: cgroupProtected.map((item) => {
|
|
const protectedItem = recordOrEmpty(item);
|
|
return { id: protectedItem.id, reason: protectedItem.reason, memoryCurrentBytes: protectedItem.memoryCurrentBytes };
|
|
}),
|
|
},
|
|
policySource: scan.policySource,
|
|
};
|
|
}
|
|
|
|
export function projectRemoteGcResult(
|
|
value: unknown,
|
|
remoteTarget: Record<string, unknown>,
|
|
context: { action: string; memoryPressureOnly: boolean; full: boolean },
|
|
): unknown {
|
|
if (context.action !== "plan" || !context.memoryPressureOnly || context.full) return value;
|
|
const payload = recordOrEmpty(value);
|
|
const scan = recordOrEmpty(payload.candidateScan);
|
|
const summary = recordOrEmpty(payload.summary);
|
|
const candidates = Array.isArray(payload.candidates) ? payload.candidates : [];
|
|
const memoryPressure = recordOrEmpty(remoteTarget.memoryPressure);
|
|
const previewLimit = positiveInteger(memoryPressure.planPreviewLimit);
|
|
const totalCandidateCount = positiveInteger(summary.candidateCount) ?? 0;
|
|
const allCandidatesEvaluated = totalCandidateCount > 0 && totalCandidateCount === candidates.length;
|
|
const reasons: string[] = [];
|
|
if (payload.ok !== true) reasons.push("plan-not-ok");
|
|
if (scan.ok !== true || scan.blocked === true) reasons.push("candidate-scan-blocked");
|
|
if (totalCandidateCount === 0) reasons.push("no-candidates");
|
|
if (!allCandidatesEvaluated) reasons.push("candidate-page-incomplete");
|
|
if (!candidates.every(memoryPressureCandidateAllowed)) reasons.push("unsupported-or-incomplete-candidate");
|
|
if (previewLimit === null) reasons.push("yaml-plan-preview-limit-invalid");
|
|
const protectedState = recordOrEmpty(payload.protected);
|
|
const compactScan = compactMemoryPressureCandidateScan(scan);
|
|
const compactCgroup = recordOrEmpty(compactScan.cgroupReclaim);
|
|
return {
|
|
ok: payload.ok,
|
|
action: payload.action,
|
|
providerId: payload.providerId,
|
|
scope: payload.scope,
|
|
dryRun: payload.dryRun,
|
|
mutation: payload.mutation,
|
|
observedAt: payload.observedAt,
|
|
memoryBefore: payload.memoryBefore,
|
|
summary: payload.summary,
|
|
candidateScan: compactScan,
|
|
runEligibility: {
|
|
allowed: reasons.length === 0,
|
|
reasons,
|
|
evaluatedCandidateCount: candidates.length,
|
|
totalCandidateCount,
|
|
allCandidatesEvaluated,
|
|
},
|
|
candidatePreview: {
|
|
limit: previewLimit,
|
|
returnedCount: previewLimit === null ? 0 : Math.min(candidates.length, previewLimit),
|
|
omittedCount: previewLimit === null ? candidates.length : Math.max(0, candidates.length - previewLimit),
|
|
candidates: previewLimit === null ? [] : candidates.slice(0, previewLimit).map(compactMemoryPressureCandidate),
|
|
},
|
|
protected: {
|
|
activeObserverIntersection: protectedState.activeObserverIntersection,
|
|
processIdentity: protectedState.processIdentity,
|
|
treeClosure: protectedState.treeClosure,
|
|
cgroupProtectedCount: compactCgroup.protectedCount,
|
|
},
|
|
policy: payload.policy,
|
|
valuesRedacted: true,
|
|
};
|
|
}
|
|
|
|
async function runRemoteGc(config: UniDeskConfig, providerId: string, action: RemoteGcAction, options: RemoteGcOptions): Promise<unknown> {
|
|
const remoteTarget = loadRemoteGcTargetConfig(providerId);
|
|
const scriptConfig = Buffer.from(JSON.stringify({ providerId, action, options, remoteTarget }), "utf8").toString("base64");
|
|
const result = await runSshCommandCapture(config, providerId, ["py"], remoteGcPython(scriptConfig));
|
|
if (result.exitCode !== 0) {
|
|
const degraded = remoteGcDegradedFailure(providerId, action, result);
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-command-failed",
|
|
providerId,
|
|
action: `gc remote ${action}`,
|
|
exitCode: result.exitCode,
|
|
degradedReason: degraded.degradedReason,
|
|
transport: degraded.transport,
|
|
safeCandidateCount: null,
|
|
runAllowed: false,
|
|
mutation: false,
|
|
degraded,
|
|
stdoutTail: result.stdout.slice(-4000),
|
|
stderrTail: result.stderr.slice(-4000),
|
|
next: degraded.next,
|
|
};
|
|
}
|
|
try {
|
|
return projectRemoteGcResult(JSON.parse(result.stdout) as unknown, remoteTarget, {
|
|
action,
|
|
memoryPressureOnly: options.memoryPressureOnly,
|
|
full: options.full,
|
|
});
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
error: "gc-remote-invalid-json",
|
|
providerId,
|
|
action: `gc remote ${action}`,
|
|
message: error instanceof Error ? error.message : String(error),
|
|
stdoutTail: result.stdout.slice(-4000),
|
|
stderrTail: result.stderr.slice(-4000),
|
|
};
|
|
}
|
|
}
|
|
|
|
function remoteGcPython(configBase64: string): string {
|
|
const template = readFileSync(rootPath(GC_REMOTE_RUNNER_RELATIVE_PATH), "utf8");
|
|
if (!template.includes(GC_REMOTE_RUNNER_CONFIG_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_RUNNER_CONFIG_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_WEB_OBSERVE_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_WEB_OBSERVE_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_MEMORY_DISTRIBUTION_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_MEMORY_DISTRIBUTION_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_KUBERNETES_RETENTION_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_KUBERNETES_RETENTION_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_MERGED_WORKTREE_PLACEHOLDER)) throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_MERGED_WORKTREE_PLACEHOLDER}`);
|
|
if (!template.includes(GC_REMOTE_CONTAINERD_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_CONTAINERD_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_PVC_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_PVC_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_GROWTH_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_GROWTH_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_REGISTRY_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_REGISTRY_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_HOST_DOCKER_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_HOST_DOCKER_PLACEHOLDER}`);
|
|
}
|
|
if (!template.includes(GC_REMOTE_POLICY_RUNNER_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_POLICY_RUNNER_PLACEHOLDER}`);
|
|
}
|
|
const webObserveHelpers = readFileSync(rootPath(GC_REMOTE_WEB_OBSERVE_RELATIVE_PATH), "utf8");
|
|
const memoryDistributionHelpers = readFileSync(rootPath(GC_REMOTE_MEMORY_DISTRIBUTION_RELATIVE_PATH), "utf8");
|
|
const kubernetesRetentionHelpers = readFileSync(rootPath(GC_REMOTE_KUBERNETES_RETENTION_RELATIVE_PATH), "utf8");
|
|
const mergedWorktreeHelpers = readFileSync(rootPath(GC_REMOTE_MERGED_WORKTREE_RELATIVE_PATH), "utf8");
|
|
const containerdHelpers = readFileSync(rootPath(GC_REMOTE_CONTAINERD_RELATIVE_PATH), "utf8");
|
|
const pvcHelpers = readFileSync(rootPath(GC_REMOTE_PVC_RELATIVE_PATH), "utf8");
|
|
const growthHelpers = readFileSync(rootPath(GC_REMOTE_GROWTH_RELATIVE_PATH), "utf8");
|
|
const registryHelpers = readFileSync(rootPath(GC_REMOTE_REGISTRY_RELATIVE_PATH), "utf8");
|
|
const hostDockerHelpers = readFileSync(rootPath(GC_REMOTE_HOST_DOCKER_RELATIVE_PATH), "utf8");
|
|
const policyRunnerTemplate = readFileSync(rootPath(GC_REMOTE_POLICY_RUNNER_RELATIVE_PATH), "utf8");
|
|
if (!policyRunnerTemplate.includes(GC_REMOTE_POLICY_KUBERNETES_RETENTION_PLACEHOLDER)) {
|
|
throw new Error(`${GC_REMOTE_POLICY_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_POLICY_KUBERNETES_RETENTION_PLACEHOLDER}`);
|
|
}
|
|
if (!policyRunnerTemplate.includes(GC_REMOTE_POLICY_MERGED_WORKTREE_PLACEHOLDER)) throw new Error(`${GC_REMOTE_POLICY_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_POLICY_MERGED_WORKTREE_PLACEHOLDER}`);
|
|
const policyRunner = policyRunnerTemplate
|
|
.replace(GC_REMOTE_POLICY_KUBERNETES_RETENTION_PLACEHOLDER, () => kubernetesRetentionHelpers.trimEnd())
|
|
.replace(GC_REMOTE_POLICY_MERGED_WORKTREE_PLACEHOLDER, () => mergedWorktreeHelpers.trimEnd());
|
|
return template
|
|
.replace(GC_REMOTE_MEMORY_DISTRIBUTION_PLACEHOLDER, () => memoryDistributionHelpers.trimEnd())
|
|
.replace(GC_REMOTE_KUBERNETES_RETENTION_PLACEHOLDER, () => kubernetesRetentionHelpers.trimEnd())
|
|
.replace(GC_REMOTE_MERGED_WORKTREE_PLACEHOLDER, () => mergedWorktreeHelpers.trimEnd())
|
|
.replace(GC_REMOTE_WEB_OBSERVE_PLACEHOLDER, () => webObserveHelpers.trimEnd())
|
|
.replace(GC_REMOTE_CONTAINERD_PLACEHOLDER, () => containerdHelpers.trimEnd())
|
|
.replace(GC_REMOTE_PVC_PLACEHOLDER, () => pvcHelpers.trimEnd())
|
|
.replace(GC_REMOTE_GROWTH_PLACEHOLDER, () => growthHelpers.trimEnd())
|
|
.replace(GC_REMOTE_REGISTRY_PLACEHOLDER, () => registryHelpers.trimEnd())
|
|
.replace(GC_REMOTE_HOST_DOCKER_PLACEHOLDER, () => hostDockerHelpers.trimEnd())
|
|
.replace(GC_REMOTE_POLICY_RUNNER_PLACEHOLDER, Buffer.from(policyRunner, "utf8").toString("base64"))
|
|
.replace(GC_REMOTE_RUNNER_CONFIG_PLACEHOLDER, configBase64);
|
|
}
|