mirrors/pikasTech-unidesk
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add NC01 registry retention GC

Browse Source
This commit is contained in:
pikastech
2026-07-21 07:14:22 +02:00
parent 9471f43b45
commit 03c96bec19
6 changed files with 529 additions and 83 deletions
Download Patch File Download Diff File Expand all files Collapse all files
config/unidesk-cli.yaml
+49
View File
@@ -154,6 +154,39 @@ gc:
remote:
targets:
NC01:
registryRetention:
enabled: true
configRef: config/hwlab-node-control-plane.yaml#nodes.NC01.registry
repositoryPrefixes:
- hwlab/
- agentrun/
- pikaoa/
- selfmedia/
- unidesk/
protectedRepositoryPrefixes:
- hwlab/cache/
- pikaoa/cache-
protectedTags:
- latest
- 16-alpine
- 20-bookworm-slim
- node22-alpine-v1
- node22-alpine-bun-v1
- sidecar
deletableTagPattern: ^[0-9a-f]{7,64}$
keepPerRepo: 20
minAgeHours: 48
previewLimit: 20
estimateBlobLimit: 5000
automatic:
enabled: false
maxDeleteManifestsPerRun: 100
requireIdleNamespaces:
- devops-infra
- agentrun-ci
- hwlab-ci
- pikaoa-ci
- selfmedia-ci
hostDockerGc:
dockerJsonLogs:
enabled: false
@@ -332,6 +365,22 @@ gc:
includeHostContainerdCache: false
includeLocalPathOrphans: false
includeKubernetesObjectRetention: true
pvcAttribution:
namespaces:
- agentrun-ci
- agentrun-v02
- devops-infra
- hwlab-ci
- pikaoa-ci
- selfmedia-ci
candidateNamespaces:
- agentrun-ci
- hwlab-ci
hwlabNode: NC01
hwlabLane: v03
agentrunNode: NC01
agentrunLane: nc01-v02
limit: 80
JD01:
hostDockerGc:
dockerJsonLogs:
docs/reference/gc.md
+12
View File
@@ -42,6 +42,18 @@
- 按目标节点 `kubernetesObjectRetention` YAML 规划和异步回收 Kubernetes 历史对象；
- `plan` 与手动、自动 `run` 必须复用同一个 fail-closed 筛选器；
- 表格形状、时间或分组无法识别时保护对象并停止删除。
- `gc remote <providerId> plan --include-registry`：
- 从 `config/unidesk-cli.yaml#gc.remote.targets.<providerId>.registryRetention` 读取留存策略；
- 通过显式 `configRef` 读取 node-local registry 的 endpoint、Deployment、namespace 和 PVC；
- 保护当前 workload tag/digest refs、近期 tag、每仓库最新 tag、保护仓库和 digest closure；
- 输出候选 manifest、保护原因、估算收益和 typed diagnosis；
- 不支持的目标、PVC 未绑定、存储布局不可读或 workload 形状漂移时必须 fail closed。
- `gc remote <providerId> run --confirm --include-registry`：
- 作为异步 job 返回，并通过 `gc remote <providerId> status --job-id <id>` 短查询；
- 只通过 Registry API 删除 plan 中不受保护的 manifest；
- 对 YAML 声明的 Registry Deployment 缩容后运行官方 `registry garbage-collect`；
- PVC 只挂载给 GC Pod，不删除 PVC、PV、blob 目录、containerd 或 k3s storage；
- 无论成功或失败都恢复原副本数，并复核 Deployment readiness 与 `/v2/` endpoint。
- `--include-tool-caches`：本机和远端都必须是显式 opt-in，只清理固定 allowlist 的可重建 npm/npx/Bun 缓存；不得扩大成清理 `~/.npm`、`~/.bun`、`node_modules`、auth/config 或运行面依赖。
所有成功和失败输出都必须是 JSON。`plan` 必须标记 `dryRun=true`、`mutation=false`；`run` 必须要求 `--confirm` 并报告 `diskBefore`、`diskAfter`、`summary`、`results` 和 `protected`。远端 GC 可用 `--target-use-percent N` 显式表达目标根盘水位；`summary.target` 必须给出目标所需释放量、候选估算、预计水位、缺口和 `safeStop` 决策，避免靠人工心算判断是否应该继续扩大清理范围。
scripts/src/gc-remote-pvc.py
+3 -1
View File
@@ -65,7 +65,9 @@ def ci_storage_snapshot():
hwlab_node = config_str(PVC_CONFIG, "hwlabNode", PROVIDER_ID)
hwlab_lane = config_str(PVC_CONFIG, "hwlabLane", "v03")
agentrun_node = config_str(PVC_CONFIG, "agentrunNode", PROVIDER_ID)
agentrun_lane = config_str(PVC_CONFIG, "agentrunLane", "v02")
agentrun_lane = config_str(PVC_CONFIG, "agentrunLane", "")
if not agentrun_lane:
raise RuntimeError("gc.remote.targets.%s.pvcAttribution.agentrunLane must be declared in YAML" % PROVIDER_ID)
limit = config_int(PVC_CONFIG, "limit", int(OPTIONS.get("limit") or 50), minimum=1, maximum=5000)
pv_data = kubectl_json(["get", "pv"], 30) or {}
pvc_data = kubectl_json(["get", "pvc", "-A"], 30) or {}
scripts/src/gc-remote-registry.py
+315 -52
View File
@@ -1,21 +1,100 @@
def active_hwlab_ci_writes():
result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pipelinerun,taskrun -n hwlab-ci --no-headers 2>/dev/null | awk '$2 != \"True\" && $2 != \"False\" {print}' | head -40"], 15)
lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()]
return {"ok": result["exitCode"] == 0, "activeCount": len(lines), "activePreview": lines, "command": bounded(result)}
def registry_config_list(key, default=None):
return config_list(REGISTRY_CONFIG, key, default or [])
def active_hwlab_ci_jobs():
result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get jobs -n hwlab-ci --no-headers 2>/dev/null | awk '$2 != \"Complete\" && $2 != \"Failed\" {print}' | head -40"], 15)
lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()]
return {"ok": result["exitCode"] == 0, "activeCount": len(lines), "activePreview": lines, "command": bounded(result)}
def registry_config_str(key, default=""):
return config_str(REGISTRY_CONFIG, key, default)
def wait_no_active_hwlab_ci(timeout=180):
def registry_spec():
value = REGISTRY_CONFIG.get("resolvedRegistry") if isinstance(REGISTRY_CONFIG, dict) else None
return value if isinstance(value, dict) else {}
def registry_identity():
spec = registry_spec()
mode = config_str(spec, "mode", "host-path" if PROVIDER_ID.upper() == "G14" else "")
namespace = config_str(spec, "namespace", "hwlab-ci" if PROVIDER_ID.upper() == "G14" else "")
deployment = config_str(spec, "deploymentName", "hwlab-registry" if PROVIDER_ID.upper() == "G14" else "")
pvc = config_str(spec, "pvcName", "")
endpoint = config_str(spec, "endpoint", "127.0.0.1:5000")
image = config_str(spec, "image", "registry:2.8.3")
return {
"mode": mode,
"namespace": namespace,
"deploymentName": deployment,
"pvcName": pvc,
"endpoint": endpoint,
"image": image,
"configRef": registry_config_str("configRef", "") if REGISTRY_CONFIG else ("legacy:G14" if PROVIDER_ID.upper() == "G14" else None),
}
REGISTRY_ROOT = "/var/lib/hwlab/registry"
REGISTRY_REPOSITORY_ROOT = os.path.join(REGISTRY_ROOT, "docker/registry/v2/repositories")
def configure_registry_storage():
global REGISTRY_ROOT, REGISTRY_REPOSITORY_ROOT
identity = registry_identity()
if REGISTRY_CONFIG and not config_bool(REGISTRY_CONFIG, "enabled", False):
return {"ok": False, "diagnosis": registry_diagnosis("registry-retention-disabled", "Registry retention is disabled by YAML", configRef=identity.get("configRef"))}
if identity.get("mode") == "host-path" and PROVIDER_ID.upper() == "G14":
REGISTRY_ROOT = "/var/lib/hwlab/registry"
REGISTRY_REPOSITORY_ROOT = os.path.join(REGISTRY_ROOT, "docker/registry/v2/repositories")
return {"ok": True, "identity": identity, "storagePath": REGISTRY_ROOT, "storageKind": "host-path"}
if identity.get("mode") != "k8s-workload":
return {"ok": False, "diagnosis": registry_diagnosis("registry-target-mode-unsupported", "Registry retention requires a YAML-declared k8s-workload registry", mode=identity.get("mode"), configRef=identity.get("configRef"))}
required = ["namespace", "deploymentName", "pvcName", "endpoint"]
missing = [key for key in required if not identity.get(key)]
if missing:
return {"ok": False, "diagnosis": registry_diagnosis("registry-target-config-incomplete", "Registry configRef is missing required workload fields", missingFields=missing, configRef=identity.get("configRef"))}
pvc = kubectl_json(["-n", identity["namespace"], "get", "pvc", identity["pvcName"]], 20)
if not pvc:
return {"ok": False, "diagnosis": registry_diagnosis("registry-pvc-unavailable", "Registry PVC could not be read", namespace=identity["namespace"], pvcName=identity["pvcName"], configRef=identity.get("configRef"))}
volume_name = str(((pvc.get("spec") or {}).get("volumeName")) or "")
if not volume_name or str(((pvc.get("status") or {}).get("phase")) or "") != "Bound":
return {"ok": False, "diagnosis": registry_diagnosis("registry-pvc-not-bound", "Registry PVC must be Bound before retention planning", namespace=identity["namespace"], pvcName=identity["pvcName"], volumeName=volume_name or None)}
pv = kubectl_json(["get", "pv", volume_name], 20)
spec = (pv or {}).get("spec") or {}
storage_path = str(((spec.get("local") or {}).get("path")) or ((spec.get("hostPath") or {}).get("path")) or "")
if not storage_path.startswith("/"):
return {"ok": False, "diagnosis": registry_diagnosis("registry-pv-storage-path-unsupported", "Registry PV does not expose a node-local absolute storage path", volumeName=volume_name, storageClass=spec.get("storageClassName"))}
storage_path = os.path.realpath(storage_path)
repository_root = os.path.join(storage_path, "docker/registry/v2/repositories")
if not os.path.isdir(repository_root):
return {"ok": False, "diagnosis": registry_diagnosis("registry-storage-layout-unavailable", "Registry repository layout is not readable from the node-local PVC path", storagePath=storage_path, repositoryRoot=repository_root)}
REGISTRY_ROOT = storage_path
REGISTRY_REPOSITORY_ROOT = repository_root
return {"ok": True, "identity": identity, "storagePath": storage_path, "storageKind": "pvc", "volumeName": volume_name}
def registry_diagnosis(code, message, **details):
return {"code": code, "message": message, "providerId": PROVIDER_ID, **details}
def active_registry_writes():
automatic = REGISTRY_CONFIG.get("automatic") if isinstance(REGISTRY_CONFIG.get("automatic"), dict) else {}
namespaces = config_list(automatic, "requireIdleNamespaces", [])
rows = []
commands = []
for namespace in namespaces:
result = kctl(["-n", namespace, "get", "pipelinerun,taskrun", "-o", "json"], 20)
commands.append(bounded(result))
if result["exitCode"] != 0:
return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace}
try:
items = json.loads(result.get("stdout") or "{}").get("items") or []
except Exception:
return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace, "error": "invalid-kubernetes-json"}
for item in items:
conditions = ((item.get("status") or {}).get("conditions")) or []
terminal = any(condition.get("type") == "Succeeded" and condition.get("status") in set(["True", "False"]) for condition in conditions)
if not terminal:
meta = item.get("metadata") or {}
rows.append("%s/%s/%s" % (namespace, item.get("kind") or "Unknown", meta.get("name") or "unknown"))
return {"ok": True, "activeCount": len(rows), "activePreview": rows[:40], "commands": commands}
def wait_no_active_registry_writes(timeout=180):
deadline = time.time() + timeout
last = None
while time.time() < deadline:
writes = active_hwlab_ci_writes()
jobs = active_hwlab_ci_jobs()
last = {"writes": writes, "jobs": jobs}
if writes.get("ok") and jobs.get("ok") and int(writes.get("activeCount") or 0) == 0 and int(jobs.get("activeCount") or 0) == 0:
writes = active_registry_writes()
last = {"writes": writes}
if writes.get("ok") and int(writes.get("activeCount") or 0) == 0:
return {"ok": True, "last": last}
time.sleep(5)
return {"ok": False, "last": last}
@@ -33,14 +112,21 @@ def kctl(args, timeout=30):
return command(["env", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml", "kubectl"] + args, timeout)
def workload_image_refs():
result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get deploy,sts,ds,pod -A -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.initContainers[*]}{.image}{\"\\n\"}{end}{range .spec.template.spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.template.spec.initContainers[*]}{.image}{\"\\n\"}{end}{end}' 2>/dev/null | sort -u"], 30)
script = """{
KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get deploy,sts,ds -A -o jsonpath='{range .items[*]}{range .spec.template.spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.template.spec.initContainers[*]}{.image}{\"\\n\"}{end}{end}' 2>/dev/null
KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pod -A --field-selector=status.phase=Running -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.initContainers[*]}{.image}{\"\\n\"}{end}{end}' 2>/dev/null
KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pod -A --field-selector=status.phase=Pending -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.initContainers[*]}{.image}{\"\\n\"}{end}{end}' 2>/dev/null
} | sort -u"""
result = command(["sh", "-lc", script], 30)
refs = set()
digests = set()
for image in (result.get("stdout") or "").splitlines():
image = image.strip()
if not image.startswith("127.0.0.1:5000/"):
endpoint = registry_identity().get("endpoint") or ""
prefix = endpoint.rstrip("/") + "/"
if not image.startswith(prefix):
continue
ref = image.split("127.0.0.1:5000/", 1)[1]
ref = image[len(prefix):]
if "@sha256:" in ref:
repo, digest = ref.split("@", 1)
refs.add((repo, "@" + digest))
@@ -51,7 +137,8 @@ def workload_image_refs():
return refs, digests, bounded(result)
def registry_request(method, path, headers=None, timeout=20):
url = "http://127.0.0.1:5000" + path
endpoint = registry_identity().get("endpoint") or ""
url = (endpoint if endpoint.startswith("http://") or endpoint.startswith("https://") else "http://" + endpoint) + path
req = urllib.request.Request(url, method=method, headers=headers or {})
with urllib.request.urlopen(req, timeout=timeout) as response:
body = response.read()
@@ -132,7 +219,11 @@ def registry_revision_rows():
return rows
def registry_retention_repo(repo):
return repo.startswith("hwlab/hwlab-") or repo.startswith("hwlab/cache/hwlab-")
prefixes = registry_config_list("repositoryPrefixes", ["hwlab/hwlab-"])
return any(repo.startswith(prefix) for prefix in prefixes)
def registry_protected_repo(repo):
return any(repo.startswith(prefix) for prefix in registry_config_list("protectedRepositoryPrefixes", ["hwlab/cache/"]))
def registry_digest_hex(digest):
if not isinstance(digest, str) or not digest.startswith("sha256:"):
@@ -168,23 +259,25 @@ def registry_manifest_json(digest):
def registry_manifest_refs(digest):
manifest = registry_manifest_json(digest)
if not isinstance(manifest, dict):
return set()
refs = set()
return {"manifests": set(), "blobs": set()}
manifests = set()
blobs = set()
config = manifest.get("config") or {}
config_digest = config.get("digest")
if isinstance(config_digest, str) and registry_digest_hex(config_digest) is not None:
refs.add(config_digest)
blobs.add(config_digest)
for item in manifest.get("layers") or []:
item_digest = (item or {}).get("digest")
if isinstance(item_digest, str) and registry_digest_hex(item_digest) is not None:
refs.add(item_digest)
blobs.add(item_digest)
for item in manifest.get("manifests") or []:
item_digest = (item or {}).get("digest")
if isinstance(item_digest, str) and registry_digest_hex(item_digest) is not None:
refs.add(item_digest)
return refs
manifests.add(item_digest)
blobs.add(item_digest)
return {"manifests": manifests, "blobs": blobs}
def registry_digest_closure(seed):
def registry_manifest_closure(seed):
seen = set()
stack = list(seed)
while stack:
@@ -192,11 +285,18 @@ def registry_digest_closure(seed):
if digest in seen or registry_digest_hex(digest) is None:
continue
seen.add(digest)
for child in registry_manifest_refs(digest):
for child in registry_manifest_refs(digest).get("manifests") or []:
if child not in seen:
stack.append(child)
return seen
def registry_blob_closure(seed):
manifests = registry_manifest_closure(seed)
blobs = set(manifests)
for digest in manifests:
blobs.update(registry_manifest_refs(digest).get("blobs") or [])
return blobs
def registry_blob_size(digest):
path = registry_blob_data_path(digest)
if path is None or not os.path.isfile(path):
@@ -207,14 +307,31 @@ def registry_blob_size(digest):
return 0
def estimate_registry_reclaim(delete_manifest_digests, kept_manifest_digests):
deleted = registry_digest_closure(delete_manifest_digests)
kept = registry_digest_closure(kept_manifest_digests)
reclaim = deleted - kept
return sum(registry_blob_size(digest) for digest in reclaim)
deleted = registry_blob_closure(delete_manifest_digests)
kept = registry_blob_closure(kept_manifest_digests)
reclaim = sorted(deleted - kept)
limit = config_int(REGISTRY_CONFIG, "estimateBlobLimit", 5000, minimum=1, maximum=50000)
sampled = reclaim[:limit]
return {
"bytes": sum(registry_blob_size(digest) for digest in sampled),
"blobCount": len(reclaim),
"sampledBlobCount": len(sampled),
"truncated": len(sampled) < len(reclaim),
"basis": "allocated-bytes-lower-bound" if len(sampled) < len(reclaim) else "allocated-bytes",
}
def plan_registry_retention():
keep_per_repo = int(OPTIONS.get("registryKeepPerRepo") if OPTIONS.get("registryKeepPerRepo") is not None else 5)
min_age_hours = float(OPTIONS.get("registryMinAgeHours") if OPTIONS.get("registryMinAgeHours") is not None else 48)
storage = configure_registry_storage()
if not storage.get("ok"):
return {"tagRows": [], "revisionRows": [], "deleteRows": [], "deleteRevisionRows": [], "diagnosis": storage.get("diagnosis"), "summary": {"configRef": registry_identity().get("configRef"), "registrySizeBytes": 0, "estimatedReclaimBytes": 0}}
keep_per_repo = config_int(REGISTRY_CONFIG, "keepPerRepo", int(OPTIONS.get("registryKeepPerRepo") or 20), minimum=1, maximum=200)
min_age_hours = config_float(REGISTRY_CONFIG, "minAgeHours", float(OPTIONS.get("registryMinAgeHours") or 48), minimum=0.0)
protected_tags = set(registry_config_list("protectedTags", ["latest"]))
tag_pattern = registry_config_str("deletableTagPattern", r"^[0-9a-f]{7,40}$")
try:
deletable_tag = re.compile(tag_pattern)
except re.error as exc:
return {"tagRows": [], "revisionRows": [], "deleteRows": [], "deleteRevisionRows": [], "diagnosis": registry_diagnosis("registry-tag-pattern-invalid", "Registry deletableTagPattern is invalid", configRef=registry_identity().get("configRef"), error=str(exc)), "summary": {"configRef": registry_identity().get("configRef"), "registrySizeBytes": 0, "estimatedReclaimBytes": 0}}
cutoff = time.time() - min_age_hours * 3600
refs, digests, refs_command = workload_image_refs()
rows = registry_tag_rows()
@@ -232,7 +349,7 @@ def plan_registry_retention():
keep_reasons[key] = "latest-per-repo"
for row in items:
key = (row["repo"], row["tag"])
if row["tag"] in REGISTRY_PROTECTED_TAGS:
if row["tag"] in protected_tags:
keep.add(key)
keep_reasons[key] = "protected-tag"
if key in refs:
@@ -241,7 +358,7 @@ def plan_registry_retention():
if row["digest"] in digests:
keep.add(key)
keep_reasons[key] = "workload-digest-ref"
if row["repo"].startswith("hwlab/cache/"):
if registry_protected_repo(row["repo"]):
keep.add(key)
keep_reasons[key] = "cache-repo"
if row["mtime"] >= cutoff:
@@ -256,8 +373,9 @@ def plan_registry_retention():
key = (row["repo"], row["tag"])
should_delete = (
key not in keep
and row["repo"].startswith("hwlab/hwlab-")
and re.match(r"^[0-9a-f]{7,40}$", row["tag"]) is not None
and registry_retention_repo(row["repo"])
and not registry_protected_repo(row["repo"])
and deletable_tag.match(row["tag"]) is not None
)
if should_delete:
delete_rows.append(row)
@@ -266,9 +384,18 @@ def plan_registry_retention():
kept_count += 1
kept_digests.add(row["digest"])
keep_by_repo[row["repo"]] = keep_by_repo.get(row["repo"], 0) + 1
shared_digest_rows = [row for row in delete_rows if row["digest"] in kept_digests]
if shared_digest_rows:
delete_rows = [row for row in delete_rows if row["digest"] not in kept_digests]
kept_count += len(shared_digest_rows)
delete_by_repo = {}
for row in delete_rows:
delete_by_repo[row["repo"]] = delete_by_repo.get(row["repo"], 0) + 1
for row in shared_digest_rows:
keep_by_repo[row["repo"]] = keep_by_repo.get(row["repo"], 0) + 1
protected_digests = kept_digests | digests
protected_digests.update(row["digest"] for row in revision_rows if not registry_retention_repo(row["repo"]))
protected_digests = registry_digest_closure(protected_digests)
protected_digests = registry_manifest_closure(protected_digests)
delete_revision_rows = []
revision_delete_by_repo = {}
for row in revision_rows:
@@ -288,7 +415,7 @@ def plan_registry_retention():
for row in delete_revision_rows:
deletable_manifests.setdefault(row["repo"], set()).add(row["digest"])
deletable_manifest_count = sum(len(items) for items in deletable_manifests.values())
registry_size = du_size(REGISTRY_ROOT, 30) or 0
registry_size = None
estimate = estimate_registry_reclaim(delete_revision_digests, kept_revision_digests)
return {
"tagRows": rows,
@@ -296,6 +423,13 @@ def plan_registry_retention():
"deleteRows": delete_rows,
"deleteRevisionRows": delete_revision_rows,
"summary": {
"configRef": registry_identity().get("configRef"),
"storageKind": storage.get("storageKind"),
"storagePath": storage.get("storagePath"),
"namespace": registry_identity().get("namespace"),
"deploymentName": registry_identity().get("deploymentName"),
"pvcName": registry_identity().get("pvcName"),
"endpoint": registry_identity().get("endpoint"),
"totalTags": len(rows),
"totalRevisions": len(revision_rows),
"repoCount": len(by_repo),
@@ -306,37 +440,58 @@ def plan_registry_retention():
"protectedDigestClosure": len(protected_digests),
"keptTags": kept_count,
"deleteTags": len(delete_rows),
"sharedDigestProtectedTags": len(shared_digest_rows),
"deleteManifests": deletable_manifest_count,
"deleteRevisions": len(delete_revision_rows),
"deleteByRepo": delete_by_repo,
"revisionDeleteByRepo": revision_delete_by_repo,
"keepByRepo": keep_by_repo,
"registrySizeBytes": registry_size,
"estimatedReclaimBytes": estimate,
"estimatedReclaimBytes": estimate.get("bytes"),
"estimatedReclaimBlobCount": estimate.get("blobCount"),
"estimatedReclaimSampledBlobCount": estimate.get("sampledBlobCount"),
"estimatedReclaimTruncated": estimate.get("truncated"),
"estimatedReclaimBasis": estimate.get("basis"),
},
"deleteManifestsByRepo": {repo: sorted(list(digests)) for repo, digests in deletable_manifests.items()},
"refsCommand": refs_command,
}
def registry_deployment_preflight():
dep = kubectl_json(["-n", "hwlab-ci", "get", "deploy", "hwlab-registry"], 20)
identity = registry_identity()
namespace = identity.get("namespace")
deployment_name = identity.get("deploymentName")
dep = kubectl_json(["-n", namespace, "get", "deploy", deployment_name], 20)
if not dep:
return {"ok": False, "reason": "registry-deployment-missing"}
return {"ok": False, "reason": "registry-deployment-missing", "diagnosis": registry_diagnosis("registry-deployment-missing", "Registry Deployment could not be read", namespace=namespace, deploymentName=deployment_name)}
spec = ((dep.get("spec") or {}).get("template") or {}).get("spec") or {}
containers = spec.get("containers") or []
volumes = spec.get("volumes") or []
registry_container = next((item for item in containers if item.get("name") == "registry"), containers[0] if containers else {})
mounts = registry_container.get("volumeMounts") or []
has_host_path = any(((vol.get("hostPath") or {}).get("path") == REGISTRY_ROOT and vol.get("name") == "storage") for vol in volumes)
has_storage = any(
vol.get("name") == "storage" and (
((vol.get("hostPath") or {}).get("path") == REGISTRY_ROOT)
or ((vol.get("persistentVolumeClaim") or {}).get("claimName") == identity.get("pvcName"))
)
for vol in volumes
)
has_mount = any((mount.get("name") == "storage" and mount.get("mountPath") == "/var/lib/registry") for mount in mounts)
image = str(registry_container.get("image") or "")
ok = bool(has_host_path and has_mount and image.startswith("registry:") and spec.get("hostNetwork") is True)
expected_image = str(identity.get("image") or "")
image_matches = image.startswith("registry:") if PROVIDER_ID.upper() == "G14" and not REGISTRY_CONFIG else image == expected_image
ok = bool(has_storage and has_mount and image_matches and spec.get("hostNetwork") is True)
return {
"ok": ok,
"reason": "ok" if ok else "unexpected-registry-deployment-shape",
"diagnosis": None if ok else registry_diagnosis("registry-deployment-shape-mismatch", "Registry Deployment does not match the YAML configRef", namespace=namespace, deploymentName=deployment_name, configRef=identity.get("configRef")),
"namespace": namespace,
"deploymentName": deployment_name,
"pvcName": identity.get("pvcName"),
"image": image,
"expectedImage": expected_image,
"hostNetwork": spec.get("hostNetwork"),
"hasExpectedHostPath": has_host_path,
"hasExpectedStorage": has_storage,
"hasExpectedMount": has_mount,
"replicas": (dep.get("spec") or {}).get("replicas"),
"readyReplicas": (dep.get("status") or {}).get("readyReplicas"),
@@ -355,10 +510,13 @@ def patch_cronjob_suspend(name, suspend):
return kctl(["-n", "hwlab-ci", "patch", "cronjob", name, "--type=merge", "-p", payload], 30)
def wait_registry_pod_count(target, timeout=90):
identity = registry_identity()
namespace = identity.get("namespace")
deployment_name = identity.get("deploymentName")
deadline = time.time() + timeout
last = None
while time.time() < deadline:
result = kctl(["-n", "hwlab-ci", "get", "pods", "-l", "app.kubernetes.io/name=hwlab-registry", "--no-headers"], 20)
result = kctl(["-n", namespace, "get", "pods", "-l", "app.kubernetes.io/name=%s" % deployment_name, "--no-headers"], 20)
last = bounded(result)
lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()]
active = []
@@ -374,10 +532,11 @@ def wait_registry_pod_count(target, timeout=90):
return {"ok": False, "lines": [], "last": last}
def wait_pod_terminal(name, timeout=900):
namespace = registry_identity().get("namespace")
deadline = time.time() + timeout
last = None
while time.time() < deadline:
data = kubectl_json(["-n", "hwlab-ci", "get", "pod", name], 20)
data = kubectl_json(["-n", namespace, "get", "pod", name], 20)
if data:
phase = ((data.get("status") or {}).get("phase")) or ""
last = {"phase": phase}
@@ -388,7 +547,108 @@ def wait_pod_terminal(name, timeout=900):
time.sleep(3)
return {"ok": False, "phase": "Timeout", "last": last}
def registry_gc_pod_overrides(identity):
storage = {"name": "storage", "persistentVolumeClaim": {"claimName": identity.get("pvcName")}}
return {
"apiVersion": "v1",
"spec": {
"restartPolicy": "Never",
"containers": [{
"name": "registry-gc",
"image": identity.get("image"),
"command": ["registry", "garbage-collect", "/etc/docker/registry/config.yml"],
"volumeMounts": [{"name": "storage", "mountPath": "/var/lib/registry"}],
}],
"volumes": [storage],
},
}
def execute_k8s_registry_maintenance(mode):
storage = configure_registry_storage()
if not storage.get("ok"):
raise RuntimeError("registry storage preflight failed: %s" % ((storage.get("diagnosis") or {}).get("code") or "unknown"))
identity = storage.get("identity") or registry_identity()
deployment = registry_deployment_preflight()
if not deployment.get("ok"):
raise RuntimeError("registry deployment preflight failed: %s" % deployment.get("reason"))
plan = plan_registry_retention()
if plan.get("diagnosis"):
raise RuntimeError("registry retention plan failed: %s" % ((plan.get("diagnosis") or {}).get("code") or "unknown"))
delete_manifests = plan.get("deleteManifestsByRepo") or {}
if mode == "retention" and not delete_manifests:
return {"reclaimedBytes": 0, "commandOutput": {"message": "no unprotected registry manifests matched YAML retention", "registryPlan": plan.get("summary")}}
idle = wait_no_active_registry_writes(180)
if not idle.get("ok"):
raise RuntimeError("refusing registry maintenance because configured writer namespaces are not idle")
namespace = identity.get("namespace")
deployment_name = identity.get("deploymentName")
before = du_size(REGISTRY_ROOT, 60) or 0
gc_name = "%s-registry-gc-%s" % (re.sub(r"[^a-z0-9]+", "-", PROVIDER_ID.lower()).strip("-") or "node", int(time.time()))
steps = [{"step": "writer-idle-preflight", "result": idle}]
deleted_manifests = []
try:
if mode == "retention":
automatic = REGISTRY_CONFIG.get("automatic") if isinstance(REGISTRY_CONFIG.get("automatic"), dict) else {}
max_delete = config_int(automatic, "maxDeleteManifestsPerRun", 100, minimum=1, maximum=1000)
selected = []
for repo in sorted(delete_manifests.keys()):
for digest in sorted(delete_manifests.get(repo) or []):
selected.append((repo, digest))
selected = selected[:max_delete]
for repo, digest in selected:
encoded_repo = "/".join(urllib.parse.quote(part, safe="") for part in repo.split("/"))
try:
result = registry_request("DELETE", "/v2/%s/manifests/%s" % (encoded_repo, urllib.parse.quote(digest, safe=":")), {"Accept": "application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json"})
deleted_manifests.append({"repo": repo, "digest": digest, "status": result.get("status")})
except urllib.error.HTTPError as exc:
if exc.code == 404:
deleted_manifests.append({"repo": repo, "digest": digest, "status": 404})
else:
raise
steps.append({"step": "registry-api-delete-manifests", "count": len(deleted_manifests), "preview": deleted_manifests[:20]})
scale_down = kctl(["-n", namespace, "scale", "deploy", deployment_name, "--replicas=0"], 60)
steps.append({"step": "scale-registry-down", "result": bounded(scale_down)})
if scale_down["exitCode"] != 0 or not wait_registry_pod_count(0, 120).get("ok"):
raise RuntimeError("registry workload did not scale down")
overrides = registry_gc_pod_overrides(identity)
run_gc = kctl(["-n", namespace, "run", gc_name, "--restart=Never", "--image=%s" % identity.get("image"), "--overrides=%s" % json.dumps(overrides)], 60)
steps.append({"step": "start-registry-gc-pod", "result": bounded(run_gc), "pod": gc_name})
if run_gc["exitCode"] != 0:
raise RuntimeError("failed to start registry GC pod")
waited_gc = wait_pod_terminal(gc_name, 900)
logs = kctl(["-n", namespace, "logs", gc_name], 120)
steps.append({"step": "registry-gc-logs", "result": bounded(logs)})
if not waited_gc.get("ok"):
raise RuntimeError("registry GC pod did not complete successfully")
finally:
cleanup_gc = kctl(["-n", namespace, "delete", "pod", gc_name, "--ignore-not-found=true"], 60)
steps.append({"step": "delete-registry-gc-pod", "result": bounded(cleanup_gc)})
scale_up = kctl(["-n", namespace, "scale", "deploy", deployment_name, "--replicas=%s" % int(deployment.get("replicas") or 1)], 60)
steps.append({"step": "scale-registry-up", "result": bounded(scale_up)})
rollout = kctl(["-n", namespace, "rollout", "status", "deploy/%s" % deployment_name, "--timeout=180s"], 200)
steps.append({"step": "wait-registry-rollout", "result": bounded(rollout)})
readiness = registry_deployment_preflight()
if not readiness.get("ok") or int(readiness.get("readyReplicas") or 0) < 1:
raise RuntimeError("registry workload did not recover readiness")
probe = registry_request("GET", "/v2/", timeout=20)
if int(probe.get("status") or 0) != 200:
raise RuntimeError("registry endpoint did not recover readiness")
after = du_size(REGISTRY_ROOT, 60) or 0
return {
"reclaimedBytes": max(0, before - after),
"commandOutput": {
"registryPlan": plan.get("summary"),
"deletedManifestCount": len(deleted_manifests),
"diskBeforeBytes": before,
"diskAfterBytes": after,
"readiness": readiness,
"steps": steps[-12:],
},
}
def execute_registry_retention():
if registry_identity().get("mode") == "k8s-workload":
return execute_k8s_registry_maintenance("retention")
if PROVIDER_ID.upper() != "G14":
raise RuntimeError("HWLAB registry retention is only supported on G14")
deployment = registry_deployment_preflight()
@@ -516,6 +776,8 @@ def execute_registry_retention():
}
def execute_registry_garbage_collect_only():
if registry_identity().get("mode") == "k8s-workload":
return execute_k8s_registry_maintenance("garbage-collect")
if PROVIDER_ID.upper() != "G14":
raise RuntimeError("HWLAB registry garbage-collect is only supported on G14")
deployment = registry_deployment_preflight()
@@ -591,7 +853,8 @@ def execute_registry_garbage_collect_only():
}
def start_registry_retention_job(mode):
job_id = "g14-registry-%s-%s" % (int(time.time()), os.getpid())
provider_slug = re.sub(r"[^a-z0-9]+", "-", PROVIDER_ID.lower()).strip("-") or "provider"
job_id = "%s-registry-%s-%s" % (provider_slug, int(time.time()), os.getpid())
paths = job_paths(job_id)
started_at = now_iso()
initial = {
@@ -600,7 +863,7 @@ def start_registry_retention_job(mode):
"providerId": PROVIDER_ID,
"jobId": job_id,
"status": "running",
"kind": "hwlab-registry-retention-gc" if mode == "retention" else "hwlab-registry-garbage-collect",
"kind": "registry-retention-gc" if mode == "retention" else "registry-garbage-collect",
"mode": mode,
"startedAt": started_at,
"statePath": paths["state"],
@@ -619,7 +882,7 @@ def start_registry_retention_job(mode):
"statePath": paths["state"],
"logPath": paths["log"],
"statusCommand": "bun scripts/cli.ts gc remote %s status --job-id %s" % (PROVIDER_ID, job_id),
"message": "registry retention GC is running as a detached remote job",
"message": "registry maintenance is running as a detached remote job",
},
}
@@ -640,7 +903,7 @@ def start_registry_retention_job(mode):
except Exception:
log_handle = None
try:
print("[%s] starting HWLAB registry %s job %s" % (now_iso(), mode, job_id), flush=True)
print("[%s] starting registry %s job %s" % (now_iso(), mode, job_id), flush=True)
result = execute_registry_retention() if mode == "retention" else execute_registry_garbage_collect_only()
payload = dict(initial)
payload.update({
@@ -651,7 +914,7 @@ def start_registry_retention_job(mode):
"clusterAfter": cluster_preflight(),
})
write_json_atomic(paths["state"], payload)
print("[%s] completed HWLAB registry %s job %s" % (now_iso(), mode, job_id), flush=True)
print("[%s] completed registry %s job %s" % (now_iso(), mode, job_id), flush=True)
os._exit(0)
except Exception as exc:
payload = dict(initial)
@@ -667,7 +930,7 @@ def start_registry_retention_job(mode):
write_json_atomic(paths["state"], payload)
except Exception:
pass
print("[%s] failed HWLAB registry %s job %s: %s" % (now_iso(), mode, job_id, exc), flush=True)
print("[%s] failed registry %s job %s: %s" % (now_iso(), mode, job_id, exc), flush=True)
os._exit(1)
finally:
try:
scripts/src/gc-remote-runner.py
+110 -26
View File
@@ -27,6 +27,7 @@ LOCAL_PATH_CONFIG = REMOTE_TARGET.get("localPathStorage") if isinstance(REMOTE_T
KUBERNETES_RETENTION_CONFIG = REMOTE_TARGET.get("kubernetesObjectRetention") if isinstance(REMOTE_TARGET.get("kubernetesObjectRetention"), dict) else {}
POLICY_TIMER_CONFIG = REMOTE_TARGET.get("policyTimer") if isinstance(REMOTE_TARGET.get("policyTimer"), dict) else {}
HOST_DOCKER_GC_CONFIG = REMOTE_TARGET.get("hostDockerGc") if isinstance(REMOTE_TARGET.get("hostDockerGc"), dict) else {}
REGISTRY_CONFIG = REMOTE_TARGET.get("registryRetention") if isinstance(REMOTE_TARGET.get("registryRetention"), dict) else {}
POLICY_RUNNER_SOURCE = base64.b64decode("__UNIDESK_GC_REMOTE_POLICY_RUNNER_BASE64__").decode("utf-8")
TMP_PREFIX_ALLOWLIST = [
@@ -88,18 +89,6 @@ TOOL_CACHE_ALLOWLIST = [
},
]
REGISTRY_REPOSITORY_ROOT = "/var/lib/hwlab/registry/docker/registry/v2/repositories"
REGISTRY_ROOT = "/var/lib/hwlab/registry"
REGISTRY_PROTECTED_TAGS = set([
"latest",
"16-alpine",
"20-bookworm-slim",
"node22-alpine-v1",
"node22-alpine-bun-v1",
"sidecar",
"1b99888d3dae",
])
EXPECTED_G14_NODE = "ubuntu-rog-zephyrus-g14-ga401iv-ga401iv"
REMOTE_GC_JOB_DIR = "/tmp/unidesk-gc-remote/jobs"
REMOTE_GROWTH_SNAPSHOT_DIR = "/tmp/unidesk-gc-remote/growth-snapshots"
@@ -1343,19 +1332,31 @@ def collect_candidates(observed_at):
"estimatedReclaimBytes": size,
"action": {"op": "rm-recursive", "allowlist": "tmp-prefix"},
})
if OPTIONS.get("hwlabRegistry", False):
if OPTIONS.get("registry", False) or OPTIONS.get("hwlabRegistry", False):
registry = plan_registry_retention()
summary = registry.get("summary") or {}
diagnosis = registry.get("diagnosis")
delete_rows = registry.get("deleteRows") or []
delete_revision_rows = registry.get("deleteRevisionRows") or []
estimate = int(summary.get("estimatedReclaimBytes") or 0)
if delete_rows or delete_revision_rows:
if diagnosis:
candidates.append({
"id": "hwlab-registry:retention-gc",
"kind": "hwlab-registry-retention-gc",
"id": "registry:%s" % str(diagnosis.get("code") or "unsupported-target"),
"kind": "registry-retention-diagnosis",
"risk": "none",
"description": str(diagnosis.get("message") or "Registry retention is unavailable for this target"),
"sizeBytes": int(summary.get("registrySizeBytes") or 0),
"estimatedReclaimBytes": 0,
"diagnosis": diagnosis,
"action": {"op": "none", "runAllowed": False},
})
elif delete_rows or delete_revision_rows:
candidates.append({
"id": "registry:retention-gc",
"kind": "registry-retention-gc",
"risk": "medium",
"description": "Conservative HWLAB registry retention: keep current workload refs, retained tags and protected repos, delete stale manifest revisions, then run official registry garbage-collect",
"path": REGISTRY_ROOT,
"description": "Conservative registry retention: keep workload refs, recent tags, per-repository newest tags and digest closure, then run official registry garbage-collect",
"path": summary.get("storagePath"),
"sizeBytes": int(summary.get("registrySizeBytes") or 0),
"estimatedReclaimBytes": estimate,
"action": {
@@ -1375,11 +1376,11 @@ def collect_candidates(observed_at):
})
elif bool(OPTIONS.get("registryGcOnly")) and int(summary.get("totalTags") or 0) > 0 and int(summary.get("deleteTags") or 0) == 0:
candidates.append({
"id": "hwlab-registry:garbage-collect-only",
"kind": "hwlab-registry-garbage-collect",
"id": "registry:garbage-collect-only",
"kind": "registry-garbage-collect",
"risk": "medium",
"description": "Run official HWLAB registry garbage-collect without deleting additional tags; useful after a previously interrupted retention run",
"path": REGISTRY_ROOT,
"description": "Run official registry garbage-collect without deleting additional manifests",
"path": summary.get("storagePath"),
"sizeBytes": int(summary.get("registrySizeBytes") or 0),
"estimatedReclaimBytes": 0,
"action": {
@@ -1636,9 +1637,11 @@ def execute(candidate):
before = allocated_file_size(path)
os.unlink(path)
return {"reclaimedBytes": before}
if kind == "hwlab-registry-retention-gc":
if kind == "registry-retention-diagnosis":
raise RuntimeError("registry retention is not runnable: %s" % ((candidate.get("diagnosis") or {}).get("code") or "unsupported-target"))
if kind in set(["registry-retention-gc", "hwlab-registry-retention-gc"]):
return start_registry_retention_job("retention")
if kind == "hwlab-registry-garbage-collect":
if kind in set(["registry-garbage-collect", "hwlab-registry-garbage-collect"]):
return start_registry_retention_job("garbage-collect")
raise RuntimeError("unsupported remote gc candidate kind: %s" % kind)
@@ -1737,7 +1740,7 @@ def plan_payload(observed_at, preflight, protected, candidates, visible):
"/var/lib/rancher/k3s/storage",
"/var/lib/kubelet",
"/var/lib/containerd",
"/var/lib/hwlab unless --include-hwlab-registry is explicitly supplied",
"registry storage unless --include-registry is explicitly supplied",
"Kubernetes Deployments/StatefulSets/Secrets/PVCs/PVs",
"HWLAB fixed source workspaces",
"Docker images, containers and volumes",
@@ -1747,7 +1750,7 @@ def plan_payload(observed_at, preflight, protected, candidates, visible):
"G14 run requires the expected native k3s node preflight before mutation.",
"HWLAB DEV runtime and local-path PVC data are protected and require HWLAB-specific retention commands.",
"Core dump cleanup only removes untracked /root/unidesk/core.<pid> regular files with no active fuser reference.",
"HWLAB registry retention is opt-in: it keeps workload tag/digest refs, all tags newer than the retention age and the newest N tags per repo before official registry garbage-collect.",
"Registry retention is opt-in: it keeps workload tag/digest refs, all tags newer than the YAML retention age and the newest N tags per repo before official registry garbage-collect.",
"When summary.target.safeStop is true, do not broaden deletion scope; choose registry retention, k3s/containerd image cache maintenance, PVC/runtime retention or capacity expansion explicitly.",
],
}
@@ -1797,6 +1800,84 @@ def plan_payload(observed_at, preflight, protected, candidates, visible):
}
return payload
def registry_plan_payload(observed_at):
registry = plan_registry_retention()
full_summary = registry.get("summary") or {}
summary = full_summary if bool(OPTIONS.get("full")) else {
key: full_summary.get(key)
for key in [
"configRef", "storageKind", "storagePath", "namespace", "deploymentName", "pvcName", "endpoint",
"totalTags", "totalRevisions", "repoCount", "keepPerRepo", "minAgeHours", "protectedWorkloadRefs",
"protectedDigestRefs", "protectedDigestClosure", "keptTags", "deleteTags", "deleteManifests",
"deleteRevisions", "registrySizeBytes", "estimatedReclaimBytes", "estimatedReclaimBlobCount",
"estimatedReclaimSampledBlobCount", "estimatedReclaimTruncated", "estimatedReclaimBasis",
]
}
diagnosis = registry.get("diagnosis")
preview_limit = min(
config_int(REGISTRY_CONFIG, "previewLimit", 20, minimum=1, maximum=200),
int(OPTIONS.get("limit") or 10),
5 if not bool(OPTIONS.get("full")) else 200,
)
delete_rows = registry.get("deleteRows") or []
delete_revision_rows = registry.get("deleteRevisionRows") or []
automatic = REGISTRY_CONFIG.get("automatic") if isinstance(REGISTRY_CONFIG.get("automatic"), dict) else {}
candidate = {
"id": "registry:retention-gc",
"kind": "registry-retention-gc",
"risk": "medium",
"estimatedReclaimBytes": int(summary.get("estimatedReclaimBytes") or 0),
"action": {
"op": "registry-retention-gc",
"runAllowed": diagnosis is None,
"deleteTags": summary.get("deleteTags"),
"deleteManifests": summary.get("deleteManifests"),
"deleteRevisions": summary.get("deleteRevisions"),
},
}
if diagnosis:
candidate.update({"kind": "registry-retention-diagnosis", "risk": "none", "diagnosis": diagnosis})
return {
"ok": diagnosis is None,
"action": "gc remote plan",
"providerId": PROVIDER_ID,
"scope": "registry-retention",
"dryRun": True,
"mutation": False,
"observedAt": observed_at,
"configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.registryRetention" % PROVIDER_ID,
"summary": summary,
"candidates": [] if diagnosis else [candidate],
"diagnosis": diagnosis,
"preview": {
"deleteTags": [
{"repo": row.get("repo"), "tag": row.get("tag"), "digest": row.get("digest"), "mtimeIso": row.get("mtimeIso")}
for row in delete_rows[:preview_limit]
],
"deleteRevisions": [
{"repo": row.get("repo"), "digest": row.get("digest"), "mtimeIso": row.get("mtimeIso")}
for row in delete_revision_rows[:preview_limit]
],
"limit": preview_limit,
},
"protected": {
"reasons": ["workload-tag-ref", "workload-digest-ref", "digest-closure", "recent-tag", "latest-per-repo", "protected-tag", "protected-repository"],
"workloadTagRefs": summary.get("protectedWorkloadRefs"),
"workloadDigestRefs": summary.get("protectedDigestRefs"),
"digestClosure": summary.get("protectedDigestClosure"),
"keepPerRepo": summary.get("keepPerRepo"),
"minAgeHours": summary.get("minAgeHours"),
},
"automaticPolicy": {
"enabled": config_bool(automatic, "enabled", False),
"maxDeleteManifestsPerRun": config_int(automatic, "maxDeleteManifestsPerRun", 100, minimum=1, maximum=1000),
"requireIdleNamespaces": config_list(automatic, "requireIdleNamespaces", []),
"mode": "bounded-rolling-manifest-delete-then-official-gc",
},
"runCommand": "bun scripts/cli.ts gc remote %s run --confirm --include-registry --limit 1" % PROVIDER_ID,
"statusCommand": "bun scripts/cli.ts gc remote %s status --job-id <job-id>" % PROVIDER_ID,
}
def safe_unit_name(value):
raw = str(value or "").strip().lower()
raw = re.sub(r"[^a-z0-9_.@-]+", "-", raw).strip("-")
@@ -2687,6 +2768,9 @@ def main():
else:
emit_json(start_memory_pressure_job(observed_at, visible), persist_large=True)
return 0
if (bool(OPTIONS.get("registry")) or bool(OPTIONS.get("hwlabRegistry"))) and ACTION == "plan":
emit_json(registry_plan_payload(observed_at), persist_large=True)
return 0
preflight = cluster_preflight()
if ACTION == "policy-plan":
emit_json(remote_policy_plan_payload(observed_at), persist_large=False)
scripts/src/gc-remote.ts
+40 -4
View File
@@ -25,6 +25,7 @@ interface RemoteGcOptions {
aptCache: boolean;
coreDumps: boolean;
coreDumpMinAgeHours: number;
registry: boolean;
hwlabRegistry: boolean;
registryGcOnly: boolean;
registryKeepPerRepo: number;
@@ -57,6 +58,7 @@ const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = {
aptCache: true,
coreDumps: true,
coreDumpMinAgeHours: 1,
registry: false,
hwlabRegistry: false,
registryGcOnly: false,
registryKeepPerRepo: 20,
@@ -96,7 +98,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
return {
ok: false,
error: "gc-remote-provider-required",
usage: "bun scripts/cli.ts gc remote <providerId> memory|memory-distribution|retention|plan|snapshot|trend|run|status|policy [--confirm]",
usage: "bun scripts/cli.ts gc remote <providerId> memory|memory-distribution|retention|plan|snapshot|trend|run|status|policy [--include-registry] [--confirm]",
};
}
const subaction = action ?? "plan";
@@ -181,14 +183,15 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri
if (subaction === "status") return await runRemoteGc(config, providerId, "status", options);
if (subaction === "run") {
if (!options.confirm) {
const registryFlag = options.registry ? " --include-registry" : "";
return {
ok: false,
error: "gc-remote-run-requires-confirm",
dryRun: true,
mutation: false,
requiredFlag: "--confirm",
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan${options.memoryPressureOnly ? " --memory-pressure-only" : ""}`,
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm${options.memoryPressureOnly ? " --memory-pressure-only" : ""}`,
planCommand: `bun scripts/cli.ts gc remote ${providerId} plan${options.memoryPressureOnly ? " --memory-pressure-only" : registryFlag}`,
runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm${options.memoryPressureOnly ? " --memory-pressure-only" : registryFlag}`,
};
}
return await runRemoteGc(config, providerId, "run", options);
@@ -280,9 +283,13 @@ function parseRemoteGcOptions(args: string[]): RemoteGcOptions {
options.aptCache = false;
} else if (arg === "--no-core-dumps") {
options.coreDumps = false;
} else if (arg === "--include-registry") {
options.registry = true;
} else if (arg === "--include-hwlab-registry") {
options.registry = true;
options.hwlabRegistry = true;
} else if (arg === "--registry-gc-only") {
options.registry = true;
options.hwlabRegistry = true;
options.registryGcOnly = true;
} else if (arg === "--full" || arg === "--raw") {
@@ -339,11 +346,40 @@ function loadRemoteGcTargetConfig(providerId: string): Record<string, unknown> {
const targets = yamlRecordOrEmpty(remote.targets, GC_REMOTE_CONFIG_REF);
const candidates = [providerId, providerId.toUpperCase(), providerId.toLowerCase()];
for (const key of candidates) {
if (Object.prototype.hasOwnProperty.call(targets, key)) return yamlRecordOrEmpty(targets[key], `${GC_REMOTE_CONFIG_REF}.${key}`);
if (Object.prototype.hasOwnProperty.call(targets, key)) {
const target = yamlRecordOrEmpty(targets[key], `${GC_REMOTE_CONFIG_REF}.${key}`);
const registryRetention = yamlRecordOrEmpty(target.registryRetention, `${GC_REMOTE_CONFIG_REF}.${key}.registryRetention`);
if (Object.keys(registryRetention).length === 0) return target;
const configRef = typeof registryRetention.configRef === "string" ? registryRetention.configRef : "";
if (configRef.length === 0) throw new Error(`${GC_REMOTE_CONFIG_REF}.${key}.registryRetention.configRef must be a file#path reference`);
return {
...target,
registryRetention: {
...registryRetention,
resolvedRegistry: resolveYamlConfigRef(configRef),
},
};
}
}
return {};
}
function resolveYamlConfigRef(configRef: string): Record<string, unknown> {
const separator = configRef.indexOf("#");
if (separator <= 0 || separator === configRef.length - 1) throw new Error(`configRef must contain file#path: ${configRef}`);
const relativePath = configRef.slice(0, separator);
const selector = configRef.slice(separator + 1);
const absolutePath = rootPath(relativePath);
if (!existsSync(absolutePath)) throw new Error(`configRef file does not exist: ${relativePath}`);
let value: unknown = Bun.YAML.parse(readFileSync(absolutePath, "utf8")) as unknown;
for (const segment of selector.split(".").filter(Boolean)) {
const record = yamlRecordOrEmpty(value, `${relativePath}#${selector}`);
if (!Object.prototype.hasOwnProperty.call(record, segment)) throw new Error(`configRef path not found: ${configRef}`);
value = record[segment];
}
return yamlRecordOrEmpty(value, configRef);
}
function recordOrEmpty(value: unknown): Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record<string, unknown> : {};
}