diff --git a/config/unidesk-cli.yaml b/config/unidesk-cli.yaml index 61c0e200..8a246743 100644 --- a/config/unidesk-cli.yaml +++ b/config/unidesk-cli.yaml @@ -154,6 +154,39 @@ gc: remote: targets: NC01: + registryRetention: + enabled: true + configRef: config/hwlab-node-control-plane.yaml#nodes.NC01.registry + repositoryPrefixes: + - hwlab/ + - agentrun/ + - pikaoa/ + - selfmedia/ + - unidesk/ + protectedRepositoryPrefixes: + - hwlab/cache/ + - pikaoa/cache- + protectedTags: + - latest + - 16-alpine + - 20-bookworm-slim + - node22-alpine-v1 + - node22-alpine-bun-v1 + - sidecar + deletableTagPattern: ^[0-9a-f]{7,64}$ + keepPerRepo: 20 + minAgeHours: 48 + previewLimit: 20 + estimateBlobLimit: 5000 + automatic: + enabled: false + maxDeleteManifestsPerRun: 100 + requireIdleNamespaces: + - devops-infra + - agentrun-ci + - hwlab-ci + - pikaoa-ci + - selfmedia-ci hostDockerGc: dockerJsonLogs: enabled: false @@ -332,6 +365,22 @@ gc: includeHostContainerdCache: false includeLocalPathOrphans: false includeKubernetesObjectRetention: true + pvcAttribution: + namespaces: + - agentrun-ci + - agentrun-v02 + - devops-infra + - hwlab-ci + - pikaoa-ci + - selfmedia-ci + candidateNamespaces: + - agentrun-ci + - hwlab-ci + hwlabNode: NC01 + hwlabLane: v03 + agentrunNode: NC01 + agentrunLane: nc01-v02 + limit: 80 JD01: hostDockerGc: dockerJsonLogs: diff --git a/docs/reference/gc.md b/docs/reference/gc.md index 3a05288f..b95eb372 100644 --- a/docs/reference/gc.md +++ b/docs/reference/gc.md @@ -42,6 +42,18 @@ - 按目标节点 `kubernetesObjectRetention` YAML 规划和异步回收 Kubernetes 历史对象； - `plan` 与手动、自动 `run` 必须复用同一个 fail-closed 筛选器； - 表格形状、时间或分组无法识别时保护对象并停止删除。 +- `gc remote plan --include-registry`： + - 从 `config/unidesk-cli.yaml#gc.remote.targets..registryRetention` 读取留存策略； + - 通过显式 `configRef` 读取 node-local registry 的 endpoint、Deployment、namespace 和 PVC； + - 保护当前 workload tag/digest refs、近期 tag、每仓库最新 tag、保护仓库和 digest closure； + - 输出候选 manifest、保护原因、估算收益和 typed diagnosis； + - 不支持的目标、PVC 未绑定、存储布局不可读或 workload 形状漂移时必须 fail closed。 +- `gc remote run --confirm --include-registry`： + - 作为异步 job 返回，并通过 `gc remote status --job-id ` 短查询； + - 只通过 Registry API 删除 plan 中不受保护的 manifest； + - 对 YAML 声明的 Registry Deployment 缩容后运行官方 `registry garbage-collect`； + - PVC 只挂载给 GC Pod，不删除 PVC、PV、blob 目录、containerd 或 k3s storage； + - 无论成功或失败都恢复原副本数，并复核 Deployment readiness 与 `/v2/` endpoint。 - `--include-tool-caches`：本机和远端都必须是显式 opt-in，只清理固定 allowlist 的可重建 npm/npx/Bun 缓存；不得扩大成清理 `~/.npm`、`~/.bun`、`node_modules`、auth/config 或运行面依赖。 所有成功和失败输出都必须是 JSON。`plan` 必须标记 `dryRun=true`、`mutation=false`；`run` 必须要求 `--confirm` 并报告 `diskBefore`、`diskAfter`、`summary`、`results` 和 `protected`。远端 GC 可用 `--target-use-percent N` 显式表达目标根盘水位；`summary.target` 必须给出目标所需释放量、候选估算、预计水位、缺口和 `safeStop` 决策，避免靠人工心算判断是否应该继续扩大清理范围。 diff --git a/scripts/src/gc-remote-pvc.py b/scripts/src/gc-remote-pvc.py index 95f5726a..6342f4cc 100644 --- a/scripts/src/gc-remote-pvc.py +++ b/scripts/src/gc-remote-pvc.py @@ -65,7 +65,9 @@ def ci_storage_snapshot(): hwlab_node = config_str(PVC_CONFIG, "hwlabNode", PROVIDER_ID) hwlab_lane = config_str(PVC_CONFIG, "hwlabLane", "v03") agentrun_node = config_str(PVC_CONFIG, "agentrunNode", PROVIDER_ID) - agentrun_lane = config_str(PVC_CONFIG, "agentrunLane", "v02") + agentrun_lane = config_str(PVC_CONFIG, "agentrunLane", "") + if not agentrun_lane: + raise RuntimeError("gc.remote.targets.%s.pvcAttribution.agentrunLane must be declared in YAML" % PROVIDER_ID) limit = config_int(PVC_CONFIG, "limit", int(OPTIONS.get("limit") or 50), minimum=1, maximum=5000) pv_data = kubectl_json(["get", "pv"], 30) or {} pvc_data = kubectl_json(["get", "pvc", "-A"], 30) or {} diff --git a/scripts/src/gc-remote-registry.py b/scripts/src/gc-remote-registry.py index 918b2a49..37a6a901 100644 --- a/scripts/src/gc-remote-registry.py +++ b/scripts/src/gc-remote-registry.py @@ -1,21 +1,100 @@ -def active_hwlab_ci_writes(): - result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pipelinerun,taskrun -n hwlab-ci --no-headers 2>/dev/null | awk '$2 != \"True\" && $2 != \"False\" {print}' | head -40"], 15) - lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()] - return {"ok": result["exitCode"] == 0, "activeCount": len(lines), "activePreview": lines, "command": bounded(result)} +def registry_config_list(key, default=None): + return config_list(REGISTRY_CONFIG, key, default or []) -def active_hwlab_ci_jobs(): - result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get jobs -n hwlab-ci --no-headers 2>/dev/null | awk '$2 != \"Complete\" && $2 != \"Failed\" {print}' | head -40"], 15) - lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()] - return {"ok": result["exitCode"] == 0, "activeCount": len(lines), "activePreview": lines, "command": bounded(result)} +def registry_config_str(key, default=""): + return config_str(REGISTRY_CONFIG, key, default) -def wait_no_active_hwlab_ci(timeout=180): +def registry_spec(): + value = REGISTRY_CONFIG.get("resolvedRegistry") if isinstance(REGISTRY_CONFIG, dict) else None + return value if isinstance(value, dict) else {} + +def registry_identity(): + spec = registry_spec() + mode = config_str(spec, "mode", "host-path" if PROVIDER_ID.upper() == "G14" else "") + namespace = config_str(spec, "namespace", "hwlab-ci" if PROVIDER_ID.upper() == "G14" else "") + deployment = config_str(spec, "deploymentName", "hwlab-registry" if PROVIDER_ID.upper() == "G14" else "") + pvc = config_str(spec, "pvcName", "") + endpoint = config_str(spec, "endpoint", "127.0.0.1:5000") + image = config_str(spec, "image", "registry:2.8.3") + return { + "mode": mode, + "namespace": namespace, + "deploymentName": deployment, + "pvcName": pvc, + "endpoint": endpoint, + "image": image, + "configRef": registry_config_str("configRef", "") if REGISTRY_CONFIG else ("legacy:G14" if PROVIDER_ID.upper() == "G14" else None), + } + +REGISTRY_ROOT = "/var/lib/hwlab/registry" +REGISTRY_REPOSITORY_ROOT = os.path.join(REGISTRY_ROOT, "docker/registry/v2/repositories") + +def configure_registry_storage(): + global REGISTRY_ROOT, REGISTRY_REPOSITORY_ROOT + identity = registry_identity() + if REGISTRY_CONFIG and not config_bool(REGISTRY_CONFIG, "enabled", False): + return {"ok": False, "diagnosis": registry_diagnosis("registry-retention-disabled", "Registry retention is disabled by YAML", configRef=identity.get("configRef"))} + if identity.get("mode") == "host-path" and PROVIDER_ID.upper() == "G14": + REGISTRY_ROOT = "/var/lib/hwlab/registry" + REGISTRY_REPOSITORY_ROOT = os.path.join(REGISTRY_ROOT, "docker/registry/v2/repositories") + return {"ok": True, "identity": identity, "storagePath": REGISTRY_ROOT, "storageKind": "host-path"} + if identity.get("mode") != "k8s-workload": + return {"ok": False, "diagnosis": registry_diagnosis("registry-target-mode-unsupported", "Registry retention requires a YAML-declared k8s-workload registry", mode=identity.get("mode"), configRef=identity.get("configRef"))} + required = ["namespace", "deploymentName", "pvcName", "endpoint"] + missing = [key for key in required if not identity.get(key)] + if missing: + return {"ok": False, "diagnosis": registry_diagnosis("registry-target-config-incomplete", "Registry configRef is missing required workload fields", missingFields=missing, configRef=identity.get("configRef"))} + pvc = kubectl_json(["-n", identity["namespace"], "get", "pvc", identity["pvcName"]], 20) + if not pvc: + return {"ok": False, "diagnosis": registry_diagnosis("registry-pvc-unavailable", "Registry PVC could not be read", namespace=identity["namespace"], pvcName=identity["pvcName"], configRef=identity.get("configRef"))} + volume_name = str(((pvc.get("spec") or {}).get("volumeName")) or "") + if not volume_name or str(((pvc.get("status") or {}).get("phase")) or "") != "Bound": + return {"ok": False, "diagnosis": registry_diagnosis("registry-pvc-not-bound", "Registry PVC must be Bound before retention planning", namespace=identity["namespace"], pvcName=identity["pvcName"], volumeName=volume_name or None)} + pv = kubectl_json(["get", "pv", volume_name], 20) + spec = (pv or {}).get("spec") or {} + storage_path = str(((spec.get("local") or {}).get("path")) or ((spec.get("hostPath") or {}).get("path")) or "") + if not storage_path.startswith("/"): + return {"ok": False, "diagnosis": registry_diagnosis("registry-pv-storage-path-unsupported", "Registry PV does not expose a node-local absolute storage path", volumeName=volume_name, storageClass=spec.get("storageClassName"))} + storage_path = os.path.realpath(storage_path) + repository_root = os.path.join(storage_path, "docker/registry/v2/repositories") + if not os.path.isdir(repository_root): + return {"ok": False, "diagnosis": registry_diagnosis("registry-storage-layout-unavailable", "Registry repository layout is not readable from the node-local PVC path", storagePath=storage_path, repositoryRoot=repository_root)} + REGISTRY_ROOT = storage_path + REGISTRY_REPOSITORY_ROOT = repository_root + return {"ok": True, "identity": identity, "storagePath": storage_path, "storageKind": "pvc", "volumeName": volume_name} + +def registry_diagnosis(code, message, **details): + return {"code": code, "message": message, "providerId": PROVIDER_ID, **details} + +def active_registry_writes(): + automatic = REGISTRY_CONFIG.get("automatic") if isinstance(REGISTRY_CONFIG.get("automatic"), dict) else {} + namespaces = config_list(automatic, "requireIdleNamespaces", []) + rows = [] + commands = [] + for namespace in namespaces: + result = kctl(["-n", namespace, "get", "pipelinerun,taskrun", "-o", "json"], 20) + commands.append(bounded(result)) + if result["exitCode"] != 0: + return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace} + try: + items = json.loads(result.get("stdout") or "{}").get("items") or [] + except Exception: + return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace, "error": "invalid-kubernetes-json"} + for item in items: + conditions = ((item.get("status") or {}).get("conditions")) or [] + terminal = any(condition.get("type") == "Succeeded" and condition.get("status") in set(["True", "False"]) for condition in conditions) + if not terminal: + meta = item.get("metadata") or {} + rows.append("%s/%s/%s" % (namespace, item.get("kind") or "Unknown", meta.get("name") or "unknown")) + return {"ok": True, "activeCount": len(rows), "activePreview": rows[:40], "commands": commands} + +def wait_no_active_registry_writes(timeout=180): deadline = time.time() + timeout last = None while time.time() < deadline: - writes = active_hwlab_ci_writes() - jobs = active_hwlab_ci_jobs() - last = {"writes": writes, "jobs": jobs} - if writes.get("ok") and jobs.get("ok") and int(writes.get("activeCount") or 0) == 0 and int(jobs.get("activeCount") or 0) == 0: + writes = active_registry_writes() + last = {"writes": writes} + if writes.get("ok") and int(writes.get("activeCount") or 0) == 0: return {"ok": True, "last": last} time.sleep(5) return {"ok": False, "last": last} @@ -33,14 +112,21 @@ def kctl(args, timeout=30): return command(["env", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml", "kubectl"] + args, timeout) def workload_image_refs(): - result = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get deploy,sts,ds,pod -A -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.initContainers[*]}{.image}{\"\\n\"}{end}{range .spec.template.spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.template.spec.initContainers[*]}{.image}{\"\\n\"}{end}{end}' 2>/dev/null | sort -u"], 30) + script = """{ +KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get deploy,sts,ds -A -o jsonpath='{range .items[*]}{range .spec.template.spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.template.spec.initContainers[*]}{.image}{\"\\n\"}{end}{end}' 2>/dev/null +KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pod -A --field-selector=status.phase=Running -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.initContainers[*]}{.image}{\"\\n\"}{end}{end}' 2>/dev/null +KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get pod -A --field-selector=status.phase=Pending -o jsonpath='{range .items[*]}{range .spec.containers[*]}{.image}{\"\\n\"}{end}{range .spec.initContainers[*]}{.image}{\"\\n\"}{end}{end}' 2>/dev/null +} | sort -u""" + result = command(["sh", "-lc", script], 30) refs = set() digests = set() for image in (result.get("stdout") or "").splitlines(): image = image.strip() - if not image.startswith("127.0.0.1:5000/"): + endpoint = registry_identity().get("endpoint") or "" + prefix = endpoint.rstrip("/") + "/" + if not image.startswith(prefix): continue - ref = image.split("127.0.0.1:5000/", 1)[1] + ref = image[len(prefix):] if "@sha256:" in ref: repo, digest = ref.split("@", 1) refs.add((repo, "@" + digest)) @@ -51,7 +137,8 @@ def workload_image_refs(): return refs, digests, bounded(result) def registry_request(method, path, headers=None, timeout=20): - url = "http://127.0.0.1:5000" + path + endpoint = registry_identity().get("endpoint") or "" + url = (endpoint if endpoint.startswith("http://") or endpoint.startswith("https://") else "http://" + endpoint) + path req = urllib.request.Request(url, method=method, headers=headers or {}) with urllib.request.urlopen(req, timeout=timeout) as response: body = response.read() @@ -132,7 +219,11 @@ def registry_revision_rows(): return rows def registry_retention_repo(repo): - return repo.startswith("hwlab/hwlab-") or repo.startswith("hwlab/cache/hwlab-") + prefixes = registry_config_list("repositoryPrefixes", ["hwlab/hwlab-"]) + return any(repo.startswith(prefix) for prefix in prefixes) + +def registry_protected_repo(repo): + return any(repo.startswith(prefix) for prefix in registry_config_list("protectedRepositoryPrefixes", ["hwlab/cache/"])) def registry_digest_hex(digest): if not isinstance(digest, str) or not digest.startswith("sha256:"): @@ -168,23 +259,25 @@ def registry_manifest_json(digest): def registry_manifest_refs(digest): manifest = registry_manifest_json(digest) if not isinstance(manifest, dict): - return set() - refs = set() + return {"manifests": set(), "blobs": set()} + manifests = set() + blobs = set() config = manifest.get("config") or {} config_digest = config.get("digest") if isinstance(config_digest, str) and registry_digest_hex(config_digest) is not None: - refs.add(config_digest) + blobs.add(config_digest) for item in manifest.get("layers") or []: item_digest = (item or {}).get("digest") if isinstance(item_digest, str) and registry_digest_hex(item_digest) is not None: - refs.add(item_digest) + blobs.add(item_digest) for item in manifest.get("manifests") or []: item_digest = (item or {}).get("digest") if isinstance(item_digest, str) and registry_digest_hex(item_digest) is not None: - refs.add(item_digest) - return refs + manifests.add(item_digest) + blobs.add(item_digest) + return {"manifests": manifests, "blobs": blobs} -def registry_digest_closure(seed): +def registry_manifest_closure(seed): seen = set() stack = list(seed) while stack: @@ -192,11 +285,18 @@ def registry_digest_closure(seed): if digest in seen or registry_digest_hex(digest) is None: continue seen.add(digest) - for child in registry_manifest_refs(digest): + for child in registry_manifest_refs(digest).get("manifests") or []: if child not in seen: stack.append(child) return seen +def registry_blob_closure(seed): + manifests = registry_manifest_closure(seed) + blobs = set(manifests) + for digest in manifests: + blobs.update(registry_manifest_refs(digest).get("blobs") or []) + return blobs + def registry_blob_size(digest): path = registry_blob_data_path(digest) if path is None or not os.path.isfile(path): @@ -207,14 +307,31 @@ def registry_blob_size(digest): return 0 def estimate_registry_reclaim(delete_manifest_digests, kept_manifest_digests): - deleted = registry_digest_closure(delete_manifest_digests) - kept = registry_digest_closure(kept_manifest_digests) - reclaim = deleted - kept - return sum(registry_blob_size(digest) for digest in reclaim) + deleted = registry_blob_closure(delete_manifest_digests) + kept = registry_blob_closure(kept_manifest_digests) + reclaim = sorted(deleted - kept) + limit = config_int(REGISTRY_CONFIG, "estimateBlobLimit", 5000, minimum=1, maximum=50000) + sampled = reclaim[:limit] + return { + "bytes": sum(registry_blob_size(digest) for digest in sampled), + "blobCount": len(reclaim), + "sampledBlobCount": len(sampled), + "truncated": len(sampled) < len(reclaim), + "basis": "allocated-bytes-lower-bound" if len(sampled) < len(reclaim) else "allocated-bytes", + } def plan_registry_retention(): - keep_per_repo = int(OPTIONS.get("registryKeepPerRepo") if OPTIONS.get("registryKeepPerRepo") is not None else 5) - min_age_hours = float(OPTIONS.get("registryMinAgeHours") if OPTIONS.get("registryMinAgeHours") is not None else 48) + storage = configure_registry_storage() + if not storage.get("ok"): + return {"tagRows": [], "revisionRows": [], "deleteRows": [], "deleteRevisionRows": [], "diagnosis": storage.get("diagnosis"), "summary": {"configRef": registry_identity().get("configRef"), "registrySizeBytes": 0, "estimatedReclaimBytes": 0}} + keep_per_repo = config_int(REGISTRY_CONFIG, "keepPerRepo", int(OPTIONS.get("registryKeepPerRepo") or 20), minimum=1, maximum=200) + min_age_hours = config_float(REGISTRY_CONFIG, "minAgeHours", float(OPTIONS.get("registryMinAgeHours") or 48), minimum=0.0) + protected_tags = set(registry_config_list("protectedTags", ["latest"])) + tag_pattern = registry_config_str("deletableTagPattern", r"^[0-9a-f]{7,40}$") + try: + deletable_tag = re.compile(tag_pattern) + except re.error as exc: + return {"tagRows": [], "revisionRows": [], "deleteRows": [], "deleteRevisionRows": [], "diagnosis": registry_diagnosis("registry-tag-pattern-invalid", "Registry deletableTagPattern is invalid", configRef=registry_identity().get("configRef"), error=str(exc)), "summary": {"configRef": registry_identity().get("configRef"), "registrySizeBytes": 0, "estimatedReclaimBytes": 0}} cutoff = time.time() - min_age_hours * 3600 refs, digests, refs_command = workload_image_refs() rows = registry_tag_rows() @@ -232,7 +349,7 @@ def plan_registry_retention(): keep_reasons[key] = "latest-per-repo" for row in items: key = (row["repo"], row["tag"]) - if row["tag"] in REGISTRY_PROTECTED_TAGS: + if row["tag"] in protected_tags: keep.add(key) keep_reasons[key] = "protected-tag" if key in refs: @@ -241,7 +358,7 @@ def plan_registry_retention(): if row["digest"] in digests: keep.add(key) keep_reasons[key] = "workload-digest-ref" - if row["repo"].startswith("hwlab/cache/"): + if registry_protected_repo(row["repo"]): keep.add(key) keep_reasons[key] = "cache-repo" if row["mtime"] >= cutoff: @@ -256,8 +373,9 @@ def plan_registry_retention(): key = (row["repo"], row["tag"]) should_delete = ( key not in keep - and row["repo"].startswith("hwlab/hwlab-") - and re.match(r"^[0-9a-f]{7,40}$", row["tag"]) is not None + and registry_retention_repo(row["repo"]) + and not registry_protected_repo(row["repo"]) + and deletable_tag.match(row["tag"]) is not None ) if should_delete: delete_rows.append(row) @@ -266,9 +384,18 @@ def plan_registry_retention(): kept_count += 1 kept_digests.add(row["digest"]) keep_by_repo[row["repo"]] = keep_by_repo.get(row["repo"], 0) + 1 + shared_digest_rows = [row for row in delete_rows if row["digest"] in kept_digests] + if shared_digest_rows: + delete_rows = [row for row in delete_rows if row["digest"] not in kept_digests] + kept_count += len(shared_digest_rows) + delete_by_repo = {} + for row in delete_rows: + delete_by_repo[row["repo"]] = delete_by_repo.get(row["repo"], 0) + 1 + for row in shared_digest_rows: + keep_by_repo[row["repo"]] = keep_by_repo.get(row["repo"], 0) + 1 protected_digests = kept_digests | digests protected_digests.update(row["digest"] for row in revision_rows if not registry_retention_repo(row["repo"])) - protected_digests = registry_digest_closure(protected_digests) + protected_digests = registry_manifest_closure(protected_digests) delete_revision_rows = [] revision_delete_by_repo = {} for row in revision_rows: @@ -288,7 +415,7 @@ def plan_registry_retention(): for row in delete_revision_rows: deletable_manifests.setdefault(row["repo"], set()).add(row["digest"]) deletable_manifest_count = sum(len(items) for items in deletable_manifests.values()) - registry_size = du_size(REGISTRY_ROOT, 30) or 0 + registry_size = None estimate = estimate_registry_reclaim(delete_revision_digests, kept_revision_digests) return { "tagRows": rows, @@ -296,6 +423,13 @@ def plan_registry_retention(): "deleteRows": delete_rows, "deleteRevisionRows": delete_revision_rows, "summary": { + "configRef": registry_identity().get("configRef"), + "storageKind": storage.get("storageKind"), + "storagePath": storage.get("storagePath"), + "namespace": registry_identity().get("namespace"), + "deploymentName": registry_identity().get("deploymentName"), + "pvcName": registry_identity().get("pvcName"), + "endpoint": registry_identity().get("endpoint"), "totalTags": len(rows), "totalRevisions": len(revision_rows), "repoCount": len(by_repo), @@ -306,37 +440,58 @@ def plan_registry_retention(): "protectedDigestClosure": len(protected_digests), "keptTags": kept_count, "deleteTags": len(delete_rows), + "sharedDigestProtectedTags": len(shared_digest_rows), "deleteManifests": deletable_manifest_count, "deleteRevisions": len(delete_revision_rows), "deleteByRepo": delete_by_repo, "revisionDeleteByRepo": revision_delete_by_repo, "keepByRepo": keep_by_repo, "registrySizeBytes": registry_size, - "estimatedReclaimBytes": estimate, + "estimatedReclaimBytes": estimate.get("bytes"), + "estimatedReclaimBlobCount": estimate.get("blobCount"), + "estimatedReclaimSampledBlobCount": estimate.get("sampledBlobCount"), + "estimatedReclaimTruncated": estimate.get("truncated"), + "estimatedReclaimBasis": estimate.get("basis"), }, "deleteManifestsByRepo": {repo: sorted(list(digests)) for repo, digests in deletable_manifests.items()}, "refsCommand": refs_command, } def registry_deployment_preflight(): - dep = kubectl_json(["-n", "hwlab-ci", "get", "deploy", "hwlab-registry"], 20) + identity = registry_identity() + namespace = identity.get("namespace") + deployment_name = identity.get("deploymentName") + dep = kubectl_json(["-n", namespace, "get", "deploy", deployment_name], 20) if not dep: - return {"ok": False, "reason": "registry-deployment-missing"} + return {"ok": False, "reason": "registry-deployment-missing", "diagnosis": registry_diagnosis("registry-deployment-missing", "Registry Deployment could not be read", namespace=namespace, deploymentName=deployment_name)} spec = ((dep.get("spec") or {}).get("template") or {}).get("spec") or {} containers = spec.get("containers") or [] volumes = spec.get("volumes") or [] registry_container = next((item for item in containers if item.get("name") == "registry"), containers[0] if containers else {}) mounts = registry_container.get("volumeMounts") or [] - has_host_path = any(((vol.get("hostPath") or {}).get("path") == REGISTRY_ROOT and vol.get("name") == "storage") for vol in volumes) + has_storage = any( + vol.get("name") == "storage" and ( + ((vol.get("hostPath") or {}).get("path") == REGISTRY_ROOT) + or ((vol.get("persistentVolumeClaim") or {}).get("claimName") == identity.get("pvcName")) + ) + for vol in volumes + ) has_mount = any((mount.get("name") == "storage" and mount.get("mountPath") == "/var/lib/registry") for mount in mounts) image = str(registry_container.get("image") or "") - ok = bool(has_host_path and has_mount and image.startswith("registry:") and spec.get("hostNetwork") is True) + expected_image = str(identity.get("image") or "") + image_matches = image.startswith("registry:") if PROVIDER_ID.upper() == "G14" and not REGISTRY_CONFIG else image == expected_image + ok = bool(has_storage and has_mount and image_matches and spec.get("hostNetwork") is True) return { "ok": ok, "reason": "ok" if ok else "unexpected-registry-deployment-shape", + "diagnosis": None if ok else registry_diagnosis("registry-deployment-shape-mismatch", "Registry Deployment does not match the YAML configRef", namespace=namespace, deploymentName=deployment_name, configRef=identity.get("configRef")), + "namespace": namespace, + "deploymentName": deployment_name, + "pvcName": identity.get("pvcName"), "image": image, + "expectedImage": expected_image, "hostNetwork": spec.get("hostNetwork"), - "hasExpectedHostPath": has_host_path, + "hasExpectedStorage": has_storage, "hasExpectedMount": has_mount, "replicas": (dep.get("spec") or {}).get("replicas"), "readyReplicas": (dep.get("status") or {}).get("readyReplicas"), @@ -355,10 +510,13 @@ def patch_cronjob_suspend(name, suspend): return kctl(["-n", "hwlab-ci", "patch", "cronjob", name, "--type=merge", "-p", payload], 30) def wait_registry_pod_count(target, timeout=90): + identity = registry_identity() + namespace = identity.get("namespace") + deployment_name = identity.get("deploymentName") deadline = time.time() + timeout last = None while time.time() < deadline: - result = kctl(["-n", "hwlab-ci", "get", "pods", "-l", "app.kubernetes.io/name=hwlab-registry", "--no-headers"], 20) + result = kctl(["-n", namespace, "get", "pods", "-l", "app.kubernetes.io/name=%s" % deployment_name, "--no-headers"], 20) last = bounded(result) lines = [line for line in (result.get("stdout") or "").splitlines() if line.strip()] active = [] @@ -374,10 +532,11 @@ def wait_registry_pod_count(target, timeout=90): return {"ok": False, "lines": [], "last": last} def wait_pod_terminal(name, timeout=900): + namespace = registry_identity().get("namespace") deadline = time.time() + timeout last = None while time.time() < deadline: - data = kubectl_json(["-n", "hwlab-ci", "get", "pod", name], 20) + data = kubectl_json(["-n", namespace, "get", "pod", name], 20) if data: phase = ((data.get("status") or {}).get("phase")) or "" last = {"phase": phase} @@ -388,7 +547,108 @@ def wait_pod_terminal(name, timeout=900): time.sleep(3) return {"ok": False, "phase": "Timeout", "last": last} +def registry_gc_pod_overrides(identity): + storage = {"name": "storage", "persistentVolumeClaim": {"claimName": identity.get("pvcName")}} + return { + "apiVersion": "v1", + "spec": { + "restartPolicy": "Never", + "containers": [{ + "name": "registry-gc", + "image": identity.get("image"), + "command": ["registry", "garbage-collect", "/etc/docker/registry/config.yml"], + "volumeMounts": [{"name": "storage", "mountPath": "/var/lib/registry"}], + }], + "volumes": [storage], + }, + } + +def execute_k8s_registry_maintenance(mode): + storage = configure_registry_storage() + if not storage.get("ok"): + raise RuntimeError("registry storage preflight failed: %s" % ((storage.get("diagnosis") or {}).get("code") or "unknown")) + identity = storage.get("identity") or registry_identity() + deployment = registry_deployment_preflight() + if not deployment.get("ok"): + raise RuntimeError("registry deployment preflight failed: %s" % deployment.get("reason")) + plan = plan_registry_retention() + if plan.get("diagnosis"): + raise RuntimeError("registry retention plan failed: %s" % ((plan.get("diagnosis") or {}).get("code") or "unknown")) + delete_manifests = plan.get("deleteManifestsByRepo") or {} + if mode == "retention" and not delete_manifests: + return {"reclaimedBytes": 0, "commandOutput": {"message": "no unprotected registry manifests matched YAML retention", "registryPlan": plan.get("summary")}} + idle = wait_no_active_registry_writes(180) + if not idle.get("ok"): + raise RuntimeError("refusing registry maintenance because configured writer namespaces are not idle") + namespace = identity.get("namespace") + deployment_name = identity.get("deploymentName") + before = du_size(REGISTRY_ROOT, 60) or 0 + gc_name = "%s-registry-gc-%s" % (re.sub(r"[^a-z0-9]+", "-", PROVIDER_ID.lower()).strip("-") or "node", int(time.time())) + steps = [{"step": "writer-idle-preflight", "result": idle}] + deleted_manifests = [] + try: + if mode == "retention": + automatic = REGISTRY_CONFIG.get("automatic") if isinstance(REGISTRY_CONFIG.get("automatic"), dict) else {} + max_delete = config_int(automatic, "maxDeleteManifestsPerRun", 100, minimum=1, maximum=1000) + selected = [] + for repo in sorted(delete_manifests.keys()): + for digest in sorted(delete_manifests.get(repo) or []): + selected.append((repo, digest)) + selected = selected[:max_delete] + for repo, digest in selected: + encoded_repo = "/".join(urllib.parse.quote(part, safe="") for part in repo.split("/")) + try: + result = registry_request("DELETE", "/v2/%s/manifests/%s" % (encoded_repo, urllib.parse.quote(digest, safe=":")), {"Accept": "application/vnd.docker.distribution.manifest.v2+json, application/vnd.oci.image.manifest.v1+json, application/vnd.oci.image.index.v1+json"}) + deleted_manifests.append({"repo": repo, "digest": digest, "status": result.get("status")}) + except urllib.error.HTTPError as exc: + if exc.code == 404: + deleted_manifests.append({"repo": repo, "digest": digest, "status": 404}) + else: + raise + steps.append({"step": "registry-api-delete-manifests", "count": len(deleted_manifests), "preview": deleted_manifests[:20]}) + scale_down = kctl(["-n", namespace, "scale", "deploy", deployment_name, "--replicas=0"], 60) + steps.append({"step": "scale-registry-down", "result": bounded(scale_down)}) + if scale_down["exitCode"] != 0 or not wait_registry_pod_count(0, 120).get("ok"): + raise RuntimeError("registry workload did not scale down") + overrides = registry_gc_pod_overrides(identity) + run_gc = kctl(["-n", namespace, "run", gc_name, "--restart=Never", "--image=%s" % identity.get("image"), "--overrides=%s" % json.dumps(overrides)], 60) + steps.append({"step": "start-registry-gc-pod", "result": bounded(run_gc), "pod": gc_name}) + if run_gc["exitCode"] != 0: + raise RuntimeError("failed to start registry GC pod") + waited_gc = wait_pod_terminal(gc_name, 900) + logs = kctl(["-n", namespace, "logs", gc_name], 120) + steps.append({"step": "registry-gc-logs", "result": bounded(logs)}) + if not waited_gc.get("ok"): + raise RuntimeError("registry GC pod did not complete successfully") + finally: + cleanup_gc = kctl(["-n", namespace, "delete", "pod", gc_name, "--ignore-not-found=true"], 60) + steps.append({"step": "delete-registry-gc-pod", "result": bounded(cleanup_gc)}) + scale_up = kctl(["-n", namespace, "scale", "deploy", deployment_name, "--replicas=%s" % int(deployment.get("replicas") or 1)], 60) + steps.append({"step": "scale-registry-up", "result": bounded(scale_up)}) + rollout = kctl(["-n", namespace, "rollout", "status", "deploy/%s" % deployment_name, "--timeout=180s"], 200) + steps.append({"step": "wait-registry-rollout", "result": bounded(rollout)}) + readiness = registry_deployment_preflight() + if not readiness.get("ok") or int(readiness.get("readyReplicas") or 0) < 1: + raise RuntimeError("registry workload did not recover readiness") + probe = registry_request("GET", "/v2/", timeout=20) + if int(probe.get("status") or 0) != 200: + raise RuntimeError("registry endpoint did not recover readiness") + after = du_size(REGISTRY_ROOT, 60) or 0 + return { + "reclaimedBytes": max(0, before - after), + "commandOutput": { + "registryPlan": plan.get("summary"), + "deletedManifestCount": len(deleted_manifests), + "diskBeforeBytes": before, + "diskAfterBytes": after, + "readiness": readiness, + "steps": steps[-12:], + }, + } + def execute_registry_retention(): + if registry_identity().get("mode") == "k8s-workload": + return execute_k8s_registry_maintenance("retention") if PROVIDER_ID.upper() != "G14": raise RuntimeError("HWLAB registry retention is only supported on G14") deployment = registry_deployment_preflight() @@ -516,6 +776,8 @@ def execute_registry_retention(): } def execute_registry_garbage_collect_only(): + if registry_identity().get("mode") == "k8s-workload": + return execute_k8s_registry_maintenance("garbage-collect") if PROVIDER_ID.upper() != "G14": raise RuntimeError("HWLAB registry garbage-collect is only supported on G14") deployment = registry_deployment_preflight() @@ -591,7 +853,8 @@ def execute_registry_garbage_collect_only(): } def start_registry_retention_job(mode): - job_id = "g14-registry-%s-%s" % (int(time.time()), os.getpid()) + provider_slug = re.sub(r"[^a-z0-9]+", "-", PROVIDER_ID.lower()).strip("-") or "provider" + job_id = "%s-registry-%s-%s" % (provider_slug, int(time.time()), os.getpid()) paths = job_paths(job_id) started_at = now_iso() initial = { @@ -600,7 +863,7 @@ def start_registry_retention_job(mode): "providerId": PROVIDER_ID, "jobId": job_id, "status": "running", - "kind": "hwlab-registry-retention-gc" if mode == "retention" else "hwlab-registry-garbage-collect", + "kind": "registry-retention-gc" if mode == "retention" else "registry-garbage-collect", "mode": mode, "startedAt": started_at, "statePath": paths["state"], @@ -619,7 +882,7 @@ def start_registry_retention_job(mode): "statePath": paths["state"], "logPath": paths["log"], "statusCommand": "bun scripts/cli.ts gc remote %s status --job-id %s" % (PROVIDER_ID, job_id), - "message": "registry retention GC is running as a detached remote job", + "message": "registry maintenance is running as a detached remote job", }, } @@ -640,7 +903,7 @@ def start_registry_retention_job(mode): except Exception: log_handle = None try: - print("[%s] starting HWLAB registry %s job %s" % (now_iso(), mode, job_id), flush=True) + print("[%s] starting registry %s job %s" % (now_iso(), mode, job_id), flush=True) result = execute_registry_retention() if mode == "retention" else execute_registry_garbage_collect_only() payload = dict(initial) payload.update({ @@ -651,7 +914,7 @@ def start_registry_retention_job(mode): "clusterAfter": cluster_preflight(), }) write_json_atomic(paths["state"], payload) - print("[%s] completed HWLAB registry %s job %s" % (now_iso(), mode, job_id), flush=True) + print("[%s] completed registry %s job %s" % (now_iso(), mode, job_id), flush=True) os._exit(0) except Exception as exc: payload = dict(initial) @@ -667,7 +930,7 @@ def start_registry_retention_job(mode): write_json_atomic(paths["state"], payload) except Exception: pass - print("[%s] failed HWLAB registry %s job %s: %s" % (now_iso(), mode, job_id, exc), flush=True) + print("[%s] failed registry %s job %s: %s" % (now_iso(), mode, job_id, exc), flush=True) os._exit(1) finally: try: diff --git a/scripts/src/gc-remote-runner.py b/scripts/src/gc-remote-runner.py index 2b353d6e..7f7d8736 100644 --- a/scripts/src/gc-remote-runner.py +++ b/scripts/src/gc-remote-runner.py @@ -27,6 +27,7 @@ LOCAL_PATH_CONFIG = REMOTE_TARGET.get("localPathStorage") if isinstance(REMOTE_T KUBERNETES_RETENTION_CONFIG = REMOTE_TARGET.get("kubernetesObjectRetention") if isinstance(REMOTE_TARGET.get("kubernetesObjectRetention"), dict) else {} POLICY_TIMER_CONFIG = REMOTE_TARGET.get("policyTimer") if isinstance(REMOTE_TARGET.get("policyTimer"), dict) else {} HOST_DOCKER_GC_CONFIG = REMOTE_TARGET.get("hostDockerGc") if isinstance(REMOTE_TARGET.get("hostDockerGc"), dict) else {} +REGISTRY_CONFIG = REMOTE_TARGET.get("registryRetention") if isinstance(REMOTE_TARGET.get("registryRetention"), dict) else {} POLICY_RUNNER_SOURCE = base64.b64decode("__UNIDESK_GC_REMOTE_POLICY_RUNNER_BASE64__").decode("utf-8") TMP_PREFIX_ALLOWLIST = [ @@ -88,18 +89,6 @@ TOOL_CACHE_ALLOWLIST = [ }, ] -REGISTRY_REPOSITORY_ROOT = "/var/lib/hwlab/registry/docker/registry/v2/repositories" -REGISTRY_ROOT = "/var/lib/hwlab/registry" -REGISTRY_PROTECTED_TAGS = set([ - "latest", - "16-alpine", - "20-bookworm-slim", - "node22-alpine-v1", - "node22-alpine-bun-v1", - "sidecar", - "1b99888d3dae", -]) - EXPECTED_G14_NODE = "ubuntu-rog-zephyrus-g14-ga401iv-ga401iv" REMOTE_GC_JOB_DIR = "/tmp/unidesk-gc-remote/jobs" REMOTE_GROWTH_SNAPSHOT_DIR = "/tmp/unidesk-gc-remote/growth-snapshots" @@ -1343,19 +1332,31 @@ def collect_candidates(observed_at): "estimatedReclaimBytes": size, "action": {"op": "rm-recursive", "allowlist": "tmp-prefix"}, }) - if OPTIONS.get("hwlabRegistry", False): + if OPTIONS.get("registry", False) or OPTIONS.get("hwlabRegistry", False): registry = plan_registry_retention() summary = registry.get("summary") or {} + diagnosis = registry.get("diagnosis") delete_rows = registry.get("deleteRows") or [] delete_revision_rows = registry.get("deleteRevisionRows") or [] estimate = int(summary.get("estimatedReclaimBytes") or 0) - if delete_rows or delete_revision_rows: + if diagnosis: candidates.append({ - "id": "hwlab-registry:retention-gc", - "kind": "hwlab-registry-retention-gc", + "id": "registry:%s" % str(diagnosis.get("code") or "unsupported-target"), + "kind": "registry-retention-diagnosis", + "risk": "none", + "description": str(diagnosis.get("message") or "Registry retention is unavailable for this target"), + "sizeBytes": int(summary.get("registrySizeBytes") or 0), + "estimatedReclaimBytes": 0, + "diagnosis": diagnosis, + "action": {"op": "none", "runAllowed": False}, + }) + elif delete_rows or delete_revision_rows: + candidates.append({ + "id": "registry:retention-gc", + "kind": "registry-retention-gc", "risk": "medium", - "description": "Conservative HWLAB registry retention: keep current workload refs, retained tags and protected repos, delete stale manifest revisions, then run official registry garbage-collect", - "path": REGISTRY_ROOT, + "description": "Conservative registry retention: keep workload refs, recent tags, per-repository newest tags and digest closure, then run official registry garbage-collect", + "path": summary.get("storagePath"), "sizeBytes": int(summary.get("registrySizeBytes") or 0), "estimatedReclaimBytes": estimate, "action": { @@ -1375,11 +1376,11 @@ def collect_candidates(observed_at): }) elif bool(OPTIONS.get("registryGcOnly")) and int(summary.get("totalTags") or 0) > 0 and int(summary.get("deleteTags") or 0) == 0: candidates.append({ - "id": "hwlab-registry:garbage-collect-only", - "kind": "hwlab-registry-garbage-collect", + "id": "registry:garbage-collect-only", + "kind": "registry-garbage-collect", "risk": "medium", - "description": "Run official HWLAB registry garbage-collect without deleting additional tags; useful after a previously interrupted retention run", - "path": REGISTRY_ROOT, + "description": "Run official registry garbage-collect without deleting additional manifests", + "path": summary.get("storagePath"), "sizeBytes": int(summary.get("registrySizeBytes") or 0), "estimatedReclaimBytes": 0, "action": { @@ -1636,9 +1637,11 @@ def execute(candidate): before = allocated_file_size(path) os.unlink(path) return {"reclaimedBytes": before} - if kind == "hwlab-registry-retention-gc": + if kind == "registry-retention-diagnosis": + raise RuntimeError("registry retention is not runnable: %s" % ((candidate.get("diagnosis") or {}).get("code") or "unsupported-target")) + if kind in set(["registry-retention-gc", "hwlab-registry-retention-gc"]): return start_registry_retention_job("retention") - if kind == "hwlab-registry-garbage-collect": + if kind in set(["registry-garbage-collect", "hwlab-registry-garbage-collect"]): return start_registry_retention_job("garbage-collect") raise RuntimeError("unsupported remote gc candidate kind: %s" % kind) @@ -1737,7 +1740,7 @@ def plan_payload(observed_at, preflight, protected, candidates, visible): "/var/lib/rancher/k3s/storage", "/var/lib/kubelet", "/var/lib/containerd", - "/var/lib/hwlab unless --include-hwlab-registry is explicitly supplied", + "registry storage unless --include-registry is explicitly supplied", "Kubernetes Deployments/StatefulSets/Secrets/PVCs/PVs", "HWLAB fixed source workspaces", "Docker images, containers and volumes", @@ -1747,7 +1750,7 @@ def plan_payload(observed_at, preflight, protected, candidates, visible): "G14 run requires the expected native k3s node preflight before mutation.", "HWLAB DEV runtime and local-path PVC data are protected and require HWLAB-specific retention commands.", "Core dump cleanup only removes untracked /root/unidesk/core. regular files with no active fuser reference.", - "HWLAB registry retention is opt-in: it keeps workload tag/digest refs, all tags newer than the retention age and the newest N tags per repo before official registry garbage-collect.", + "Registry retention is opt-in: it keeps workload tag/digest refs, all tags newer than the YAML retention age and the newest N tags per repo before official registry garbage-collect.", "When summary.target.safeStop is true, do not broaden deletion scope; choose registry retention, k3s/containerd image cache maintenance, PVC/runtime retention or capacity expansion explicitly.", ], } @@ -1797,6 +1800,84 @@ def plan_payload(observed_at, preflight, protected, candidates, visible): } return payload +def registry_plan_payload(observed_at): + registry = plan_registry_retention() + full_summary = registry.get("summary") or {} + summary = full_summary if bool(OPTIONS.get("full")) else { + key: full_summary.get(key) + for key in [ + "configRef", "storageKind", "storagePath", "namespace", "deploymentName", "pvcName", "endpoint", + "totalTags", "totalRevisions", "repoCount", "keepPerRepo", "minAgeHours", "protectedWorkloadRefs", + "protectedDigestRefs", "protectedDigestClosure", "keptTags", "deleteTags", "deleteManifests", + "deleteRevisions", "registrySizeBytes", "estimatedReclaimBytes", "estimatedReclaimBlobCount", + "estimatedReclaimSampledBlobCount", "estimatedReclaimTruncated", "estimatedReclaimBasis", + ] + } + diagnosis = registry.get("diagnosis") + preview_limit = min( + config_int(REGISTRY_CONFIG, "previewLimit", 20, minimum=1, maximum=200), + int(OPTIONS.get("limit") or 10), + 5 if not bool(OPTIONS.get("full")) else 200, + ) + delete_rows = registry.get("deleteRows") or [] + delete_revision_rows = registry.get("deleteRevisionRows") or [] + automatic = REGISTRY_CONFIG.get("automatic") if isinstance(REGISTRY_CONFIG.get("automatic"), dict) else {} + candidate = { + "id": "registry:retention-gc", + "kind": "registry-retention-gc", + "risk": "medium", + "estimatedReclaimBytes": int(summary.get("estimatedReclaimBytes") or 0), + "action": { + "op": "registry-retention-gc", + "runAllowed": diagnosis is None, + "deleteTags": summary.get("deleteTags"), + "deleteManifests": summary.get("deleteManifests"), + "deleteRevisions": summary.get("deleteRevisions"), + }, + } + if diagnosis: + candidate.update({"kind": "registry-retention-diagnosis", "risk": "none", "diagnosis": diagnosis}) + return { + "ok": diagnosis is None, + "action": "gc remote plan", + "providerId": PROVIDER_ID, + "scope": "registry-retention", + "dryRun": True, + "mutation": False, + "observedAt": observed_at, + "configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.registryRetention" % PROVIDER_ID, + "summary": summary, + "candidates": [] if diagnosis else [candidate], + "diagnosis": diagnosis, + "preview": { + "deleteTags": [ + {"repo": row.get("repo"), "tag": row.get("tag"), "digest": row.get("digest"), "mtimeIso": row.get("mtimeIso")} + for row in delete_rows[:preview_limit] + ], + "deleteRevisions": [ + {"repo": row.get("repo"), "digest": row.get("digest"), "mtimeIso": row.get("mtimeIso")} + for row in delete_revision_rows[:preview_limit] + ], + "limit": preview_limit, + }, + "protected": { + "reasons": ["workload-tag-ref", "workload-digest-ref", "digest-closure", "recent-tag", "latest-per-repo", "protected-tag", "protected-repository"], + "workloadTagRefs": summary.get("protectedWorkloadRefs"), + "workloadDigestRefs": summary.get("protectedDigestRefs"), + "digestClosure": summary.get("protectedDigestClosure"), + "keepPerRepo": summary.get("keepPerRepo"), + "minAgeHours": summary.get("minAgeHours"), + }, + "automaticPolicy": { + "enabled": config_bool(automatic, "enabled", False), + "maxDeleteManifestsPerRun": config_int(automatic, "maxDeleteManifestsPerRun", 100, minimum=1, maximum=1000), + "requireIdleNamespaces": config_list(automatic, "requireIdleNamespaces", []), + "mode": "bounded-rolling-manifest-delete-then-official-gc", + }, + "runCommand": "bun scripts/cli.ts gc remote %s run --confirm --include-registry --limit 1" % PROVIDER_ID, + "statusCommand": "bun scripts/cli.ts gc remote %s status --job-id " % PROVIDER_ID, + } + def safe_unit_name(value): raw = str(value or "").strip().lower() raw = re.sub(r"[^a-z0-9_.@-]+", "-", raw).strip("-") @@ -2687,6 +2768,9 @@ def main(): else: emit_json(start_memory_pressure_job(observed_at, visible), persist_large=True) return 0 + if (bool(OPTIONS.get("registry")) or bool(OPTIONS.get("hwlabRegistry"))) and ACTION == "plan": + emit_json(registry_plan_payload(observed_at), persist_large=True) + return 0 preflight = cluster_preflight() if ACTION == "policy-plan": emit_json(remote_policy_plan_payload(observed_at), persist_large=False) diff --git a/scripts/src/gc-remote.ts b/scripts/src/gc-remote.ts index 11733861..3390a16f 100644 --- a/scripts/src/gc-remote.ts +++ b/scripts/src/gc-remote.ts @@ -25,6 +25,7 @@ interface RemoteGcOptions { aptCache: boolean; coreDumps: boolean; coreDumpMinAgeHours: number; + registry: boolean; hwlabRegistry: boolean; registryGcOnly: boolean; registryKeepPerRepo: number; @@ -57,6 +58,7 @@ const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = { aptCache: true, coreDumps: true, coreDumpMinAgeHours: 1, + registry: false, hwlabRegistry: false, registryGcOnly: false, registryKeepPerRepo: 20, @@ -96,7 +98,7 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri return { ok: false, error: "gc-remote-provider-required", - usage: "bun scripts/cli.ts gc remote memory|memory-distribution|retention|plan|snapshot|trend|run|status|policy [--confirm]", + usage: "bun scripts/cli.ts gc remote memory|memory-distribution|retention|plan|snapshot|trend|run|status|policy [--include-registry] [--confirm]", }; } const subaction = action ?? "plan"; @@ -181,14 +183,15 @@ export async function runRemoteGcCommand(config: UniDeskConfig, providerId: stri if (subaction === "status") return await runRemoteGc(config, providerId, "status", options); if (subaction === "run") { if (!options.confirm) { + const registryFlag = options.registry ? " --include-registry" : ""; return { ok: false, error: "gc-remote-run-requires-confirm", dryRun: true, mutation: false, requiredFlag: "--confirm", - planCommand: `bun scripts/cli.ts gc remote ${providerId} plan${options.memoryPressureOnly ? " --memory-pressure-only" : ""}`, - runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm${options.memoryPressureOnly ? " --memory-pressure-only" : ""}`, + planCommand: `bun scripts/cli.ts gc remote ${providerId} plan${options.memoryPressureOnly ? " --memory-pressure-only" : registryFlag}`, + runCommand: `bun scripts/cli.ts gc remote ${providerId} run --confirm${options.memoryPressureOnly ? " --memory-pressure-only" : registryFlag}`, }; } return await runRemoteGc(config, providerId, "run", options); @@ -280,9 +283,13 @@ function parseRemoteGcOptions(args: string[]): RemoteGcOptions { options.aptCache = false; } else if (arg === "--no-core-dumps") { options.coreDumps = false; + } else if (arg === "--include-registry") { + options.registry = true; } else if (arg === "--include-hwlab-registry") { + options.registry = true; options.hwlabRegistry = true; } else if (arg === "--registry-gc-only") { + options.registry = true; options.hwlabRegistry = true; options.registryGcOnly = true; } else if (arg === "--full" || arg === "--raw") { @@ -339,11 +346,40 @@ function loadRemoteGcTargetConfig(providerId: string): Record { const targets = yamlRecordOrEmpty(remote.targets, GC_REMOTE_CONFIG_REF); const candidates = [providerId, providerId.toUpperCase(), providerId.toLowerCase()]; for (const key of candidates) { - if (Object.prototype.hasOwnProperty.call(targets, key)) return yamlRecordOrEmpty(targets[key], `${GC_REMOTE_CONFIG_REF}.${key}`); + if (Object.prototype.hasOwnProperty.call(targets, key)) { + const target = yamlRecordOrEmpty(targets[key], `${GC_REMOTE_CONFIG_REF}.${key}`); + const registryRetention = yamlRecordOrEmpty(target.registryRetention, `${GC_REMOTE_CONFIG_REF}.${key}.registryRetention`); + if (Object.keys(registryRetention).length === 0) return target; + const configRef = typeof registryRetention.configRef === "string" ? registryRetention.configRef : ""; + if (configRef.length === 0) throw new Error(`${GC_REMOTE_CONFIG_REF}.${key}.registryRetention.configRef must be a file#path reference`); + return { + ...target, + registryRetention: { + ...registryRetention, + resolvedRegistry: resolveYamlConfigRef(configRef), + }, + }; + } } return {}; } +function resolveYamlConfigRef(configRef: string): Record { + const separator = configRef.indexOf("#"); + if (separator <= 0 || separator === configRef.length - 1) throw new Error(`configRef must contain file#path: ${configRef}`); + const relativePath = configRef.slice(0, separator); + const selector = configRef.slice(separator + 1); + const absolutePath = rootPath(relativePath); + if (!existsSync(absolutePath)) throw new Error(`configRef file does not exist: ${relativePath}`); + let value: unknown = Bun.YAML.parse(readFileSync(absolutePath, "utf8")) as unknown; + for (const segment of selector.split(".").filter(Boolean)) { + const record = yamlRecordOrEmpty(value, `${relativePath}#${selector}`); + if (!Object.prototype.hasOwnProperty.call(record, segment)) throw new Error(`configRef path not found: ${configRef}`); + value = record[segment]; + } + return yamlRecordOrEmpty(value, configRef); +} + function recordOrEmpty(value: unknown): Record { return typeof value === "object" && value !== null && !Array.isArray(value) ? value as Record : {}; }