diff --git a/config/unidesk-cli.yaml b/config/unidesk-cli.yaml index e774d46b..ab3cf793 100644 --- a/config/unidesk-cli.yaml +++ b/config/unidesk-cli.yaml @@ -209,6 +209,19 @@ gc: enabled: true until: 24h reason: 仅回收超过 YAML 年龄阈值且 Docker 判定可回收的 BuildKit cache；不清理 images、containers 或 volumes。 + automatic: + enabled: true + until: 168h + imagePrune: + enabled: true + minAgeHours: 168 + maxDeletePerRun: 20 + previewLimit: 20 + reason: 仅删除超过 YAML 年龄阈值且未被任何运行、停止或 Created 容器引用的 host Docker image。 + automatic: + enabled: true + minAgeHours: 336 + maxDeletePerRun: 5 memoryPressure: planPreviewLimit: 6 processPatterns: diff --git a/project-management/PJ2026-01/specs/PJ2026-01060106-hwlab-dev-production-lanes.md b/project-management/PJ2026-01/specs/PJ2026-01060106-hwlab-dev-production-lanes.md index a5446297..1ca9f88f 100644 --- a/project-management/PJ2026-01/specs/PJ2026-01060106-hwlab-dev-production-lanes.md +++ b/project-management/PJ2026-01/specs/PJ2026-01060106-hwlab-dev-production-lanes.md @@ -144,6 +144,11 @@ owning YAML 至少声明： - 两个 lane 可以复用同一 NC01 host PostgreSQL 服务进程，但必须使用不同 database、role、Secret consumer 和 migration ledger。 - migration 命令必须显式选中 lane，只能迁移其 database；禁止用共享 DSN、共享 schema 或默认 database 回退。 +- migration 执行边界： + - lane 自有 schema migration 可以作为 owning workload 的 init 阶段执行； + - 禁止使用会在 migration 失败时停止整个 Argo Application 资源应用的 Sync hook； + - migration 失败只能使依赖该 schema 的组件保持未就绪； + - 失败状态必须保留具名组件、lane 和失败阶段，不能阻止同一 lane 的无关服务滚动。 - production 初始化不复制 development 会话、用户业务数据或 migration history；需要数据导入时必须另立受控任务。 - development 与 production 使用不同 Kafka consumer group identity。运行状态和 lag 必须带 lane 标签，禁止同名 group 导致 offset 互相推进。 - 数据库、Kafka 或 schema warning 不得触发功能关闭、authority 切换、HTTP fallback 或阻塞其他 lane 滚动上线。 diff --git a/scripts/src/gc-remote-host-docker.py b/scripts/src/gc-remote-host-docker.py new file mode 100644 index 00000000..eda3fcf2 --- /dev/null +++ b/scripts/src/gc-remote-host-docker.py @@ -0,0 +1,153 @@ +from datetime import datetime, timezone + + +def parse_docker_human_size(raw): + raw = str(raw).split("(")[0].strip() + match = re.match(r"^([0-9.]+)\s*([KMGT]?B)$", raw, re.I) + if not match: + return None + multiplier = {"B": 1, "KB": 1000, "MB": 1000**2, "GB": 1000**3, "TB": 1000**4}.get(match.group(2).upper(), 1) + return int(float(match.group(1)) * multiplier) + + +def parse_docker_build_cache(text): + for line in text.splitlines(): + if not line.startswith("Build Cache"): + continue + match = re.match(r"^Build Cache\s+\S+\s+\S+\s+(\S+)\s+(\S+)", line.strip()) + if not match: + continue + size = parse_docker_human_size(match.group(1)) + reclaim = parse_docker_human_size(match.group(2)) + if size is None or reclaim is None: + return None + return {"sizeBytes": size, "reclaimableBytes": reclaim} + return None + + +def build_cache_only_payload(observed_at): + disk_before = df_snapshot() + section = host_docker_gc_section("buildCachePrune") + until = str(OPTIONS.get("buildCacheUntil") or section.get("until") or "24h") + if not host_docker_gc_enabled("buildCachePrune"): + return {"ok": False, "action": "gc remote %s" % ACTION, "providerId": PROVIDER_ID, "scope": "build-cache-only", "dryRun": ACTION == "plan", "mutation": False, "diagnosis": host_docker_gc_reason("buildCachePrune", "BuildKit cache GC is disabled by YAML")} + system_df = command(["docker", "system", "df"], 8) + if system_df["exitCode"] != 0: + return {"ok": False, "action": "gc remote %s" % ACTION, "providerId": PROVIDER_ID, "scope": "build-cache-only", "dryRun": ACTION == "plan", "mutation": False, "diagnosis": bounded(system_df)} + cache = parse_docker_build_cache(system_df["stdout"]) + candidate = None + if cache is not None and cache["reclaimableBytes"] > 0: + candidate = {"id": "docker-builder:prune", "kind": "docker-build-cache-prune", "risk": "low", "sizeBytes": cache["sizeBytes"], "estimatedReclaimBytes": cache["reclaimableBytes"], "action": {"command": ["docker", "builder", "prune", "--all", "--force", "--filter", "until=%s" % until]}} + payload = {"ok": True, "action": "gc remote %s" % ACTION, "providerId": PROVIDER_ID, "scope": "build-cache-only", "configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.hostDockerGc.buildCachePrune" % PROVIDER_ID, "dryRun": ACTION == "plan", "mutation": ACTION == "run" and candidate is not None, "observedAt": observed_at, "diskBefore": disk_before, "until": until, "candidates": [candidate] if candidate else []} + if ACTION == "run" and candidate is not None: + payload["result"] = execute(candidate) + payload["diskAfter"] = df_snapshot() + return payload + + +def docker_containers(): + ps = command(["docker", "ps", "-qa", "--no-trunc"], 5) + if ps["exitCode"] != 0 or not ps["stdout"].strip(): + return [] + ids = ps["stdout"].split() + inspect = command(["docker", "inspect"] + ids, 10) + if inspect["exitCode"] != 0 or not inspect["stdout"].strip(): + return [] + try: + data = json.loads(inspect["stdout"]) + except Exception: + return [] + rows = [] + for item in data: + cfg = item.get("Config") or {} + rows.append({"id": str(item.get("Id") or ""), "name": str(item.get("Name") or "").lstrip("/"), "image": str(cfg.get("Image") or item.get("Image") or ""), "imageId": str(item.get("Image") or ""), "logPath": str(item.get("LogPath") or "")}) + return [row for row in rows if row["id"]] + + +def parse_docker_created(raw): + value = str(raw or "").strip() + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc).timestamp() + except Exception: + return None + + +def collect_host_docker_image_candidates(observed_at): + global HOST_DOCKER_IMAGE_SCAN + section = host_docker_gc_section("imagePrune") + min_age_hours = config_float(section, "minAgeHours", 168.0, minimum=0.0) + max_delete = config_int(section, "maxDeletePerRun", 20, minimum=1, maximum=500) + preview_limit = config_int(section, "previewLimit", max_delete, minimum=1, maximum=500) + image_ids = command(["docker", "image", "ls", "-aq", "--no-trunc"], 10) + if image_ids["exitCode"] != 0: + HOST_DOCKER_IMAGE_SCAN = {"ok": False, "error": "docker-image-list-failed", "command": bounded(image_ids)} + return [] + ids = sorted(set(image_ids["stdout"].split())) + referenced = set(row.get("imageId") for row in docker_containers() if row.get("imageId")) + if not ids: + HOST_DOCKER_IMAGE_SCAN = {"ok": True, "minAgeHours": min_age_hours, "maxDeletePerRun": max_delete, "imageCount": 0, "referencedImageCount": len(referenced), "eligibleCount": 0, "selectedCount": 0} + return [] + inspected = command(["docker", "image", "inspect"] + ids, 30) + if inspected["exitCode"] != 0: + HOST_DOCKER_IMAGE_SCAN = {"ok": False, "error": "docker-image-inspect-failed", "command": bounded(inspected)} + return [] + try: + images = json.loads(inspected["stdout"]) + except Exception as exc: + HOST_DOCKER_IMAGE_SCAN = {"ok": False, "error": "docker-image-inspect-invalid-json", "details": str(exc)} + return [] + cutoff = time.time() - min_age_hours * 3600.0 + eligible = [] + protected_rows = [] + for image in images: + image_id = str(image.get("Id") or "") + created_at = str(image.get("Created") or "") + created_epoch = parse_docker_created(created_at) + tags = [str(item) for item in (image.get("RepoTags") or []) if item] + if image_id in referenced: + protected_rows.append({"imageId": image_id, "tags": tags, "reason": "referenced-by-container"}) + continue + if created_epoch is None or created_epoch > cutoff: + protected_rows.append({"imageId": image_id, "tags": tags, "reason": "younger-than-min-age-or-age-unknown", "createdAt": created_at}) + continue + size = int(image.get("Size") or 0) + eligible.append({"id": "docker-image:%s" % image_id, "kind": "docker-image-delete", "risk": "medium", "description": "Delete old host Docker image unreferenced by any container", "imageId": image_id, "tags": tags, "createdAt": created_at, "ageHours": round(max(0.0, (time.time() - created_epoch) / 3600.0), 1), "minAgeHours": min_age_hours, "sizeBytes": size, "estimatedReclaimBytes": size, "protection": {"allContainerStatesInspected": True, "unreferencedByAnyContainer": True}, "action": {"command": ["docker", "image", "rm", image_id]}}) + eligible.sort(key=lambda item: (item.get("createdAt") or "", item.get("imageId") or "")) + selected = eligible[:max_delete] + HOST_DOCKER_IMAGE_SCAN = {"ok": True, "configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.hostDockerGc.imagePrune" % PROVIDER_ID, "minAgeHours": min_age_hours, "maxDeletePerRun": max_delete, "imageCount": len(images), "referencedImageCount": len(referenced), "protectedCount": len(protected_rows), "eligibleCount": len(eligible), "selectedCount": len(selected), "protectedPreview": protected_rows[:preview_limit]} + return selected + + +def host_docker_only_payload(observed_at): + disk_before = df_snapshot() + candidates = [] + build_section = host_docker_gc_section("buildCachePrune") + build_until = str(OPTIONS.get("buildCacheUntil") or build_section.get("until") or "24h") + if host_docker_gc_enabled("buildCachePrune"): + system_df = command(["docker", "system", "df"], 8) + if system_df["exitCode"] != 0: + return {"ok": False, "action": "gc remote %s" % ACTION, "providerId": PROVIDER_ID, "scope": "host-docker-only", "dryRun": ACTION == "plan", "mutation": False, "diagnosis": bounded(system_df)} + cache = parse_docker_build_cache(system_df["stdout"]) + if cache is not None and cache["reclaimableBytes"] > 0: + candidates.append({"id": "docker-builder:prune", "kind": "docker-build-cache-prune", "risk": "low", "description": "Prune Docker BuildKit cache unused for %s" % build_until, "sizeBytes": cache["sizeBytes"], "estimatedReclaimBytes": cache["reclaimableBytes"], "action": {"command": ["docker", "builder", "prune", "--all", "--force", "--filter", "until=%s" % build_until]}}) + if host_docker_gc_enabled("imagePrune", False): + candidates.extend(collect_host_docker_image_candidates(observed_at)) + visible = visible_items(candidates) + payload = {"ok": HOST_DOCKER_IMAGE_SCAN.get("ok") is not False, "action": "gc remote %s" % ACTION, "providerId": PROVIDER_ID, "scope": "host-docker-only", "configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.hostDockerGc" % PROVIDER_ID, "dryRun": ACTION == "plan", "mutation": ACTION == "run" and bool(visible), "observedAt": observed_at, "diskBefore": disk_before, "buildCacheUntil": build_until, "imageRetention": HOST_DOCKER_IMAGE_SCAN, "summary": summarize(candidates, visible, disk_before), "candidates": visible, "protected": {"containerStates": ["running", "stopped", "created"], "referencedImages": "excluded-and-rechecked-before-delete", "containers": "never-deleted", "volumes": "never-deleted"}} + if ACTION == "run": + results = [] + for candidate in visible: + try: + execution = execute(candidate) + results.append({**candidate, "status": "succeeded", "reclaimedBytes": execution.get("reclaimedBytes"), "commandOutput": execution.get("commandOutput")}) + except Exception as exc: + results.append({**candidate, "status": "failed", "reclaimedBytes": None, "error": str(exc)}) + break + payload["results"] = results + payload["ok"] = payload["ok"] and all(item.get("status") == "succeeded" for item in results) + payload["diskAfter"] = df_snapshot() + if payload["diskBefore"] and payload["diskAfter"]: + payload["actualDiskReclaimBytes"] = max(0, payload["diskAfter"]["availableBytes"] - payload["diskBefore"]["availableBytes"]) + return payload diff --git a/scripts/src/gc-remote-host-docker.test.ts b/scripts/src/gc-remote-host-docker.test.ts new file mode 100644 index 00000000..c8793828 --- /dev/null +++ b/scripts/src/gc-remote-host-docker.test.ts @@ -0,0 +1,30 @@ +import assert from "node:assert/strict"; +import { readFileSync } from "node:fs"; +import { test } from "bun:test"; + +import { rootPath } from "./config"; + +const runner = readFileSync(rootPath("scripts/src/gc-remote-runner.py"), "utf8"); +const hostDocker = readFileSync(rootPath("scripts/src/gc-remote-host-docker.py"), "utf8"); +const policyRunner = readFileSync(rootPath("scripts/src/gc-remote-policy-runner.py"), "utf8"); +const yaml = readFileSync(rootPath("config/unidesk-cli.yaml"), "utf8"); + +test("host Docker image GC protects every container state and uses exact image removal", () => { + assert.match(hostDocker, /docker", "ps", "-qa", "--no-trunc/u); + assert.match(hostDocker, /unreferencedByAnyContainer/u); + assert.match(runner, /docker", "image", "rm", image_id/u); + assert.match(hostDocker, /scope": "host-docker-only/u); + assert.doesNotMatch(runner, /docker", "system", "prune/u); + assert.doesNotMatch(runner, /docker", "image", "prune/u); +}); + +test("policy stages use conservative YAML-owned host Docker limits", () => { + assert.match(yaml, /imagePrune:[\s\S]*minAgeHours: 168[\s\S]*automatic:[\s\S]*minAgeHours: 336[\s\S]*maxDeletePerRun: 5/u); + assert.match(yaml, /buildCachePrune:[\s\S]*automatic:[\s\S]*until: 168h/u); + assert.match(policyRunner, /run_host_docker_build_cache/u); + assert.match(policyRunner, /run_host_docker_images/u); + assert.match(policyRunner, /became-referenced/u); + assert.match(policyRunner, /command_lines\(\["docker", "ps", "-qa", "--no-trunc"\]/u); + assert.match(policyRunner, /actualDiskReclaimBytes/u); + assert.match(runner, /"hostDockerGc": rendered\["policyConfig"\]\["hostDockerGc"\]/u); +}); diff --git a/scripts/src/gc-remote-policy-runner.py b/scripts/src/gc-remote-policy-runner.py index 72c7521b..1f37cb0d 100644 --- a/scripts/src/gc-remote-policy-runner.py +++ b/scripts/src/gc-remote-policy-runner.py @@ -126,6 +126,36 @@ def run_json(args, timeout=45): return None, result +def command_lines(args, timeout=30, limit=10000): + started = time.time() + try: + process = subprocess.run(args, text=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=timeout) + except subprocess.TimeoutExpired as exc: + return None, { + "exitCode": 124, + "elapsedMs": int((time.time() - started) * 1000), + "stderrTail": (exc.stderr or "")[-1200:] if isinstance(exc.stderr, str) else "", + "timedOut": True, + } + lines = process.stdout.split() + result = {"exitCode": process.returncode, "elapsedMs": int((time.time() - started) * 1000), "stderrTail": process.stderr[-1200:], "lineCount": len(lines)} + if process.returncode != 0 or len(lines) > limit: + if len(lines) > limit: + result["error"] = "line-limit-exceeded" + return None, result + return lines, result + + +def root_available_bytes(): + result = command(["df", "-B1", "--output=avail", "/"], 10) + if result["exitCode"] != 0: + return None + try: + return int(result["stdoutTail"].splitlines()[-1].strip()) + except Exception: + return None + + def write_state(config, payload): state_path = config.get("statePath") if not state_path: @@ -146,6 +176,88 @@ def run_journal(config): return {"id": "journal", "ok": result["exitCode"] == 0, "command": result} +def parse_docker_created(raw): + value = str(raw or "").strip() + if not value: + return None + try: + return datetime.fromisoformat(value.replace("Z", "+00:00")).astimezone(timezone.utc).timestamp() + except Exception: + return None + + +def docker_container_image_ids(): + container_ids, ids = command_lines(["docker", "ps", "-qa", "--no-trunc"], 10) + if container_ids is None: + return None, ids + if not container_ids: + return set(), ids + data, inspect = run_json(["docker", "inspect"] + container_ids, 30) + if data is None: + return None, inspect + return set(str(item.get("Image") or "") for item in data if item.get("Image")), inspect + + +def run_host_docker_build_cache(config): + cfg = ((config.get("hostDockerGc") or {}).get("buildCachePrune") or {}) + if not cfg.get("enabled"): + return {"id": "host-docker-build-cache", "ok": True, "skipped": True} + until = str(cfg.get("until") or "") + if not re.match(r"^[0-9]+(?:s|m|h|d)$", until): + return {"id": "host-docker-build-cache", "ok": False, "error": "invalid-until"} + before = root_available_bytes() + result = command(["docker", "builder", "prune", "--all", "--force", "--filter", "until=%s" % until], 90) + after = root_available_bytes() + return {"id": "host-docker-build-cache", "ok": result["exitCode"] == 0, "until": until, "actualDiskReclaimBytes": max(0, after - before) if before is not None and after is not None else None, "command": result} + + +def run_host_docker_images(config): + cfg = ((config.get("hostDockerGc") or {}).get("imagePrune") or {}) + if not cfg.get("enabled"): + return {"id": "host-docker-images", "ok": True, "skipped": True} + min_age_hours = float(cfg.get("minAgeHours") or 0) + limit = max(1, min(safe_int(cfg.get("maxDeletePerRun"), 1), 500)) + referenced, container_command = docker_container_image_ids() + if referenced is None: + return {"id": "host-docker-images", "ok": False, "error": "container-reference-scan-failed", "command": container_command} + image_ids, ids = command_lines(["docker", "image", "ls", "-aq", "--no-trunc"], 10) + if image_ids is None: + return {"id": "host-docker-images", "ok": False, "error": "image-list-failed", "command": ids} + image_ids = sorted(set(image_ids)) + if not image_ids: + return {"id": "host-docker-images", "ok": True, "candidateCount": 0, "deletedCount": 0, "protectedReferencedCount": len(referenced), "reclaimedBytes": 0} + images, inspect = run_json(["docker", "image", "inspect"] + image_ids, 45) + if images is None: + return {"id": "host-docker-images", "ok": False, "error": "image-inspect-failed", "command": inspect} + cutoff = time.time() - min_age_hours * 3600.0 + candidates = [] + for image in images: + image_id = str(image.get("Id") or "") + created = parse_docker_created(image.get("Created")) + if not image_id or image_id in referenced or created is None or created > cutoff: + continue + candidates.append({"imageId": image_id, "createdAt": image.get("Created"), "sizeBytes": safe_int(image.get("Size"))}) + candidates.sort(key=lambda item: (item.get("createdAt") or "", item.get("imageId") or "")) + deleted = 0 + before = root_available_bytes() + failures = [] + for candidate in candidates[:limit]: + current_referenced, reference_command = docker_container_image_ids() + if current_referenced is None: + failures.append({"imageId": candidate["imageId"], "error": "container-reference-rescan-failed", "command": reference_command}) + break + if candidate["imageId"] in current_referenced: + failures.append({"imageId": candidate["imageId"], "error": "became-referenced"}) + continue + result = command(["docker", "image", "rm", candidate["imageId"]], 45) + if result["exitCode"] != 0: + failures.append({"imageId": candidate["imageId"], "error": "image-rm-failed", "command": result}) + continue + deleted += 1 + after = root_available_bytes() + return {"id": "host-docker-images", "ok": len(failures) == 0, "minAgeHours": min_age_hours, "maxDeletePerRun": limit, "candidateCount": len(candidates), "deletedCount": deleted, "protectedReferencedCount": len(referenced), "reclaimedBytes": max(0, after - before) if before is not None and after is not None else 0, "actualDiskReclaimBytes": max(0, after - before) if before is not None and after is not None else None, "failures": failures} + + def run_apt(config): if not config.get("includeAptCache"): return {"id": "apt-cache", "ok": True, "skipped": True} @@ -463,7 +575,7 @@ def main(): config_path = sys.argv[1] if len(sys.argv) > 1 else "" config = load_config(config_path) results = [] - for fn in [run_journal, run_apt, run_tmp, run_tool_caches, run_web_observe, run_session_pvcs, run_k3s_images, run_host_containerd, run_local_path_orphans, run_kubernetes_object_retention_policy, lambda value: run_merged_worktrees(value, automatic=True)]: + for fn in [run_journal, run_apt, run_tmp, run_tool_caches, run_web_observe, run_session_pvcs, run_k3s_images, run_host_containerd, run_local_path_orphans, run_kubernetes_object_retention_policy, lambda value: run_merged_worktrees(value, automatic=True), run_host_docker_build_cache, run_host_docker_images]: try: results.append(fn(config)) except Exception as exc: diff --git a/scripts/src/gc-remote-runner.py b/scripts/src/gc-remote-runner.py index d37c0093..42a95251 100644 --- a/scripts/src/gc-remote-runner.py +++ b/scripts/src/gc-remote-runner.py @@ -30,6 +30,7 @@ POLICY_TIMER_CONFIG = REMOTE_TARGET.get("policyTimer") if isinstance(REMOTE_TARG HOST_DOCKER_GC_CONFIG = REMOTE_TARGET.get("hostDockerGc") if isinstance(REMOTE_TARGET.get("hostDockerGc"), dict) else {} REGISTRY_CONFIG = REMOTE_TARGET.get("registryRetention") if isinstance(REMOTE_TARGET.get("registryRetention"), dict) else {} POLICY_RUNNER_SOURCE = base64.b64decode("__UNIDESK_GC_REMOTE_POLICY_RUNNER_BASE64__").decode("utf-8") +HOST_DOCKER_IMAGE_SCAN = {} # __UNIDESK_GC_REMOTE_MERGED_WORKTREE_HELPERS__ @@ -1052,69 +1053,7 @@ def parse_journal_usage(text): mult = {"": 1, "K": 1024, "M": 1024**2, "G": 1024**3, "T": 1024**4}.get(m.group(2).upper(), 1) return int(float(m.group(1)) * mult) -def parse_docker_human_size(raw): - raw = str(raw).split("(")[0].strip() - m = re.match(r"^([0-9.]+)\s*([KMGT]?B)$", raw, re.I) - if not m: - return None - mult = {"B": 1, "KB": 1000, "MB": 1000**2, "GB": 1000**3, "TB": 1000**4}.get(m.group(2).upper(), 1) - return int(float(m.group(1)) * mult) - -def parse_docker_build_cache(text): - for line in text.splitlines(): - if not line.startswith("Build Cache"): - continue - match = re.match(r"^Build Cache\s+\S+\s+\S+\s+(\S+)\s+(\S+)", line.strip()) - if not match: - continue - size = parse_docker_human_size(match.group(1)) - reclaim = parse_docker_human_size(match.group(2)) - if size is None or reclaim is None: - return None - return {"sizeBytes": size, "reclaimableBytes": reclaim} - return None - -def build_cache_only_payload(observed_at): - disk_before = df_snapshot() - section = host_docker_gc_section("buildCachePrune") - until = str(OPTIONS.get("buildCacheUntil") or section.get("until") or "24h") - if not host_docker_gc_enabled("buildCachePrune"): - return {"ok": False, "action": "gc remote %s" % ACTION, "providerId": PROVIDER_ID, "scope": "build-cache-only", "dryRun": ACTION == "plan", "mutation": False, "diagnosis": host_docker_gc_reason("buildCachePrune", "BuildKit cache GC is disabled by YAML")} - system_df = command(["docker", "system", "df"], 8) - if system_df["exitCode"] != 0: - return {"ok": False, "action": "gc remote %s" % ACTION, "providerId": PROVIDER_ID, "scope": "build-cache-only", "dryRun": ACTION == "plan", "mutation": False, "diagnosis": bounded(system_df)} - cache = parse_docker_build_cache(system_df["stdout"]) - candidate = None - if cache is not None and cache["reclaimableBytes"] > 0: - candidate = {"id": "docker-builder:prune", "kind": "docker-build-cache-prune", "risk": "low", "sizeBytes": cache["sizeBytes"], "estimatedReclaimBytes": cache["reclaimableBytes"], "action": {"command": ["docker", "builder", "prune", "--all", "--force", "--filter", "until=%s" % until]}} - payload = {"ok": True, "action": "gc remote %s" % ACTION, "providerId": PROVIDER_ID, "scope": "build-cache-only", "configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.hostDockerGc.buildCachePrune" % PROVIDER_ID, "dryRun": ACTION == "plan", "mutation": ACTION == "run" and candidate is not None, "observedAt": observed_at, "diskBefore": disk_before, "until": until, "candidates": [candidate] if candidate else []} - if ACTION == "run" and candidate is not None: - payload["result"] = execute(candidate) - payload["diskAfter"] = df_snapshot() - return payload - -def docker_containers(): - ps = command(["docker", "ps", "-qa", "--no-trunc"], 5) - if ps["exitCode"] != 0 or not ps["stdout"].strip(): - return [] - ids = ps["stdout"].split() - inspect = command(["docker", "inspect"] + ids, 10) - if inspect["exitCode"] != 0 or not inspect["stdout"].strip(): - return [] - try: - data = json.loads(inspect["stdout"]) - except Exception: - return [] - rows = [] - for item in data: - cfg = item.get("Config") or {} - rows.append({ - "id": str(item.get("Id") or ""), - "name": str(item.get("Name") or "").lstrip("/"), - "image": str(cfg.get("Image") or item.get("Image") or ""), - "logPath": str(item.get("LogPath") or ""), - }) - return [row for row in rows if row["id"]] +# __UNIDESK_GC_REMOTE_HOST_DOCKER_HELPERS__ def cluster_preflight(): node_cmd = command(["sh", "-lc", "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get nodes -o jsonpath='{range .items[*]}{.metadata.name}{\"\\n\"}{end}' 2>/dev/null"], 10) @@ -1225,6 +1164,9 @@ def collect_candidates(observed_at): "action": {"command": ["docker", "builder", "prune", "--all", "--force", "--filter", "until=%s" % until], "estimate": "docker-system-df-reclaimable-upper-bound"}, }) + if host_docker_gc_enabled("imagePrune", False): + candidates.extend(collect_host_docker_image_candidates(observed_at)) + if OPTIONS.get("aptCache", True): apt_path = "/var/cache/apt/archives" size = du_size(apt_path) or 0 @@ -1627,6 +1569,22 @@ def execute(candidate): if result["exitCode"] != 0: raise RuntimeError((result["stderr"] or "docker builder prune failed").strip()) return {"reclaimedBytes": None, "commandOutput": bounded(result)} + if kind == "docker-image-delete": + if not host_docker_gc_enabled("imagePrune", False): + raise RuntimeError(host_docker_gc_reason("imagePrune", "refusing host Docker image deletion on this provider")) + image_id = str(candidate.get("imageId") or "") + if not image_id.startswith("sha256:"): + raise RuntimeError("refusing invalid Docker image id") + referenced = set(row.get("imageId") for row in docker_containers() if row.get("imageId")) + if image_id in referenced: + raise RuntimeError("refusing Docker image now referenced by a container") + inspect = command(["docker", "image", "inspect", image_id], 10) + if inspect["exitCode"] != 0: + raise RuntimeError((inspect["stderr"] or "docker image inspect failed").strip()) + result = command(["docker", "image", "rm", image_id], 30) + if result["exitCode"] != 0: + raise RuntimeError((result["stderr"] or "docker image rm failed").strip()) + return {"reclaimedBytes": None, "commandOutput": bounded(result)} if kind == "apt-cache-clean": before = du_size("/var/cache/apt/archives") or 0 result = command(["apt-get", "clean"], 30) @@ -1785,7 +1743,7 @@ def plan_payload(observed_at, preflight, protected, candidates, visible): "registry storage unless --include-registry is explicitly supplied", "Kubernetes Deployments/StatefulSets/Secrets/PVCs/PVs", "HWLAB fixed source workspaces", - "Docker images, containers and volumes", + "Docker containers and volumes; host images are eligible only when old and unreferenced by every container state", ], "notes": [ "Remote gc only executes the returned candidate page unless --full or a larger --limit is supplied.", @@ -1800,7 +1758,7 @@ def plan_payload(observed_at, preflight, protected, candidates, visible): policy = { "requiresRunConfirm": True, "runCommand": "bun scripts/cli.ts gc remote %s run --confirm" % PROVIDER_ID, - "neverTouches": ["k3s runtime", "PVC/PV/local-path data", "Secrets/auth/config", "Docker volumes/images"], + "neverTouches": ["k3s runtime", "PVC/PV/local-path data", "Secrets/auth/config", "Docker containers/volumes and referenced images"], "notes": [ "Default plan is compact; rerun with --full for complete policy notes and protected rows.", "When summary.target.safeStop is true, stop at protected boundaries and choose an owner-aware retention or capacity decision.", @@ -1820,6 +1778,7 @@ def plan_payload(observed_at, preflight, protected, candidates, visible): "candidates": visible, "protected": protected if bool(OPTIONS.get("full")) else protected[:3], "policy": policy, + "hostDockerImageRetention": HOST_DOCKER_IMAGE_SCAN, } if bool(OPTIONS.get("full")): payload.update({ @@ -1948,6 +1907,12 @@ def render_remote_policy(): include_host_containerd = required_policy_bool("includeHostContainerdCache") include_local_path_orphans = required_policy_bool("includeLocalPathOrphans") include_kubernetes_retention = required_policy_bool("includeKubernetesObjectRetention") + build_cache_section = host_docker_gc_section("buildCachePrune") + build_cache_automatic = build_cache_section.get("automatic") if isinstance(build_cache_section.get("automatic"), dict) else {} + image_section = host_docker_gc_section("imagePrune") + image_automatic = image_section.get("automatic") if isinstance(image_section.get("automatic"), dict) else {} + include_host_docker_build_cache = host_docker_gc_enabled("buildCachePrune") and bool(build_cache_automatic.get("enabled")) + include_host_docker_images = host_docker_gc_enabled("imagePrune", False) and bool(image_automatic.get("enabled")) include_merged_worktrees = required_policy_bool("includeMergedWorktrees") state_dir = config_str(POLICY_TIMER_CONFIG, "stateDir", "/var/lib/unidesk-gc") config_dir = config_str(POLICY_TIMER_CONFIG, "configDir", "/etc/unidesk-gc") @@ -1979,6 +1944,14 @@ def render_remote_policy(): ), "mergedWorktrees": ({**MERGED_WORKTREE_CONFIG, "enabled": True} if include_merged_worktrees else {"enabled": False}), "agentrunSessionPvcs": POLICY_TIMER_CONFIG.get("agentrunSessionPvcs") if isinstance(POLICY_TIMER_CONFIG.get("agentrunSessionPvcs"), dict) else {"enabled": False}, + "hostDockerGc": { + "buildCachePrune": {"enabled": include_host_docker_build_cache, "until": config_str(build_cache_automatic, "until", config_str(build_cache_section, "until", "168h"))}, + "imagePrune": { + "enabled": include_host_docker_images, + "minAgeHours": config_float(image_automatic, "minAgeHours", config_float(image_section, "minAgeHours", 336.0, minimum=0.0), minimum=0.0), + "maxDeletePerRun": config_int(image_automatic, "maxDeletePerRun", config_int(image_section, "maxDeletePerRun", 5, minimum=1, maximum=500), minimum=1, maximum=500), + }, + }, } stages = [ {"id": "journal", "enabled": include_journal, "risk": "low", "mode": "journalctl-vacuum"}, @@ -1991,6 +1964,8 @@ def render_remote_policy(): {"id": "host-containerd-orphans", "enabled": include_host_containerd, "risk": "medium", "mode": "ctr-metadata-empty-yaml-orphan-state"}, {"id": "local-path-orphans", "enabled": include_local_path_orphans, "risk": "medium", "mode": "pv-unreferenced-direct-child"}, {"id": "kubernetes-object-retention", "enabled": include_kubernetes_retention, "risk": "medium", "mode": "terminal-owner-cascade-bounded"}, + {"id": "host-docker-build-cache", "enabled": include_host_docker_build_cache, "risk": "low", "mode": "docker-builder-prune-yaml-until"}, + {"id": "host-docker-images", "enabled": include_host_docker_images, "risk": "medium", "mode": "old-unreferenced-all-container-states-bounded"}, {"id": "merged-worktrees", "enabled": include_merged_worktrees, "risk": "medium", "mode": "git-registered-clean-inactive-absorbed-bounded"}, ] service = "\n".join([ @@ -2042,6 +2017,8 @@ def render_remote_policy(): "includeHostContainerdCache": include_host_containerd, "includeLocalPathOrphans": include_local_path_orphans, "includeKubernetesObjectRetention": include_kubernetes_retention, + "includeHostDockerBuildCache": include_host_docker_build_cache, + "includeHostDockerImages": include_host_docker_images, "includeMergedWorktrees": include_merged_worktrees, "stages": stages, "policyConfig": policy_config, @@ -2063,6 +2040,7 @@ def remote_policy_plan_payload(observed_at): "enabled": config_bool(POLICY_TIMER_CONFIG, "enabled", False), "timer": {key: rendered.get(key) for key in ["unitName", "scriptPath", "configPath", "statePath", "servicePath", "timerPath", "onCalendar", "randomizedDelaySec", "journalTargetBytes", "journalTarget", "tmpMinAgeHours", "includeJournal", "includeTmp", "includeAptCache", "includeToolCaches", "includeWebObserveArtifacts", "includeK3sImageCache", "includeHostContainerdCache", "includeLocalPathOrphans", "includeKubernetesObjectRetention", "includeMergedWorktrees"]}, "stages": rendered.get("stages"), + "hostDockerGc": rendered["policyConfig"]["hostDockerGc"], "servicePreview": rendered["service"] if bool(OPTIONS.get("full")) else None, "timerPreview": rendered["timer"] if bool(OPTIONS.get("full")) else None, "installCommand": "bun scripts/cli.ts gc remote %s policy install --confirm" % PROVIDER_ID, @@ -2072,7 +2050,7 @@ def remote_policy_plan_payload(observed_at): "neverTouches": [ "k3s runtime metadata unless a dedicated CRI prune stage is enabled", "active PVC/PV/local-path data", - "Docker images, containers, volumes or Docker build cache", + "Docker containers, volumes and referenced images; host Docker GC is limited to the displayed YAML stages", "Secret/auth/config state", "active Web observe runners or Chrome processes", ], @@ -2106,6 +2084,8 @@ def remote_policy_status_payload(observed_at): "service": bounded(service), }, "policySync": policy_sync, + "desiredStages": rendered["stages"], + "hostDockerGc": rendered["policyConfig"]["hostDockerGc"], "lastRun": last, "next": { "plan": "bun scripts/cli.ts gc remote %s policy plan" % PROVIDER_ID, @@ -2832,6 +2812,9 @@ def main(): if bool(OPTIONS.get("buildCacheOnly")) and ACTION in set(["plan", "run"]): emit_json(build_cache_only_payload(observed_at), persist_large=False) return 0 + if bool(OPTIONS.get("hostDockerOnly")) and ACTION in set(["plan", "run"]): + emit_json(host_docker_only_payload(observed_at), persist_large=False) + return 0 if (bool(OPTIONS.get("registry")) or bool(OPTIONS.get("hwlabRegistry"))) and ACTION == "plan": emit_json(registry_plan_payload(observed_at), persist_large=True) return 0 diff --git a/scripts/src/gc-remote.ts b/scripts/src/gc-remote.ts index c803b1b8..39f61ac0 100644 --- a/scripts/src/gc-remote.ts +++ b/scripts/src/gc-remote.ts @@ -39,6 +39,7 @@ interface RemoteGcOptions { saveSnapshot: boolean; memoryPressureOnly: boolean; buildCacheOnly: boolean; + hostDockerOnly: boolean; } const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = { @@ -71,6 +72,7 @@ const DEFAULT_REMOTE_OPTIONS: RemoteGcOptions = { saveSnapshot: true, memoryPressureOnly: false, buildCacheOnly: false, + hostDockerOnly: false, }; const GC_CONFIG_RELATIVE_PATH = "config/unidesk-cli.yaml"; @@ -95,6 +97,8 @@ const GC_REMOTE_GROWTH_RELATIVE_PATH = "scripts/src/gc-remote-growth.py"; const GC_REMOTE_GROWTH_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_GROWTH_HELPERS__"; const GC_REMOTE_REGISTRY_RELATIVE_PATH = "scripts/src/gc-remote-registry.py"; const GC_REMOTE_REGISTRY_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_REGISTRY_HELPERS__"; +const GC_REMOTE_HOST_DOCKER_RELATIVE_PATH = "scripts/src/gc-remote-host-docker.py"; +const GC_REMOTE_HOST_DOCKER_PLACEHOLDER = "# __UNIDESK_GC_REMOTE_HOST_DOCKER_HELPERS__"; const GC_REMOTE_POLICY_RUNNER_RELATIVE_PATH = "scripts/src/gc-remote-policy-runner.py"; const GC_REMOTE_POLICY_RUNNER_PLACEHOLDER = "__UNIDESK_GC_REMOTE_POLICY_RUNNER_BASE64__"; @@ -315,6 +319,8 @@ function parseRemoteGcOptions(args: string[]): RemoteGcOptions { options.memoryPressureOnly = true; } else if (arg === "--build-cache-only") { options.buildCacheOnly = true; + } else if (arg === "--host-docker-only") { + options.hostDockerOnly = true; } else { throw new Error(`unknown gc remote option: ${arg}`); } @@ -655,6 +661,9 @@ function remoteGcPython(configBase64: string): string { if (!template.includes(GC_REMOTE_REGISTRY_PLACEHOLDER)) { throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_REGISTRY_PLACEHOLDER}`); } + if (!template.includes(GC_REMOTE_HOST_DOCKER_PLACEHOLDER)) { + throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_HOST_DOCKER_PLACEHOLDER}`); + } if (!template.includes(GC_REMOTE_POLICY_RUNNER_PLACEHOLDER)) { throw new Error(`${GC_REMOTE_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_POLICY_RUNNER_PLACEHOLDER}`); } @@ -666,6 +675,7 @@ function remoteGcPython(configBase64: string): string { const pvcHelpers = readFileSync(rootPath(GC_REMOTE_PVC_RELATIVE_PATH), "utf8"); const growthHelpers = readFileSync(rootPath(GC_REMOTE_GROWTH_RELATIVE_PATH), "utf8"); const registryHelpers = readFileSync(rootPath(GC_REMOTE_REGISTRY_RELATIVE_PATH), "utf8"); + const hostDockerHelpers = readFileSync(rootPath(GC_REMOTE_HOST_DOCKER_RELATIVE_PATH), "utf8"); const policyRunnerTemplate = readFileSync(rootPath(GC_REMOTE_POLICY_RUNNER_RELATIVE_PATH), "utf8"); if (!policyRunnerTemplate.includes(GC_REMOTE_POLICY_KUBERNETES_RETENTION_PLACEHOLDER)) { throw new Error(`${GC_REMOTE_POLICY_RUNNER_RELATIVE_PATH} missing ${GC_REMOTE_POLICY_KUBERNETES_RETENTION_PLACEHOLDER}`); @@ -683,6 +693,7 @@ function remoteGcPython(configBase64: string): string { .replace(GC_REMOTE_PVC_PLACEHOLDER, () => pvcHelpers.trimEnd()) .replace(GC_REMOTE_GROWTH_PLACEHOLDER, () => growthHelpers.trimEnd()) .replace(GC_REMOTE_REGISTRY_PLACEHOLDER, () => registryHelpers.trimEnd()) + .replace(GC_REMOTE_HOST_DOCKER_PLACEHOLDER, () => hostDockerHelpers.trimEnd()) .replace(GC_REMOTE_POLICY_RUNNER_PLACEHOLDER, Buffer.from(policyRunner, "utf8").toString("base64")) .replace(GC_REMOTE_RUNNER_CONFIG_PLACEHOLDER, configBase64); }