From ba2a6219ec02f8364d91bf7e90271cde6f322a67 Mon Sep 17 00:00:00 2001 From: pikastech Date: Tue, 21 Jul 2026 07:30:25 +0200 Subject: [PATCH] fix: ignore stale registry writer runs --- config/unidesk-cli.yaml | 1 + docs/reference/gc.md | 1 + scripts/src/gc-remote-registry.py | 38 +++++++++++++++++++++++++++++-- scripts/src/gc-remote-runner.py | 8 +++++++ 4 files changed, 46 insertions(+), 2 deletions(-) diff --git a/config/unidesk-cli.yaml b/config/unidesk-cli.yaml index 8a246743..8b3aefe0 100644 --- a/config/unidesk-cli.yaml +++ b/config/unidesk-cli.yaml @@ -181,6 +181,7 @@ gc: automatic: enabled: false maxDeleteManifestsPerRun: 100 + writerGraceMinutes: 30 requireIdleNamespaces: - devops-infra - agentrun-ci diff --git a/docs/reference/gc.md b/docs/reference/gc.md index b95eb372..09b6dba4 100644 --- a/docs/reference/gc.md +++ b/docs/reference/gc.md @@ -54,6 +54,7 @@ - 对 YAML 声明的 Registry Deployment 缩容后运行官方 `registry garbage-collect`； - PVC 只挂载给 GC Pod，不删除 PVC、PV、blob 目录、containerd 或 k3s storage； - 无论成功或失败都恢复原副本数，并复核 Deployment readiness 与 `/v2/` endpoint。 + - writer idle guard 只阻塞 YAML grace 窗口内的新任务，或仍有 `Pending`/`Running` Pod 的 Tekton run；超过 grace 且没有活动 Pod 的残留 run 只报告为 stale，不得永久阻塞维护。 - `--include-tool-caches`：本机和远端都必须是显式 opt-in，只清理固定 allowlist 的可重建 npm/npx/Bun 缓存；不得扩大成清理 `~/.npm`、`~/.bun`、`node_modules`、auth/config 或运行面依赖。 所有成功和失败输出都必须是 JSON。`plan` 必须标记 `dryRun=true`、`mutation=false`；`run` 必须要求 `--confirm` 并报告 `diskBefore`、`diskAfter`、`summary`、`results` 和 `protected`。远端 GC 可用 `--target-use-percent N` 显式表达目标根盘水位；`summary.target` 必须给出目标所需释放量、候选估算、预计水位、缺口和 `safeStop` 决策，避免靠人工心算判断是否应该继续扩大清理范围。 diff --git a/scripts/src/gc-remote-registry.py b/scripts/src/gc-remote-registry.py index 37a6a901..60768e81 100644 --- a/scripts/src/gc-remote-registry.py +++ b/scripts/src/gc-remote-registry.py @@ -69,24 +69,58 @@ def registry_diagnosis(code, message, **details): def active_registry_writes(): automatic = REGISTRY_CONFIG.get("automatic") if isinstance(REGISTRY_CONFIG.get("automatic"), dict) else {} namespaces = config_list(automatic, "requireIdleNamespaces", []) + writer_grace_minutes = config_int(automatic, "writerGraceMinutes", 30, minimum=1, maximum=1440) rows = [] + stale_rows = [] commands = [] for namespace in namespaces: result = kctl(["-n", namespace, "get", "pipelinerun,taskrun", "-o", "json"], 20) commands.append(bounded(result)) if result["exitCode"] != 0: return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace} + pods_result = kctl(["-n", namespace, "get", "pod", "-o", "json"], 20) + commands.append(bounded(pods_result)) + if pods_result["exitCode"] != 0: + return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace} try: items = json.loads(result.get("stdout") or "{}").get("items") or [] + pods = json.loads(pods_result.get("stdout") or "{}").get("items") or [] except Exception: return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace, "error": "invalid-kubernetes-json"} + active_pods_by_run = {} + for pod in pods: + phase = str(((pod.get("status") or {}).get("phase")) or "") + if phase not in set(["Pending", "Running", "Unknown"]): + continue + labels = (pod.get("metadata") or {}).get("labels") or {} + for label in ["tekton.dev/pipelineRun", "tekton.dev/taskRun"]: + run_name = str(labels.get(label) or "") + if run_name: + active_pods_by_run.setdefault(run_name, []).append(str((pod.get("metadata") or {}).get("name") or "unknown")) for item in items: conditions = ((item.get("status") or {}).get("conditions")) or [] terminal = any(condition.get("type") == "Succeeded" and condition.get("status") in set(["True", "False"]) for condition in conditions) if not terminal: meta = item.get("metadata") or {} - rows.append("%s/%s/%s" % (namespace, item.get("kind") or "Unknown", meta.get("name") or "unknown")) - return {"ok": True, "activeCount": len(rows), "activePreview": rows[:40], "commands": commands} + name = str(meta.get("name") or "unknown") + active_pods = active_pods_by_run.get(name) or [] + created = parse_iso_timestamp(meta.get("creationTimestamp")) + age_seconds = max(0, time.time() - created) if created is not None else None + within_grace = age_seconds is None or age_seconds < writer_grace_minutes * 60 + row = "%s/%s/%s" % (namespace, item.get("kind") or "Unknown", name) + if active_pods or within_grace: + rows.append(row) + else: + stale_rows.append(row) + return { + "ok": True, + "activeCount": len(rows), + "activePreview": rows[:40], + "staleIgnoredCount": len(stale_rows), + "staleIgnoredPreview": stale_rows[:40], + "writerGraceMinutes": writer_grace_minutes, + "commands": commands, + } def wait_no_active_registry_writes(timeout=180): deadline = time.time() + timeout diff --git a/scripts/src/gc-remote-runner.py b/scripts/src/gc-remote-runner.py index 7f7d8736..acaa15e7 100644 --- a/scripts/src/gc-remote-runner.py +++ b/scripts/src/gc-remote-runner.py @@ -167,6 +167,14 @@ def parse_size_value(value, default=None): def now_iso(): return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime()) +def parse_iso_timestamp(value): + if not isinstance(value, str) or not value: + return None + try: + return calendar.timegm(time.strptime(value, "%Y-%m-%dT%H:%M:%SZ")) + except (TypeError, ValueError, OverflowError): + return None + def command(cmd, timeout=10): try: p = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout)