fix: ignore stale registry writer runs

This commit is contained in:
pikastech
2026-07-21 07:30:25 +02:00
parent 3992329388
commit ba2a6219ec
4 changed files with 46 additions and 2 deletions
+1
View File
@@ -181,6 +181,7 @@ gc:
automatic:
enabled: false
maxDeleteManifestsPerRun: 100
writerGraceMinutes: 30
requireIdleNamespaces:
- devops-infra
- agentrun-ci
+1
View File
@@ -54,6 +54,7 @@
- 对 YAML 声明的 Registry Deployment 缩容后运行官方 `registry garbage-collect`
- PVC 只挂载给 GC Pod,不删除 PVC、PV、blob 目录、containerd 或 k3s storage
- 无论成功或失败都恢复原副本数,并复核 Deployment readiness 与 `/v2/` endpoint。
- writer idle guard 只阻塞 YAML grace 窗口内的新任务,或仍有 `Pending`/`Running` Pod 的 Tekton run;超过 grace 且没有活动 Pod 的残留 run 只报告为 stale,不得永久阻塞维护。
- `--include-tool-caches`:本机和远端都必须是显式 opt-in,只清理固定 allowlist 的可重建 npm/npx/Bun 缓存;不得扩大成清理 `~/.npm``~/.bun``node_modules`、auth/config 或运行面依赖。
所有成功和失败输出都必须是 JSON。`plan` 必须标记 `dryRun=true``mutation=false``run` 必须要求 `--confirm` 并报告 `diskBefore``diskAfter``summary``results``protected`。远端 GC 可用 `--target-use-percent N` 显式表达目标根盘水位;`summary.target` 必须给出目标所需释放量、候选估算、预计水位、缺口和 `safeStop` 决策,避免靠人工心算判断是否应该继续扩大清理范围。
+36 -2
View File
@@ -69,24 +69,58 @@ def registry_diagnosis(code, message, **details):
def active_registry_writes():
automatic = REGISTRY_CONFIG.get("automatic") if isinstance(REGISTRY_CONFIG.get("automatic"), dict) else {}
namespaces = config_list(automatic, "requireIdleNamespaces", [])
writer_grace_minutes = config_int(automatic, "writerGraceMinutes", 30, minimum=1, maximum=1440)
rows = []
stale_rows = []
commands = []
for namespace in namespaces:
result = kctl(["-n", namespace, "get", "pipelinerun,taskrun", "-o", "json"], 20)
commands.append(bounded(result))
if result["exitCode"] != 0:
return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace}
pods_result = kctl(["-n", namespace, "get", "pod", "-o", "json"], 20)
commands.append(bounded(pods_result))
if pods_result["exitCode"] != 0:
return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace}
try:
items = json.loads(result.get("stdout") or "{}").get("items") or []
pods = json.loads(pods_result.get("stdout") or "{}").get("items") or []
except Exception:
return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace, "error": "invalid-kubernetes-json"}
active_pods_by_run = {}
for pod in pods:
phase = str(((pod.get("status") or {}).get("phase")) or "")
if phase not in set(["Pending", "Running", "Unknown"]):
continue
labels = (pod.get("metadata") or {}).get("labels") or {}
for label in ["tekton.dev/pipelineRun", "tekton.dev/taskRun"]:
run_name = str(labels.get(label) or "")
if run_name:
active_pods_by_run.setdefault(run_name, []).append(str((pod.get("metadata") or {}).get("name") or "unknown"))
for item in items:
conditions = ((item.get("status") or {}).get("conditions")) or []
terminal = any(condition.get("type") == "Succeeded" and condition.get("status") in set(["True", "False"]) for condition in conditions)
if not terminal:
meta = item.get("metadata") or {}
rows.append("%s/%s/%s" % (namespace, item.get("kind") or "Unknown", meta.get("name") or "unknown"))
return {"ok": True, "activeCount": len(rows), "activePreview": rows[:40], "commands": commands}
name = str(meta.get("name") or "unknown")
active_pods = active_pods_by_run.get(name) or []
created = parse_iso_timestamp(meta.get("creationTimestamp"))
age_seconds = max(0, time.time() - created) if created is not None else None
within_grace = age_seconds is None or age_seconds < writer_grace_minutes * 60
row = "%s/%s/%s" % (namespace, item.get("kind") or "Unknown", name)
if active_pods or within_grace:
rows.append(row)
else:
stale_rows.append(row)
return {
"ok": True,
"activeCount": len(rows),
"activePreview": rows[:40],
"staleIgnoredCount": len(stale_rows),
"staleIgnoredPreview": stale_rows[:40],
"writerGraceMinutes": writer_grace_minutes,
"commands": commands,
}
def wait_no_active_registry_writes(timeout=180):
deadline = time.time() + timeout
+8
View File
@@ -167,6 +167,14 @@ def parse_size_value(value, default=None):
def now_iso():
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
def parse_iso_timestamp(value):
if not isinstance(value, str) or not value:
return None
try:
return calendar.timegm(time.strptime(value, "%Y-%m-%dT%H:%M:%SZ"))
except (TypeError, ValueError, OverflowError):
return None
def command(cmd, timeout=10):
try:
p = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout)