fix: ignore stale registry writer runs
This commit is contained in:
@@ -181,6 +181,7 @@ gc:
|
||||
automatic:
|
||||
enabled: false
|
||||
maxDeleteManifestsPerRun: 100
|
||||
writerGraceMinutes: 30
|
||||
requireIdleNamespaces:
|
||||
- devops-infra
|
||||
- agentrun-ci
|
||||
|
||||
@@ -54,6 +54,7 @@
|
||||
- 对 YAML 声明的 Registry Deployment 缩容后运行官方 `registry garbage-collect`;
|
||||
- PVC 只挂载给 GC Pod,不删除 PVC、PV、blob 目录、containerd 或 k3s storage;
|
||||
- 无论成功或失败都恢复原副本数,并复核 Deployment readiness 与 `/v2/` endpoint。
|
||||
- writer idle guard 只阻塞 YAML grace 窗口内的新任务,或仍有 `Pending`/`Running` Pod 的 Tekton run;超过 grace 且没有活动 Pod 的残留 run 只报告为 stale,不得永久阻塞维护。
|
||||
- `--include-tool-caches`:本机和远端都必须是显式 opt-in,只清理固定 allowlist 的可重建 npm/npx/Bun 缓存;不得扩大成清理 `~/.npm`、`~/.bun`、`node_modules`、auth/config 或运行面依赖。
|
||||
|
||||
所有成功和失败输出都必须是 JSON。`plan` 必须标记 `dryRun=true`、`mutation=false`;`run` 必须要求 `--confirm` 并报告 `diskBefore`、`diskAfter`、`summary`、`results` 和 `protected`。远端 GC 可用 `--target-use-percent N` 显式表达目标根盘水位;`summary.target` 必须给出目标所需释放量、候选估算、预计水位、缺口和 `safeStop` 决策,避免靠人工心算判断是否应该继续扩大清理范围。
|
||||
|
||||
@@ -69,24 +69,58 @@ def registry_diagnosis(code, message, **details):
|
||||
def active_registry_writes():
|
||||
automatic = REGISTRY_CONFIG.get("automatic") if isinstance(REGISTRY_CONFIG.get("automatic"), dict) else {}
|
||||
namespaces = config_list(automatic, "requireIdleNamespaces", [])
|
||||
writer_grace_minutes = config_int(automatic, "writerGraceMinutes", 30, minimum=1, maximum=1440)
|
||||
rows = []
|
||||
stale_rows = []
|
||||
commands = []
|
||||
for namespace in namespaces:
|
||||
result = kctl(["-n", namespace, "get", "pipelinerun,taskrun", "-o", "json"], 20)
|
||||
commands.append(bounded(result))
|
||||
if result["exitCode"] != 0:
|
||||
return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace}
|
||||
pods_result = kctl(["-n", namespace, "get", "pod", "-o", "json"], 20)
|
||||
commands.append(bounded(pods_result))
|
||||
if pods_result["exitCode"] != 0:
|
||||
return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace}
|
||||
try:
|
||||
items = json.loads(result.get("stdout") or "{}").get("items") or []
|
||||
pods = json.loads(pods_result.get("stdout") or "{}").get("items") or []
|
||||
except Exception:
|
||||
return {"ok": False, "activeCount": None, "activePreview": rows, "commands": commands, "namespace": namespace, "error": "invalid-kubernetes-json"}
|
||||
active_pods_by_run = {}
|
||||
for pod in pods:
|
||||
phase = str(((pod.get("status") or {}).get("phase")) or "")
|
||||
if phase not in set(["Pending", "Running", "Unknown"]):
|
||||
continue
|
||||
labels = (pod.get("metadata") or {}).get("labels") or {}
|
||||
for label in ["tekton.dev/pipelineRun", "tekton.dev/taskRun"]:
|
||||
run_name = str(labels.get(label) or "")
|
||||
if run_name:
|
||||
active_pods_by_run.setdefault(run_name, []).append(str((pod.get("metadata") or {}).get("name") or "unknown"))
|
||||
for item in items:
|
||||
conditions = ((item.get("status") or {}).get("conditions")) or []
|
||||
terminal = any(condition.get("type") == "Succeeded" and condition.get("status") in set(["True", "False"]) for condition in conditions)
|
||||
if not terminal:
|
||||
meta = item.get("metadata") or {}
|
||||
rows.append("%s/%s/%s" % (namespace, item.get("kind") or "Unknown", meta.get("name") or "unknown"))
|
||||
return {"ok": True, "activeCount": len(rows), "activePreview": rows[:40], "commands": commands}
|
||||
name = str(meta.get("name") or "unknown")
|
||||
active_pods = active_pods_by_run.get(name) or []
|
||||
created = parse_iso_timestamp(meta.get("creationTimestamp"))
|
||||
age_seconds = max(0, time.time() - created) if created is not None else None
|
||||
within_grace = age_seconds is None or age_seconds < writer_grace_minutes * 60
|
||||
row = "%s/%s/%s" % (namespace, item.get("kind") or "Unknown", name)
|
||||
if active_pods or within_grace:
|
||||
rows.append(row)
|
||||
else:
|
||||
stale_rows.append(row)
|
||||
return {
|
||||
"ok": True,
|
||||
"activeCount": len(rows),
|
||||
"activePreview": rows[:40],
|
||||
"staleIgnoredCount": len(stale_rows),
|
||||
"staleIgnoredPreview": stale_rows[:40],
|
||||
"writerGraceMinutes": writer_grace_minutes,
|
||||
"commands": commands,
|
||||
}
|
||||
|
||||
def wait_no_active_registry_writes(timeout=180):
|
||||
deadline = time.time() + timeout
|
||||
|
||||
@@ -167,6 +167,14 @@ def parse_size_value(value, default=None):
|
||||
def now_iso():
|
||||
return time.strftime("%Y-%m-%dT%H:%M:%SZ", time.gmtime())
|
||||
|
||||
def parse_iso_timestamp(value):
|
||||
if not isinstance(value, str) or not value:
|
||||
return None
|
||||
try:
|
||||
return calendar.timegm(time.strptime(value, "%Y-%m-%dT%H:%M:%SZ"))
|
||||
except (TypeError, ValueError, OverflowError):
|
||||
return None
|
||||
|
||||
def command(cmd, timeout=10):
|
||||
try:
|
||||
p = subprocess.run(cmd, text=True, capture_output=True, timeout=timeout)
|
||||
|
||||
Reference in New Issue
Block a user