diff --git a/config/hwlab-node-control-plane.yaml b/config/hwlab-node-control-plane.yaml index f9376e2f..91d416bc 100644 --- a/config/hwlab-node-control-plane.yaml +++ b/config/hwlab-node-control-plane.yaml @@ -553,10 +553,11 @@ targets: - COPY --from=golang-toolchain /usr/local/go /usr/local/go - COPY --from=docker-cli /usr/bin/kubectl /usr/local/bin/kubectl - ENV PATH=/usr/local/go/bin:$PATH + - RUN apt-get update && apt-get install -y --no-install-recommends tini && rm -rf /var/lib/apt/lists/* - RUN ln -sf /usr/local/bin/bun /usr/local/bin/bunx - ENV HWLAB_CI_NODE_DEPS=/opt/hwlab-ci-node-deps/node_modules - RUN set -eu; export HTTP_PROXY="${HTTP_PROXY:-${http_proxy:-}}"; export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-$HTTP_PROXY}}"; export ALL_PROXY="${ALL_PROXY:-${all_proxy:-}}"; export NO_PROXY="${NO_PROXY:-${no_proxy:-}}"; export http_proxy="$HTTP_PROXY"; export https_proxy="$HTTPS_PROXY"; export all_proxy="$ALL_PROXY"; export no_proxy="$NO_PROXY"; export npm_config_registry="https://registry.npmmirror.com/"; export BUN_CONFIG_REGISTRY="https://registry.npmmirror.com/"; export npm_config_noproxy="$NO_PROXY"; if [ -n "$HTTP_PROXY" ]; then export npm_config_proxy="$HTTP_PROXY"; fi; if [ -n "$HTTPS_PROXY" ]; then export npm_config_https_proxy="$HTTPS_PROXY"; fi; export npm_config_fetch_retries=2; export npm_config_fetch_retry_mintimeout=2000; export npm_config_fetch_retry_maxtimeout=16000; export npm_config_fetch_timeout=120000; proxy_label="${HTTP_PROXY:+HTTP_PROXY}"; proxy_label="${proxy_label:-none}"; mkdir -p /opt/hwlab-ci-node-deps; cd /opt/hwlab-ci-node-deps; printf '{"private":true,"dependencies":{}}\n' > package.json; ok=0; delay=2; for attempt in 1 2 3 4 5; do echo "{\"event\":\"tools-yaml-node-npm-install\",\"attempt\":\"$attempt/5\",\"registry\":\"$npm_config_registry\",\"proxy\":\"$proxy_label\"}" >&2; if timeout 180s npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev yaml@2.8.3; then ok=1; break; fi; if [ "$attempt" = 5 ]; then break; fi; echo "{\"event\":\"tools-yaml-node-npm-install\",\"status\":\"retrying\",\"attempt\":\"$attempt/5\",\"sleepSeconds\":$delay}" >&2; sleep "$delay"; delay=$((delay * 2)); done; test "$ok" = 1; node --input-type=module -e 'import("/opt/hwlab-ci-node-deps/node_modules/yaml/browser/dist/index.js").then((yaml)=>console.log("yaml-ok", typeof yaml.parse))' - - RUN node --version && npm --version && bun --version && git --version && python3 --version && docker --version && kubectl version --client=true && ssh -V && go version + - RUN node --version && npm --version && bun --version && git --version && python3 --version && docker --version && kubectl version --client=true && ssh -V && go version && tini --version buildArgs: {} buildNetwork: host publicBaseImages: @@ -771,7 +772,8 @@ targets: - RUN ln -sf /usr/local/bin/bun /usr/local/bin/bunx - ENV HWLAB_CI_NODE_DEPS=/opt/hwlab-ci-node-deps/node_modules - RUN set -eu; export HTTP_PROXY="${HTTP_PROXY:-${http_proxy:-}}"; export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-$HTTP_PROXY}}"; export ALL_PROXY="${ALL_PROXY:-${all_proxy:-}}"; export NO_PROXY="${NO_PROXY:-${no_proxy:-}}"; export http_proxy="$HTTP_PROXY"; export https_proxy="$HTTPS_PROXY"; export all_proxy="$ALL_PROXY"; export no_proxy="$NO_PROXY"; export npm_config_registry="https://registry.npmmirror.com/"; export BUN_CONFIG_REGISTRY="https://registry.npmmirror.com/"; export npm_config_noproxy="$NO_PROXY"; if [ -n "$HTTP_PROXY" ]; then export npm_config_proxy="$HTTP_PROXY"; fi; if [ -n "$HTTPS_PROXY" ]; then export npm_config_https_proxy="$HTTPS_PROXY"; fi; export npm_config_fetch_retries=2; export npm_config_fetch_retry_mintimeout=2000; export npm_config_fetch_retry_maxtimeout=16000; export npm_config_fetch_timeout=120000; proxy_label="${HTTP_PROXY:+HTTP_PROXY}"; proxy_label="${proxy_label:-none}"; mkdir -p /opt/hwlab-ci-node-deps; cd /opt/hwlab-ci-node-deps; printf '{"private":true,"dependencies":{}}\n' > package.json; ok=0; delay=2; for attempt in 1 2 3 4 5; do echo "{\"event\":\"tools-yaml-node-npm-install\",\"attempt\":\"$attempt/5\",\"registry\":\"$npm_config_registry\",\"proxy\":\"$proxy_label\"}" >&2; if timeout 180s npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev yaml@2.8.3; then ok=1; break; fi; if [ "$attempt" = 5 ]; then break; fi; echo "{\"event\":\"tools-yaml-node-npm-install\",\"status\":\"retrying\",\"attempt\":\"$attempt/5\",\"sleepSeconds\":$delay}" >&2; sleep "$delay"; delay=$((delay * 2)); done; test "$ok" = 1; node --input-type=module -e 'import("/opt/hwlab-ci-node-deps/node_modules/yaml/browser/dist/index.js").then((yaml)=>console.log("yaml-ok", typeof yaml.parse))' - - RUN node --version && npm --version && bun --version && git --version && python3 --version && docker --version && kubectl version --client=true && ssh -V && go version + - RUN apt-get update && apt-get install -y --no-install-recommends tini && rm -rf /var/lib/apt/lists/* + - RUN node --version && npm --version && bun --version && git --version && python3 --version && docker --version && kubectl version --client=true && ssh -V && go version && tini --version buildArgs: {} buildNetwork: host publicBaseImages: @@ -991,7 +993,8 @@ targets: - RUN ln -sf /usr/local/bin/bun /usr/local/bin/bunx - ENV HWLAB_CI_NODE_DEPS=/opt/hwlab-ci-node-deps/node_modules - RUN set -eu; export HTTP_PROXY="${HTTP_PROXY:-${http_proxy:-}}"; export HTTPS_PROXY="${HTTPS_PROXY:-${https_proxy:-$HTTP_PROXY}}"; export ALL_PROXY="${ALL_PROXY:-${all_proxy:-}}"; export NO_PROXY="${NO_PROXY:-${no_proxy:-}}"; export http_proxy="$HTTP_PROXY"; export https_proxy="$HTTPS_PROXY"; export all_proxy="$ALL_PROXY"; export no_proxy="$NO_PROXY"; export npm_config_registry="https://registry.npmmirror.com/"; export BUN_CONFIG_REGISTRY="https://registry.npmmirror.com/"; export npm_config_noproxy="$NO_PROXY"; if [ -n "$HTTP_PROXY" ]; then export npm_config_proxy="$HTTP_PROXY"; fi; if [ -n "$HTTPS_PROXY" ]; then export npm_config_https_proxy="$HTTPS_PROXY"; fi; export npm_config_fetch_retries=2; export npm_config_fetch_retry_mintimeout=2000; export npm_config_fetch_retry_maxtimeout=16000; export npm_config_fetch_timeout=120000; proxy_label="${HTTP_PROXY:+HTTP_PROXY}"; proxy_label="${proxy_label:-none}"; mkdir -p /opt/hwlab-ci-node-deps; cd /opt/hwlab-ci-node-deps; printf '{"private":true,"dependencies":{}}\n' > package.json; ok=0; delay=2; for attempt in 1 2 3 4 5; do echo "{\"event\":\"tools-yaml-node-npm-install\",\"attempt\":\"$attempt/5\",\"registry\":\"$npm_config_registry\",\"proxy\":\"$proxy_label\"}" >&2; if timeout 180s npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev yaml@2.8.3; then ok=1; break; fi; if [ "$attempt" = 5 ]; then break; fi; echo "{\"event\":\"tools-yaml-node-npm-install\",\"status\":\"retrying\",\"attempt\":\"$attempt/5\",\"sleepSeconds\":$delay}" >&2; sleep "$delay"; delay=$((delay * 2)); done; test "$ok" = 1; node --input-type=module -e 'import("/opt/hwlab-ci-node-deps/node_modules/yaml/browser/dist/index.js").then((yaml)=>console.log("yaml-ok", typeof yaml.parse))' - - RUN node --version && npm --version && bun --version && git --version && python3 --version && docker --version && kubectl version --client=true && ssh -V && go version + - RUN apt-get update && apt-get install -y --no-install-recommends tini && rm -rf /var/lib/apt/lists/* + - RUN node --version && npm --version && bun --version && git --version && python3 --version && docker --version && kubectl version --client=true && ssh -V && go version && tini --version buildArgs: {} buildNetwork: host publicBaseImages: @@ -1252,7 +1255,8 @@ targets: - RUN ln -sf /usr/local/bin/bun /usr/local/bin/bunx - ENV HWLAB_CI_NODE_DEPS=/opt/hwlab-ci-node-deps/node_modules - RUN set -eu; unset HTTP_PROXY HTTPS_PROXY ALL_PROXY http_proxy https_proxy all_proxy; export npm_config_registry="https://registry.npmmirror.com/"; export BUN_CONFIG_REGISTRY="https://registry.npmmirror.com/"; export npm_config_fetch_retries=2; export npm_config_fetch_retry_mintimeout=2000; export npm_config_fetch_retry_maxtimeout=16000; export npm_config_fetch_timeout=120000; mkdir -p /opt/hwlab-ci-node-deps; cd /opt/hwlab-ci-node-deps; printf '{"private":true,"dependencies":{}}\n' > package.json; ok=0; delay=2; for attempt in 1 2 3 4 5; do echo "{\"event\":\"tools-yaml-node-npm-install\",\"attempt\":\"$attempt/5\",\"registry\":\"$npm_config_registry\",\"proxy\":\"none\"}" >&2; if timeout 180s npm install --package-lock=false --no-save --ignore-scripts --no-audit --no-fund --omit=dev yaml@2.8.3; then ok=1; break; fi; if [ "$attempt" = 5 ]; then break; fi; echo "{\"event\":\"tools-yaml-node-npm-install\",\"status\":\"retrying\",\"attempt\":\"$attempt/5\",\"sleepSeconds\":$delay}" >&2; sleep "$delay"; delay=$((delay * 2)); done; test "$ok" = 1; node --input-type=module -e 'import("/opt/hwlab-ci-node-deps/node_modules/yaml/browser/dist/index.js").then((yaml)=>console.log("yaml-ok", typeof yaml.parse))' - - RUN node --version && npm --version && bun --version && git --version && python3 --version && docker --version && kubectl version --client=true && ssh -V && go version + - RUN apt-get update && apt-get install -y --no-install-recommends tini && rm -rf /var/lib/apt/lists/* + - RUN node --version && npm --version && bun --version && git --version && python3 --version && docker --version && kubectl version --client=true && ssh -V && go version && tini --version buildArgs: {} buildNetwork: host publicBaseImages: diff --git a/config/unidesk-cli.yaml b/config/unidesk-cli.yaml index ab3cf793..af1015b1 100644 --- a/config/unidesk-cli.yaml +++ b/config/unidesk-cli.yaml @@ -223,7 +223,7 @@ gc: minAgeHours: 336 maxDeletePerRun: 5 memoryPressure: - planPreviewLimit: 6 + planPreviewLimit: 4 processPatterns: - k3s-server - containerd @@ -291,12 +291,86 @@ gc: targetMemAvailableBytes: 5368709120 minimumSwapFreeBytesAfterReclaim: 34359738368 targets: - - id: kubepods-inactive-anon - path: /sys/fs/cgroup/kubepods.slice - reclaimBytes: 3221225472 + - id: kafka-broker-inactive-anon + workloadSelector: + namespace: platform-infra + matchLabels: + strimzi.io/name: platform-infra-kafka-kafka + maxMatchedPods: 1 + reclaimBytes: 268435456 swappiness: 200 - minimumInactiveAnonBytes: 3221225472 + minimumInactiveAnonBytes: 268435456 priority: 10 + - id: kafka-entity-operator-inactive-anon + workloadSelector: + namespace: platform-infra + matchLabels: + strimzi.io/name: platform-infra-kafka-entity-operator + maxMatchedPods: 1 + reclaimBytes: 134217728 + swappiness: 200 + minimumInactiveAnonBytes: 134217728 + priority: 20 + - id: argocd-controller-inactive-anon + workloadSelector: + namespace: argocd + matchLabels: + app.kubernetes.io/name: argocd-application-controller + maxMatchedPods: 1 + reclaimBytes: 134217728 + swappiness: 200 + minimumInactiveAnonBytes: 134217728 + priority: 30 + - id: gitea-inactive-anon + workloadSelector: + namespace: devops-infra + matchLabels: + app.kubernetes.io/name: gitea + maxMatchedPods: 1 + reclaimBytes: 134217728 + swappiness: 200 + minimumInactiveAnonBytes: 134217728 + priority: 40 + - id: hwlab-v03-hwpod-worker-inactive-anon + workloadSelector: + namespace: hwlab-v03 + matchLabels: + app.kubernetes.io/name: hwlab-hwpod-worker + maxMatchedPods: 2 + reclaimBytes: 134217728 + swappiness: 200 + minimumInactiveAnonBytes: 134217728 + priority: 50 + - id: sub2rank-development-worker-inactive-anon + workloadSelector: + namespace: platform-infra-development + matchLabels: + app.kubernetes.io/name: sub2rank-worker + maxMatchedPods: 1 + reclaimBytes: 134217728 + swappiness: 200 + minimumInactiveAnonBytes: 134217728 + priority: 60 + - id: hwlab-v03-harnessrl-worker-inactive-anon + workloadSelector: + namespace: hwlab-v03 + matchLabels: + app.kubernetes.io/name: hwlab-harnessrl-worker + maxMatchedPods: 2 + reclaimBytes: 134217728 + swappiness: 200 + minimumInactiveAnonBytes: 134217728 + priority: 70 + - id: hwlab-v03-workbench-worker-inactive-anon + workloadSelector: + namespace: hwlab-v03 + matchLabels: + app.kubernetes.io/name: hwlab-workbench-worker + maxMatchedPods: 2 + reclaimBytes: 134217728 + swappiness: 200 + minimumInactiveAnonBytes: 134217728 + priority: 80 kubernetesObjectRetention: enabled: true aggregatePreviewLimit: 1 diff --git a/project-management/PJ2026-01/specs/PJ2026-01060107-nc01-ci-resource-governance.md b/project-management/PJ2026-01/specs/PJ2026-01060107-nc01-ci-resource-governance.md index 7209a6af..bb40cb6c 100644 --- a/project-management/PJ2026-01/specs/PJ2026-01060107-nc01-ci-resource-governance.md +++ b/project-management/PJ2026-01/specs/PJ2026-01060107-nc01-ci-resource-governance.md @@ -96,10 +96,17 @@ CoreDNS 和 k3s 控制面使用 Kubernetes 系统关键优先级。Sub2API 和 中心 PaC owning YAML 必须声明 `deliveryTrigger.mode=manual-plan-confirm` 和 `deliveryTrigger.mechanism=pac-webhook`。PaC webhook 接收端继续作为唯一 PipelineRun 创建入口，但 Gitea push hook、PR merge callback、branch follower、poller 和其他自动发送方不得调用它；PR merge 后只允许 source mirror 或 authority 更新。受控 apply 必须删除或禁用 Gitea 自动 hook，并在 status 中把自动发送方启用视为配置漂移。 +### 5.7 CI-RESOURCE-REQ-007 子进程回收 + +长期运行且会启动 Git、SSH、编译器或其他子进程的容器必须具有明确的子进程回收者。业务进程不得在没有 init/reaper 的情况下直接作为容器 PID 1；已经正确等待直接子进程的实现仍须处理其退出后被 PID 1 接管的孤儿后代。init/reaper 必须只负责信号转发和子进程回收，不得成为第二业务生命周期 authority。 + +运行面诊断必须按父 PID 聚合 zombie，并映射到 namespace、Pod、稳定 workload owner 和修复类别。修复通过正常自动交付滚动 owner；直接终止 zombie、批量杀进程或只重启 Pod 不能作为根因修复。 + ## 6. 验收 - L0：配置解析、资源 quantity、requests/limits 关系、matrix 收敛、有界 worker 队列和依赖保持通过轻量验证。 - L1：受控 `plan/status/diagnosis` 显示 YAML 来源、并发预算、批次、资源和保护状态，不依赖裸 Kubernetes 命令。 +- L1：所有会启动子进程的常驻 Pod 均显示 init/reaper 为 PID 1；在真实 Git/SSH 操作前后，owner 的 zombie 数不增长。 - L2：正常 HWLAB source PR merge 后不产生 PipelineRun。受控 `plan` 显示 env reuse、镜像构建数量、精确构建与 rollout 范围且没有非预期扩大；随后使用手动 `trigger` 向 PaC 发送一次 webhook，并由 PaC 创建一次 PipelineRun。观察到同一时刻构建 TaskRun 不超过 YAML 预算，相同发布意图不产生第二次构建，所有构建 Pod 带资源合同，并且 NC01 Node、CoreDNS、Sub2API、Public Edge 和公开 API 在构建期间保持健康。 验收失败时保留计划、手动触发链和运行证据，并修复 owning YAML、planner 或 renderer。重启 k3s、删除 Pod/PipelineRun、手工 Argo sync、运行时 patch、mirror flush 或裸补跑均不能作为最终通过证据。 diff --git a/project-management/PJ2026-01/specs/PJ2026-010603-yaml-first-ops.md b/project-management/PJ2026-01/specs/PJ2026-010603-yaml-first-ops.md index 524ce9de..281147bb 100644 --- a/project-management/PJ2026-01/specs/PJ2026-010603-yaml-first-ops.md +++ b/project-management/PJ2026-01/specs/PJ2026-010603-yaml-first-ops.md @@ -71,6 +71,8 @@ YAML运维负责 HWLAB/UniDesk 自有平台配置的真相源、解析、渲染 | 有界默认输出 | CLI 在未显式请求机器输出或完整输出时返回的短摘要、表格和 drill-down 命令集合。 | | host proxy client | 新节点 k3s 安装前由 `trans` 分发的 0 依赖出网代理客户端，用于给 Docker、k3s、containerd、包管理器、Git 和 git-mirror 提供初始出网。 | | 配置组合 | 通过 baseline、configRef、继承、覆盖和变量渲染生成有效计划的配置方式；组合结果是执行视图，不是第二份可编辑真相源。 | +| 稳定工作负载选择器 | 由 namespace 和 Kubernetes labels 组成、跨 Pod 重建保持不变的工作负载身份；Pod UID 只能是运行面解析结果。 | +| 精确 cgroup 回收 | 受控 CLI 将稳定工作负载选择器解析为当前 Running Pod cgroup，并在执行前重新解析身份后请求有界 `memory.reclaim`。 | ## 4. 系统边界和接口 @@ -103,6 +105,7 @@ YAML运维负责 HWLAB/UniDesk 自有平台配置的真相源、解析、渲染 | PJ2026-01060312 | 配置组合 | 本规格 6.10 | baseline、configRef、继承、覆盖、变量渲染和重复块治理 | 配置真相、目标解析、公共原语 | 多节点 YAML 复用、长期维护 | | PJ2026-01060313 | K3s本地Registry | 本规格 6.11 | node-local registry 的 k3s workload/PVC、零种子迁移、镜像预加载和 Docker 依赖分类 | Host引导代理、目标解析、发布流水 | 纯净 k3s bring-up、HWLAB/AgentRun 构建 | | PJ2026-01060314 | Hyper-V Ubuntu虚拟机 | [PJ2026-01060314 Hyper-V Ubuntu虚拟机](PJ2026-01060314-hyperv-ubuntu-vm.md) | Windows Hyper-V 宿主上的 Ubuntu VM、网络、初始化、常驻和 Provider 接入 | 配置真相、目标解析、Host引导代理 | Docker、k3s、provider-gateway、trans | +| PJ2026-01060315 | 精确工作负载回收 | 本规格 6.12 | 稳定工作负载选择器、当前 Pod cgroup 解析和有界内存回收 | 配置真相、目标解析、公共原语 | 平台 GC、核心运行面保护 | ## 6. 原子需求 @@ -239,3 +242,15 @@ YAML运维应允许新节点在没有旧 registry 数据和没有长期 host Doc 当迁移目标是证明从零部署能力时，受控 CLI 应先停止旧 host Docker registry，并在状态中明确 `seededFromOldRegistry=false`、`zeroSeeded=true`、PVC bound、workload ready 和 endpoint ready。迁移后所需的 BuildKit、runtime/base、tools image 和 service build artifact 必须通过 YAML-first `runtime-image preload`、tools image build/status 或等价 k3s 构建入口补齐，不能通过旧 registry 拷贝伪装成从零成功。 YAML运维状态输出应区分三类依赖：必须保留的 host bootstrap 面（provider-gateway/trans、0 依赖 host proxy、k3s installer/systemd）、待迁移的 host Docker/one-shot build 面，以及已经进入 k3s 的长期服务。后续减少 Docker 依赖时，应优先把长期服务、registry、git-mirror cache、web-probe/哨兵和构建执行迁入 k3s；host proxy client 不属于迁移目标。 + +### 6.12 OPS-YAML-REQ-012 精确工作负载内存回收 + +| 编号 | 短名 | 主责模块 | 关联模块 | +| --- | --- | --- | --- | +| OPS-YAML-REQ-012 | 精确工作负载回收 | PJ2026-01060315 精确工作负载回收 | [平台运维](PJ2026-0106-platform-ops.md)、[NC01 CI资源治理](PJ2026-01060107-nc01-ci-resource-governance.md) | + +平台 GC 应允许 owning YAML 使用稳定工作负载选择器声明精确 cgroup 回收目标。选择器至少包含 namespace、非空 labels 和最大匹配 Pod 数；目标可用内存、单 Pod 回收上限、swappiness、inactive anonymous memory 下限和 swap 保留线继续由 YAML 声明。YAML 不得保存 Pod UID、容器 ID、当前 cgroup 路径或从运行面反解出的可调数值。 + +受控 plan 只选择 Running Pod，并披露 selector、解析出的 namespace、Pod、UID、cgroup、当前内存、inactive anonymous memory 和保护原因。零匹配、超过最大匹配数、API 不可读、cgroup 缺失、symlink、内存下限不足或 swap 保留线不足时必须保护，禁止回退到父 cgroup、按名称猜测或扩大 selector。 + +执行前必须使用同一 YAML selector 重新解析当前 Running Pod，并逐项核对 Pod UID、cgroup 路径和全部回收参数。身份变化、目标已满足或候选不再合格时跳过且结构化披露；只有身份闭合的当前 cgroup 才能写入有界 `memory.reclaim`。执行不得停止、重启、驱逐或迁移 workload，也不得把内核请求上限宣称为实际收益；终态继续分别披露 cgroup 下降、宿主 `MemAvailable` 变化、目标缺口和 swap 排除状态。 diff --git a/scripts/src/agentrun-managed-repository-reconciler.test.ts b/scripts/src/agentrun-managed-repository-reconciler.test.ts index 96283194..dcd01aed 100644 --- a/scripts/src/agentrun-managed-repository-reconciler.test.ts +++ b/scripts/src/agentrun-managed-repository-reconciler.test.ts @@ -59,6 +59,7 @@ describe("AgentRun YAML-owned managed repository reconciler", () => { expect(documents[2].spec.template.spec.automountServiceAccountToken).toBe(false); expect(documents[2].spec.template.spec.hostNetwork).toBe(spec.gitMirror.repositoryReconciler.hostNetwork); expect(documents[2].spec.template.spec.dnsPolicy).toBe(spec.gitMirror.repositoryReconciler.dnsPolicy); + expect(documents[2].spec.template.spec.containers[0].command.slice(0, 2)).toEqual(["tini", "--"]); expect(documents[2].spec.template.spec.containers[0].readinessProbe.exec.command).toContain("--health"); const authVolume = documents[2].spec.template.spec.volumes.find((item: any) => item.name === "git-auth"); expect(authVolume.projected.sources).toHaveLength(2); diff --git a/scripts/src/agentrun-managed-repository-reconciler.ts b/scripts/src/agentrun-managed-repository-reconciler.ts index 0cbdb782..8e72c166 100644 --- a/scripts/src/agentrun-managed-repository-reconciler.ts +++ b/scripts/src/agentrun-managed-repository-reconciler.ts @@ -235,6 +235,8 @@ function managedRepositoryObjects(spec: AgentRunLaneSpec): Record { + assert.match(webObserveSource, /_CGROUP_RECLAIM_RUNTIME_INVENTORY = None/u); + assert.match(webObserveSource, /cgroup reclaim candidate identity changed before execution/u); + assert.match(webObserveSource, /item\.get\("path"\) == path/u); + assert.match(webObserveSource, /item\.get\("pod"\).*get\("uid"\).*resolved_uid/u); +}); + test("memory-pressure plan includes only YAML allowlisted cgroup reclaim stages with swap reserve", () => { const result = runPythonFixture(String.raw` MEMORY_CONFIG = { @@ -239,7 +246,7 @@ test("default memory-pressure plan projects an actionable YAML-bounded preview b assert.equal(projected.runEligibility.allowed, true); assert.equal(projected.runEligibility.allCandidatesEvaluated, true); assert.equal(projected.candidatePreview.limit, remoteTarget.memoryPressure.planPreviewLimit); - assert.equal(projected.candidatePreview.candidates.length, 6); + assert.equal(projected.candidatePreview.candidates.length, Math.min(candidates.length, remoteTarget.memoryPressure.planPreviewLimit)); assert.equal(projected.candidatePreview.candidates[2].identityClosure.identitiesComplete, true); assert.equal(projected.candidatePreview.candidates[2].processIdentities, undefined); assert.equal(projected.policy.runCommand, "bun scripts/cli.ts gc remote NC01 run --confirm --memory-pressure-only"); diff --git a/scripts/src/gc-remote-web-observe.py b/scripts/src/gc-remote-web-observe.py index 23646366..8f0c1ce3 100644 --- a/scripts/src/gc-remote-web-observe.py +++ b/scripts/src/gc-remote-web-observe.py @@ -359,15 +359,40 @@ def configured_cgroup_reclaim_targets(): return [] target_id = item.get("id") path = item.get("path") + workload_selector = item.get("workloadSelector") reclaim_bytes = item.get("reclaimBytes") swappiness = item.get("swappiness") minimum_inactive_anon = item.get("minimumInactiveAnonBytes", 0) if not isinstance(target_id, str) or not re.match(r"^[A-Za-z0-9._-]{1,80}$", target_id) or target_id in seen_ids: _MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-target-id-invalid:%s" % index return [] - if not isinstance(path, str) or (path != "/sys/fs/cgroup" and not path.startswith("/sys/fs/cgroup/")) or os.path.normpath(path) != path: - _MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-target-path-invalid:%s" % target_id + if (path is None) == (workload_selector is None): + _MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-target-identity-invalid:%s" % target_id return [] + if path is not None: + if not isinstance(path, str) or (path != "/sys/fs/cgroup" and not path.startswith("/sys/fs/cgroup/")) or os.path.normpath(path) != path: + _MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-target-path-invalid:%s" % target_id + return [] + if workload_selector is not None: + if not isinstance(workload_selector, dict): + _MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-workload-selector-invalid:%s" % target_id + return [] + namespace = workload_selector.get("namespace") + match_labels = workload_selector.get("matchLabels") + max_matched_pods = workload_selector.get("maxMatchedPods") + if not isinstance(namespace, str) or not re.match(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", namespace): + _MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-workload-namespace-invalid:%s" % target_id + return [] + if not isinstance(match_labels, dict) or not match_labels or any( + not isinstance(key, str) or not re.match(r"^[A-Za-z0-9./_-]{1,253}$", key) + or not isinstance(value, str) or not re.match(r"^[A-Za-z0-9._-]{1,63}$", value) + for key, value in match_labels.items() + ): + _MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-workload-labels-invalid:%s" % target_id + return [] + if isinstance(max_matched_pods, bool) or not isinstance(max_matched_pods, int) or max_matched_pods < 1 or max_matched_pods > 20: + _MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-workload-max-pods-invalid:%s" % target_id + return [] if isinstance(reclaim_bytes, bool) or not isinstance(reclaim_bytes, int) or reclaim_bytes <= 0 or reclaim_bytes > 8 * 1024 * 1024 * 1024: _MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-bytes-invalid:%s" % target_id return [] @@ -388,6 +413,7 @@ def configured_cgroup_reclaim_targets(): validated.append({ "id": target_id, "path": path, + "workloadSelector": workload_selector, "reclaimBytes": reclaim_bytes, "swappiness": swappiness, "minimumInactiveAnonBytes": minimum_inactive_anon, @@ -397,6 +423,92 @@ def configured_cgroup_reclaim_targets(): }) return sorted(validated, key=lambda item: item.get("priority")) +_CGROUP_RECLAIM_RUNTIME_INVENTORY = None + +def cgroup_reclaim_runtime_inventory(): + global _CGROUP_RECLAIM_RUNTIME_INVENTORY + if _CGROUP_RECLAIM_RUNTIME_INVENTORY is not None: + return _CGROUP_RECLAIM_RUNTIME_INVENTORY + attribution = MEMORY_CONFIG.get("attribution") or {} + timeout = safe_int(attribution.get("commandTimeoutSeconds")) + runtime = command(["k3s", "crictl", "pods", "-o", "json"], timeout) + if runtime.get("exitCode") != 0: + _CGROUP_RECLAIM_RUNTIME_INVENTORY = { + "ok": False, + "reason": "workload-runtime-inventory-failed", + "command": bounded(runtime), + "pods": [], + } + return _CGROUP_RECLAIM_RUNTIME_INVENTORY + try: + payload = json.loads(runtime.get("stdout") or "{}") + except (TypeError, ValueError, json.JSONDecodeError): + _CGROUP_RECLAIM_RUNTIME_INVENTORY = {"ok": False, "reason": "workload-runtime-inventory-invalid", "pods": []} + return _CGROUP_RECLAIM_RUNTIME_INVENTORY + cgroup_root = attribution.get("cgroupRoot") + pod_paths = {} + scan_limit = safe_int(attribution.get("cgroupScanLimit")) + scanned = 0 + for current, directories, files in os.walk(os.path.join(cgroup_root, "kubepods.slice"), topdown=True, followlinks=False): + directories[:] = sorted(name for name in directories if not os.path.islink(os.path.join(current, name))) + scanned += 1 + if scanned > scan_limit: + _CGROUP_RECLAIM_RUNTIME_INVENTORY = {"ok": False, "reason": "workload-cgroup-scan-truncated", "pods": []} + return _CGROUP_RECLAIM_RUNTIME_INVENTORY + match = re.search(r"pod([0-9a-f_]{36})\.slice$", os.path.basename(current)) + if match is not None and "memory.current" in files: + pod_paths[match.group(1).replace("_", "-")] = current + directories[:] = [] + pods = [] + for pod in payload.get("items") or []: + metadata = pod.get("metadata") if isinstance(pod, dict) else None + uid = metadata.get("uid") if isinstance(metadata, dict) else None + name = metadata.get("name") if isinstance(metadata, dict) else None + namespace = metadata.get("namespace") if isinstance(metadata, dict) else None + labels = pod.get("labels") if isinstance(pod, dict) else None + if pod.get("state") != "SANDBOX_READY" or not all(isinstance(value, str) and value for value in [uid, name, namespace]): + continue + pods.append({ + "namespace": namespace, + "name": name, + "uid": uid, + "labels": labels if isinstance(labels, dict) else {}, + "path": pod_paths.get(uid), + }) + _CGROUP_RECLAIM_RUNTIME_INVENTORY = {"ok": True, "pods": pods} + return _CGROUP_RECLAIM_RUNTIME_INVENTORY + +def resolve_cgroup_reclaim_target(item): + path = item.get("path") + if isinstance(path, str): + return {"ok": True, "matches": [{"path": path, "pod": None}]} + selector = item.get("workloadSelector") or {} + labels = selector.get("matchLabels") or {} + label_selector = ",".join("%s=%s" % pair for pair in sorted(labels.items())) + inventory = cgroup_reclaim_runtime_inventory() + if inventory.get("ok") is not True: + return {"ok": False, "reason": inventory.get("reason"), "command": inventory.get("command"), "matches": []} + pods = [pod for pod in inventory.get("pods") or [] if + pod.get("namespace") == selector.get("namespace") + and all((pod.get("labels") or {}).get(key) == value for key, value in labels.items()) + ] + if len(pods) == 0: + return {"ok": False, "reason": "workload-selector-no-running-pods", "matches": []} + if len(pods) > safe_int(selector.get("maxMatchedPods")): + return {"ok": False, "reason": "workload-selector-max-pods-exceeded", "matchedPodCount": len(pods), "matches": []} + matches = [] + for pod in pods: + uid = pod.get("uid") + name = pod.get("name") + resolved_path = pod.get("path") + if not isinstance(uid, str) or not isinstance(name, str) or not isinstance(resolved_path, str): + return {"ok": False, "reason": "workload-pod-cgroup-unavailable", "matches": []} + matches.append({ + "path": resolved_path, + "pod": {"namespace": selector.get("namespace"), "name": name, "uid": uid, "labelSelector": label_selector}, + }) + return {"ok": True, "matches": matches} + def collect_cgroup_reclaim_candidates(): global _MEMORY_RECLAIM_CONFIG_ERROR, _CGROUP_RECLAIM_DIAGNOSTICS targets = configured_cgroup_reclaim_targets() @@ -412,58 +524,77 @@ def collect_cgroup_reclaim_candidates(): swap_free = safe_int((host.get("swap") or {}).get("freeBytes")) candidates = [] for item in targets: - path = item.get("path") - reclaim_path = os.path.join(path, "memory.reclaim") - protected_reason = None - if os.path.realpath(path) != path or not os.path.isdir(path) or os.path.islink(path): - protected_reason = "cgroup-path-unavailable-or-redirected" - elif not os.path.exists(reclaim_path) or not os.access(reclaim_path, os.W_OK): - protected_reason = "memory-reclaim-unavailable" - elif available > safe_int(item.get("targetMemAvailableBytes")): - protected_reason = "target-memavailable-already-met" - elif safe_int(item.get("swappiness")) > 0 and swap_free - safe_int(item.get("reclaimBytes")) < safe_int(item.get("minimumSwapFreeBytesAfterReclaim")): - protected_reason = "minimum-swap-free-reserve" - current = read_cgroup_integer(path, "memory.current") - memory_stat = read_cgroup_memory_stat(path) - inactive_anon = safe_int(memory_stat.get("inactive_anon")) - if protected_reason is None and safe_int(item.get("swappiness")) > 0 and inactive_anon < safe_int(item.get("minimumInactiveAnonBytes")): - protected_reason = "minimum-inactive-anon-not-met" - if protected_reason is not None: + resolution = resolve_cgroup_reclaim_target(item) + if resolution.get("ok") is not True: diagnostics["protected"].append({ "id": item.get("id"), - "path": path, - "reason": protected_reason, - "memoryCurrentBytes": current, - "inactiveAnonBytes": inactive_anon, + "workloadSelector": item.get("workloadSelector"), + "reason": resolution.get("reason"), + "matchedPodCount": resolution.get("matchedPodCount"), }) continue - reclaim_bytes = safe_int(item.get("reclaimBytes")) - candidates.append({ - "id": "cgroup-memory-reclaim:%s" % item.get("id"), - "configId": item.get("id"), - "kind": "cgroup-memory-reclaim", - "risk": "medium", - "description": "Request bounded cgroup v2 memory reclaim from an exact YAML allowlist path", - "reason": "host-memavailable-below-yaml-target", - "path": path, - "priority": safe_int(item.get("priority")), - "reclaimBytes": reclaim_bytes, - "swappiness": safe_int(item.get("swappiness")), - "minimumInactiveAnonBytes": safe_int(item.get("minimumInactiveAnonBytes")), - "targetMemAvailableBytes": safe_int(item.get("targetMemAvailableBytes")), - "minimumSwapFreeBytesAfterReclaim": safe_int(item.get("minimumSwapFreeBytesAfterReclaim")), - "memoryCurrentBytes": current, - "memorySwapCurrentBytes": read_cgroup_integer(path, "memory.swap.current"), - "memoryStat": memory_stat, - "requestedMemoryReclaimBytes": reclaim_bytes, - "requestedMemoryReclaim": fmt_bytes(reclaim_bytes), - "estimateKind": "kernel-request-upper-bound", - "expectedMemoryRssBytes": reclaim_bytes, - "expectedMemoryRss": fmt_bytes(reclaim_bytes), - "estimatedDiskBytes": 0, - "estimatedReclaimBytes": 0, - "action": {"op": "cgroup-v2-memory-reclaim", "allowlist": "yaml-exact-cgroup-path"}, - }) + for resolved in resolution.get("matches") or []: + path = resolved.get("path") + pod = resolved.get("pod") + reclaim_path = os.path.join(path, "memory.reclaim") + protected_reason = None + if os.path.realpath(path) != path or not os.path.isdir(path) or os.path.islink(path): + protected_reason = "cgroup-path-unavailable-or-redirected" + elif not os.path.exists(reclaim_path) or not os.access(reclaim_path, os.W_OK): + protected_reason = "memory-reclaim-unavailable" + elif available > safe_int(item.get("targetMemAvailableBytes")): + protected_reason = "target-memavailable-already-met" + elif safe_int(item.get("swappiness")) > 0 and swap_free - safe_int(item.get("reclaimBytes")) < safe_int(item.get("minimumSwapFreeBytesAfterReclaim")): + protected_reason = "minimum-swap-free-reserve" + current = read_cgroup_integer(path, "memory.current") + memory_stat = read_cgroup_memory_stat(path) + inactive_anon = safe_int(memory_stat.get("inactive_anon")) + if protected_reason is None and safe_int(item.get("swappiness")) > 0 and inactive_anon < safe_int(item.get("minimumInactiveAnonBytes")): + protected_reason = "minimum-inactive-anon-not-met" + if protected_reason is not None: + diagnostics["protected"].append({ + "id": item.get("id"), + "path": path, + "workloadSelector": item.get("workloadSelector"), + "resolvedPod": pod, + "reason": protected_reason, + "memoryCurrentBytes": current, + "inactiveAnonBytes": inactive_anon, + }) + continue + reclaim_bytes = safe_int(item.get("reclaimBytes")) + identity_suffix = (pod or {}).get("uid") + candidate_id = "cgroup-memory-reclaim:%s" % item.get("id") + if identity_suffix: + candidate_id += ":%s" % identity_suffix + candidates.append({ + "id": candidate_id, + "configId": item.get("id"), + "kind": "cgroup-memory-reclaim", + "risk": "medium", + "description": "Request bounded cgroup v2 memory reclaim from a YAML-resolved workload", + "reason": "host-memavailable-below-yaml-target", + "path": path, + "workloadSelector": item.get("workloadSelector"), + "resolvedPod": pod, + "priority": safe_int(item.get("priority")), + "reclaimBytes": reclaim_bytes, + "swappiness": safe_int(item.get("swappiness")), + "minimumInactiveAnonBytes": safe_int(item.get("minimumInactiveAnonBytes")), + "targetMemAvailableBytes": safe_int(item.get("targetMemAvailableBytes")), + "minimumSwapFreeBytesAfterReclaim": safe_int(item.get("minimumSwapFreeBytesAfterReclaim")), + "memoryCurrentBytes": current, + "memorySwapCurrentBytes": read_cgroup_integer(path, "memory.swap.current"), + "memoryStat": memory_stat, + "requestedMemoryReclaimBytes": reclaim_bytes, + "requestedMemoryReclaim": fmt_bytes(reclaim_bytes), + "estimateKind": "kernel-request-upper-bound", + "expectedMemoryRssBytes": reclaim_bytes, + "expectedMemoryRss": fmt_bytes(reclaim_bytes), + "estimatedDiskBytes": 0, + "estimatedReclaimBytes": 0, + "action": {"op": "cgroup-v2-memory-reclaim", "allowlist": "yaml-workload-selector-or-exact-path"}, + }) diagnostics["eligibleCount"] = len(candidates) return candidates @@ -679,14 +810,23 @@ def extend_with_allowlisted_session_members(identities, processes, sid, browser_ return added def execute_cgroup_reclaim_candidate(candidate): + global _CGROUP_RECLAIM_RUNTIME_INVENTORY config_id = candidate.get("configId") configured = next((item for item in configured_cgroup_reclaim_targets() if item.get("id") == config_id), None) if configured is None: raise RuntimeError("cgroup reclaim candidate is no longer present in YAML allowlist") - for key in ["path", "reclaimBytes", "swappiness", "minimumInactiveAnonBytes", "targetMemAvailableBytes", "minimumSwapFreeBytesAfterReclaim"]: + for key in ["workloadSelector", "reclaimBytes", "swappiness", "minimumInactiveAnonBytes", "targetMemAvailableBytes", "minimumSwapFreeBytesAfterReclaim"]: if configured.get(key) != candidate.get(key): raise RuntimeError("cgroup reclaim candidate no longer matches YAML: %s" % key) - path = configured.get("path") + _CGROUP_RECLAIM_RUNTIME_INVENTORY = None + resolution = resolve_cgroup_reclaim_target(configured) + if resolution.get("ok") is not True: + raise RuntimeError("cgroup reclaim workload selector no longer resolves: %s" % resolution.get("reason")) + path = candidate.get("path") + resolved_uid = ((candidate.get("resolvedPod") or {}).get("uid")) + current_match = next((item for item in resolution.get("matches") or [] if item.get("path") == path and ((item.get("pod") or {}).get("uid")) == resolved_uid), None) + if current_match is None: + raise RuntimeError("cgroup reclaim candidate identity changed before execution") reclaim_path = os.path.join(path, "memory.reclaim") if os.path.realpath(path) != path or not os.path.isdir(path) or os.path.islink(path): raise RuntimeError("cgroup reclaim path is unavailable or redirected") diff --git a/scripts/src/gc-remote.ts b/scripts/src/gc-remote.ts index 39f61ac0..e7c5b04f 100644 --- a/scripts/src/gc-remote.ts +++ b/scripts/src/gc-remote.ts @@ -411,6 +411,23 @@ function positiveInteger(value: unknown): number | null { return typeof value === "number" && Number.isInteger(value) && value > 0 ? value : null; } +function compactWorkloadSelector(value: unknown): string | undefined { + const selector = recordOrEmpty(value); + const namespace = typeof selector.namespace === "string" ? selector.namespace : null; + const labels = recordOrEmpty(selector.matchLabels); + const pairs = Object.entries(labels) + .filter((entry): entry is [string, string] => typeof entry[1] === "string") + .sort(([left], [right]) => left.localeCompare(right)) + .map(([key, label]) => `${key}=${label}`); + return namespace !== null && pairs.length > 0 ? `${namespace}:${pairs.join(",")}` : undefined; +} + +function compactResolvedPod(value: unknown): Record | undefined { + const pod = recordOrEmpty(value); + if (typeof pod.namespace !== "string" || typeof pod.name !== "string" || typeof pod.uid !== "string") return undefined; + return { name: `${pod.namespace}/${pod.name}`, uid: pod.uid }; +} + function compactMemoryPressureCandidate(value: unknown): Record { const candidate = recordOrEmpty(value); const action = recordOrEmpty(candidate.action); @@ -426,14 +443,14 @@ function compactMemoryPressureCandidate(value: unknown): Record ...base, configId: candidate.configId, path: candidate.path, + workloadSelector: compactWorkloadSelector(candidate.workloadSelector), + resolvedPod: compactResolvedPod(candidate.resolvedPod), reclaimBytes: candidate.reclaimBytes, requestedMemoryReclaimBytes: candidate.requestedMemoryReclaimBytes, requestedMemoryReclaim: candidate.requestedMemoryReclaim, estimateKind: candidate.estimateKind, swappiness: candidate.swappiness, minimumInactiveAnonBytes: candidate.minimumInactiveAnonBytes, - targetMemAvailableBytes: candidate.targetMemAvailableBytes, - minimumSwapFreeBytesAfterReclaim: candidate.minimumSwapFreeBytesAfterReclaim, memoryCurrentBytes: candidate.memoryCurrentBytes, memorySwapCurrentBytes: candidate.memorySwapCurrentBytes, }; @@ -478,7 +495,10 @@ function memoryPressureCandidateAllowed(value: unknown): boolean { && swappiness !== null && swappiness >= 0 && swappiness <= 200 && (swappiness === 0 || positiveInteger(candidate.minimumInactiveAnonBytes) !== null) && action.op === "cgroup-v2-memory-reclaim" - && action.allowlist === "yaml-exact-cgroup-path"; + && (action.allowlist === "yaml-exact-cgroup-path" + || (action.allowlist === "yaml-workload-selector-or-exact-path" + && typeof recordOrEmpty(candidate.workloadSelector).namespace === "string" + && typeof recordOrEmpty(candidate.resolvedPod).uid === "string")); } if (candidate.kind !== "browser-orphan-process-tree" && candidate.kind !== "stale-web-observer-process-tree") return false; const compact = compactMemoryPressureCandidate(candidate); @@ -526,7 +546,15 @@ function compactMemoryPressureCandidateScan(value: unknown): Record { const protectedItem = recordOrEmpty(item); - return { id: protectedItem.id, reason: protectedItem.reason, memoryCurrentBytes: protectedItem.memoryCurrentBytes }; + return { + id: protectedItem.id, + reason: protectedItem.reason, + matchedPodCount: protectedItem.matchedPodCount, + memoryCurrentBytes: protectedItem.memoryCurrentBytes, + inactiveAnonBytes: protectedItem.inactiveAnonBytes, + workloadSelector: compactWorkloadSelector(protectedItem.workloadSelector), + resolvedPod: compactResolvedPod(protectedItem.resolvedPod), + }; }), }, policySource: scan.policySource, diff --git a/scripts/src/hwlab-node-control-plane.ts b/scripts/src/hwlab-node-control-plane.ts index 8436f35f..f228ca4d 100644 --- a/scripts/src/hwlab-node-control-plane.ts +++ b/scripts/src/hwlab-node-control-plane.ts @@ -2423,7 +2423,7 @@ function gitMirrorDeployment(name: string, namespace: string, labels: Record