feat: add bounded root cgroup file reclaim
This commit is contained in:
@@ -223,7 +223,7 @@ gc:
|
||||
minAgeHours: 336
|
||||
maxDeletePerRun: 5
|
||||
memoryPressure:
|
||||
planPreviewLimit: 4
|
||||
planPreviewLimit: 2
|
||||
processPatterns:
|
||||
- k3s-server
|
||||
- containerd
|
||||
@@ -291,6 +291,14 @@ gc:
|
||||
targetMemAvailableBytes: 3221225472
|
||||
minimumSwapFreeBytesAfterReclaim: 34359738368
|
||||
targets:
|
||||
- id: root-cgroup-file-cache
|
||||
rootCgroupSelector:
|
||||
scope: root
|
||||
reclaimBytes: 805306368
|
||||
swappiness: 0
|
||||
minimumInactiveFileBytes: 268435456
|
||||
minimumReclaimableFileSlabBytes: 1073741824
|
||||
priority: 2
|
||||
- id: k3s-service-inactive-anon
|
||||
systemdUnitSelector:
|
||||
unit: k3s.service
|
||||
|
||||
@@ -200,6 +200,106 @@ print(json.dumps({"ids": [item["id"] for item in candidates], "swappiness": [ite
|
||||
assert.match(webObserveSource, /memory\.reclaim/u);
|
||||
});
|
||||
|
||||
test("root cgroup file-cache reclaim requires cgroup2 identity and fresh dual-source lower bounds", () => {
|
||||
const result = runPythonFixture(String.raw`
|
||||
MEMORY_CONFIG = {
|
||||
"observeStateRoots": ["/tmp/observe"], "minimumReadableObserveRoots": 1,
|
||||
"observeRunDepth": 6, "observeScanLimit": 100, "staleRunMaxAgeHours": 1,
|
||||
"processScanLimit": 100, "processStaleMinAgeSeconds": 3600,
|
||||
"maxMemoryReclaimCandidates": 20, "terminationGraceSeconds": 1,
|
||||
"terminationPollMilliseconds": 50, "observerRootPatterns": ["observer-runner.mjs"],
|
||||
"browserRootPatterns": ["cliDaemon.mjs"], "browserProcessPatterns": ["chromium"],
|
||||
"attribution": {"cgroupRoot": "/sys/fs/cgroup", "commandTimeoutSeconds": 20},
|
||||
"cgroupReclaim": {
|
||||
"enabled": True, "targetMemAvailableBytes": 4294967296,
|
||||
"minimumSwapFreeBytesAfterReclaim": 402653184,
|
||||
"targets": [{
|
||||
"id": "root-file-cache", "rootCgroupSelector": {"scope": "root"},
|
||||
"reclaimBytes": 805306368, "swappiness": 0,
|
||||
"minimumInactiveFileBytes": 268435456,
|
||||
"minimumReclaimableFileSlabBytes": 1073741824,
|
||||
"priority": 1,
|
||||
}],
|
||||
},
|
||||
}
|
||||
identity = {
|
||||
"filesystemType": "cgroup2", "mountRoot": "/", "mountPoint": "/sys/fs/cgroup",
|
||||
"majorMinor": "0:29", "mountId": 31, "parentMountId": 30, "inode": 1,
|
||||
"path": "/sys/fs/cgroup",
|
||||
}
|
||||
root_stat = {
|
||||
"anon": 1024, "file": 1610612736, "inactive_anon": 0,
|
||||
"inactive_file": 805306368, "slab_reclaimable": 536870912,
|
||||
}
|
||||
host_evidence = {
|
||||
"ok": True, "source": "/proc/meminfo", "cachedBytes": 1610612736,
|
||||
"buffersBytes": 134217728, "slabReclaimableBytes": 536870912,
|
||||
"cachedAndBuffersBytes": 1744830464,
|
||||
"reclaimableFileSlabUpperBoundBytes": 2281701376,
|
||||
"estimateKind": "kernel-reclaimable-upper-bound",
|
||||
}
|
||||
fresh_evidence = lambda: {
|
||||
"ok": True, "host": host_evidence, "rootMemoryStat": root_stat,
|
||||
"rootReclaimableFileSlabUpperBoundBytes": 2147483648,
|
||||
"estimateKind": "kernel-reclaimable-upper-bound",
|
||||
}
|
||||
read_proc_process_table = lambda: {1: {"pid": 1, "ppid": 0, "sid": 1, "startTicks": 1, "ageSeconds": 1, "rssBytes": 1, "commandLine": "init", "commandPreview": "init"}}
|
||||
observer_run_records = lambda: []
|
||||
collect_memory_snapshot = lambda: {"ok": True, "memory": {"availableBytes": 3 * 1024 * 1024 * 1024}, "swap": {"freeBytes": 2 * 1024 * 1024 * 1024}}
|
||||
resolve_root_cgroup_identity = lambda: {"ok": True, "path": "/sys/fs/cgroup", "identity": identity}
|
||||
collect_root_file_reclaim_evidence = lambda path: fresh_evidence()
|
||||
read_cgroup_memory_stat = lambda path: root_stat
|
||||
read_cgroup_integer = lambda path, name: 8 * 1024 * 1024 * 1024
|
||||
os.path.realpath = lambda path: path
|
||||
os.path.isdir = lambda path: True
|
||||
os.path.islink = lambda path: False
|
||||
os.path.exists = lambda path: True
|
||||
os.access = lambda path, mode: True
|
||||
candidates = collect_memory_reclaim_candidates()
|
||||
memory_reclaim_write_attempts = []
|
||||
original_open = open
|
||||
def tracked_open(path, *args, **kwargs):
|
||||
if str(path).endswith("/memory.reclaim"):
|
||||
memory_reclaim_write_attempts.append(path)
|
||||
return original_open(path, *args, **kwargs)
|
||||
open = tracked_open
|
||||
collect_root_file_reclaim_evidence = lambda path: {
|
||||
**fresh_evidence(),
|
||||
"rootMemoryStat": {**root_stat, "inactive_file": 134217728},
|
||||
}
|
||||
try:
|
||||
execute_cgroup_reclaim_candidate(candidates[0])
|
||||
revalidation_error = None
|
||||
except RuntimeError as error:
|
||||
revalidation_error = str(error)
|
||||
print(json.dumps({
|
||||
"candidate": candidates[0],
|
||||
"revalidationError": revalidation_error,
|
||||
"memoryReclaimWriteAttempts": memory_reclaim_write_attempts,
|
||||
"diagnostics": memory_reclaim_scan_diagnostics(),
|
||||
}))
|
||||
`);
|
||||
const candidate = result.candidate as Record<string, any>;
|
||||
assert.equal(candidate.rootCgroupSelector.scope, "root");
|
||||
assert.equal(candidate.resolvedRootCgroup.filesystemType, "cgroup2");
|
||||
assert.equal(candidate.resolvedRootCgroup.mountRoot, "/");
|
||||
assert.equal(candidate.swappiness, 0);
|
||||
assert.equal(candidate.estimateKind, "kernel-request-upper-bound");
|
||||
assert.equal(candidate.rootFileReclaimEvidence.host.source, "/proc/meminfo");
|
||||
assert.equal(result.revalidationError, "minimum root inactive file memory is no longer available");
|
||||
assert.deepEqual(result.memoryReclaimWriteAttempts, []);
|
||||
assert.equal(((result.diagnostics as Record<string, any>).cgroupReclaim).eligibleCount, 1);
|
||||
const projected = projectRemoteGcResult({
|
||||
ok: true,
|
||||
summary: { candidateCount: 1 },
|
||||
candidateScan: result.diagnostics,
|
||||
candidates: [candidate],
|
||||
}, { memoryPressure: { planPreviewLimit: 4 } }, { action: "plan", memoryPressureOnly: true, full: false }) as Record<string, any>;
|
||||
assert.equal(projected.runEligibility.allowed, true);
|
||||
assert.equal(projected.candidatePreview.candidates[0].rootCgroupSelector, "root");
|
||||
assert.equal(projected.candidatePreview.candidates[0].resolvedRootCgroup.inode, 1);
|
||||
});
|
||||
|
||||
test("default memory-pressure plan projects an actionable YAML-bounded preview below stdout limit", () => {
|
||||
const processCandidate = (index: number): Record<string, unknown> => ({
|
||||
id: `browser-orphan-process-tree:${1000 + index}:12345`,
|
||||
@@ -289,8 +389,8 @@ test("default memory-pressure plan projects an actionable YAML-bounded preview b
|
||||
assert.equal(projected.runEligibility.allCandidatesEvaluated, true);
|
||||
assert.equal(projected.candidatePreview.limit, remoteTarget.memoryPressure.planPreviewLimit);
|
||||
assert.equal(projected.candidatePreview.candidates.length, Math.min(candidates.length, remoteTarget.memoryPressure.planPreviewLimit));
|
||||
assert.equal(projected.candidatePreview.candidates[2].identityClosure.identitiesComplete, true);
|
||||
assert.equal(projected.candidatePreview.candidates[2].processIdentities, undefined);
|
||||
assert.equal(projected.candidatePreview.omittedCount, candidates.length - projected.candidatePreview.candidates.length);
|
||||
assert.ok(projected.candidatePreview.candidates.every((candidate: Record<string, unknown>) => candidate.processIdentities === undefined));
|
||||
assert.equal(projected.policy.runCommand, "bun scripts/cli.ts gc remote NC01 run --confirm --memory-pressure-only");
|
||||
assert.equal(projected.valuesRedacted, true);
|
||||
const renderedEnvelope = `${JSON.stringify({
|
||||
|
||||
@@ -323,12 +323,98 @@ def read_cgroup_memory_stat(path):
|
||||
with open(os.path.join(path, "memory.stat"), "r", encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
parts = line.split()
|
||||
if len(parts) == 2 and parts[0] in set(["anon", "file", "inactive_anon", "inactive_file"]):
|
||||
if len(parts) == 2 and parts[0] in set(["anon", "file", "inactive_anon", "inactive_file", "slab_reclaimable"]):
|
||||
selected[parts[0]] = safe_int(parts[1])
|
||||
except OSError:
|
||||
return {}
|
||||
return selected
|
||||
|
||||
def read_host_file_reclaim_evidence():
|
||||
selected = {}
|
||||
try:
|
||||
with open("/proc/meminfo", "r", encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
key, separator, raw = line.partition(":")
|
||||
if separator and key in set(["Cached", "Buffers", "SReclaimable"]):
|
||||
value = raw.strip().split()[0]
|
||||
selected[key] = int(value) * 1024
|
||||
except (OSError, ValueError, IndexError):
|
||||
return {"ok": False, "reason": "host-file-reclaim-evidence-unavailable"}
|
||||
if any(key not in selected for key in ["Cached", "Buffers", "SReclaimable"]):
|
||||
return {"ok": False, "reason": "host-file-reclaim-evidence-incomplete"}
|
||||
cached_and_buffers = selected["Cached"] + selected["Buffers"]
|
||||
return {
|
||||
"ok": True,
|
||||
"source": "/proc/meminfo",
|
||||
"cachedBytes": selected["Cached"],
|
||||
"buffersBytes": selected["Buffers"],
|
||||
"slabReclaimableBytes": selected["SReclaimable"],
|
||||
"cachedAndBuffersBytes": cached_and_buffers,
|
||||
"reclaimableFileSlabUpperBoundBytes": cached_and_buffers + selected["SReclaimable"],
|
||||
"estimateKind": "kernel-reclaimable-upper-bound",
|
||||
}
|
||||
|
||||
def resolve_root_cgroup_identity():
|
||||
cgroup_root = (MEMORY_CONFIG.get("attribution") or {}).get("cgroupRoot")
|
||||
if not isinstance(cgroup_root, str) or not os.path.isabs(cgroup_root) or os.path.normpath(cgroup_root) != cgroup_root:
|
||||
return {"ok": False, "reason": "root-cgroup-config-path-invalid"}
|
||||
if os.path.realpath(cgroup_root) != cgroup_root or not os.path.isdir(cgroup_root) or os.path.islink(cgroup_root):
|
||||
return {"ok": False, "reason": "root-cgroup-path-unavailable-or-redirected"}
|
||||
matches = []
|
||||
try:
|
||||
with open("/proc/self/mountinfo", "r", encoding="utf-8") as handle:
|
||||
for line in handle:
|
||||
before, separator, after = line.strip().partition(" - ")
|
||||
if not separator:
|
||||
continue
|
||||
left = before.split()
|
||||
right = after.split()
|
||||
if len(left) < 6 or len(right) < 3 or right[0] != "cgroup2" or left[4] != cgroup_root:
|
||||
continue
|
||||
matches.append({
|
||||
"mountId": safe_int(left[0], None),
|
||||
"parentMountId": safe_int(left[1], None),
|
||||
"majorMinor": left[2],
|
||||
"mountRoot": left[3],
|
||||
"mountPoint": left[4],
|
||||
"filesystemType": right[0],
|
||||
})
|
||||
except OSError:
|
||||
return {"ok": False, "reason": "root-cgroup-mountinfo-unavailable"}
|
||||
if len(matches) != 1:
|
||||
return {"ok": False, "reason": "root-cgroup-v2-mount-not-unique"}
|
||||
identity = matches[0]
|
||||
if identity.get("mountRoot") != "/" or identity.get("mountId") is None or identity.get("parentMountId") is None:
|
||||
return {"ok": False, "reason": "configured-cgroup-path-is-not-mount-root"}
|
||||
try:
|
||||
stat = os.stat(cgroup_root, follow_symlinks=False)
|
||||
except OSError:
|
||||
return {"ok": False, "reason": "root-cgroup-stat-unavailable"}
|
||||
stat_major_minor = "%s:%s" % (os.major(stat.st_dev), os.minor(stat.st_dev))
|
||||
if stat_major_minor != identity.get("majorMinor") or stat.st_ino <= 0:
|
||||
return {"ok": False, "reason": "root-cgroup-mount-identity-mismatch"}
|
||||
identity["inode"] = stat.st_ino
|
||||
identity["path"] = cgroup_root
|
||||
return {"ok": True, "path": cgroup_root, "identity": identity}
|
||||
|
||||
def collect_root_file_reclaim_evidence(path):
|
||||
host = read_host_file_reclaim_evidence()
|
||||
stat = read_cgroup_memory_stat(path)
|
||||
if host.get("ok") is not True:
|
||||
return {"ok": False, "reason": host.get("reason"), "host": host, "rootMemoryStat": stat}
|
||||
if any(key not in stat for key in ["file", "inactive_file", "slab_reclaimable"]):
|
||||
return {"ok": False, "reason": "root-memory-stat-file-evidence-incomplete", "host": host, "rootMemoryStat": stat}
|
||||
if safe_int(stat.get("inactive_file")) > safe_int(stat.get("file")):
|
||||
return {"ok": False, "reason": "root-memory-stat-file-evidence-inconsistent", "host": host, "rootMemoryStat": stat}
|
||||
root_upper_bound = safe_int(stat.get("file")) + safe_int(stat.get("slab_reclaimable"))
|
||||
return {
|
||||
"ok": True,
|
||||
"host": host,
|
||||
"rootMemoryStat": stat,
|
||||
"rootReclaimableFileSlabUpperBoundBytes": root_upper_bound,
|
||||
"estimateKind": "kernel-reclaimable-upper-bound",
|
||||
}
|
||||
|
||||
def configured_cgroup_reclaim_targets():
|
||||
global _MEMORY_RECLAIM_CONFIG_ERROR
|
||||
policy = MEMORY_CONFIG.get("cgroupReclaim")
|
||||
@@ -361,18 +447,25 @@ def configured_cgroup_reclaim_targets():
|
||||
path = item.get("path")
|
||||
workload_selector = item.get("workloadSelector")
|
||||
systemd_unit_selector = item.get("systemdUnitSelector")
|
||||
root_cgroup_selector = item.get("rootCgroupSelector")
|
||||
reclaim_bytes = item.get("reclaimBytes")
|
||||
swappiness = item.get("swappiness")
|
||||
minimum_inactive_anon = item.get("minimumInactiveAnonBytes", 0)
|
||||
minimum_inactive_file = item.get("minimumInactiveFileBytes", 0)
|
||||
minimum_file_slab = item.get("minimumReclaimableFileSlabBytes", 0)
|
||||
if not isinstance(target_id, str) or not re.match(r"^[A-Za-z0-9._-]{1,80}$", target_id) or target_id in seen_ids:
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-target-id-invalid:%s" % index
|
||||
return []
|
||||
if path is not None:
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-path-selector-unsupported:%s" % target_id
|
||||
return []
|
||||
if (workload_selector is None) == (systemd_unit_selector is None):
|
||||
if sum(selector is not None for selector in [workload_selector, systemd_unit_selector, root_cgroup_selector]) != 1:
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-target-identity-invalid:%s" % target_id
|
||||
return []
|
||||
if root_cgroup_selector is not None:
|
||||
if not isinstance(root_cgroup_selector, dict) or root_cgroup_selector != {"scope": "root"}:
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-root-selector-invalid:%s" % target_id
|
||||
return []
|
||||
if systemd_unit_selector is not None:
|
||||
if not isinstance(systemd_unit_selector, dict) or set(systemd_unit_selector.keys()) != set(["unit"]):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-systemd-selector-invalid:%s" % target_id
|
||||
@@ -413,6 +506,20 @@ def configured_cgroup_reclaim_targets():
|
||||
if swappiness > 0 and minimum_inactive_anon < reclaim_bytes:
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-minimum-inactive-anon-too-small:%s" % target_id
|
||||
return []
|
||||
if any(isinstance(value, bool) or not isinstance(value, int) or value < 0 for value in [minimum_inactive_file, minimum_file_slab]):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-file-lower-bound-invalid:%s" % target_id
|
||||
return []
|
||||
if root_cgroup_selector is not None and (
|
||||
swappiness != 0
|
||||
or minimum_inactive_anon != 0
|
||||
or minimum_inactive_file <= 0
|
||||
or minimum_file_slab < reclaim_bytes
|
||||
):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "yaml-root-cgroup-file-reclaim-policy-invalid:%s" % target_id
|
||||
return []
|
||||
if root_cgroup_selector is None and (minimum_inactive_file != 0 or minimum_file_slab != 0):
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "yaml-file-lower-bound-requires-root-selector:%s" % target_id
|
||||
return []
|
||||
priority = item.get("priority", index + 1)
|
||||
if isinstance(priority, bool) or not isinstance(priority, int) or priority < 1 or priority > 1000:
|
||||
_MEMORY_RECLAIM_CONFIG_ERROR = "yaml-cgroup-reclaim-priority-invalid:%s" % target_id
|
||||
@@ -422,9 +529,12 @@ def configured_cgroup_reclaim_targets():
|
||||
"id": target_id,
|
||||
"workloadSelector": workload_selector,
|
||||
"systemdUnitSelector": systemd_unit_selector,
|
||||
"rootCgroupSelector": root_cgroup_selector,
|
||||
"reclaimBytes": reclaim_bytes,
|
||||
"swappiness": swappiness,
|
||||
"minimumInactiveAnonBytes": minimum_inactive_anon,
|
||||
"minimumInactiveFileBytes": minimum_inactive_file,
|
||||
"minimumReclaimableFileSlabBytes": minimum_file_slab,
|
||||
"priority": priority,
|
||||
"targetMemAvailableBytes": target_available,
|
||||
"minimumSwapFreeBytesAfterReclaim": swap_reserve,
|
||||
@@ -487,6 +597,17 @@ def cgroup_reclaim_runtime_inventory():
|
||||
return _CGROUP_RECLAIM_RUNTIME_INVENTORY
|
||||
|
||||
def resolve_cgroup_reclaim_target(item):
|
||||
root_selector = item.get("rootCgroupSelector")
|
||||
if isinstance(root_selector, dict):
|
||||
resolved = resolve_root_cgroup_identity()
|
||||
if resolved.get("ok") is not True:
|
||||
return {"ok": False, "reason": resolved.get("reason"), "matches": []}
|
||||
return {"ok": True, "matches": [{
|
||||
"path": resolved.get("path"),
|
||||
"pod": None,
|
||||
"systemdUnit": None,
|
||||
"rootCgroup": resolved.get("identity"),
|
||||
}]}
|
||||
systemd_selector = item.get("systemdUnitSelector")
|
||||
if isinstance(systemd_selector, dict):
|
||||
unit = systemd_selector.get("unit")
|
||||
@@ -528,7 +649,7 @@ def resolve_cgroup_reclaim_target(item):
|
||||
"mainPid": main_pid,
|
||||
"startTicks": safe_int(start_ticks),
|
||||
}
|
||||
return {"ok": True, "matches": [{"path": path, "pod": None, "systemdUnit": identity}]}
|
||||
return {"ok": True, "matches": [{"path": path, "pod": None, "systemdUnit": identity, "rootCgroup": None}]}
|
||||
selector = item.get("workloadSelector") or {}
|
||||
labels = selector.get("matchLabels") or {}
|
||||
label_selector = ",".join("%s=%s" % pair for pair in sorted(labels.items()))
|
||||
@@ -554,6 +675,7 @@ def resolve_cgroup_reclaim_target(item):
|
||||
"path": resolved_path,
|
||||
"pod": {"namespace": selector.get("namespace"), "name": name, "uid": uid, "labelSelector": label_selector},
|
||||
"systemdUnit": None,
|
||||
"rootCgroup": None,
|
||||
})
|
||||
return {"ok": True, "matches": matches}
|
||||
|
||||
@@ -578,6 +700,7 @@ def collect_cgroup_reclaim_candidates():
|
||||
"id": item.get("id"),
|
||||
"workloadSelector": item.get("workloadSelector"),
|
||||
"systemdUnitSelector": item.get("systemdUnitSelector"),
|
||||
"rootCgroupSelector": item.get("rootCgroupSelector"),
|
||||
"reason": resolution.get("reason"),
|
||||
"matchedPodCount": resolution.get("matchedPodCount"),
|
||||
})
|
||||
@@ -586,6 +709,7 @@ def collect_cgroup_reclaim_candidates():
|
||||
path = resolved.get("path")
|
||||
pod = resolved.get("pod")
|
||||
systemd_unit = resolved.get("systemdUnit")
|
||||
root_cgroup = resolved.get("rootCgroup")
|
||||
reclaim_path = os.path.join(path, "memory.reclaim")
|
||||
protected_reason = None
|
||||
if os.path.realpath(path) != path or not os.path.isdir(path) or os.path.islink(path):
|
||||
@@ -601,17 +725,31 @@ def collect_cgroup_reclaim_candidates():
|
||||
inactive_anon = safe_int(memory_stat.get("inactive_anon"))
|
||||
if protected_reason is None and safe_int(item.get("swappiness")) > 0 and inactive_anon < safe_int(item.get("minimumInactiveAnonBytes")):
|
||||
protected_reason = "minimum-inactive-anon-not-met"
|
||||
root_file_evidence = None
|
||||
if protected_reason is None and item.get("rootCgroupSelector") is not None:
|
||||
root_file_evidence = collect_root_file_reclaim_evidence(path)
|
||||
if root_file_evidence.get("ok") is not True:
|
||||
protected_reason = root_file_evidence.get("reason")
|
||||
elif safe_int((root_file_evidence.get("rootMemoryStat") or {}).get("inactive_file")) < safe_int(item.get("minimumInactiveFileBytes")):
|
||||
protected_reason = "minimum-root-inactive-file-not-met"
|
||||
elif safe_int(root_file_evidence.get("rootReclaimableFileSlabUpperBoundBytes")) < safe_int(item.get("minimumReclaimableFileSlabBytes")):
|
||||
protected_reason = "minimum-root-file-slab-upper-bound-not-met"
|
||||
elif safe_int((root_file_evidence.get("host") or {}).get("reclaimableFileSlabUpperBoundBytes")) < safe_int(item.get("minimumReclaimableFileSlabBytes")):
|
||||
protected_reason = "minimum-host-file-slab-upper-bound-not-met"
|
||||
if protected_reason is not None:
|
||||
diagnostics["protected"].append({
|
||||
"id": item.get("id"),
|
||||
"path": path,
|
||||
"workloadSelector": item.get("workloadSelector"),
|
||||
"systemdUnitSelector": item.get("systemdUnitSelector"),
|
||||
"rootCgroupSelector": item.get("rootCgroupSelector"),
|
||||
"resolvedPod": pod,
|
||||
"resolvedSystemdUnit": systemd_unit,
|
||||
"resolvedRootCgroup": root_cgroup,
|
||||
"reason": protected_reason,
|
||||
"memoryCurrentBytes": current,
|
||||
"inactiveAnonBytes": inactive_anon,
|
||||
"rootFileReclaimEvidence": root_file_evidence,
|
||||
})
|
||||
continue
|
||||
reclaim_bytes = safe_int(item.get("reclaimBytes"))
|
||||
@@ -629,17 +767,22 @@ def collect_cgroup_reclaim_candidates():
|
||||
"path": path,
|
||||
"workloadSelector": item.get("workloadSelector"),
|
||||
"systemdUnitSelector": item.get("systemdUnitSelector"),
|
||||
"rootCgroupSelector": item.get("rootCgroupSelector"),
|
||||
"resolvedPod": pod,
|
||||
"resolvedSystemdUnit": systemd_unit,
|
||||
"resolvedRootCgroup": root_cgroup,
|
||||
"priority": safe_int(item.get("priority")),
|
||||
"reclaimBytes": reclaim_bytes,
|
||||
"swappiness": safe_int(item.get("swappiness")),
|
||||
"minimumInactiveAnonBytes": safe_int(item.get("minimumInactiveAnonBytes")),
|
||||
"minimumInactiveFileBytes": safe_int(item.get("minimumInactiveFileBytes")),
|
||||
"minimumReclaimableFileSlabBytes": safe_int(item.get("minimumReclaimableFileSlabBytes")),
|
||||
"targetMemAvailableBytes": safe_int(item.get("targetMemAvailableBytes")),
|
||||
"minimumSwapFreeBytesAfterReclaim": safe_int(item.get("minimumSwapFreeBytesAfterReclaim")),
|
||||
"memoryCurrentBytes": current,
|
||||
"memorySwapCurrentBytes": read_cgroup_integer(path, "memory.swap.current"),
|
||||
"memoryStat": memory_stat,
|
||||
"rootFileReclaimEvidence": root_file_evidence,
|
||||
"requestedMemoryReclaimBytes": reclaim_bytes,
|
||||
"requestedMemoryReclaim": fmt_bytes(reclaim_bytes),
|
||||
"estimateKind": "kernel-request-upper-bound",
|
||||
@@ -647,7 +790,7 @@ def collect_cgroup_reclaim_candidates():
|
||||
"expectedMemoryRss": fmt_bytes(reclaim_bytes),
|
||||
"estimatedDiskBytes": 0,
|
||||
"estimatedReclaimBytes": 0,
|
||||
"action": {"op": "cgroup-v2-memory-reclaim", "allowlist": "yaml-stable-cgroup-selector"},
|
||||
"action": {"op": "cgroup-v2-memory-reclaim", "allowlist": "yaml-root-cgroup-file-cache" if root_cgroup else "yaml-stable-cgroup-selector"},
|
||||
})
|
||||
diagnostics["eligibleCount"] = len(candidates)
|
||||
return candidates
|
||||
@@ -869,7 +1012,11 @@ def execute_cgroup_reclaim_candidate(candidate):
|
||||
configured = next((item for item in configured_cgroup_reclaim_targets() if item.get("id") == config_id), None)
|
||||
if configured is None:
|
||||
raise RuntimeError("cgroup reclaim candidate is no longer present in YAML allowlist")
|
||||
for key in ["workloadSelector", "systemdUnitSelector", "reclaimBytes", "swappiness", "minimumInactiveAnonBytes", "targetMemAvailableBytes", "minimumSwapFreeBytesAfterReclaim", "priority"]:
|
||||
for key in [
|
||||
"workloadSelector", "systemdUnitSelector", "rootCgroupSelector", "reclaimBytes", "swappiness",
|
||||
"minimumInactiveAnonBytes", "minimumInactiveFileBytes", "minimumReclaimableFileSlabBytes",
|
||||
"targetMemAvailableBytes", "minimumSwapFreeBytesAfterReclaim", "priority",
|
||||
]:
|
||||
if configured.get(key) != candidate.get(key):
|
||||
raise RuntimeError("cgroup reclaim candidate no longer matches YAML: %s" % key)
|
||||
_CGROUP_RECLAIM_RUNTIME_INVENTORY = None
|
||||
@@ -879,10 +1026,12 @@ def execute_cgroup_reclaim_candidate(candidate):
|
||||
path = candidate.get("path")
|
||||
resolved_uid = ((candidate.get("resolvedPod") or {}).get("uid"))
|
||||
resolved_systemd_unit = candidate.get("resolvedSystemdUnit")
|
||||
resolved_root_cgroup = candidate.get("resolvedRootCgroup")
|
||||
current_match = next((item for item in resolution.get("matches") or []
|
||||
if item.get("path") == path
|
||||
and ((item.get("pod") or {}).get("uid")) == resolved_uid
|
||||
and item.get("systemdUnit") == resolved_systemd_unit
|
||||
and item.get("rootCgroup") == resolved_root_cgroup
|
||||
), None)
|
||||
if current_match is None:
|
||||
raise RuntimeError("cgroup reclaim candidate identity changed before execution")
|
||||
@@ -895,7 +1044,7 @@ def execute_cgroup_reclaim_candidate(candidate):
|
||||
before_available = safe_int((before.get("memory") or {}).get("availableBytes"), None)
|
||||
if before_available is None:
|
||||
raise RuntimeError("MemAvailable is unavailable before cgroup reclaim")
|
||||
if before_available > safe_int(configured.get("targetMemAvailableBytes")):
|
||||
if before_available >= safe_int(configured.get("targetMemAvailableBytes")):
|
||||
return {
|
||||
"status": "succeeded",
|
||||
"outcome": "target-memavailable-already-met",
|
||||
@@ -905,6 +1054,8 @@ def execute_cgroup_reclaim_candidate(candidate):
|
||||
"cgroupMemoryBeforeBytes": read_cgroup_integer(path, "memory.current"),
|
||||
"cgroupMemoryAfterBytes": read_cgroup_integer(path, "memory.current"),
|
||||
"kernelWriteAttempted": False,
|
||||
"targetGapBeforeBytes": 0,
|
||||
"targetGapAfterBytes": 0,
|
||||
}
|
||||
swap_free = safe_int((before.get("swap") or {}).get("freeBytes"), None)
|
||||
if safe_int(configured.get("swappiness")) > 0:
|
||||
@@ -916,6 +1067,19 @@ def execute_cgroup_reclaim_candidate(candidate):
|
||||
inactive_anon = safe_int(read_cgroup_memory_stat(path).get("inactive_anon"))
|
||||
if inactive_anon < safe_int(configured.get("minimumInactiveAnonBytes")):
|
||||
raise RuntimeError("minimum inactive anonymous memory is no longer available")
|
||||
root_evidence_before = None
|
||||
if configured.get("rootCgroupSelector") is not None:
|
||||
if safe_int(configured.get("swappiness")) != 0 or path != (resolved_root_cgroup or {}).get("mountPoint"):
|
||||
raise RuntimeError("root cgroup file reclaim identity or swappiness is invalid")
|
||||
root_evidence_before = collect_root_file_reclaim_evidence(path)
|
||||
if root_evidence_before.get("ok") is not True:
|
||||
raise RuntimeError("root cgroup file reclaim evidence is unavailable: %s" % root_evidence_before.get("reason"))
|
||||
if safe_int((root_evidence_before.get("rootMemoryStat") or {}).get("inactive_file")) < safe_int(configured.get("minimumInactiveFileBytes")):
|
||||
raise RuntimeError("minimum root inactive file memory is no longer available")
|
||||
if safe_int(root_evidence_before.get("rootReclaimableFileSlabUpperBoundBytes")) < safe_int(configured.get("minimumReclaimableFileSlabBytes")):
|
||||
raise RuntimeError("minimum root file/slab reclaim upper bound is no longer available")
|
||||
if safe_int((root_evidence_before.get("host") or {}).get("reclaimableFileSlabUpperBoundBytes")) < safe_int(configured.get("minimumReclaimableFileSlabBytes")):
|
||||
raise RuntimeError("minimum host file/slab reclaim upper bound is no longer available")
|
||||
cgroup_before = read_cgroup_integer(path, "memory.current")
|
||||
write_error = None
|
||||
try:
|
||||
@@ -927,6 +1091,7 @@ def execute_cgroup_reclaim_candidate(candidate):
|
||||
after = collect_memory_snapshot()
|
||||
after_available = safe_int((after.get("memory") or {}).get("availableBytes"), None)
|
||||
cgroup_after = read_cgroup_integer(path, "memory.current")
|
||||
root_evidence_after = collect_root_file_reclaim_evidence(path) if configured.get("rootCgroupSelector") is not None else None
|
||||
cgroup_delta = max(0, cgroup_before - cgroup_after) if cgroup_before is not None and cgroup_after is not None else 0
|
||||
host_delta = max(0, after_available - before_available) if after_available is not None else 0
|
||||
succeeded = write_error is None or cgroup_delta > 0 or host_delta > 0
|
||||
@@ -949,6 +1114,10 @@ def execute_cgroup_reclaim_candidate(candidate):
|
||||
"kernelWriteError": write_error,
|
||||
"memoryBefore": before,
|
||||
"memoryAfter": after,
|
||||
"rootFileReclaimEvidenceBefore": root_evidence_before,
|
||||
"rootFileReclaimEvidenceAfter": root_evidence_after,
|
||||
"targetGapBeforeBytes": max(0, safe_int(configured.get("targetMemAvailableBytes")) - before_available),
|
||||
"targetGapAfterBytes": max(0, safe_int(configured.get("targetMemAvailableBytes")) - after_available) if after_available is not None else None,
|
||||
}
|
||||
|
||||
def execute_memory_process_candidate(candidate):
|
||||
|
||||
@@ -453,6 +453,45 @@ function compactResolvedSystemdUnit(value: unknown): Record<string, unknown> | u
|
||||
};
|
||||
}
|
||||
|
||||
function compactRootCgroupSelector(value: unknown): string | undefined {
|
||||
const selector = recordOrEmpty(value);
|
||||
return selector.scope === "root" && Object.keys(selector).length === 1 ? "root" : undefined;
|
||||
}
|
||||
|
||||
function compactResolvedRootCgroup(value: unknown): Record<string, unknown> | undefined {
|
||||
const identity = recordOrEmpty(value);
|
||||
if (identity.filesystemType !== "cgroup2"
|
||||
|| identity.mountRoot !== "/"
|
||||
|| typeof identity.mountPoint !== "string"
|
||||
|| typeof identity.majorMinor !== "string"
|
||||
|| positiveInteger(identity.mountId) === null
|
||||
|| positiveInteger(identity.parentMountId) === null
|
||||
|| positiveInteger(identity.inode) === null) return undefined;
|
||||
return {
|
||||
filesystemType: identity.filesystemType,
|
||||
mountRoot: identity.mountRoot,
|
||||
mountPoint: identity.mountPoint,
|
||||
majorMinor: identity.majorMinor,
|
||||
mountId: identity.mountId,
|
||||
parentMountId: identity.parentMountId,
|
||||
inode: identity.inode,
|
||||
};
|
||||
}
|
||||
|
||||
function compactRootFileReclaimEvidence(value: unknown): Record<string, unknown> | undefined {
|
||||
const evidence = recordOrEmpty(value);
|
||||
const host = recordOrEmpty(evidence.host);
|
||||
const rootMemoryStat = recordOrEmpty(evidence.rootMemoryStat);
|
||||
if (evidence.ok !== true || host.source !== "/proc/meminfo") return undefined;
|
||||
return {
|
||||
source: host.source,
|
||||
estimateKind: evidence.estimateKind,
|
||||
hostFileSlabUpperBoundBytes: host.reclaimableFileSlabUpperBoundBytes,
|
||||
rootFileSlabUpperBoundBytes: evidence.rootReclaimableFileSlabUpperBoundBytes,
|
||||
rootInactiveFileBytes: rootMemoryStat.inactive_file,
|
||||
};
|
||||
}
|
||||
|
||||
function compactMemoryPressureCandidate(value: unknown): Record<string, unknown> {
|
||||
const candidate = recordOrEmpty(value);
|
||||
const action = recordOrEmpty(candidate.action);
|
||||
@@ -476,14 +515,22 @@ function compactMemoryPressureCandidate(value: unknown): Record<string, unknown>
|
||||
resolvedPod: compactResolvedPod(candidate.resolvedPod),
|
||||
systemdUnitSelector: compactSystemdUnitSelector(candidate.systemdUnitSelector),
|
||||
resolvedSystemdUnit: compactResolvedSystemdUnit(candidate.resolvedSystemdUnit),
|
||||
rootCgroupSelector: compactRootCgroupSelector(candidate.rootCgroupSelector),
|
||||
resolvedRootCgroup: compactResolvedRootCgroup(candidate.resolvedRootCgroup),
|
||||
requestedMemoryReclaimBytes: candidate.requestedMemoryReclaimBytes,
|
||||
requestedMemoryReclaim: candidate.requestedMemoryReclaim,
|
||||
estimateKind: candidate.estimateKind,
|
||||
swappiness: candidate.swappiness,
|
||||
minimumInactiveAnonBytes: candidate.minimumInactiveAnonBytes,
|
||||
minimumInactiveFileBytes: candidate.minimumInactiveFileBytes,
|
||||
minimumReclaimableFileSlabBytes: candidate.minimumReclaimableFileSlabBytes,
|
||||
memoryCurrentBytes: candidate.memoryCurrentBytes,
|
||||
memorySwapCurrentBytes: candidate.memorySwapCurrentBytes,
|
||||
inactiveAnonBytes: memoryStat.inactive_anon,
|
||||
inactiveFileBytes: memoryStat.inactive_file,
|
||||
fileBytes: memoryStat.file,
|
||||
slabReclaimableBytes: memoryStat.slab_reclaimable,
|
||||
rootFileReclaimEvidence: compactRootFileReclaimEvidence(candidate.rootFileReclaimEvidence),
|
||||
};
|
||||
}
|
||||
const identities = Array.isArray(candidate.processIdentities) ? candidate.processIdentities.map(recordOrEmpty) : [];
|
||||
@@ -524,6 +571,11 @@ function memoryPressureCandidateAllowed(value: unknown): boolean {
|
||||
const resolvedPod = recordOrEmpty(candidate.resolvedPod);
|
||||
const systemdUnitSelector = recordOrEmpty(candidate.systemdUnitSelector);
|
||||
const resolvedSystemdUnit = recordOrEmpty(candidate.resolvedSystemdUnit);
|
||||
const rootCgroupSelector = compactRootCgroupSelector(candidate.rootCgroupSelector);
|
||||
const resolvedRootCgroup = compactResolvedRootCgroup(candidate.resolvedRootCgroup);
|
||||
const reclaimBytes = positiveInteger(candidate.reclaimBytes);
|
||||
const minimumInactiveFileBytes = positiveInteger(candidate.minimumInactiveFileBytes);
|
||||
const minimumReclaimableFileSlabBytes = positiveInteger(candidate.minimumReclaimableFileSlabBytes);
|
||||
const workloadIdentityClosed = typeof workloadSelector.namespace === "string" && typeof resolvedPod.uid === "string";
|
||||
const systemdIdentityClosed = typeof systemdUnitSelector.unit === "string"
|
||||
&& resolvedSystemdUnit.unit === systemdUnitSelector.unit
|
||||
@@ -535,14 +587,23 @@ function memoryPressureCandidateAllowed(value: unknown): boolean {
|
||||
&& candidate.path === `/sys/fs/cgroup${resolvedSystemdUnit.controlGroup}`
|
||||
&& positiveInteger(resolvedSystemdUnit.mainPid) !== null
|
||||
&& positiveInteger(resolvedSystemdUnit.startTicks) !== null;
|
||||
const rootIdentityClosed = rootCgroupSelector === "root"
|
||||
&& resolvedRootCgroup !== undefined
|
||||
&& candidate.path === resolvedRootCgroup.mountPoint
|
||||
&& swappiness === 0
|
||||
&& reclaimBytes !== null
|
||||
&& minimumInactiveFileBytes !== null
|
||||
&& minimumReclaimableFileSlabBytes !== null
|
||||
&& minimumReclaimableFileSlabBytes >= reclaimBytes;
|
||||
return typeof candidate.configId === "string" && candidate.configId.length > 0
|
||||
&& typeof candidate.path === "string" && candidate.path.startsWith("/")
|
||||
&& positiveInteger(candidate.reclaimBytes) !== null
|
||||
&& reclaimBytes !== null
|
||||
&& swappiness !== null && swappiness >= 0 && swappiness <= 200
|
||||
&& (swappiness === 0 || positiveInteger(candidate.minimumInactiveAnonBytes) !== null)
|
||||
&& action.op === "cgroup-v2-memory-reclaim"
|
||||
&& action.allowlist === "yaml-stable-cgroup-selector"
|
||||
&& workloadIdentityClosed !== systemdIdentityClosed;
|
||||
&& (action.allowlist === "yaml-stable-cgroup-selector" || action.allowlist === "yaml-root-cgroup-file-cache")
|
||||
&& [workloadIdentityClosed, systemdIdentityClosed, rootIdentityClosed].filter(Boolean).length === 1
|
||||
&& (rootIdentityClosed === (action.allowlist === "yaml-root-cgroup-file-cache"));
|
||||
}
|
||||
if (candidate.kind !== "browser-orphan-process-tree" && candidate.kind !== "stale-web-observer-process-tree") return false;
|
||||
const compact = compactMemoryPressureCandidate(candidate);
|
||||
@@ -596,6 +657,9 @@ function compactMemoryPressureCandidateScan(value: unknown): Record<string, unkn
|
||||
matchedPodCount: protectedItem.matchedPodCount,
|
||||
memoryCurrentBytes: protectedItem.memoryCurrentBytes,
|
||||
inactiveAnonBytes: protectedItem.inactiveAnonBytes,
|
||||
rootCgroupSelector: compactRootCgroupSelector(protectedItem.rootCgroupSelector),
|
||||
resolvedRootCgroup: compactResolvedRootCgroup(protectedItem.resolvedRootCgroup),
|
||||
rootFileReclaimEvidence: compactRootFileReclaimEvidence(protectedItem.rootFileReclaimEvidence),
|
||||
workloadSelector: compactWorkloadSelector(protectedItem.workloadSelector),
|
||||
resolvedPod: compactResolvedPod(protectedItem.resolvedPod),
|
||||
systemdUnitSelector: compactSystemdUnitSelector(protectedItem.systemdUnitSelector),
|
||||
|
||||
Reference in New Issue
Block a user