Codex
538 lines
24 KiB
Python
538 lines
24 KiB
Python
def memory_distribution_config():
|
|
config = MEMORY_CONFIG.get("attribution")
|
|
if not isinstance(config, dict):
|
|
raise RuntimeError("required YAML memory attribution policy is missing")
|
|
required_integers = {
|
|
"cgroupScanLimit": (1, 1000000),
|
|
"topLevelLimit": (1, 100),
|
|
"systemServiceLimit": (1, 100),
|
|
"podLimit": (1, 100),
|
|
"processLimit": (1, 100),
|
|
"processPssLimit": (1, 50),
|
|
"processSwapLimit": (1, 100),
|
|
"commandTimeoutSeconds": (1, 60),
|
|
"targetMemAvailableBytes": (1, 1 << 50),
|
|
"namespaceCountConcurrency": (1, 32),
|
|
}
|
|
for key, (minimum, maximum) in required_integers.items():
|
|
value = config.get(key)
|
|
if isinstance(value, bool) or not isinstance(value, int) or value < minimum or value > maximum:
|
|
raise RuntimeError("invalid YAML memory attribution integer: %s" % key)
|
|
cgroup_root = config.get("cgroupRoot")
|
|
kubeconfig = config.get("kubeconfigPath")
|
|
resources = config.get("objectCountResources")
|
|
namespaces = config.get("namespaceCountNamespaces")
|
|
if not isinstance(cgroup_root, str) or not cgroup_root.startswith("/") or os.path.normpath(cgroup_root) != cgroup_root:
|
|
raise RuntimeError("invalid YAML memory attribution cgroupRoot")
|
|
if not isinstance(kubeconfig, str) or not kubeconfig.startswith("/") or os.path.normpath(kubeconfig) != kubeconfig:
|
|
raise RuntimeError("invalid YAML memory attribution kubeconfigPath")
|
|
if not isinstance(resources, list) or not resources or any(
|
|
not isinstance(item, str) or not re.match(r"^[a-z0-9.-]{1,100}$", item) for item in resources
|
|
):
|
|
raise RuntimeError("invalid YAML memory attribution objectCountResources")
|
|
if not isinstance(namespaces, list) or not namespaces or any(
|
|
not isinstance(item, str) or not re.match(r"^[a-z0-9]([-a-z0-9]*[a-z0-9])?$", item) for item in namespaces
|
|
):
|
|
raise RuntimeError("invalid YAML memory attribution namespaceCountNamespaces")
|
|
return config
|
|
|
|
def read_memory_distribution_key_values(path, multiplier=1):
|
|
result = {}
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
for line in handle:
|
|
parts = line.replace(":", "", 1).split()
|
|
if len(parts) < 2:
|
|
continue
|
|
try:
|
|
result[parts[0]] = int(parts[1]) * multiplier
|
|
except ValueError:
|
|
continue
|
|
except OSError:
|
|
return {}
|
|
return result
|
|
|
|
def read_memory_distribution_integer(path):
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
return int(handle.read().strip())
|
|
except (OSError, ValueError):
|
|
return None
|
|
|
|
def memory_distribution_cgroup_row(root, path):
|
|
stat = read_memory_distribution_key_values(os.path.join(path, "memory.stat"))
|
|
file_bytes = safe_int(stat.get("file"))
|
|
slab_reclaimable = safe_int(stat.get("slab_reclaimable"))
|
|
return {
|
|
"path": os.path.relpath(path, root),
|
|
"currentBytes": read_memory_distribution_integer(os.path.join(path, "memory.current")),
|
|
"swapCurrentBytes": read_memory_distribution_integer(os.path.join(path, "memory.swap.current")),
|
|
"anonBytes": stat.get("anon"),
|
|
"fileBytes": stat.get("file"),
|
|
"inactiveAnonBytes": stat.get("inactive_anon"),
|
|
"inactiveFileBytes": stat.get("inactive_file"),
|
|
"reclaimableFileSlabUpperBoundBytes": file_bytes + slab_reclaimable,
|
|
}
|
|
|
|
def memory_distribution_limit(config, key):
|
|
configured = safe_int(config.get(key))
|
|
return min(100, configured * 5) if bool(OPTIONS.get("full")) else configured
|
|
|
|
def memory_distribution_top(rows, limit):
|
|
return sorted(rows, key=lambda item: safe_int(item.get("currentBytes")), reverse=True)[:limit]
|
|
|
|
def memory_distribution_command_summary(result):
|
|
return {
|
|
"ok": result.get("exitCode") == 0,
|
|
"exitCode": result.get("exitCode"),
|
|
"timedOut": result.get("timedOut") is True,
|
|
"durationMs": result.get("durationMs"),
|
|
}
|
|
|
|
def memory_distribution_runtime_maps(config):
|
|
timeout = safe_int(config.get("commandTimeoutSeconds"))
|
|
docker = command(["docker", "ps", "--no-trunc", "--format", "{{.ID}}\t{{.Names}}"], timeout)
|
|
docker_names = {}
|
|
if docker.get("exitCode") == 0:
|
|
for line in docker.get("stdout", "").splitlines():
|
|
parts = line.split("\t", 1)
|
|
if len(parts) == 2 and re.match(r"^[a-f0-9]{64}$", parts[0]):
|
|
docker_names[parts[0]] = parts[1]
|
|
cri = command(["k3s", "crictl", "pods", "-o", "json"], timeout)
|
|
pod_names = {}
|
|
if cri.get("exitCode") == 0:
|
|
try:
|
|
payload = json.loads(cri.get("stdout") or "{}")
|
|
for item in payload.get("items") or []:
|
|
metadata = item.get("metadata") or {}
|
|
uid = str(metadata.get("uid") or "")
|
|
if uid:
|
|
pod_names[uid] = {
|
|
"namespace": metadata.get("namespace"),
|
|
"name": metadata.get("name"),
|
|
"state": item.get("state"),
|
|
}
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
pass
|
|
return docker_names, pod_names, {
|
|
"docker": memory_distribution_command_summary(docker),
|
|
"cri": memory_distribution_command_summary(cri),
|
|
}
|
|
|
|
def collect_memory_distribution_cgroups(config, docker_names, pod_names):
|
|
root = config.get("cgroupRoot")
|
|
scan_limit = safe_int(config.get("cgroupScanLimit"))
|
|
top_level = []
|
|
system_services = []
|
|
pods = []
|
|
scanned = 0
|
|
truncated = False
|
|
pod_pattern = re.compile(r"pod([0-9a-f_]{36})\.slice$")
|
|
docker_pattern = re.compile(r"docker-([a-f0-9]{64})\.scope$")
|
|
for current, directories, files in os.walk(root, topdown=True, followlinks=False):
|
|
directories[:] = sorted(name for name in directories if not os.path.islink(os.path.join(current, name)))
|
|
scanned += 1
|
|
if scanned > scan_limit:
|
|
truncated = True
|
|
break
|
|
if "memory.current" not in files:
|
|
continue
|
|
row = memory_distribution_cgroup_row(root, current)
|
|
parent = os.path.dirname(current)
|
|
if parent == root:
|
|
top_level.append(row)
|
|
if parent == os.path.join(root, "system.slice"):
|
|
match = docker_pattern.search(os.path.basename(current))
|
|
if match:
|
|
row["owner"] = {"kind": "docker", "name": docker_names.get(match.group(1)), "id": match.group(1)}
|
|
system_services.append(row)
|
|
match = pod_pattern.search(os.path.basename(current))
|
|
if match:
|
|
uid = match.group(1).replace("_", "-")
|
|
row["owner"] = {"kind": "kubernetes-pod", "uid": uid, **(pod_names.get(uid) or {})}
|
|
pods.append(row)
|
|
directories[:] = []
|
|
return {
|
|
"scan": {"root": root, "scannedCount": scanned, "limit": scan_limit, "truncated": truncated},
|
|
"topLevel": memory_distribution_top(top_level, memory_distribution_limit(config, "topLevelLimit")),
|
|
"systemServices": memory_distribution_top(system_services, memory_distribution_limit(config, "systemServiceLimit")),
|
|
"pods": memory_distribution_top(pods, memory_distribution_limit(config, "podLimit")),
|
|
"podCgroupCount": len(pods),
|
|
}
|
|
|
|
def memory_distribution_cgroup_for_pid(pid):
|
|
try:
|
|
with open("/proc/%s/cgroup" % int(pid), "r", encoding="utf-8") as handle:
|
|
for line in handle:
|
|
parts = line.rstrip("\n").split(":", 2)
|
|
if len(parts) == 3 and parts[0] == "0" and parts[1] == "":
|
|
return parts[2]
|
|
except (OSError, ValueError):
|
|
pass
|
|
return None
|
|
|
|
def memory_distribution_pss(pid):
|
|
values = read_memory_distribution_key_values("/proc/%s/smaps_rollup" % int(pid), 1024)
|
|
if not values:
|
|
return None
|
|
return {
|
|
"pssBytes": values.get("Pss"),
|
|
"pssAnonBytes": values.get("Pss_Anon"),
|
|
"pssFileBytes": values.get("Pss_File"),
|
|
"swapPssBytes": values.get("SwapPss"),
|
|
}
|
|
|
|
def memory_distribution_process_swap(pid):
|
|
values = read_memory_distribution_key_values("/proc/%s/status" % int(pid), 1024)
|
|
return values.get("VmSwap")
|
|
|
|
def memory_distribution_process_owner(cgroup_path, docker_names, pod_names):
|
|
if not isinstance(cgroup_path, str):
|
|
return None
|
|
docker_match = re.search(r"docker-([a-f0-9]{64})\.scope(?:/|$)", cgroup_path)
|
|
if docker_match is not None:
|
|
container_id = docker_match.group(1)
|
|
return {"kind": "docker", "id": container_id, "name": docker_names.get(container_id)}
|
|
match = re.search(r"pod([0-9a-f_]{36})\.slice(?:/|$)", cgroup_path)
|
|
if match is None:
|
|
return None
|
|
uid = match.group(1).replace("_", "-")
|
|
return {"kind": "kubernetes-pod", "uid": uid, **(pod_names.get(uid) or {})}
|
|
|
|
def collect_memory_distribution_processes(config, docker_names, pod_names):
|
|
result = command(["ps", "-eo", "pid=,ppid=,rss=,stat=,comm=,args="], safe_int(config.get("commandTimeoutSeconds")))
|
|
if result.get("exitCode") != 0:
|
|
return {"ok": False, "command": memory_distribution_command_summary(result), "top": [], "zombies": {}}
|
|
rows = []
|
|
zombie_by_comm = {}
|
|
zombie_by_parent = {}
|
|
for line in result.get("stdout", "").splitlines():
|
|
parts = line.strip().split(None, 5)
|
|
if len(parts) < 5:
|
|
continue
|
|
pid = safe_int(parts[0])
|
|
ppid = safe_int(parts[1])
|
|
rss = safe_int(parts[2]) * 1024
|
|
state = parts[3]
|
|
comm = parts[4]
|
|
args = parts[5] if len(parts) > 5 else comm
|
|
row = {
|
|
"pid": pid,
|
|
"ppid": ppid,
|
|
"comm": comm,
|
|
"state": state,
|
|
"rssBytes": rss,
|
|
"swapBytes": memory_distribution_process_swap(pid),
|
|
"commandPreview": redact_command_preview(args)[:120],
|
|
}
|
|
rows.append(row)
|
|
if state.startswith("Z"):
|
|
zombie_by_comm[comm] = zombie_by_comm.get(comm, 0) + 1
|
|
zombie_by_parent[ppid] = zombie_by_parent.get(ppid, 0) + 1
|
|
by_pid = {item.get("pid"): item for item in rows}
|
|
top = sorted(rows, key=lambda item: safe_int(item.get("rssBytes")), reverse=True)[:memory_distribution_limit(config, "processLimit")]
|
|
top_swap = sorted(rows, key=lambda item: safe_int(item.get("swapBytes")), reverse=True)[:memory_distribution_limit(config, "processSwapLimit")]
|
|
pss_limit = memory_distribution_limit(config, "processPssLimit")
|
|
for index, item in enumerate(top):
|
|
item["cgroupPath"] = memory_distribution_cgroup_for_pid(item.get("pid"))
|
|
item["owner"] = memory_distribution_process_owner(item.get("cgroupPath"), docker_names, pod_names)
|
|
if index < pss_limit:
|
|
item.update(memory_distribution_pss(item.get("pid")) or {})
|
|
for item in top_swap:
|
|
item["cgroupPath"] = memory_distribution_cgroup_for_pid(item.get("pid"))
|
|
item["owner"] = memory_distribution_process_owner(item.get("cgroupPath"), docker_names, pod_names)
|
|
parent_rows = []
|
|
for ppid, count in sorted(zombie_by_parent.items(), key=lambda item: item[1], reverse=True)[:5]:
|
|
parent = by_pid.get(ppid) or {}
|
|
cgroup_path = memory_distribution_cgroup_for_pid(ppid)
|
|
owner = memory_distribution_process_owner(cgroup_path, docker_names, pod_names)
|
|
parent_rows.append({
|
|
"ppid": ppid,
|
|
"parentComm": parent.get("comm"),
|
|
"zombieCount": count,
|
|
"cgroupPath": cgroup_path,
|
|
"owner": owner,
|
|
"classification": "unreaped-child-process-leak",
|
|
"remediation": "fix-child-reaping-and-rollout-owner",
|
|
})
|
|
return {
|
|
"ok": True,
|
|
"command": memory_distribution_command_summary(result),
|
|
"processCount": len(rows),
|
|
"rssTotalBytes": sum(safe_int(item.get("rssBytes")) for item in rows),
|
|
"swapTotalBytes": sum(safe_int(item.get("swapBytes")) for item in rows),
|
|
"top": top,
|
|
"topSwap": top_swap,
|
|
"zombies": {
|
|
"count": sum(zombie_by_comm.values()),
|
|
"byComm": [{"comm": comm, "count": count} for comm, count in sorted(zombie_by_comm.items(), key=lambda item: item[1], reverse=True)[:5]],
|
|
"byParent": parent_rows,
|
|
},
|
|
}
|
|
|
|
def parse_memory_distribution_pressure(path):
|
|
result = {}
|
|
try:
|
|
with open(path, "r", encoding="utf-8") as handle:
|
|
for line in handle:
|
|
parts = line.split()
|
|
if not parts:
|
|
continue
|
|
result[parts[0]] = {
|
|
key: float(value) if key != "total" else safe_int(value)
|
|
for key, value in (item.split("=", 1) for item in parts[1:] if "=" in item)
|
|
}
|
|
except OSError:
|
|
pass
|
|
return result
|
|
|
|
def collect_memory_distribution_metrics(config):
|
|
timeout = safe_int(config.get("commandTimeoutSeconds"))
|
|
kubeconfig = config.get("kubeconfigPath")
|
|
result = command(["kubectl", "--kubeconfig", kubeconfig, "get", "--raw", "/metrics"], timeout)
|
|
selected = set(config.get("objectCountResources") or [])
|
|
objects = {}
|
|
runtime = {}
|
|
if result.get("exitCode") == 0:
|
|
object_pattern = re.compile(r'^apiserver_storage_objects\{resource="([^"]+)"\}\s+([0-9.eE+-]+)$')
|
|
runtime_names = set([
|
|
"process_resident_memory_bytes", "process_virtual_memory_bytes", "go_goroutines",
|
|
"go_memstats_heap_alloc_bytes", "go_memstats_heap_inuse_bytes", "go_memstats_heap_sys_bytes",
|
|
])
|
|
for line in result.get("stdout", "").splitlines():
|
|
match = object_pattern.match(line)
|
|
if match and match.group(1) in selected:
|
|
objects[match.group(1)] = int(float(match.group(2)))
|
|
continue
|
|
parts = line.split()
|
|
if len(parts) == 2 and parts[0] in runtime_names:
|
|
try:
|
|
runtime[parts[0]] = float(parts[1])
|
|
except ValueError:
|
|
pass
|
|
return {"command": memory_distribution_command_summary(result), "objects": objects, "runtime": runtime}
|
|
|
|
def memory_distribution_namespace_resource_path(resource, namespace):
|
|
encoded_namespace = urllib.parse.quote(namespace, safe="")
|
|
paths = {
|
|
"pods": "/api/v1/namespaces/%s/pods?limit=1" % encoded_namespace,
|
|
"events": "/api/v1/namespaces/%s/events?limit=1" % encoded_namespace,
|
|
"secrets": "/api/v1/namespaces/%s/secrets?limit=1" % encoded_namespace,
|
|
"persistentvolumeclaims": "/api/v1/namespaces/%s/persistentvolumeclaims?limit=1" % encoded_namespace,
|
|
"pipelineruns.tekton.dev": "/apis/tekton.dev/v1/namespaces/%s/pipelineruns?limit=1" % encoded_namespace,
|
|
"taskruns.tekton.dev": "/apis/tekton.dev/v1/namespaces/%s/taskruns?limit=1" % encoded_namespace,
|
|
}
|
|
return paths.get(resource)
|
|
|
|
def memory_distribution_namespace_count(config, namespace, resource):
|
|
path = memory_distribution_namespace_resource_path(resource, namespace)
|
|
if path is None:
|
|
return namespace, resource, None, "unsupported-resource"
|
|
result = command([
|
|
"kubectl", "--kubeconfig", config.get("kubeconfigPath"), "get", "--raw", path,
|
|
], safe_int(config.get("commandTimeoutSeconds")))
|
|
if result.get("exitCode") != 0:
|
|
return namespace, resource, None, "read-failed"
|
|
try:
|
|
payload = json.loads(result.get("stdout") or "{}")
|
|
metadata = payload.get("metadata") or {}
|
|
count = len(payload.get("items") or []) + safe_int(metadata.get("remainingItemCount"))
|
|
return namespace, resource, count, None
|
|
except (TypeError, ValueError, json.JSONDecodeError):
|
|
return namespace, resource, None, "parse-failed"
|
|
|
|
def collect_memory_distribution_namespace_counts(config):
|
|
import concurrent.futures
|
|
|
|
namespaces = list(config.get("namespaceCountNamespaces") or [])
|
|
resources = [
|
|
item for item in (config.get("objectCountResources") or [])
|
|
if memory_distribution_namespace_resource_path(item, namespaces[0]) is not None
|
|
]
|
|
jobs = [(namespace, resource) for namespace in namespaces for resource in resources]
|
|
rows = {namespace: {"namespace": namespace} for namespace in namespaces}
|
|
errors = []
|
|
with concurrent.futures.ThreadPoolExecutor(max_workers=safe_int(config.get("namespaceCountConcurrency"))) as pool:
|
|
futures = [pool.submit(memory_distribution_namespace_count, config, namespace, resource) for namespace, resource in jobs]
|
|
for future in concurrent.futures.as_completed(futures):
|
|
namespace, resource, count, error = future.result()
|
|
if error is None:
|
|
rows[namespace][resource] = count
|
|
else:
|
|
errors.append({"namespace": namespace, "resource": resource, "error": error})
|
|
ordered = sorted(
|
|
rows.values(),
|
|
key=lambda item: sum(safe_int(value) for key, value in item.items() if key != "namespace"),
|
|
reverse=True,
|
|
)
|
|
return {"rows": ordered, "errors": errors, "complete": not errors}
|
|
|
|
def compact_memory_distribution_cgroup(item):
|
|
compact = {
|
|
key: item.get(key)
|
|
for key in ["currentBytes", "swapCurrentBytes", "anonBytes"]
|
|
if item.get(key) is not None
|
|
}
|
|
owner = item.get("owner") if isinstance(item.get("owner"), dict) else None
|
|
if owner is not None:
|
|
compact["owner"] = {
|
|
key: owner.get(key)
|
|
for key in ["kind", "namespace", "name"]
|
|
if owner.get(key) is not None
|
|
}
|
|
else:
|
|
compact["path"] = item.get("path")
|
|
return compact
|
|
|
|
def compact_memory_distribution_process(item):
|
|
compact = {
|
|
key: item.get(key)
|
|
for key in ["pid", "comm", "rssBytes", "swapBytes", "pssBytes"]
|
|
if item.get(key) is not None
|
|
}
|
|
owner = item.get("owner") if isinstance(item.get("owner"), dict) else None
|
|
if owner is not None:
|
|
compact["owner"] = {
|
|
key: owner.get(key)
|
|
for key in ["kind", "namespace", "name"]
|
|
if owner.get(key) is not None
|
|
}
|
|
else:
|
|
compact["cgroupPath"] = item.get("cgroupPath")
|
|
return compact
|
|
|
|
def compact_memory_distribution_pressure(pressure):
|
|
return {
|
|
level: {
|
|
key: values.get(key)
|
|
for key in ["avg10", "avg60"]
|
|
if values.get(key) is not None
|
|
}
|
|
for level, values in pressure.items()
|
|
if isinstance(values, dict)
|
|
}
|
|
|
|
def compact_memory_distribution_zombies(zombies):
|
|
by_parent = []
|
|
for item in zombies.get("byParent") or []:
|
|
compact = {
|
|
key: item.get(key)
|
|
for key in ["ppid", "parentComm", "zombieCount", "classification", "remediation"]
|
|
if item.get(key) is not None
|
|
}
|
|
owner = item.get("owner") if isinstance(item.get("owner"), dict) else None
|
|
if owner is not None:
|
|
compact["owner"] = {
|
|
key: owner.get(key)
|
|
for key in ["kind", "namespace", "name"]
|
|
if owner.get(key) is not None
|
|
}
|
|
by_parent.append(compact)
|
|
return {
|
|
"count": zombies.get("count"),
|
|
"byComm": (zombies.get("byComm") or [])[:3],
|
|
"byParent": by_parent[:3],
|
|
}
|
|
|
|
def collect_memory_distribution(observed_at):
|
|
try:
|
|
config = memory_distribution_config()
|
|
except RuntimeError as exc:
|
|
return {"ok": False, "action": "gc remote memory-distribution", "providerId": PROVIDER_ID, "dryRun": True, "mutation": False, "error": str(exc)}
|
|
started = time.time()
|
|
docker_names, pod_names, runtime_commands = memory_distribution_runtime_maps(config)
|
|
host_snapshot = collect_memory_snapshot()
|
|
meminfo = read_memory_distribution_key_values("/proc/meminfo", 1024)
|
|
cgroups = collect_memory_distribution_cgroups(config, docker_names, pod_names)
|
|
processes = collect_memory_distribution_processes(config, docker_names, pod_names)
|
|
metrics = collect_memory_distribution_metrics(config)
|
|
namespace_counts = collect_memory_distribution_namespace_counts(config)
|
|
available = safe_int((host_snapshot.get("memory") or {}).get("availableBytes"))
|
|
target = safe_int(config.get("targetMemAvailableBytes"))
|
|
cached = safe_int(meminfo.get("Cached")) + safe_int(meminfo.get("Buffers"))
|
|
slab_reclaimable = safe_int(meminfo.get("SReclaimable"))
|
|
payload = {
|
|
"ok": cgroups.get("scan", {}).get("truncated") is not True,
|
|
"action": "gc remote memory-distribution",
|
|
"providerId": PROVIDER_ID,
|
|
"dryRun": True,
|
|
"mutation": False,
|
|
"observedAt": observed_at,
|
|
"durationMs": int((time.time() - started) * 1000),
|
|
"summary": {
|
|
"metric": "MemAvailable",
|
|
"source": "/proc/meminfo",
|
|
"swapExcludedFromAvailability": True,
|
|
"availableBytes": available,
|
|
"targetAvailableBytes": target,
|
|
"targetGapBytes": max(0, target - available + 1),
|
|
"anonPagesBytes": meminfo.get("AnonPages"),
|
|
"cachedAndBuffersBytes": cached,
|
|
"slabBytes": meminfo.get("Slab"),
|
|
"reclaimableFileSlabUpperBoundBytes": cached + slab_reclaimable,
|
|
"swapFreeBytes": safe_int((host_snapshot.get("swap") or {}).get("freeBytes")),
|
|
"memoryPsiFullAvg60": (parse_memory_distribution_pressure("/proc/pressure/memory").get("full") or {}).get("avg60"),
|
|
},
|
|
"host": {
|
|
"snapshot": host_snapshot,
|
|
"pages": {key: meminfo.get(key) for key in [
|
|
"MemFree", "Buffers", "Cached", "Active", "Inactive", "Active(anon)", "Inactive(anon)",
|
|
"Active(file)", "Inactive(file)", "AnonPages", "Mapped", "Shmem", "Slab", "SReclaimable",
|
|
"SUnreclaim", "KernelStack", "PageTables", "SwapCached",
|
|
]},
|
|
"pressure": {resource: parse_memory_distribution_pressure("/proc/pressure/%s" % resource) for resource in ["memory", "io", "cpu"]},
|
|
},
|
|
"cgroups": cgroups,
|
|
"processes": processes,
|
|
"kubernetes": {**metrics, "namespaceCounts": namespace_counts},
|
|
"collection": {
|
|
"configSource": "config/unidesk-cli.yaml#gc.remote.targets.%s.memoryPressure.attribution" % PROVIDER_ID,
|
|
"runtimeMaps": runtime_commands,
|
|
"podRuntimeMapCount": len(pod_names),
|
|
"dockerRuntimeMapCount": len(docker_names),
|
|
"full": bool(OPTIONS.get("full")),
|
|
},
|
|
"policy": {
|
|
"analysisOnly": True,
|
|
"next": "Use this distribution before designing or running memory-pressure GC candidates.",
|
|
},
|
|
}
|
|
if bool(OPTIONS.get("full")):
|
|
return payload
|
|
payload["host"] = {
|
|
"pressure": {"memory": compact_memory_distribution_pressure(parse_memory_distribution_pressure("/proc/pressure/memory"))},
|
|
}
|
|
payload["cgroups"] = {
|
|
"scan": cgroups.get("scan"),
|
|
"podCgroupCount": cgroups.get("podCgroupCount"),
|
|
"topLevel": [compact_memory_distribution_cgroup(item) for item in cgroups.get("topLevel") or []],
|
|
"systemServices": [compact_memory_distribution_cgroup(item) for item in cgroups.get("systemServices") or []],
|
|
"pods": [compact_memory_distribution_cgroup(item) for item in cgroups.get("pods") or []],
|
|
}
|
|
payload["processes"] = {
|
|
"ok": processes.get("ok"),
|
|
"processCount": processes.get("processCount"),
|
|
"rssTotalBytes": processes.get("rssTotalBytes"),
|
|
"swapTotalBytes": processes.get("swapTotalBytes"),
|
|
"top": [compact_memory_distribution_process(item) for item in processes.get("top") or []],
|
|
"topSwap": [compact_memory_distribution_process(item) for item in processes.get("topSwap") or []],
|
|
"zombies": compact_memory_distribution_zombies(processes.get("zombies") or {}),
|
|
}
|
|
payload["kubernetes"] = {
|
|
"objects": metrics.get("objects"),
|
|
"namespaceCounts": namespace_counts,
|
|
}
|
|
if (metrics.get("command", {}).get("ok") is not True):
|
|
payload["kubernetes"]["commandError"] = metrics.get("command")
|
|
payload["collection"] = {
|
|
"configSource": payload.get("collection", {}).get("configSource"),
|
|
"podRuntimeMapCount": len(pod_names),
|
|
"dockerRuntimeMapCount": len(docker_names),
|
|
"runtimeMapErrors": {
|
|
key: value for key, value in runtime_commands.items()
|
|
if value.get("ok") is not True
|
|
},
|
|
"full": False,
|
|
}
|
|
return payload
|