import { execFile } from "node:child_process"; import { promisify } from "node:util"; const execFileAsync = promisify(execFile); const defaultKubeconfig = "/etc/rancher/k3s/k3s.yaml"; const defaultNamespace = "hwlab-dev"; const defaultDeployment = "hwlab-cloud-api"; const defaultConfigMap = "hwlab-cloud-api-code-agent-hotfix"; const defaultMountPath = "/app/internal/cloud/code-agent-chat.ts"; const defaultSubPath = "code-agent-chat.ts"; const defaultMarker = "runtime-hotfix-pc-gateway-shell"; const defaultAnnotationKey = "hwlab.pikastech.local/pc-gateway-shell-hotfix"; const defaultPodLabelSelector = "app.kubernetes.io/name=hwlab-cloud-api"; const forbiddenKubectlVerbs = new Set([ "apply", "patch", "rollout", "restart", "delete", "create", "replace", "scale", "set", "edit", "annotate", "label" ]); const allowedKubectlVerbs = new Set(["get", "describe", "exec", "auth", "config", "version"]); const classificationOrder = Object.freeze([ "hotfix-configmap-present", "deployment-mounts-hotfix", "pod-loads-hotfix-marker", "source-artifact-expected", "rollback-required", "no-hotfix-detected", "unknown-needs-manual-readonly-check" ]); export async function runDevRuntimeHotfixAuditCli(argv = process.argv.slice(2), options = {}) { const args = parseArgs(argv); if (args.help) { process.stdout.write(`${usage()}\n`); return 0; } const report = await buildDevRuntimeHotfixAuditReport(args, options); process.stdout.write(`${JSON.stringify(report, null, args.pretty ? 2 : 0)}\n`); return 0; } export function parseArgs(argv = []) { const args = { collectReadonly: false, pretty: false, timeoutMs: 5000, kubeconfig: defaultKubeconfig, namespace: defaultNamespace, deployment: defaultDeployment, configMap: defaultConfigMap, mountPath: defaultMountPath, subPath: defaultSubPath, marker: defaultMarker, annotationKey: defaultAnnotationKey, podLabelSelector: defaultPodLabelSelector, help: false }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === "--collect-readonly") args.collectReadonly = true; else if (arg === "--plan" || arg === "--dry-run") args.collectReadonly = false; else if (arg === "--pretty") args.pretty = true; else if (arg === "--timeout-ms") args.timeoutMs = parsePositiveInt(requireValue(argv[++index], arg), arg); else if (arg === "--kubeconfig") args.kubeconfig = requireValue(argv[++index], arg); else if (arg === "--namespace") args.namespace = requireValue(argv[++index], arg); else if (arg === "--deployment") args.deployment = requireValue(argv[++index], arg); else if (arg === "--configmap") args.configMap = requireValue(argv[++index], arg); else if (arg === "--mount-path") args.mountPath = requireValue(argv[++index], arg); else if (arg === "--sub-path") args.subPath = requireValue(argv[++index], arg); else if (arg === "--marker") args.marker = requireValue(argv[++index], arg); else if (arg === "--annotation-key") args.annotationKey = requireValue(argv[++index], arg); else if (arg === "--pod-label-selector") args.podLabelSelector = requireValue(argv[++index], arg); else if (arg === "--help" || arg === "-h") args.help = true; else throw new Error(`unknown argument: ${arg}`); } return args; } export async function buildDevRuntimeHotfixAuditReport(args = {}, options = {}) { const parsed = { ...parseArgs([]), ...args }; const report = { issueRefs: ["pikasTech/HWLAB#462", "pikasTech/HWLAB#465", "pikasTech/HWLAB#7", "pikasTech/HWLAB#239"], relatedDivisionOfWork: { capabilityRouting: "pikasTech/HWLAB#460", sourceRunner: "pikasTech/HWLAB#461", runtimeHotfixRunbookAndAudit: "pikasTech/HWLAB#462", serviceIdDiagnostics: "pikasTech/HWLAB#463", devLiveSmoke: "pikasTech/HWLAB#464" }, mode: parsed.collectReadonly ? "collect-readonly" : "plan", target: { environment: "DEV", namespace: parsed.namespace, deployment: parsed.deployment, configMap: parsed.configMap, annotationKey: parsed.annotationKey, mountPath: parsed.mountPath, subPath: parsed.subPath, marker: parsed.marker, kubeconfig: parsed.kubeconfig, requiredNode: "d601" }, safety: { prodAllowed: false, secretValuesRead: false, secretResourcesRead: false, k8sWriteAttempted: false, k8sWriteCommandsExecuted: [], defaultExecutesKubectl: parsed.collectReadonly, allowedKubectlVerbs: [...allowedKubectlVerbs].sort(), forbiddenKubectlVerbs: [...forbiddenKubectlVerbs].sort() }, requiredOperatorEvidence: [ "Deployment", "pod", "ConfigMap", "annotation", "volume", "mountPath", "subPath", "traceId", "operationId", "validationCommand", "rollbackMethod", "sourceFollowUp" ], commandPlan: buildCommandPlan(parsed), checks: null, classifications: [], conclusion: null }; if (!parsed.collectReadonly) { report.classifications = orderClassifications([ "source-artifact-expected", "unknown-needs-manual-readonly-check" ]); report.conclusion = { status: "unknown", summary: "Plan mode did not query D601; run --collect-readonly on D601 with the native k3s kubeconfig to classify the current hotfix state." }; return report; } const checks = await collectReadonlyChecks(parsed, options); report.checks = checks; report.classifications = classifyHotfixState(checks); report.conclusion = summarizeClassifications(report.classifications); return report; } async function collectReadonlyChecks(args, options) { const runCommand = options.runCommand ?? defaultRunCommand; const nodeProbe = await runKubectl(args, ["get", "nodes", "-o", "json"], runCommand); const nodeJson = parseJson(nodeProbe); const nodeNames = (nodeJson?.items ?? []).map((item) => item?.metadata?.name).filter(Boolean); const configMapProbe = await runKubectl(args, ["-n", args.namespace, "get", "configmap", args.configMap, "-o", "json"], runCommand); const configMapJson = parseJson(configMapProbe); const deploymentProbe = await runKubectl(args, ["-n", args.namespace, "get", "deployment", args.deployment, "-o", "json"], runCommand); const deploymentJson = parseJson(deploymentProbe); const deployment = summarizeDeployment(deploymentJson, args); const podsProbe = await runKubectl(args, [ "-n", args.namespace, "get", "pods", "-l", args.podLabelSelector, "--field-selector=status.phase=Running", "--sort-by=.metadata.creationTimestamp", "-o", "json" ], runCommand); const podsJson = parseJson(podsProbe); const pod = summarizeSelectedPod(podsJson); let markerProbe = null; let syntaxProbe = null; if (pod?.name) { markerProbe = await runKubectl(args, [ "-n", args.namespace, "exec", pod.name, "--", "sh", "-lc", `grep -n ${shellQuote(args.marker)} ${shellQuote(args.mountPath)} | head` ], runCommand); syntaxProbe = await runKubectl(args, [ "-n", args.namespace, "exec", pod.name, "--", "sh", "-lc", `node --check ${shellQuote(args.mountPath)}` ], runCommand); } return { node: { ok: nodeProbe.ok, command: nodeProbe.command, names: nodeNames, requiredNodePresent: nodeNames.includes("d601"), probe: compactProbe(nodeProbe, { includeStdout: false }) }, configMap: { command: configMapProbe.command, present: configMapProbe.ok && Boolean(configMapJson), unknown: !configMapProbe.ok && !looksNotFound(configMapProbe), keys: configMapJson ? Object.keys(configMapJson.data ?? {}).sort() : [], probe: compactProbe(configMapProbe, { includeStdout: false }) }, deployment: { command: deploymentProbe.command, present: deploymentProbe.ok && Boolean(deploymentJson), unknown: !deploymentProbe.ok || !deploymentJson, ...deployment, probe: compactProbe(deploymentProbe, { includeStdout: false }) }, pod: { command: podsProbe.command, selected: pod, unknown: !podsProbe.ok || !podsJson, marker: markerProbe ? { command: markerProbe.command, found: markerProbe.ok, matchCount: markerProbe.ok ? markerProbe.stdout.split("\n").filter(Boolean).length : 0, probe: compactProbe(markerProbe, { includeStdout: false }) } : null, syntax: syntaxProbe ? { command: syntaxProbe.command, ok: syntaxProbe.ok, probe: compactProbe(syntaxProbe) } : null, probe: compactProbe(podsProbe, { includeStdout: false }) } }; } function summarizeDeployment(document, args) { if (!document) { return { generation: null, annotations: {}, hotfixVolumes: [], hotfixMounts: [], mountsHotfix: false }; } const volumes = document.spec?.template?.spec?.volumes ?? []; const containers = document.spec?.template?.spec?.containers ?? []; const hotfixVolumeNames = new Set( volumes .filter((volume) => volume?.configMap?.name === args.configMap) .map((volume) => volume.name) .filter(Boolean) ); const hotfixMounts = containers.flatMap((container, containerIndex) => (container.volumeMounts ?? []) .map((mount, mountIndex) => ({ container, containerIndex, mount, mountIndex })) .filter(({ mount }) => hotfixVolumeNames.has(mount.name) || mount.mountPath === args.mountPath || mount.subPath === args.subPath ) .map(({ container, containerIndex, mount, mountIndex }) => ({ container: container.name ?? `container-${containerIndex}`, containerIndex, mountIndex, name: mount.name, mountPath: mount.mountPath ?? null, subPath: mount.subPath ?? null, readOnly: mount.readOnly === true })) ); const annotations = document.spec?.template?.metadata?.annotations ?? {}; const hotfixAnnotations = Object.fromEntries( Object.entries(annotations).filter(([key]) => key === args.annotationKey || key.includes("hotfix")) ); return { generation: document.metadata?.generation ?? null, observedImageTags: containers.map((container) => ({ container: container.name, image: container.image })), annotations: hotfixAnnotations, hotfixVolumes: volumes .filter((volume) => hotfixVolumeNames.has(volume.name)) .map((volume, volumeIndex) => ({ name: volume.name, volumeIndex, configMap: volume.configMap?.name, keys: (volume.configMap?.items ?? []).map((item) => item.key).filter(Boolean) })), hotfixMounts, mountsHotfix: hotfixMounts.some((mount) => mount.mountPath === args.mountPath && mount.subPath === args.subPath ) }; } function summarizeSelectedPod(document) { const pods = document?.items ?? []; if (pods.length === 0) return null; const pod = pods[pods.length - 1]; return { name: pod.metadata?.name ?? null, phase: pod.status?.phase ?? null, nodeName: pod.spec?.nodeName ?? null, creationTimestamp: pod.metadata?.creationTimestamp ?? null }; } export function classifyHotfixState(checks) { const classifications = new Set(["source-artifact-expected"]); const unknown = !checks || checks.node?.requiredNodePresent !== true || checks.configMap?.unknown === true || checks.deployment?.unknown === true || checks.pod?.unknown === true; if (checks?.configMap?.present === true) classifications.add("hotfix-configmap-present"); if (checks?.deployment?.mountsHotfix === true) classifications.add("deployment-mounts-hotfix"); if (checks?.pod?.marker?.found === true) classifications.add("pod-loads-hotfix-marker"); const hotfixDetected = classifications.has("hotfix-configmap-present") || classifications.has("deployment-mounts-hotfix") || classifications.has("pod-loads-hotfix-marker"); if (hotfixDetected) classifications.add("rollback-required"); if (!hotfixDetected && !unknown) classifications.add("no-hotfix-detected"); if (unknown) classifications.add("unknown-needs-manual-readonly-check"); return orderClassifications([...classifications]); } function summarizeClassifications(classifications) { if (classifications.includes("rollback-required")) { return { status: "hotfix-detected", summary: "DEV runtime still shows hotfix residue or active coverage; remove the override or replace it with a source artifact before treating source/CD as the runtime truth." }; } if (classifications.includes("no-hotfix-detected")) { return { status: "clear", summary: "No ConfigMap or Deployment mount hotfix was detected by the read-only audit; keep official fixes in source/PR/artifact/CD." }; } return { status: "unknown", summary: "The audit could not fully classify current DEV hotfix state; run the listed read-only checks on D601 before rollback or source verification." }; } function orderClassifications(values) { const set = new Set(values); return classificationOrder.filter((item) => set.has(item)); } function buildCommandPlan(args) { return { readonly: [ `${kubePrefix(args)} kubectl get nodes -o jsonpath='{.items[*].metadata.name}'`, `${kubePrefix(args)} kubectl -n ${args.namespace} get configmap ${args.configMap} -o json`, `${kubePrefix(args)} kubectl -n ${args.namespace} get deployment ${args.deployment} -o json`, `${kubePrefix(args)} kubectl -n ${args.namespace} get pods -l ${shellQuote(args.podLabelSelector)} --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp -o json`, `${kubePrefix(args)} kubectl -n ${args.namespace} exec -- sh -lc ${shellQuote(`grep -n ${shellQuote(args.marker)} ${shellQuote(args.mountPath)} | head`)}`, `${kubePrefix(args)} kubectl -n ${args.namespace} exec -- sh -lc ${shellQuote(`node --check ${shellQuote(args.mountPath)}`)}` ], rollbackNotExecutedByThisScript: [ "Use source/PR/artifact/CD to replace the runtime override whenever possible.", `If an authorized DEV rollback must remove the override directly, first locate the hotfix volumeMount, volume, and annotation indexes from the read-only deployment JSON for ${args.deployment}.`, "Then execute the smallest explicit DEV-only rollback outside this audit script, with KUBECONFIG pinned to D601 native k3s, and re-run this audit in --collect-readonly mode." ], forbiddenDefaultOperations: [ "kubectl apply", "kubectl patch", "kubectl rollout", "kubectl rollout restart", "kubectl delete", "kubectl create", "kubectl set image" ] }; } async function runKubectl(args, kubectlArgs, runCommand) { assertReadonlyKubectlArgs(kubectlArgs); const displayCommand = `${kubePrefix(args)} kubectl ${kubectlArgs.map(shellDisplay).join(" ")}`; const result = await runCommand({ command: "env", args: [`KUBECONFIG=${args.kubeconfig}`, "kubectl", ...kubectlArgs], timeoutMs: args.timeoutMs, displayCommand }); const probe = normalizeCommandResult(result, displayCommand); Object.defineProperty(probe, "rawStdout", { value: String(result.stdout ?? "").trim(), enumerable: false }); return probe; } export function assertReadonlyKubectlArgs(kubectlArgs) { const verb = kubectlArgs.find((arg) => allowedKubectlVerbs.has(arg) || forbiddenKubectlVerbs.has(arg)); const forbidden = kubectlArgs.find((arg) => forbiddenKubectlVerbs.has(arg)); if (forbidden) { throw new Error(`kubectl write verb is forbidden in hotfix audit: ${forbidden}`); } if (!verb || !allowedKubectlVerbs.has(verb)) { throw new Error(`kubectl command is not in the read-only allowlist: ${kubectlArgs.join(" ")}`); } } async function defaultRunCommand({ command, args, timeoutMs }) { try { const result = await execFileAsync(command, args, { timeout: timeoutMs, maxBuffer: 1024 * 1024 }); return { ok: true, exitCode: 0, stdout: result.stdout, stderr: result.stderr }; } catch (error) { return { ok: false, exitCode: typeof error.code === "number" ? error.code : 1, stdout: String(error.stdout ?? ""), stderr: String(error.stderr ?? ""), error: error.message }; } } function normalizeCommandResult(result, command) { return { ok: result.ok === true || result.exitCode === 0, command, exitCode: Number.isInteger(result.exitCode) ? result.exitCode : result.ok === true ? 0 : 1, stdout: redact(String(result.stdout ?? "").trim()), stderr: redact(String(result.stderr ?? "").trim()), error: result.error ? redact(String(result.error)) : undefined }; } function compactProbe(probe, { includeStdout = true } = {}) { return { ok: probe.ok, command: probe.command, exitCode: probe.exitCode, stdout: includeStdout && probe.stdout.length <= 200 ? probe.stdout : undefined, stderr: probe.stderr ? oneLine(probe.stderr) : undefined, error: probe.error ? oneLine(probe.error) : undefined }; } function parseJson(probe) { if (!probe.ok) return null; const raw = probe.rawStdout ?? probe.stdout; try { return raw ? JSON.parse(raw) : null; } catch { return null; } } function looksNotFound(probe) { return /notfound|not found|notfound/i.test(`${probe.stderr ?? ""} ${probe.error ?? ""}`); } function requireValue(value, flag) { if (!value) throw new Error(`${flag} requires a value`); return value; } function parsePositiveInt(value, flag) { const parsed = Number.parseInt(value, 10); if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${flag} must be a positive integer`); return parsed; } function kubePrefix(args) { return `KUBECONFIG=${args.kubeconfig}`; } function shellQuote(value) { return `'${String(value).replaceAll("'", "'\\''")}'`; } function shellDisplay(value) { return /\s|["'\\]/u.test(String(value)) ? shellQuote(value) : String(value); } function oneLine(value) { return redact(String(value)).replace(/\s+/g, " ").trim().slice(0, 500); } function redact(value) { return String(value) .replace(/(bearer\s+)[A-Za-z0-9._~+/=-]+/giu, "$1[REDACTED]") .replace(/((?:token|password|client-key-data|client-certificate-data|certificate-authority-data)\s*[:=]\s*)\S+/giu, "$1[REDACTED]") .replace(/(authorization:\s*)\S+/giu, "$1[REDACTED]") .slice(0, 4000); } function usage() { return [ "Usage: node scripts/dev-runtime-hotfix-audit.mjs [--plan|--collect-readonly] [options]", "", "Default mode is --plan: it prints the read-only audit plan and performs no kubectl calls.", "", "Options:", " --collect-readonly Run read-only kubectl get/exec checks; never apply/patch/rollout/restart", " --plan, --dry-run Print plan only (default)", " --pretty Pretty-print JSON", " --timeout-ms Per kubectl command timeout", " --kubeconfig Kubeconfig path, default /etc/rancher/k3s/k3s.yaml", " --namespace Namespace, default hwlab-dev", " --deployment Deployment, default hwlab-cloud-api", " --configmap Hotfix ConfigMap name", " --mount-path Expected mounted file path", " --sub-path Expected subPath key", " --marker Runtime marker to grep in the mounted file", " --annotation-key Deployment hotfix annotation key", " --pod-label-selector Pod selector for the Deployment", " --help Show this help" ].join("\n"); } export function formatDevRuntimeHotfixAuditFailure(error) { return { status: "failed", script: "dev-runtime-hotfix-audit", error: { message: error instanceof Error ? error.message : String(error) }, safety: { k8sWriteAttempted: false, secretValuesRead: false } }; }