Files
pikasTech-HWLAB/scripts/src/dev-runtime-hotfix-audit.mjs
T

566 lines
20 KiB
JavaScript

import { execFile } from "node:child_process";
import { promisify } from "node:util";
const execFileAsync = promisify(execFile);
const defaultKubeconfig = "/etc/rancher/k3s/k3s.yaml";
const defaultNamespace = "hwlab-dev";
const defaultDeployment = "hwlab-cloud-api";
const defaultConfigMap = "hwlab-cloud-api-code-agent-hotfix";
const defaultMountPath = "/app/internal/cloud/code-agent-chat.ts";
const defaultSubPath = "code-agent-chat.ts";
const defaultMarker = "runtime-hotfix-pc-gateway-shell";
const defaultAnnotationKey = "hwlab.pikastech.local/pc-gateway-shell-hotfix";
const defaultPodLabelSelector = "app.kubernetes.io/name=hwlab-cloud-api";
const forbiddenKubectlVerbs = new Set([
"apply",
"patch",
"rollout",
"restart",
"delete",
"create",
"replace",
"scale",
"set",
"edit",
"annotate",
"label"
]);
const allowedKubectlVerbs = new Set(["get", "describe", "exec", "auth", "config", "version"]);
const classificationOrder = Object.freeze([
"hotfix-configmap-present",
"deployment-mounts-hotfix",
"pod-loads-hotfix-marker",
"source-artifact-expected",
"rollback-required",
"no-hotfix-detected",
"unknown-needs-manual-readonly-check"
]);
export async function runDevRuntimeHotfixAuditCli(argv = process.argv.slice(2), options = {}) {
const args = parseArgs(argv);
if (args.help) {
process.stdout.write(`${usage()}\n`);
return 0;
}
const report = await buildDevRuntimeHotfixAuditReport(args, options);
process.stdout.write(`${JSON.stringify(report, null, args.pretty ? 2 : 0)}\n`);
return 0;
}
export function parseArgs(argv = []) {
const args = {
collectReadonly: false,
pretty: false,
timeoutMs: 5000,
kubeconfig: defaultKubeconfig,
namespace: defaultNamespace,
deployment: defaultDeployment,
configMap: defaultConfigMap,
mountPath: defaultMountPath,
subPath: defaultSubPath,
marker: defaultMarker,
annotationKey: defaultAnnotationKey,
podLabelSelector: defaultPodLabelSelector,
help: false
};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--collect-readonly") args.collectReadonly = true;
else if (arg === "--plan" || arg === "--dry-run") args.collectReadonly = false;
else if (arg === "--pretty") args.pretty = true;
else if (arg === "--timeout-ms") args.timeoutMs = parsePositiveInt(requireValue(argv[++index], arg), arg);
else if (arg === "--kubeconfig") args.kubeconfig = requireValue(argv[++index], arg);
else if (arg === "--namespace") args.namespace = requireValue(argv[++index], arg);
else if (arg === "--deployment") args.deployment = requireValue(argv[++index], arg);
else if (arg === "--configmap") args.configMap = requireValue(argv[++index], arg);
else if (arg === "--mount-path") args.mountPath = requireValue(argv[++index], arg);
else if (arg === "--sub-path") args.subPath = requireValue(argv[++index], arg);
else if (arg === "--marker") args.marker = requireValue(argv[++index], arg);
else if (arg === "--annotation-key") args.annotationKey = requireValue(argv[++index], arg);
else if (arg === "--pod-label-selector") args.podLabelSelector = requireValue(argv[++index], arg);
else if (arg === "--help" || arg === "-h") args.help = true;
else throw new Error(`unknown argument: ${arg}`);
}
return args;
}
export async function buildDevRuntimeHotfixAuditReport(args = {}, options = {}) {
const parsed = { ...parseArgs([]), ...args };
const report = {
issueRefs: ["pikasTech/HWLAB#462", "pikasTech/HWLAB#465", "pikasTech/HWLAB#7", "pikasTech/HWLAB#239"],
relatedDivisionOfWork: {
capabilityRouting: "pikasTech/HWLAB#460",
sourceRunner: "pikasTech/HWLAB#461",
runtimeHotfixRunbookAndAudit: "pikasTech/HWLAB#462",
serviceIdDiagnostics: "pikasTech/HWLAB#463",
devLiveSmoke: "pikasTech/HWLAB#464"
},
mode: parsed.collectReadonly ? "collect-readonly" : "plan",
target: {
environment: "DEV",
namespace: parsed.namespace,
deployment: parsed.deployment,
configMap: parsed.configMap,
annotationKey: parsed.annotationKey,
mountPath: parsed.mountPath,
subPath: parsed.subPath,
marker: parsed.marker,
kubeconfig: parsed.kubeconfig,
requiredNode: "d601"
},
safety: {
prodAllowed: false,
secretValuesRead: false,
secretResourcesRead: false,
k8sWriteAttempted: false,
k8sWriteCommandsExecuted: [],
defaultExecutesKubectl: parsed.collectReadonly,
allowedKubectlVerbs: [...allowedKubectlVerbs].sort(),
forbiddenKubectlVerbs: [...forbiddenKubectlVerbs].sort()
},
requiredOperatorEvidence: [
"Deployment",
"pod",
"ConfigMap",
"annotation",
"volume",
"mountPath",
"subPath",
"traceId",
"operationId",
"validationCommand",
"rollbackMethod",
"sourceFollowUp"
],
commandPlan: buildCommandPlan(parsed),
checks: null,
classifications: [],
conclusion: null
};
if (!parsed.collectReadonly) {
report.classifications = orderClassifications([
"source-artifact-expected",
"unknown-needs-manual-readonly-check"
]);
report.conclusion = {
status: "unknown",
summary: "Plan mode did not query D601; run --collect-readonly on D601 with the native k3s kubeconfig to classify the current hotfix state."
};
return report;
}
const checks = await collectReadonlyChecks(parsed, options);
report.checks = checks;
report.classifications = classifyHotfixState(checks);
report.conclusion = summarizeClassifications(report.classifications);
return report;
}
async function collectReadonlyChecks(args, options) {
const runCommand = options.runCommand ?? defaultRunCommand;
const nodeProbe = await runKubectl(args, ["get", "nodes", "-o", "json"], runCommand);
const nodeJson = parseJson(nodeProbe);
const nodeNames = (nodeJson?.items ?? []).map((item) => item?.metadata?.name).filter(Boolean);
const configMapProbe = await runKubectl(args, ["-n", args.namespace, "get", "configmap", args.configMap, "-o", "json"], runCommand);
const configMapJson = parseJson(configMapProbe);
const deploymentProbe = await runKubectl(args, ["-n", args.namespace, "get", "deployment", args.deployment, "-o", "json"], runCommand);
const deploymentJson = parseJson(deploymentProbe);
const deployment = summarizeDeployment(deploymentJson, args);
const podsProbe = await runKubectl(args, [
"-n",
args.namespace,
"get",
"pods",
"-l",
args.podLabelSelector,
"--field-selector=status.phase=Running",
"--sort-by=.metadata.creationTimestamp",
"-o",
"json"
], runCommand);
const podsJson = parseJson(podsProbe);
const pod = summarizeSelectedPod(podsJson);
let markerProbe = null;
let syntaxProbe = null;
if (pod?.name) {
markerProbe = await runKubectl(args, [
"-n",
args.namespace,
"exec",
pod.name,
"--",
"sh",
"-lc",
`grep -n ${shellQuote(args.marker)} ${shellQuote(args.mountPath)} | head`
], runCommand);
syntaxProbe = await runKubectl(args, [
"-n",
args.namespace,
"exec",
pod.name,
"--",
"sh",
"-lc",
`node --check ${shellQuote(args.mountPath)}`
], runCommand);
}
return {
node: {
ok: nodeProbe.ok,
command: nodeProbe.command,
names: nodeNames,
requiredNodePresent: nodeNames.includes("d601"),
probe: compactProbe(nodeProbe, { includeStdout: false })
},
configMap: {
command: configMapProbe.command,
present: configMapProbe.ok && Boolean(configMapJson),
unknown: !configMapProbe.ok && !looksNotFound(configMapProbe),
keys: configMapJson ? Object.keys(configMapJson.data ?? {}).sort() : [],
probe: compactProbe(configMapProbe, { includeStdout: false })
},
deployment: {
command: deploymentProbe.command,
present: deploymentProbe.ok && Boolean(deploymentJson),
unknown: !deploymentProbe.ok || !deploymentJson,
...deployment,
probe: compactProbe(deploymentProbe, { includeStdout: false })
},
pod: {
command: podsProbe.command,
selected: pod,
unknown: !podsProbe.ok || !podsJson,
marker: markerProbe ? {
command: markerProbe.command,
found: markerProbe.ok,
matchCount: markerProbe.ok ? markerProbe.stdout.split("\n").filter(Boolean).length : 0,
probe: compactProbe(markerProbe, { includeStdout: false })
} : null,
syntax: syntaxProbe ? {
command: syntaxProbe.command,
ok: syntaxProbe.ok,
probe: compactProbe(syntaxProbe)
} : null,
probe: compactProbe(podsProbe, { includeStdout: false })
}
};
}
function summarizeDeployment(document, args) {
if (!document) {
return {
generation: null,
annotations: {},
hotfixVolumes: [],
hotfixMounts: [],
mountsHotfix: false
};
}
const volumes = document.spec?.template?.spec?.volumes ?? [];
const containers = document.spec?.template?.spec?.containers ?? [];
const hotfixVolumeNames = new Set(
volumes
.filter((volume) => volume?.configMap?.name === args.configMap)
.map((volume) => volume.name)
.filter(Boolean)
);
const hotfixMounts = containers.flatMap((container, containerIndex) =>
(container.volumeMounts ?? [])
.map((mount, mountIndex) => ({ container, containerIndex, mount, mountIndex }))
.filter(({ mount }) =>
hotfixVolumeNames.has(mount.name) ||
mount.mountPath === args.mountPath ||
mount.subPath === args.subPath
)
.map(({ container, containerIndex, mount, mountIndex }) => ({
container: container.name ?? `container-${containerIndex}`,
containerIndex,
mountIndex,
name: mount.name,
mountPath: mount.mountPath ?? null,
subPath: mount.subPath ?? null,
readOnly: mount.readOnly === true
}))
);
const annotations = document.spec?.template?.metadata?.annotations ?? {};
const hotfixAnnotations = Object.fromEntries(
Object.entries(annotations).filter(([key]) => key === args.annotationKey || key.includes("hotfix"))
);
return {
generation: document.metadata?.generation ?? null,
observedImageTags: containers.map((container) => ({
container: container.name,
image: container.image
})),
annotations: hotfixAnnotations,
hotfixVolumes: volumes
.filter((volume) => hotfixVolumeNames.has(volume.name))
.map((volume, volumeIndex) => ({
name: volume.name,
volumeIndex,
configMap: volume.configMap?.name,
keys: (volume.configMap?.items ?? []).map((item) => item.key).filter(Boolean)
})),
hotfixMounts,
mountsHotfix: hotfixMounts.some((mount) =>
mount.mountPath === args.mountPath &&
mount.subPath === args.subPath
)
};
}
function summarizeSelectedPod(document) {
const pods = document?.items ?? [];
if (pods.length === 0) return null;
const pod = pods[pods.length - 1];
return {
name: pod.metadata?.name ?? null,
phase: pod.status?.phase ?? null,
nodeName: pod.spec?.nodeName ?? null,
creationTimestamp: pod.metadata?.creationTimestamp ?? null
};
}
export function classifyHotfixState(checks) {
const classifications = new Set(["source-artifact-expected"]);
const unknown = !checks ||
checks.node?.requiredNodePresent !== true ||
checks.configMap?.unknown === true ||
checks.deployment?.unknown === true ||
checks.pod?.unknown === true;
if (checks?.configMap?.present === true) classifications.add("hotfix-configmap-present");
if (checks?.deployment?.mountsHotfix === true) classifications.add("deployment-mounts-hotfix");
if (checks?.pod?.marker?.found === true) classifications.add("pod-loads-hotfix-marker");
const hotfixDetected = classifications.has("hotfix-configmap-present") ||
classifications.has("deployment-mounts-hotfix") ||
classifications.has("pod-loads-hotfix-marker");
if (hotfixDetected) classifications.add("rollback-required");
if (!hotfixDetected && !unknown) classifications.add("no-hotfix-detected");
if (unknown) classifications.add("unknown-needs-manual-readonly-check");
return orderClassifications([...classifications]);
}
function summarizeClassifications(classifications) {
if (classifications.includes("rollback-required")) {
return {
status: "hotfix-detected",
summary: "DEV runtime still shows hotfix residue or active coverage; remove the override or replace it with a source artifact before treating source/CD as the runtime truth."
};
}
if (classifications.includes("no-hotfix-detected")) {
return {
status: "clear",
summary: "No ConfigMap or Deployment mount hotfix was detected by the read-only audit; keep official fixes in source/PR/artifact/CD."
};
}
return {
status: "unknown",
summary: "The audit could not fully classify current DEV hotfix state; run the listed read-only checks on D601 before rollback or source verification."
};
}
function orderClassifications(values) {
const set = new Set(values);
return classificationOrder.filter((item) => set.has(item));
}
function buildCommandPlan(args) {
return {
readonly: [
`${kubePrefix(args)} kubectl get nodes -o jsonpath='{.items[*].metadata.name}'`,
`${kubePrefix(args)} kubectl -n ${args.namespace} get configmap ${args.configMap} -o json`,
`${kubePrefix(args)} kubectl -n ${args.namespace} get deployment ${args.deployment} -o json`,
`${kubePrefix(args)} kubectl -n ${args.namespace} get pods -l ${shellQuote(args.podLabelSelector)} --field-selector=status.phase=Running --sort-by=.metadata.creationTimestamp -o json`,
`${kubePrefix(args)} kubectl -n ${args.namespace} exec <pod> -- sh -lc ${shellQuote(`grep -n ${shellQuote(args.marker)} ${shellQuote(args.mountPath)} | head`)}`,
`${kubePrefix(args)} kubectl -n ${args.namespace} exec <pod> -- sh -lc ${shellQuote(`node --check ${shellQuote(args.mountPath)}`)}`
],
rollbackNotExecutedByThisScript: [
"Use source/PR/artifact/CD to replace the runtime override whenever possible.",
`If an authorized DEV rollback must remove the override directly, first locate the hotfix volumeMount, volume, and annotation indexes from the read-only deployment JSON for ${args.deployment}.`,
"Then execute the smallest explicit DEV-only rollback outside this audit script, with KUBECONFIG pinned to D601 native k3s, and re-run this audit in --collect-readonly mode."
],
forbiddenDefaultOperations: [
"kubectl apply",
"kubectl patch",
"kubectl rollout",
"kubectl rollout restart",
"kubectl delete",
"kubectl create",
"kubectl set image"
]
};
}
async function runKubectl(args, kubectlArgs, runCommand) {
assertReadonlyKubectlArgs(kubectlArgs);
const displayCommand = `${kubePrefix(args)} kubectl ${kubectlArgs.map(shellDisplay).join(" ")}`;
const result = await runCommand({
command: "env",
args: [`KUBECONFIG=${args.kubeconfig}`, "kubectl", ...kubectlArgs],
timeoutMs: args.timeoutMs,
displayCommand
});
const probe = normalizeCommandResult(result, displayCommand);
Object.defineProperty(probe, "rawStdout", {
value: String(result.stdout ?? "").trim(),
enumerable: false
});
return probe;
}
export function assertReadonlyKubectlArgs(kubectlArgs) {
const verb = kubectlArgs.find((arg) => allowedKubectlVerbs.has(arg) || forbiddenKubectlVerbs.has(arg));
const forbidden = kubectlArgs.find((arg) => forbiddenKubectlVerbs.has(arg));
if (forbidden) {
throw new Error(`kubectl write verb is forbidden in hotfix audit: ${forbidden}`);
}
if (!verb || !allowedKubectlVerbs.has(verb)) {
throw new Error(`kubectl command is not in the read-only allowlist: ${kubectlArgs.join(" ")}`);
}
}
async function defaultRunCommand({ command, args, timeoutMs }) {
try {
const result = await execFileAsync(command, args, {
timeout: timeoutMs,
maxBuffer: 1024 * 1024
});
return {
ok: true,
exitCode: 0,
stdout: result.stdout,
stderr: result.stderr
};
} catch (error) {
return {
ok: false,
exitCode: typeof error.code === "number" ? error.code : 1,
stdout: String(error.stdout ?? ""),
stderr: String(error.stderr ?? ""),
error: error.message
};
}
}
function normalizeCommandResult(result, command) {
return {
ok: result.ok === true || result.exitCode === 0,
command,
exitCode: Number.isInteger(result.exitCode) ? result.exitCode : result.ok === true ? 0 : 1,
stdout: redact(String(result.stdout ?? "").trim()),
stderr: redact(String(result.stderr ?? "").trim()),
error: result.error ? redact(String(result.error)) : undefined
};
}
function compactProbe(probe, { includeStdout = true } = {}) {
return {
ok: probe.ok,
command: probe.command,
exitCode: probe.exitCode,
stdout: includeStdout && probe.stdout.length <= 200 ? probe.stdout : undefined,
stderr: probe.stderr ? oneLine(probe.stderr) : undefined,
error: probe.error ? oneLine(probe.error) : undefined
};
}
function parseJson(probe) {
if (!probe.ok) return null;
const raw = probe.rawStdout ?? probe.stdout;
try {
return raw ? JSON.parse(raw) : null;
} catch {
return null;
}
}
function looksNotFound(probe) {
return /notfound|not found|notfound/i.test(`${probe.stderr ?? ""} ${probe.error ?? ""}`);
}
function requireValue(value, flag) {
if (!value) throw new Error(`${flag} requires a value`);
return value;
}
function parsePositiveInt(value, flag) {
const parsed = Number.parseInt(value, 10);
if (!Number.isInteger(parsed) || parsed <= 0) throw new Error(`${flag} must be a positive integer`);
return parsed;
}
function kubePrefix(args) {
return `KUBECONFIG=${args.kubeconfig}`;
}
function shellQuote(value) {
return `'${String(value).replaceAll("'", "'\\''")}'`;
}
function shellDisplay(value) {
return /\s|["'\\]/u.test(String(value)) ? shellQuote(value) : String(value);
}
function oneLine(value) {
return redact(String(value)).replace(/\s+/g, " ").trim().slice(0, 500);
}
function redact(value) {
return String(value)
.replace(/(bearer\s+)[A-Za-z0-9._~+/=-]+/giu, "$1[REDACTED]")
.replace(/((?:token|password|client-key-data|client-certificate-data|certificate-authority-data)\s*[:=]\s*)\S+/giu, "$1[REDACTED]")
.replace(/(authorization:\s*)\S+/giu, "$1[REDACTED]")
.slice(0, 4000);
}
function usage() {
return [
"Usage: node scripts/dev-runtime-hotfix-audit.mjs [--plan|--collect-readonly] [options]",
"",
"Default mode is --plan: it prints the read-only audit plan and performs no kubectl calls.",
"",
"Options:",
" --collect-readonly Run read-only kubectl get/exec checks; never apply/patch/rollout/restart",
" --plan, --dry-run Print plan only (default)",
" --pretty Pretty-print JSON",
" --timeout-ms <ms> Per kubectl command timeout",
" --kubeconfig <path> Kubeconfig path, default /etc/rancher/k3s/k3s.yaml",
" --namespace <name> Namespace, default hwlab-dev",
" --deployment <name> Deployment, default hwlab-cloud-api",
" --configmap <name> Hotfix ConfigMap name",
" --mount-path <path> Expected mounted file path",
" --sub-path <name> Expected subPath key",
" --marker <text> Runtime marker to grep in the mounted file",
" --annotation-key <key> Deployment hotfix annotation key",
" --pod-label-selector <sel> Pod selector for the Deployment",
" --help Show this help"
].join("\n");
}
export function formatDevRuntimeHotfixAuditFailure(error) {
return {
status: "failed",
script: "dev-runtime-hotfix-audit",
error: {
message: error instanceof Error ? error.message : String(error)
},
safety: {
k8sWriteAttempted: false,
secretValuesRead: false
}
};
}