733 lines
26 KiB
JavaScript
733 lines
26 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { constants as fsConstants } from "node:fs";
|
|
import { access, mkdir, stat, writeFile } from "node:fs/promises";
|
|
import { execFile } from "node:child_process";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { promisify } from "node:util";
|
|
|
|
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT } from "../../internal/protocol/index.mjs";
|
|
import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs";
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
const namespace = "hwlab-dev";
|
|
const defaultReportPath = tempReportPath("d601-k3s-readonly-observability.json");
|
|
const issue = "pikasTech/HWLAB#46";
|
|
const supports = ["#7", "#33", "#36", "#38", "#39", "#42"].map((id) => `pikasTech/HWLAB${id}`);
|
|
const forbiddenActions = [
|
|
"prod-access",
|
|
"secret-material-read",
|
|
"cluster-mutation",
|
|
"runtime-restart",
|
|
"heavy-e2e",
|
|
"master-server-heavy-check"
|
|
];
|
|
const kubeconfigCandidatePaths = [
|
|
"~/.kube/config",
|
|
"/etc/rancher/k3s/k3s.yaml",
|
|
"/var/lib/rancher/k3s/server/cred/admin.kubeconfig"
|
|
];
|
|
const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
|
|
const publicEndpointProbeTargets = [
|
|
{ id: "frontend-16666", url: `${DEV_FRONTEND_ENDPOINT}/` },
|
|
{ id: "api-health-16667", url: `${DEV_ENDPOINT}/health` },
|
|
{ id: "api-live-16667", url: `${DEV_ENDPOINT}/health/live` }
|
|
];
|
|
const sshEnvVars = [
|
|
"HWLAB_D601_SSH_HOST",
|
|
"HWLAB_D601_SSH_TARGET",
|
|
"HWLAB_D601_MAINTENANCE_HOST",
|
|
"D601_SSH_HOST",
|
|
"HWLAB_MAINTENANCE_SSH"
|
|
];
|
|
const blockerTypes = new Set(["environment_blocker", "observability_blocker", "safety_blocker"]);
|
|
|
|
function parseArgs(argv) {
|
|
const args = {
|
|
reportPath: defaultReportPath,
|
|
timeoutMs: 5000,
|
|
writeReport: true,
|
|
pretty: false,
|
|
failOnBlocked: false
|
|
};
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const arg = argv[index];
|
|
if (arg === "--report") {
|
|
args.reportPath = argv[++index];
|
|
} else if (arg === "--timeout-ms") {
|
|
args.timeoutMs = Number.parseInt(argv[++index], 10);
|
|
} else if (arg === "--no-write") {
|
|
args.writeReport = false;
|
|
} else if (arg === "--pretty") {
|
|
args.pretty = true;
|
|
} else if (arg === "--fail-on-blocked") {
|
|
args.failOnBlocked = true;
|
|
} else if (arg === "--help") {
|
|
args.help = true;
|
|
} else {
|
|
throw new Error(`unknown argument ${arg}`);
|
|
}
|
|
}
|
|
|
|
assert.ok(Number.isInteger(args.timeoutMs) && args.timeoutMs > 0, "--timeout-ms must be positive");
|
|
return args;
|
|
}
|
|
|
|
function usage() {
|
|
return [
|
|
"Usage: node scripts/d601-k3s-readonly-observability.mjs [options]",
|
|
"",
|
|
"Options:",
|
|
" --report <path> Write report path",
|
|
" --timeout-ms <ms> Per-command timeout",
|
|
" --no-write Print only, do not write report",
|
|
" --pretty Pretty-print the full report",
|
|
" --fail-on-blocked Exit 2 when conclusion is blocked",
|
|
" --help Show this help"
|
|
].join("\n");
|
|
}
|
|
|
|
function redacted(value) {
|
|
return String(value)
|
|
.replace(/(bearer\s+)[A-Za-z0-9._~+/=-]+/giu, "$1[REDACTED]")
|
|
.replace(/((?:token|password|client-key-data|client-certificate-data|certificate-authority-data)\s*[:=]\s*)\S+/giu, "$1[REDACTED]")
|
|
.replace(/(authorization:\s*)\S+/giu, "$1[REDACTED]")
|
|
.slice(0, 4000);
|
|
}
|
|
|
|
function oneLine(value) {
|
|
return redacted(value).replace(/\s+/g, " ").trim();
|
|
}
|
|
|
|
function expandHome(filePath) {
|
|
return filePath.startsWith("~/") ? path.join(os.homedir(), filePath.slice(2)) : filePath;
|
|
}
|
|
|
|
function displayPath(filePath) {
|
|
const home = os.homedir();
|
|
return filePath.startsWith(`${home}${path.sep}`) ? `~/${filePath.slice(home.length + 1)}` : filePath;
|
|
}
|
|
|
|
function commandText(command, args) {
|
|
return [command, ...args].join(" ");
|
|
}
|
|
|
|
function runnerKubeconfigCommandText(args) {
|
|
return [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args].join(" ");
|
|
}
|
|
|
|
async function run(command, args = [], options = {}) {
|
|
try {
|
|
const result = await execFileAsync(command, args, {
|
|
cwd: repoRoot,
|
|
timeout: options.timeoutMs ?? 5000,
|
|
maxBuffer: 1024 * 1024
|
|
});
|
|
const probe = {
|
|
ok: true,
|
|
command: commandText(command, args),
|
|
exitCode: 0,
|
|
stdout: redacted(result.stdout.trim()),
|
|
stderr: redacted(result.stderr.trim())
|
|
};
|
|
Object.defineProperty(probe, "rawStdout", {
|
|
value: result.stdout.trim(),
|
|
enumerable: false
|
|
});
|
|
return probe;
|
|
} catch (error) {
|
|
const probe = {
|
|
ok: false,
|
|
command: commandText(command, args),
|
|
exitCode: typeof error.code === "number" ? error.code : 1,
|
|
stdout: redacted(String(error.stdout ?? "").trim()),
|
|
stderr: redacted(String(error.stderr ?? "").trim()),
|
|
error: oneLine(error.message)
|
|
};
|
|
Object.defineProperty(probe, "rawStdout", {
|
|
value: String(error.stdout ?? "").trim(),
|
|
enumerable: false
|
|
});
|
|
return probe;
|
|
}
|
|
}
|
|
|
|
async function findExecutable(name) {
|
|
for (const dir of (process.env.PATH ?? "").split(path.delimiter)) {
|
|
if (!dir) continue;
|
|
const candidate = path.join(dir, name);
|
|
try {
|
|
await access(candidate, fsConstants.X_OK);
|
|
return candidate;
|
|
} catch {}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function fileProbe(filePath) {
|
|
const absolutePath = expandHome(filePath);
|
|
try {
|
|
const info = await stat(absolutePath);
|
|
await access(absolutePath, fsConstants.R_OK);
|
|
return {
|
|
path: displayPath(absolutePath),
|
|
exists: true,
|
|
readable: true,
|
|
type: info.isDirectory() ? "directory" : info.isFile() ? "file" : "other",
|
|
sizeBytes: info.isFile() ? info.size : undefined
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
path: displayPath(absolutePath),
|
|
exists: error?.code !== "ENOENT",
|
|
readable: false,
|
|
error: error?.code ?? "UNKNOWN"
|
|
};
|
|
}
|
|
}
|
|
|
|
async function collectRunnerKubeconfigProbe(timeoutMs) {
|
|
const metadata = await fileProbe(runnerK3sKubeconfigPath);
|
|
const args = ["-n", namespace, "get", "pods", "-o", "name"];
|
|
const probe = await run("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], {
|
|
timeoutMs
|
|
});
|
|
const stdoutLines = probe.stdout
|
|
? probe.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
|
|
: [];
|
|
|
|
return {
|
|
path: runnerK3sKubeconfigPath,
|
|
mode: "read-only",
|
|
resource: "pods",
|
|
command: runnerKubeconfigCommandText(args),
|
|
runnerKubeconfigReadable: metadata.readable === true,
|
|
runnerKubeconfigProbeOk: probe.ok,
|
|
runnerKubeconfigProbeExitCode: probe.exitCode,
|
|
runnerKubeconfigProbeStderr: oneLine(probe.stderr || probe.error || "") || "empty",
|
|
stdoutSummary: probe.ok ? `${stdoutLines.length} pod name(s) observed` : undefined,
|
|
classification: metadata.readable
|
|
? probe.ok
|
|
? "readable"
|
|
: "readonly_observability_gap"
|
|
: "runner_permission_mount_gap",
|
|
sourceIssue: issue,
|
|
secretValuesRead: false,
|
|
secretResourcesRead: false,
|
|
note: "This probe forces KUBECONFIG=/etc/rancher/k3s/k3s.yaml for a read-only kubectl get pods command and records only exit code plus redacted stderr/stdout summary. A successful kubectl exit code does not make the kubeconfig file readable; kubectl may use an alternate in-cluster fallback."
|
|
};
|
|
}
|
|
|
|
async function probePublicEndpoint(target, timeoutMs) {
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
try {
|
|
const response = await fetch(target.url, {
|
|
method: "GET",
|
|
signal: controller.signal,
|
|
redirect: "manual"
|
|
});
|
|
const body = await response.text();
|
|
let json = null;
|
|
try {
|
|
json = body ? JSON.parse(body) : null;
|
|
} catch {}
|
|
return {
|
|
id: target.id,
|
|
url: target.url,
|
|
reached: true,
|
|
ok: response.status >= 200 && response.status < 400,
|
|
status: response.status,
|
|
serviceId: json?.serviceId,
|
|
environment: json?.environment,
|
|
bodySummary: json
|
|
? `json serviceId=${json.serviceId ?? "unknown"} status=${json.status ?? "unknown"}`
|
|
: oneLine(body.slice(0, 160))
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
id: target.id,
|
|
url: target.url,
|
|
reached: false,
|
|
ok: false,
|
|
error: error instanceof Error ? error.message : String(error)
|
|
};
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
async function collectPublicEndpoints(timeoutMs) {
|
|
const probes = await Promise.all(
|
|
publicEndpointProbeTargets.map((target) => probePublicEndpoint(target, timeoutMs))
|
|
);
|
|
return {
|
|
activeFrontendEndpoint: DEV_FRONTEND_ENDPOINT,
|
|
activeApiEndpoint: DEV_ENDPOINT,
|
|
d601PublicEndpointsReachable: probes.every((probe) => probe.ok === true),
|
|
probes
|
|
};
|
|
}
|
|
|
|
async function collectBinary(name, versionArgs) {
|
|
const executablePath = await findExecutable(name);
|
|
const result = {
|
|
name,
|
|
available: executablePath !== null,
|
|
path: executablePath ? displayPath(executablePath) : null
|
|
};
|
|
if (executablePath && versionArgs) {
|
|
const version = await run(name, versionArgs, { timeoutMs: 5000 });
|
|
result.version = {
|
|
ok: version.ok,
|
|
command: version.command,
|
|
output: oneLine(version.stdout || version.stderr || version.error)
|
|
};
|
|
}
|
|
return result;
|
|
}
|
|
|
|
async function collectBinaries() {
|
|
const entries = await Promise.all([
|
|
collectBinary("kubectl", ["version", "--client=true"]),
|
|
collectBinary("k3s", ["--version"]),
|
|
collectBinary("k3sctl", ["version"]),
|
|
collectBinary("ssh", ["-V"])
|
|
]);
|
|
return Object.fromEntries(entries.map((entry) => [entry.name, entry]));
|
|
}
|
|
|
|
function kubeconfigEnvPaths() {
|
|
const raw = process.env.KUBECONFIG;
|
|
if (!raw) return [];
|
|
return raw.split(path.delimiter).filter(Boolean);
|
|
}
|
|
|
|
async function collectKubeconfigPaths() {
|
|
const envPaths = kubeconfigEnvPaths();
|
|
const candidates = [
|
|
...envPaths.map((item) => ({ source: "KUBECONFIG", path: item })),
|
|
...kubeconfigCandidatePaths.map((item) => ({ source: "default", path: item }))
|
|
];
|
|
const seen = new Set();
|
|
const uniqueCandidates = candidates.filter((item) => {
|
|
const key = expandHome(item.path);
|
|
if (seen.has(key)) return false;
|
|
seen.add(key);
|
|
return true;
|
|
});
|
|
return Promise.all(uniqueCandidates.map(async (item) => ({
|
|
source: item.source,
|
|
...(await fileProbe(item.path))
|
|
})));
|
|
}
|
|
|
|
async function collectSshBridge(binaries) {
|
|
const env = sshEnvVars.map((name) => ({
|
|
name,
|
|
set: Object.hasOwn(process.env, name),
|
|
valueRedacted: Object.hasOwn(process.env, name) ? "[REDACTED]" : null
|
|
}));
|
|
const paths = await Promise.all(["~/.ssh", "~/.ssh/config", "~/.ssh/known_hosts"].map(fileProbe));
|
|
return {
|
|
sshBinaryAvailable: binaries.ssh.available,
|
|
env,
|
|
paths,
|
|
configured: binaries.ssh.available && (
|
|
env.some((item) => item.set) ||
|
|
paths.some((item) => item.path.endsWith("/.ssh/config") && item.readable)
|
|
),
|
|
note: "SSH bridge discovery checks only binary, environment variable presence, and file metadata; it does not read SSH config or private keys."
|
|
};
|
|
}
|
|
|
|
function parseJsonProbe(probe) {
|
|
const raw = probe.rawStdout ?? probe.stdout;
|
|
if (!probe.ok || !raw) return null;
|
|
try {
|
|
return JSON.parse(raw);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function summarizePods(document) {
|
|
const pods = document?.items ?? [];
|
|
return {
|
|
count: pods.length,
|
|
items: pods.map((pod) => {
|
|
const statuses = pod.status?.containerStatuses ?? [];
|
|
return {
|
|
name: pod.metadata?.name,
|
|
phase: pod.status?.phase,
|
|
readyContainers: statuses.filter((item) => item.ready).length,
|
|
totalContainers: statuses.length,
|
|
restartCount: statuses.reduce((total, item) => total + (item.restartCount ?? 0), 0),
|
|
nodeName: pod.spec?.nodeName ?? null
|
|
};
|
|
})
|
|
};
|
|
}
|
|
|
|
function summarizeServices(document) {
|
|
const services = document?.items ?? [];
|
|
return {
|
|
count: services.length,
|
|
items: services.map((service) => ({
|
|
name: service.metadata?.name,
|
|
type: service.spec?.type,
|
|
clusterIP: service.spec?.clusterIP ?? null,
|
|
ports: (service.spec?.ports ?? []).map((port) => ({
|
|
name: port.name ?? null,
|
|
port: port.port,
|
|
targetPort: port.targetPort,
|
|
protocol: port.protocol
|
|
}))
|
|
}))
|
|
};
|
|
}
|
|
|
|
function summarizeConfigMaps(document) {
|
|
const configMaps = document?.items ?? [];
|
|
return {
|
|
count: configMaps.length,
|
|
items: configMaps.map((configMap) => ({
|
|
name: configMap.metadata?.name,
|
|
keys: Object.keys(configMap.data ?? {}).sort()
|
|
}))
|
|
};
|
|
}
|
|
|
|
function readonlyExecutor(name, command, prefixArgs = []) {
|
|
return { name, command, prefixArgs };
|
|
}
|
|
|
|
async function runExecutorProbe(executor, args, timeoutMs) {
|
|
return run(executor.command, [...executor.prefixArgs, ...args], { timeoutMs });
|
|
}
|
|
|
|
function authAllowed(probe) {
|
|
return probe.ok && probe.stdout.trim().toLowerCase() === "yes";
|
|
}
|
|
|
|
function compactProbe(probe, { includeStdout = true } = {}) {
|
|
return {
|
|
ok: probe.ok,
|
|
command: probe.command,
|
|
exitCode: probe.exitCode,
|
|
stdout: includeStdout && probe.ok && probe.stdout.length <= 200 ? probe.stdout : undefined,
|
|
stderr: probe.stderr ? oneLine(probe.stderr) : undefined,
|
|
error: probe.error
|
|
};
|
|
}
|
|
|
|
async function collectClusterForExecutor(executor, timeoutMs) {
|
|
const version = await runExecutorProbe(executor, ["version", "--client=true"], timeoutMs);
|
|
const context = await runExecutorProbe(executor, ["config", "current-context"], timeoutMs);
|
|
const namespaceProbe = await runExecutorProbe(executor, ["get", "namespace", namespace, "-o", "json"], timeoutMs);
|
|
const auth = {};
|
|
for (const resource of ["pods", "services", "configmaps"]) {
|
|
auth[resource] = await runExecutorProbe(executor, ["auth", "can-i", "get", resource, "-n", namespace], timeoutMs);
|
|
}
|
|
|
|
const podsProbe = await runExecutorProbe(executor, ["-n", namespace, "get", "pods", "-o", "json"], timeoutMs);
|
|
const servicesProbe = await runExecutorProbe(executor, ["-n", namespace, "get", "services", "-o", "json"], timeoutMs);
|
|
const configMapsProbe = await runExecutorProbe(executor, ["-n", namespace, "get", "configmaps", "-o", "json"], timeoutMs);
|
|
const podsJson = parseJsonProbe(podsProbe);
|
|
const servicesJson = parseJsonProbe(servicesProbe);
|
|
const configMapsJson = parseJsonProbe(configMapsProbe);
|
|
const ready = Boolean(namespaceProbe.ok && podsJson && servicesJson && configMapsJson);
|
|
|
|
return {
|
|
executor: executor.name,
|
|
ready,
|
|
probes: {
|
|
clientVersion: compactProbe(version),
|
|
currentContext: compactProbe(context),
|
|
namespace: compactProbe(namespaceProbe, { includeStdout: false }),
|
|
auth: Object.fromEntries(Object.entries(auth).map(([resource, probe]) => [
|
|
resource,
|
|
{
|
|
allowed: authAllowed(probe),
|
|
...compactProbe(probe)
|
|
}
|
|
])),
|
|
pods: compactProbe(podsProbe, { includeStdout: false }),
|
|
services: compactProbe(servicesProbe, { includeStdout: false }),
|
|
configMaps: compactProbe(configMapsProbe, { includeStdout: false })
|
|
},
|
|
summaries: ready ? {
|
|
pods: summarizePods(podsJson),
|
|
services: summarizeServices(servicesJson),
|
|
configMaps: summarizeConfigMaps(configMapsJson)
|
|
} : null
|
|
};
|
|
}
|
|
|
|
async function collectClusterObservability(binaries, timeoutMs) {
|
|
const executors = [];
|
|
if (binaries.kubectl.available) {
|
|
executors.push(readonlyExecutor("kubectl", "kubectl"));
|
|
}
|
|
if (binaries.k3s.available) {
|
|
executors.push(readonlyExecutor("k3s-kubectl", "k3s", ["kubectl"]));
|
|
}
|
|
const results = [];
|
|
for (const executor of executors) {
|
|
results.push(await collectClusterForExecutor(executor, timeoutMs));
|
|
}
|
|
return {
|
|
namespace,
|
|
attemptedExecutors: results.map((item) => item.executor),
|
|
readable: results.some((item) => item.ready),
|
|
executors: results
|
|
};
|
|
}
|
|
|
|
function probeLooksLikeClusterServerUnavailable(probe) {
|
|
const text = oneLine(`${probe?.stderr ?? ""} ${probe?.error ?? ""}`).toLowerCase();
|
|
return [
|
|
"connection refused",
|
|
"no route to host",
|
|
"i/o timeout",
|
|
"context deadline exceeded",
|
|
"unable to connect to the server",
|
|
"the server is currently unable to handle the request"
|
|
].some((needle) => text.includes(needle));
|
|
}
|
|
|
|
function inferD601K3sUnavailable({ cluster, publicEndpoints }) {
|
|
if (cluster.readable === true || publicEndpoints.d601PublicEndpointsReachable === true) {
|
|
return false;
|
|
}
|
|
|
|
return (cluster.executors ?? []).some((executor) =>
|
|
Object.values(executor.probes ?? {}).some((probe) => {
|
|
if (probe && typeof probe === "object" && !Array.isArray(probe) && Object.hasOwn(probe, "ok")) {
|
|
return probeLooksLikeClusterServerUnavailable(probe);
|
|
}
|
|
return Object.values(probe ?? {}).some((nested) => probeLooksLikeClusterServerUnavailable(nested));
|
|
})
|
|
);
|
|
}
|
|
|
|
function d601StatusInference({ runnerKubeconfig, publicEndpoints, d601K3sUnavailable }) {
|
|
return {
|
|
runnerKubeconfigReadable: runnerKubeconfig.runnerKubeconfigReadable,
|
|
runnerKubeconfigProbeOk: runnerKubeconfig.runnerKubeconfigProbeOk,
|
|
runnerKubeconfigProbeExitCode: runnerKubeconfig.runnerKubeconfigProbeExitCode,
|
|
runnerKubeconfigProbeStderr: runnerKubeconfig.runnerKubeconfigProbeStderr,
|
|
d601PublicEndpointsReachable: publicEndpoints.d601PublicEndpointsReachable,
|
|
d601K3sUnavailable,
|
|
conclusion: d601K3sUnavailable
|
|
? "Direct read-only probes indicate D601 k3s may be unavailable."
|
|
: "Runner kubeconfig readability is an observability signal only; it is not used to infer D601 k3s or public DEV endpoint availability.",
|
|
blockerClassification: runnerKubeconfig.runnerKubeconfigReadable
|
|
? "none"
|
|
: "runner_permission_mount_gap",
|
|
sourceIssue: issue
|
|
};
|
|
}
|
|
|
|
function addBlocker(blockers, blocker) {
|
|
assert.ok(blockerTypes.has(blocker.type), `unknown blocker type ${blocker.type}`);
|
|
const key = `${blocker.type}:${blocker.scope}`;
|
|
if (!blockers.some((item) => `${item.type}:${item.scope}` === key)) {
|
|
blockers.push({ status: "open", ...blocker });
|
|
}
|
|
}
|
|
|
|
function buildBlockers({ binaries, kubeconfig, runnerKubeconfig, sshBridge, cluster }) {
|
|
const blockers = [];
|
|
const hasDirectClient = binaries.kubectl.available || binaries.k3s.available;
|
|
if (!hasDirectClient) {
|
|
addBlocker(blockers, {
|
|
type: "environment_blocker",
|
|
scope: "d601-k3s-client-binary",
|
|
summary: "Neither kubectl nor k3s is installed in this runner PATH.",
|
|
nextTask: "Install kubectl in the D601 Code Queue runner image, or mount an approved k3s kubectl client path for read-only hwlab-dev observation."
|
|
});
|
|
}
|
|
if (!kubeconfig.some((item) => item.readable)) {
|
|
addBlocker(blockers, {
|
|
type: "observability_blocker",
|
|
scope: "runner-kubeconfig-readonly-gap",
|
|
classification: "runner_permission_mount_gap",
|
|
sourceIssue: issue,
|
|
summary: "No readable KUBECONFIG/default k3s kubeconfig path was found by metadata checks; this is a runner permission/mount gap and does not prove D601 k3s is unavailable.",
|
|
nextTask: "Mount a read-only kubeconfig for hwlab-dev, or document the approved k3s local kubeconfig path without exposing token material."
|
|
});
|
|
}
|
|
if (!runnerKubeconfig.runnerKubeconfigReadable) {
|
|
addBlocker(blockers, {
|
|
type: "observability_blocker",
|
|
scope: "runner-kubeconfig-readonly-gap",
|
|
classification: "runner_permission_mount_gap",
|
|
sourceIssue: issue,
|
|
summary: "The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately.",
|
|
nextTask: "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report."
|
|
});
|
|
}
|
|
if (!sshBridge.configured && !cluster.readable) {
|
|
addBlocker(blockers, {
|
|
type: "environment_blocker",
|
|
scope: "d601-maintenance-ssh-bridge",
|
|
summary: "No SSH maintenance bridge was detected from ssh binary plus known environment variables or ~/.ssh/config metadata.",
|
|
nextTask: "Provide a documented read-only maintenance bridge variable or config path if kubectl cannot be mounted directly in the runner."
|
|
});
|
|
}
|
|
if (hasDirectClient && !cluster.readable) {
|
|
const attempted = cluster.attemptedExecutors.length > 0 ? cluster.attemptedExecutors.join(", ") : "none";
|
|
addBlocker(blockers, {
|
|
type: "observability_blocker",
|
|
scope: "hwlab-dev-readonly-rbac",
|
|
classification: "readonly_observability_gap",
|
|
sourceIssue: issue,
|
|
summary: `Read-only runner probes did not return pods, services, and ConfigMaps for ${namespace}; attempted executors: ${attempted}. This is an observability gap unless independent evidence proves D601 k3s unavailable.`,
|
|
nextTask: "Grant or mount read-only access that can get namespace, pods, services, and ConfigMaps in hwlab-dev, then rerun this preflight."
|
|
});
|
|
}
|
|
return blockers;
|
|
}
|
|
|
|
function runnerImageFollowUp(blockers) {
|
|
const scopes = new Set(blockers.map((blocker) => blocker.scope));
|
|
const needed = scopes.has("d601-k3s-client-binary") || scopes.has("runner-kubeconfig-readonly-gap");
|
|
return {
|
|
needed,
|
|
suggestedUniDeskIssue: needed
|
|
? "Code Queue runner image should include kubectl and a documented read-only hwlab-dev kubeconfig mount/permission path; keep Secret resources and token material inaccessible to reports."
|
|
: null
|
|
};
|
|
}
|
|
|
|
async function writeReport(report, reportPath) {
|
|
const absoluteReportPath = ensureNotRepoReportsPath(repoRoot, reportPath, "--report");
|
|
await mkdir(path.dirname(absoluteReportPath), { recursive: true });
|
|
await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
}
|
|
|
|
async function buildReport(args) {
|
|
const binaries = await collectBinaries();
|
|
const [kubeconfig, runnerKubeconfig, sshBridge, cluster, publicEndpoints] = await Promise.all([
|
|
collectKubeconfigPaths(),
|
|
collectRunnerKubeconfigProbe(args.timeoutMs),
|
|
collectSshBridge(binaries),
|
|
collectClusterObservability(binaries, args.timeoutMs),
|
|
collectPublicEndpoints(args.timeoutMs)
|
|
]);
|
|
const d601K3sUnavailable = inferD601K3sUnavailable({ cluster, publicEndpoints });
|
|
const blockers = buildBlockers({ binaries, kubeconfig, runnerKubeconfig, sshBridge, cluster });
|
|
const conclusion = cluster.readable && blockers.length === 0 ? "ready" : "blocked";
|
|
const statusInference = d601StatusInference({
|
|
runnerKubeconfig,
|
|
publicEndpoints,
|
|
d601K3sUnavailable
|
|
});
|
|
return {
|
|
$schema: "https://hwlab.pikastech.local/schemas/d601-k3s-readonly-observability-report.schema.json",
|
|
$id: "https://hwlab.pikastech.local/dev-gate/d601-k3s-readonly-observability.json",
|
|
reportVersion: "v1",
|
|
reportKind: "d601-k3s-readonly-observability",
|
|
issue,
|
|
supports,
|
|
generatedAt: new Date().toISOString(),
|
|
mode: "read-only",
|
|
devOnly: true,
|
|
prodDisabled: true,
|
|
namespace,
|
|
runtimeTarget: {
|
|
runner: "D601 Code Queue",
|
|
cluster: "D601 k3s",
|
|
namespace,
|
|
resources: ["pods", "services", "configmaps"]
|
|
},
|
|
safety: {
|
|
noProd: true,
|
|
noSecretsPrinted: true,
|
|
noSecretResourcesRead: true,
|
|
noClusterMutation: true,
|
|
noRuntimeRestart: true,
|
|
noHeavyE2E: true
|
|
},
|
|
forbiddenActions,
|
|
validationCommands: [
|
|
"node --check scripts/d601-k3s-readonly-observability.mjs",
|
|
"node --check scripts/src/d601-k3s-readonly-observability.mjs",
|
|
"node scripts/d601-k3s-readonly-observability.mjs"
|
|
],
|
|
conclusion,
|
|
runnerKubeconfigReadable: statusInference.runnerKubeconfigReadable,
|
|
runnerKubeconfigProbeOk: statusInference.runnerKubeconfigProbeOk,
|
|
runnerKubeconfigProbeExitCode: statusInference.runnerKubeconfigProbeExitCode,
|
|
runnerKubeconfigProbeStderr: statusInference.runnerKubeconfigProbeStderr,
|
|
d601PublicEndpointsReachable: statusInference.d601PublicEndpointsReachable,
|
|
d601K3sUnavailable: statusInference.d601K3sUnavailable,
|
|
statusInference,
|
|
environment: {
|
|
binaries,
|
|
kubeconfig,
|
|
runnerKubeconfig,
|
|
sshBridge
|
|
},
|
|
publicEndpoints,
|
|
cluster,
|
|
blockers,
|
|
runnerImageFollowUp: runnerImageFollowUp(blockers),
|
|
notes: "This report uses command availability checks, file metadata checks, public DEV endpoint GET probes, kubectl/k3s kubectl read-only get probes, and auth can-i checks only. It does not print kubeconfig contents, token material, Secret resources, ConfigMap values, SSH config contents, or private keys. A failed runner /etc/rancher/k3s/k3s.yaml probe is classified as a runner permission/mount or read-only observability gap and is not evidence that D601 k3s is globally unavailable."
|
|
};
|
|
}
|
|
|
|
function printSummary(args, report) {
|
|
const summary = {
|
|
issue,
|
|
reportKind: report.reportKind,
|
|
conclusion: report.conclusion,
|
|
namespace: report.namespace,
|
|
report: args.writeReport ? args.reportPath : null,
|
|
readable: report.cluster.readable,
|
|
runnerKubeconfigReadable: report.runnerKubeconfigReadable,
|
|
runnerKubeconfigProbeOk: report.runnerKubeconfigProbeOk,
|
|
runnerKubeconfigProbeExitCode: report.runnerKubeconfigProbeExitCode,
|
|
d601PublicEndpointsReachable: report.d601PublicEndpointsReachable,
|
|
d601K3sUnavailable: report.d601K3sUnavailable,
|
|
attemptedExecutors: report.cluster.attemptedExecutors,
|
|
blockers: report.blockers.map((blocker) => ({
|
|
type: blocker.type,
|
|
scope: blocker.scope,
|
|
nextTask: blocker.nextTask
|
|
}))
|
|
};
|
|
process.stdout.write(`${JSON.stringify(args.pretty ? report : summary, null, args.pretty ? 2 : 0)}\n`);
|
|
}
|
|
|
|
export function formatFailure(error) {
|
|
return {
|
|
issue,
|
|
reportKind: "d601-k3s-readonly-observability",
|
|
conclusion: "blocked",
|
|
error: error instanceof Error ? error.message : String(error)
|
|
};
|
|
}
|
|
|
|
export async function runD601K3sReadonlyObservability(argv) {
|
|
const args = parseArgs(argv);
|
|
if (args.help) {
|
|
process.stdout.write(`${usage()}\n`);
|
|
return 0;
|
|
}
|
|
|
|
const report = await buildReport(args);
|
|
if (args.writeReport) {
|
|
await writeReport(report, args.reportPath);
|
|
}
|
|
printSummary(args, report);
|
|
if (report.conclusion === "blocked" && args.failOnBlocked) {
|
|
return 2;
|
|
}
|
|
return 0;
|
|
}
|