Codex
748 lines
28 KiB
JavaScript
748 lines
28 KiB
JavaScript
import { execFileSync } from "node:child_process";
|
|
import { request as httpRequest } from "node:http";
|
|
import { request as httpsRequest } from "node:https";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import {
|
|
AGENT_MGR_SERVICE_ID,
|
|
AGENT_SKILLS_SERVICE_ID,
|
|
AGENT_WORKER_SERVICE_ID
|
|
} from "../../internal/agent/index.mjs";
|
|
import {
|
|
RUNTIME_DURABLE_ADAPTER_MISSING,
|
|
RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
|
|
RUNTIME_STORE_KIND_POSTGRES
|
|
} from "../../internal/db/runtime-store.ts";
|
|
import { CLOUD_CORE_MIGRATION_ID } from "../../internal/db/schema.ts";
|
|
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
|
|
export const d601Kubeconfig = "/etc/rancher/k3s/k3s.yaml";
|
|
export const devNamespace = "hwlab-dev";
|
|
export const d601K3sReadonlyCommand =
|
|
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods,svc,deploy,job,cm -o wide";
|
|
export const workerServerDryRunCommand =
|
|
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json";
|
|
|
|
export function isDbLiveReady(body) {
|
|
const db = body?.db;
|
|
return Boolean(db?.ready && db?.connected && db?.liveConnected !== false && db?.liveDbEvidence);
|
|
}
|
|
|
|
export function isDbLiveBlocked(body) {
|
|
const db = body?.db;
|
|
if (!db || typeof db !== "object") {
|
|
return false;
|
|
}
|
|
if (isDbLiveReady(body)) {
|
|
return false;
|
|
}
|
|
return db.connected === false ||
|
|
db.ready === false ||
|
|
db.liveConnected === false ||
|
|
db.liveDbEvidence === false ||
|
|
["blocked", "degraded", "failed"].includes(db.status);
|
|
}
|
|
|
|
export function summarizeDbBlocker(body) {
|
|
const db = body?.db ?? {};
|
|
const missing = Array.isArray(db.missingEnv) && db.missingEnv.length > 0
|
|
? ` missing ${db.missingEnv.join(", ")}`
|
|
: "";
|
|
return `cloud-api /health/live reports DB ${db.status ?? "not ready"}; connected=${db.connected ?? "unknown"}; ready=${db.ready ?? "unknown"}; liveDbEvidence=${db.liveDbEvidence ?? "unknown"}.${missing}`;
|
|
}
|
|
|
|
export function runtimeDurabilityFromHealth(body) {
|
|
const runtime = body?.runtime ?? {};
|
|
const durability = body?.readiness?.durability ?? {};
|
|
const migration = runtime.migration ?? {};
|
|
const migrationGate = runtime.gates?.migration ?? durability.gates?.migration ?? {};
|
|
const blocker = runtime.blocker ??
|
|
durability.blocker ??
|
|
(Array.isArray(body?.blockerCodes)
|
|
? body.blockerCodes.find((code) => String(code).startsWith("runtime_durable_adapter_"))
|
|
: null) ??
|
|
RUNTIME_DURABLE_ADAPTER_MISSING;
|
|
const durableRequested = Boolean(
|
|
runtime.durableRequested ||
|
|
durability.durableRequested ||
|
|
runtime.adapter === RUNTIME_STORE_KIND_POSTGRES ||
|
|
durability.adapter === RUNTIME_STORE_KIND_POSTGRES
|
|
);
|
|
const ready = Boolean(
|
|
runtime.durable === true &&
|
|
runtime.ready === true &&
|
|
runtime.liveRuntimeEvidence === true &&
|
|
durability.ready !== false
|
|
);
|
|
|
|
return {
|
|
adapter: runtime.adapter ?? durability.adapter ?? "unknown",
|
|
durable: Boolean(runtime.durable),
|
|
durableRequested,
|
|
ready,
|
|
runtimeStatus: runtime.status ?? durability.runtimeStatus ?? "unknown",
|
|
blocker,
|
|
blockedLayer: runtime.durabilityContract?.blockedLayer ?? durability.blockedLayer ?? null,
|
|
requiredEvidence: runtime.durabilityContract?.requiredEvidence ??
|
|
durability.requiredEvidence ??
|
|
RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
|
|
migration: {
|
|
checked: migration.checked ?? migrationGate.checked ?? false,
|
|
ready: migration.ready ?? migrationGate.ready ?? false,
|
|
missing: migration.missing ?? (migration.ready === false ? true : null),
|
|
requiredMigrationId: migration.requiredMigrationId ?? CLOUD_CORE_MIGRATION_ID,
|
|
appliedMigrationId: migration.appliedMigrationId ?? null
|
|
},
|
|
queryAttempted: Boolean(runtime.connection?.queryAttempted ?? durability.queryAttempted),
|
|
queryResult: runtime.connection?.queryResult ?? durability.queryResult ?? "unknown"
|
|
};
|
|
}
|
|
|
|
export function isRuntimeDurableAdapterBlocked(body) {
|
|
const runtime = runtimeDurabilityFromHealth(body);
|
|
if (!runtime.durableRequested) {
|
|
return false;
|
|
}
|
|
return !runtime.ready ||
|
|
["blocked", "degraded", "failed"].includes(runtime.runtimeStatus) ||
|
|
String(runtime.blocker).startsWith("runtime_durable_adapter_");
|
|
}
|
|
|
|
export function summarizeRuntimeDurableBlocker(body) {
|
|
const runtime = runtimeDurabilityFromHealth(body);
|
|
const migration = runtime.migration;
|
|
return `cloud-api durable runtime adapter is blocked: ${runtime.blocker}; adapter=${runtime.adapter}; durable=${runtime.durable}; ready=${runtime.ready}; migration=${migration.requiredMigrationId} checked=${migration.checked} ready=${migration.ready} missing=${migration.missing ?? "unknown"}; queryResult=${runtime.queryResult}.`;
|
|
}
|
|
|
|
export function publicDbLiveSummary(body) {
|
|
const db = body?.db ?? null;
|
|
if (!db || typeof db !== "object") {
|
|
return null;
|
|
}
|
|
return {
|
|
status: db.status ?? null,
|
|
connected: db.connected ?? null,
|
|
liveConnected: db.liveConnected ?? null,
|
|
ready: db.ready ?? null,
|
|
configReady: db.configReady ?? null,
|
|
connectionChecked: db.connectionChecked ?? null,
|
|
connectionAttempted: db.connectionAttempted ?? null,
|
|
connectionResult: db.connectionResult ?? null,
|
|
endpointSource: db.endpointSource ?? db.connection?.endpointSource ?? null,
|
|
liveDbEvidence: db.liveDbEvidence ?? null,
|
|
evidence: db.evidence ?? null
|
|
};
|
|
}
|
|
|
|
export function publicRuntimeDurabilitySummary(body) {
|
|
if (!body?.runtime && !body?.readiness?.durability) {
|
|
return null;
|
|
}
|
|
const runtime = runtimeDurabilityFromHealth(body);
|
|
return {
|
|
adapter: runtime.adapter,
|
|
durable: runtime.durable,
|
|
durableRequested: runtime.durableRequested,
|
|
ready: runtime.ready,
|
|
status: runtime.runtimeStatus,
|
|
blocker: runtime.blocker,
|
|
blockedLayer: runtime.blockedLayer,
|
|
requiredEvidence: runtime.requiredEvidence,
|
|
liveRuntimeEvidence: Boolean(runtime.ready),
|
|
migration: runtime.migration,
|
|
queryAttempted: runtime.queryAttempted,
|
|
queryResult: runtime.queryResult
|
|
};
|
|
}
|
|
|
|
export function classifyCloudApiLiveReadiness(body) {
|
|
if (isDbLiveBlocked(body)) {
|
|
return {
|
|
blocked: true,
|
|
blockerType: "runtime_blocker",
|
|
blockerScope: "db-live",
|
|
sourceIssue: "pikasTech/HWLAB#49",
|
|
blockedClassification: "DB live",
|
|
blockerSummary: summarizeDbBlocker(body),
|
|
summary: "Blocked at DB live readiness before scheduling a DEV agent task.",
|
|
evidenceSuffix: "db-blocked",
|
|
dbLiveReady: false,
|
|
runtimeDurableReady: false
|
|
};
|
|
}
|
|
|
|
if (isRuntimeDurableAdapterBlocked(body)) {
|
|
const runtime = runtimeDurabilityFromHealth(body);
|
|
return {
|
|
blocked: true,
|
|
blockerType: "runtime_blocker",
|
|
blockerScope: "runtime-durable-adapter",
|
|
sourceIssue: "pikasTech/HWLAB#164",
|
|
blockedClassification: runtime.blocker,
|
|
blockerSummary: summarizeRuntimeDurableBlocker(body),
|
|
summary: "Blocked at runtime durable adapter readiness before scheduling a DEV agent task.",
|
|
evidenceSuffix: `runtime-durable-adapter-blocked:${runtime.blocker}`,
|
|
dbLiveReady: isDbLiveReady(body),
|
|
runtimeDurableReady: false
|
|
};
|
|
}
|
|
|
|
return {
|
|
blocked: false,
|
|
blockerType: null,
|
|
blockerScope: null,
|
|
blockedClassification: "none",
|
|
blockerSummary: "",
|
|
summary: "Live DEV preflight observed DB live and durable runtime adapter readiness.",
|
|
evidenceSuffix: null,
|
|
dbLiveReady: isDbLiveReady(body),
|
|
runtimeDurableReady: true
|
|
};
|
|
}
|
|
|
|
export function requestJson(urlString, { method = "GET", headers = {}, body, timeoutMs = 5000 } = {}) {
|
|
const url = new URL(urlString);
|
|
const client = url.protocol === "https:" ? httpsRequest : httpRequest;
|
|
const payload = body === undefined ? null : Buffer.from(body);
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const request = client(
|
|
{
|
|
protocol: url.protocol,
|
|
hostname: url.hostname,
|
|
port: url.port,
|
|
path: `${url.pathname}${url.search}`,
|
|
method,
|
|
headers
|
|
},
|
|
(response) => {
|
|
let raw = "";
|
|
response.setEncoding("utf8");
|
|
response.on("data", (chunk) => {
|
|
raw += chunk;
|
|
});
|
|
response.on("end", () => {
|
|
let bodyValue = null;
|
|
if (raw.length > 0) {
|
|
try {
|
|
bodyValue = JSON.parse(raw);
|
|
} catch (error) {
|
|
reject(new Error(`invalid JSON response from ${urlString}: ${error.message}`));
|
|
return;
|
|
}
|
|
}
|
|
resolve({
|
|
statusCode: response.statusCode ?? 0,
|
|
headers: response.headers,
|
|
body: bodyValue,
|
|
raw
|
|
});
|
|
});
|
|
}
|
|
);
|
|
|
|
request.on("error", reject);
|
|
request.setTimeout(timeoutMs, () => {
|
|
request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`));
|
|
});
|
|
|
|
if (payload) request.write(payload);
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
function requestHead(urlString, { timeoutMs = 5000 } = {}) {
|
|
const url = new URL(urlString);
|
|
const client = url.protocol === "https:" ? httpsRequest : httpRequest;
|
|
|
|
return new Promise((resolve, reject) => {
|
|
const request = client(
|
|
{
|
|
protocol: url.protocol,
|
|
hostname: url.hostname,
|
|
port: url.port,
|
|
path: `${url.pathname}${url.search}`,
|
|
method: "HEAD"
|
|
},
|
|
(response) => {
|
|
response.resume();
|
|
response.on("end", () => {
|
|
resolve({
|
|
statusCode: response.statusCode ?? 0,
|
|
headers: response.headers
|
|
});
|
|
});
|
|
}
|
|
);
|
|
request.on("error", reject);
|
|
request.setTimeout(timeoutMs, () => {
|
|
request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`));
|
|
});
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
function runKubectl(args, { input, timeoutMs = 5000 } = {}) {
|
|
try {
|
|
return {
|
|
ok: true,
|
|
command: `KUBECONFIG=${d601Kubeconfig} kubectl ${args.join(" ")}`,
|
|
exitCode: 0,
|
|
stdout: execFileSync("kubectl", args, {
|
|
cwd: repoRoot,
|
|
encoding: "utf8",
|
|
env: { ...process.env, KUBECONFIG: d601Kubeconfig },
|
|
input,
|
|
stdio: ["pipe", "pipe", "pipe"],
|
|
timeout: timeoutMs
|
|
})
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
ok: false,
|
|
command: `KUBECONFIG=${d601Kubeconfig} kubectl ${args.join(" ")}`,
|
|
exitCode: typeof error.status === "number" ? error.status : 1,
|
|
stdout: String(error.stdout ?? ""),
|
|
stderr: String(error.stderr ?? ""),
|
|
error: error instanceof Error ? error.message : String(error)
|
|
};
|
|
}
|
|
}
|
|
|
|
function parseProbeJson(probe) {
|
|
if (!probe.ok || !probe.stdout.trim()) return null;
|
|
try {
|
|
return JSON.parse(probe.stdout);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function compactError(probe) {
|
|
return (probe.stderr || probe.error || probe.stdout || "unknown error")
|
|
.replace(/\s+/g, " ")
|
|
.trim()
|
|
.slice(0, 400);
|
|
}
|
|
|
|
function items(document) {
|
|
if (!document) return [];
|
|
return Array.isArray(document.items) ? document.items : [document];
|
|
}
|
|
|
|
function named(document, name) {
|
|
return items(document).find((item) => item?.metadata?.name === name) ?? null;
|
|
}
|
|
|
|
function firstContainer(workload) {
|
|
return workload?.spec?.template?.spec?.containers?.[0] ?? null;
|
|
}
|
|
|
|
function envMap(container) {
|
|
const values = {};
|
|
const valueFrom = [];
|
|
for (const entry of container?.env ?? []) {
|
|
if (Object.hasOwn(entry, "value")) values[entry.name] = entry.value;
|
|
else if (entry.valueFrom) valueFrom.push(entry.name);
|
|
}
|
|
return { values, valueFrom };
|
|
}
|
|
|
|
function condition(workload, type) {
|
|
return workload?.status?.conditions?.find((entry) => entry.type === type)?.status ?? "Unknown";
|
|
}
|
|
|
|
function deploymentSummary(deployment) {
|
|
const container = firstContainer(deployment);
|
|
const replicas = deployment?.status?.replicas ?? deployment?.spec?.replicas ?? 0;
|
|
const readyReplicas = deployment?.status?.readyReplicas ?? 0;
|
|
return {
|
|
name: deployment?.metadata?.name ?? null,
|
|
image: container?.image ?? null,
|
|
readyReplicas,
|
|
replicas,
|
|
available: condition(deployment, "Available"),
|
|
ready: replicas > 0 && readyReplicas >= replicas && condition(deployment, "Available") === "True",
|
|
env: envMap(container)
|
|
};
|
|
}
|
|
|
|
function workerTemplateSummary(job) {
|
|
const container = firstContainer(job);
|
|
return {
|
|
name: job?.metadata?.name ?? null,
|
|
suspend: job?.spec?.suspend ?? null,
|
|
image: container?.image ?? null,
|
|
restartPolicy: job?.spec?.template?.spec?.restartPolicy ?? null,
|
|
env: envMap(container)
|
|
};
|
|
}
|
|
|
|
function serviceUrl(service, fallbackPort) {
|
|
const clusterIP = service?.spec?.clusterIP;
|
|
if (!clusterIP || clusterIP === "None") return null;
|
|
return `http://${clusterIP}:${service?.spec?.ports?.[0]?.port ?? fallbackPort}`;
|
|
}
|
|
|
|
function endpointCount(endpoint) {
|
|
return (endpoint?.subsets ?? []).reduce((total, subset) => total + (subset.addresses?.length ?? 0), 0);
|
|
}
|
|
|
|
function buildWorkerDryRunJob(templateJob, fixture, liveSkillsEnv) {
|
|
const templateContainer = firstContainer(templateJob);
|
|
const labels = {
|
|
"app.kubernetes.io/name": AGENT_WORKER_SERVICE_ID,
|
|
"hwlab.pikastech.local/environment": ENVIRONMENT_DEV,
|
|
"hwlab.pikastech.local/profile": ENVIRONMENT_DEV,
|
|
"hwlab.pikastech.local/service-id": AGENT_WORKER_SERVICE_ID,
|
|
"hwlab.pikastech.local/smoke": "dev-m4-agent-loop"
|
|
};
|
|
const reservedEnv = new Set([
|
|
"HWLAB_AGENT_SESSION_ID",
|
|
"HWLAB_PROJECT_ID",
|
|
"HWLAB_SKILL_COMMIT_ID",
|
|
"HWLAB_SKILL_VERSION",
|
|
"HWLAB_WORKSPACE_ROOT"
|
|
]);
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: {
|
|
name: "hwlab-agent-worker-smoke-dryrun-20260522",
|
|
namespace: devNamespace,
|
|
labels
|
|
},
|
|
spec: {
|
|
suspend: true,
|
|
backoffLimit: 0,
|
|
template: {
|
|
metadata: { labels },
|
|
spec: {
|
|
restartPolicy: templateJob?.spec?.template?.spec?.restartPolicy ?? "Never",
|
|
containers: [
|
|
{
|
|
name: templateContainer?.name ?? AGENT_WORKER_SERVICE_ID,
|
|
image: templateContainer?.image ?? "unknown",
|
|
env: [
|
|
...(templateContainer?.env ?? []).filter((entry) => !reservedEnv.has(entry.name)),
|
|
{ name: "HWLAB_AGENT_SESSION_ID", value: fixture.agentSessionId },
|
|
{ name: "HWLAB_PROJECT_ID", value: fixture.projectId },
|
|
{ name: "HWLAB_SKILL_COMMIT_ID", value: liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? "" },
|
|
{ name: "HWLAB_SKILL_VERSION", value: liveSkillsEnv.HWLAB_SKILLS_VERSION ?? "" },
|
|
{ name: "HWLAB_WORKSPACE_ROOT", value: `/workspace/${fixture.projectId}/${fixture.agentSessionId}` }
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
function k3sBlockers(report) {
|
|
const blockers = [];
|
|
const push = (type, scope, summary, classification, nextFix) => {
|
|
blockers.push({ type, scope, status: "open", summary, classification, nextFix });
|
|
};
|
|
if (!report.clusterReadable) {
|
|
push(
|
|
"environment_blocker",
|
|
"d601-k3s-readonly",
|
|
"D601 native k3s hwlab-dev resources were not readable with /etc/rancher/k3s/k3s.yaml.",
|
|
"k3s readonly",
|
|
"Mount or authorize read-only hwlab-dev access through the D601 native k3s kubeconfig."
|
|
);
|
|
return blockers;
|
|
}
|
|
if (report.agentManager.health.status !== "ok") {
|
|
push(
|
|
"agent_blocker",
|
|
"agent-mgr-health",
|
|
`hwlab-agent-mgr /health/live is ${report.agentManager.health.status}.`,
|
|
"agent runtime",
|
|
"Inject explicit skill commit and version into agent-mgr health configuration, then redeploy DEV."
|
|
);
|
|
}
|
|
if (report.skillsInjection.status !== "pass") {
|
|
push(
|
|
"agent_blocker",
|
|
"skills-commit-version-injection",
|
|
`DEV skills injection is incomplete: missing ${report.skillsInjection.missing.join(", ")}.`,
|
|
"skills injection",
|
|
"Add HWLAB_SKILLS_VERSION to hwlab-agent-skills and ensure scheduled worker jobs receive HWLAB_SKILL_COMMIT_ID plus HWLAB_SKILL_VERSION."
|
|
);
|
|
}
|
|
if (report.workerJob.serverDryRun.status !== "pass") {
|
|
push(
|
|
"agent_blocker",
|
|
"worker-job-admission",
|
|
report.workerJob.serverDryRun.summary,
|
|
"worker job",
|
|
"Fix the hwlab-agent-worker-template image/env/job spec until server-side dry-run admission succeeds."
|
|
);
|
|
}
|
|
return blockers;
|
|
}
|
|
|
|
export async function collectPublicEntrypoints() {
|
|
const apiHealth = await requestJson(`${DEV_ENDPOINT}/health`).catch((error) => ({
|
|
statusCode: 0,
|
|
body: null,
|
|
error: error instanceof Error ? error.message : String(error)
|
|
}));
|
|
const apiLive = await requestJson(`${DEV_ENDPOINT}/health/live`).catch((error) => ({
|
|
statusCode: 0,
|
|
body: null,
|
|
error: error instanceof Error ? error.message : String(error)
|
|
}));
|
|
const frontend = await requestHead(`${DEV_FRONTEND_ENDPOINT}/`).catch((error) => ({
|
|
statusCode: 0,
|
|
headers: {},
|
|
error: error instanceof Error ? error.message : String(error)
|
|
}));
|
|
return {
|
|
status: apiHealth.statusCode >= 200 &&
|
|
apiHealth.statusCode < 300 &&
|
|
apiLive.statusCode >= 200 &&
|
|
apiLive.statusCode < 300 &&
|
|
frontend.statusCode >= 200 &&
|
|
frontend.statusCode < 400
|
|
? "pass"
|
|
: "blocked",
|
|
commands: [
|
|
`curl -fsS --max-time 10 ${DEV_ENDPOINT}/health`,
|
|
`curl -fsS --max-time 10 ${DEV_ENDPOINT}/health/live`,
|
|
`curl -fsS -I --max-time 10 ${DEV_FRONTEND_ENDPOINT}/`
|
|
],
|
|
api: {
|
|
endpoint: DEV_ENDPOINT,
|
|
health: {
|
|
statusCode: apiHealth.statusCode,
|
|
serviceId: apiHealth.body?.serviceId ?? null,
|
|
environment: apiHealth.body?.environment ?? null,
|
|
status: apiHealth.body?.status ?? null,
|
|
error: apiHealth.error ?? null
|
|
},
|
|
live: {
|
|
statusCode: apiLive.statusCode,
|
|
serviceId: apiLive.body?.serviceId ?? null,
|
|
environment: apiLive.body?.environment ?? null,
|
|
status: apiLive.body?.status ?? null,
|
|
db: publicDbLiveSummary(apiLive.body),
|
|
runtime: publicRuntimeDurabilitySummary(apiLive.body),
|
|
blockerCodes: Array.isArray(apiLive.body?.blockerCodes) ? apiLive.body.blockerCodes : [],
|
|
error: apiLive.error ?? null
|
|
}
|
|
},
|
|
frontend: {
|
|
endpoint: DEV_FRONTEND_ENDPOINT,
|
|
statusCode: frontend.statusCode,
|
|
contentType: frontend.headers?.["content-type"] ?? null,
|
|
contentLength: frontend.headers?.["content-length"] ?? null,
|
|
error: frontend.error ?? null
|
|
},
|
|
summary: `Public DEV API ${DEV_ENDPOINT} health=${apiHealth.statusCode} live=${apiLive.statusCode}; frontend ${DEV_FRONTEND_ENDPOINT}/=${frontend.statusCode}.`
|
|
};
|
|
}
|
|
|
|
export async function collectD601K3sNativeEvidence(fixture) {
|
|
const namespaceProbe = runKubectl(["get", "namespace", devNamespace, "-o", "json"]);
|
|
const deploymentsProbe = runKubectl([
|
|
"-n",
|
|
devNamespace,
|
|
"get",
|
|
"deploy",
|
|
AGENT_MGR_SERVICE_ID,
|
|
AGENT_SKILLS_SERVICE_ID,
|
|
"-o",
|
|
"json"
|
|
]);
|
|
const workerTemplateProbe = runKubectl([
|
|
"-n",
|
|
devNamespace,
|
|
"get",
|
|
"job",
|
|
"hwlab-agent-worker-template",
|
|
"-o",
|
|
"json"
|
|
]);
|
|
const servicesProbe = runKubectl([
|
|
"-n",
|
|
devNamespace,
|
|
"get",
|
|
"svc",
|
|
AGENT_MGR_SERVICE_ID,
|
|
AGENT_SKILLS_SERVICE_ID,
|
|
"-o",
|
|
"json"
|
|
]);
|
|
const endpointsProbe = runKubectl([
|
|
"-n",
|
|
devNamespace,
|
|
"get",
|
|
"endpoints",
|
|
AGENT_MGR_SERVICE_ID,
|
|
AGENT_SKILLS_SERVICE_ID,
|
|
"-o",
|
|
"json"
|
|
]);
|
|
const canCreateJobs = runKubectl(["auth", "can-i", "create", "jobs", "-n", devNamespace]);
|
|
const canDeleteJobs = runKubectl(["auth", "can-i", "delete", "jobs", "-n", devNamespace]);
|
|
const canGetSecrets = runKubectl(["auth", "can-i", "get", "secrets", "-n", devNamespace]);
|
|
|
|
const deployments = parseProbeJson(deploymentsProbe);
|
|
const services = parseProbeJson(servicesProbe);
|
|
const endpoints = parseProbeJson(endpointsProbe);
|
|
const workerTemplate = parseProbeJson(workerTemplateProbe);
|
|
const agentMgrDeployment = named(deployments, AGENT_MGR_SERVICE_ID);
|
|
const skillsDeployment = named(deployments, AGENT_SKILLS_SERVICE_ID);
|
|
const agentMgrService = named(services, AGENT_MGR_SERVICE_ID);
|
|
const skillsService = named(services, AGENT_SKILLS_SERVICE_ID);
|
|
const agentMgrEndpoint = named(endpoints, AGENT_MGR_SERVICE_ID);
|
|
const skillsEndpoint = named(endpoints, AGENT_SKILLS_SERVICE_ID);
|
|
const agentMgrSummary = deploymentSummary(agentMgrDeployment);
|
|
const skillsSummary = deploymentSummary(skillsDeployment);
|
|
const workerSummary = workerTemplateSummary(workerTemplate);
|
|
const liveSkillsEnv = skillsSummary.env.values;
|
|
const agentMgrUrl = serviceUrl(agentMgrService, 7410);
|
|
const skillsUrl = serviceUrl(skillsService, 7430);
|
|
const agentMgrHealth = agentMgrUrl ? await requestJson(`${agentMgrUrl}/health/live`).catch((error) => ({
|
|
statusCode: 0,
|
|
body: null,
|
|
error: error instanceof Error ? error.message : String(error)
|
|
})) : null;
|
|
const skillsHealth = skillsUrl ? await requestJson(`${skillsUrl}/health/live`).catch((error) => ({
|
|
statusCode: 0,
|
|
body: null,
|
|
error: error instanceof Error ? error.message : String(error)
|
|
})) : null;
|
|
const workerDryRunManifest = workerTemplate ? buildWorkerDryRunJob(workerTemplate, fixture, liveSkillsEnv) : null;
|
|
const workerDryRunProbe = workerDryRunManifest
|
|
? runKubectl(["-n", devNamespace, "create", "-f", "-", "--dry-run=server", "-o", "json"], {
|
|
input: `${JSON.stringify(workerDryRunManifest)}\n`
|
|
})
|
|
: { ok: false, stdout: "", stderr: "worker template was not readable" };
|
|
const workerDryRun = parseProbeJson(workerDryRunProbe);
|
|
const workerInjectedEnv = envMap(workerDryRunManifest?.spec?.template?.spec?.containers?.[0] ?? null).values;
|
|
const missingSkills = [];
|
|
if (!liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID) missingSkills.push("hwlab-agent-skills.HWLAB_SKILLS_COMMIT_ID");
|
|
if (!liveSkillsEnv.HWLAB_SKILLS_VERSION) missingSkills.push("hwlab-agent-skills.HWLAB_SKILLS_VERSION");
|
|
if (!workerInjectedEnv.HWLAB_SKILL_COMMIT_ID) missingSkills.push("worker-dry-run.HWLAB_SKILL_COMMIT_ID");
|
|
if (!workerInjectedEnv.HWLAB_SKILL_VERSION) missingSkills.push("worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV");
|
|
const clusterReadable = namespaceProbe.ok && deploymentsProbe.ok && workerTemplateProbe.ok && servicesProbe.ok;
|
|
const evidence = [
|
|
`k3s:namespace=${devNamespace}:read=${namespaceProbe.ok}`,
|
|
`k3s:deploy/${AGENT_MGR_SERVICE_ID}:ready=${agentMgrSummary.readyReplicas}/${agentMgrSummary.replicas}:available=${agentMgrSummary.available}`,
|
|
`k3s:deploy/${AGENT_SKILLS_SERVICE_ID}:ready=${skillsSummary.readyReplicas}/${skillsSummary.replicas}:available=${skillsSummary.available}`,
|
|
`k3s:svc/${AGENT_MGR_SERVICE_ID}:endpoints=${endpointCount(agentMgrEndpoint)}`,
|
|
`k3s:svc/${AGENT_SKILLS_SERVICE_ID}:endpoints=${endpointCount(skillsEndpoint)}`,
|
|
`agent-mgr:/health/live:${agentMgrHealth?.body?.health?.status ?? agentMgrHealth?.body?.status ?? "unreachable"}`,
|
|
`agent-skills:/health/live:${skillsHealth?.body?.status ?? "unreachable"}:${skillsHealth?.body?.revision ?? "unknown"}`,
|
|
`worker-template:suspend=${workerSummary.suspend}:image=${workerSummary.image ?? "unknown"}`,
|
|
`worker-job:server-dry-run=${workerDryRunProbe.ok}:persisted=false`,
|
|
`skills-injection:commit=${liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? "missing"}:version=${liveSkillsEnv.HWLAB_SKILLS_VERSION ?? "missing"}`,
|
|
`secret-rbac:get=${canGetSecrets.stdout.trim() || "unknown"}:secretResourcesRead=false`
|
|
];
|
|
const report = {
|
|
status: "pass",
|
|
mode: "dev-read-only-plus-server-dry-run",
|
|
generatedAt: new Date().toISOString(),
|
|
commands: [
|
|
`KUBECONFIG=${d601Kubeconfig} kubectl get namespace ${devNamespace} -o json`,
|
|
d601K3sReadonlyCommand,
|
|
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get deploy ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`,
|
|
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get job hwlab-agent-worker-template -o json`,
|
|
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get svc ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`,
|
|
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get endpoints ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`,
|
|
`KUBECONFIG=${d601Kubeconfig} kubectl auth can-i create jobs -n ${devNamespace}`,
|
|
`KUBECONFIG=${d601Kubeconfig} kubectl auth can-i delete jobs -n ${devNamespace}`,
|
|
`KUBECONFIG=${d601Kubeconfig} kubectl auth can-i get secrets -n ${devNamespace}`,
|
|
workerServerDryRunCommand
|
|
],
|
|
safety: {
|
|
namespace: devNamespace,
|
|
kubeconfig: d601Kubeconfig,
|
|
readOnly: true,
|
|
prodTouched: false,
|
|
secretResourcesRead: false,
|
|
secretMaterialPrinted: false,
|
|
servicesRestarted: false,
|
|
workerJobPersisted: false,
|
|
longRunningAgentStarted: false,
|
|
note: "The smoke checks Secret RBAC only with kubectl auth can-i; it never reads Secret resources or kubeconfig contents."
|
|
},
|
|
clusterReadable,
|
|
namespace: {
|
|
name: devNamespace,
|
|
readable: namespaceProbe.ok,
|
|
error: namespaceProbe.ok ? null : compactError(namespaceProbe)
|
|
},
|
|
rbac: {
|
|
createJobs: canCreateJobs.stdout.trim() === "yes",
|
|
deleteJobs: canDeleteJobs.stdout.trim() === "yes",
|
|
getSecrets: canGetSecrets.stdout.trim() === "yes",
|
|
secretReadNotPerformed: true
|
|
},
|
|
agentManager: {
|
|
deployment: agentMgrSummary,
|
|
endpointCount: endpointCount(agentMgrEndpoint),
|
|
health: {
|
|
statusCode: agentMgrHealth?.statusCode ?? 0,
|
|
status: agentMgrHealth?.body?.health?.status ?? agentMgrHealth?.body?.status ?? "unreachable",
|
|
serviceId: agentMgrHealth?.body?.serviceId ?? null,
|
|
missingSkills: agentMgrHealth?.body?.health?.skills?.missing ?? [],
|
|
error: agentMgrHealth?.error ?? null
|
|
}
|
|
},
|
|
agentSkills: {
|
|
deployment: skillsSummary,
|
|
endpointCount: endpointCount(skillsEndpoint),
|
|
health: {
|
|
statusCode: skillsHealth?.statusCode ?? 0,
|
|
status: skillsHealth?.body?.status ?? "unreachable",
|
|
serviceId: skillsHealth?.body?.serviceId ?? null,
|
|
revision: skillsHealth?.body?.revision ?? null,
|
|
error: skillsHealth?.error ?? null
|
|
}
|
|
},
|
|
workerJob: {
|
|
template: workerSummary,
|
|
serverDryRun: {
|
|
status: workerDryRunProbe.ok ? "pass" : "blocked",
|
|
accepted: workerDryRunProbe.ok,
|
|
command: workerServerDryRunCommand,
|
|
persisted: false,
|
|
generatedName: workerDryRunManifest?.metadata?.name ?? null,
|
|
injectedEnvNames: Object.keys(workerInjectedEnv).sort(),
|
|
dryRunKind: workerDryRun?.kind ?? null,
|
|
dryRunSuspend: workerDryRun?.spec?.suspend ?? null,
|
|
summary: workerDryRunProbe.ok
|
|
? "Server-side admission accepted a suspended worker Job generated from the template with session/workspace/skills env injected; no Job was persisted."
|
|
: compactError(workerDryRunProbe)
|
|
}
|
|
},
|
|
skillsInjection: {
|
|
status: missingSkills.length === 0 ? "pass" : "blocked",
|
|
source: "k3s deployment env plus generated worker dry-run manifest",
|
|
serviceCommitId: liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? null,
|
|
serviceVersion: liveSkillsEnv.HWLAB_SKILLS_VERSION ?? null,
|
|
workerDryRunCommitId: workerInjectedEnv.HWLAB_SKILL_COMMIT_ID ?? null,
|
|
workerDryRunVersion: workerInjectedEnv.HWLAB_SKILL_VERSION ?? null,
|
|
workerDryRunSource: "derived-from-live-dev-skills-env",
|
|
missing: missingSkills
|
|
},
|
|
traceEvidenceCleanup: {
|
|
status: "dry-run-only",
|
|
summary: "Live worker execution was not started. Trace/evidence/cleanup closure is covered by the bounded local dry-run and remains explicitly non-M3 DEV-LIVE."
|
|
},
|
|
evidence
|
|
};
|
|
report.blockers = k3sBlockers(report);
|
|
report.status = report.blockers.length === 0 ? "pass" : "blocked";
|
|
report.minimumFixRecommendations = report.blockers.map((blocker) => blocker.nextFix).filter(Boolean);
|
|
return report;
|
|
}
|