test: refresh DEV M4 agent smoke evidence (#139)
Commander review: direction is correct. This PR records DEV M4 agent-loop evidence as a bounded preflight, keeps M4 subordinate to the M3 trusted-loop mainline, and preserves blockers for DB live readiness and skill version injection. No PROD/service restart is included.
This commit is contained in:
Lyon
@@ -0,0 +1,567 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { request as httpRequest } from "node:http";
|
||||
import { request as httpsRequest } from "node:https";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
AGENT_MGR_SERVICE_ID,
|
||||
AGENT_SKILLS_SERVICE_ID,
|
||||
AGENT_WORKER_SERVICE_ID
|
||||
} from "../../internal/agent/index.mjs";
|
||||
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
|
||||
export const d601Kubeconfig = "/etc/rancher/k3s/k3s.yaml";
|
||||
export const devNamespace = "hwlab-dev";
|
||||
export const d601K3sReadonlyCommand =
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods,svc,deploy,job,cm -o wide";
|
||||
export const workerServerDryRunCommand =
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json";
|
||||
|
||||
export function requestJson(urlString, { method = "GET", headers = {}, body, timeoutMs = 5000 } = {}) {
|
||||
const url = new URL(urlString);
|
||||
const client = url.protocol === "https:" ? httpsRequest : httpRequest;
|
||||
const payload = body === undefined ? null : Buffer.from(body);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = client(
|
||||
{
|
||||
protocol: url.protocol,
|
||||
hostname: url.hostname,
|
||||
port: url.port,
|
||||
path: `${url.pathname}${url.search}`,
|
||||
method,
|
||||
headers
|
||||
},
|
||||
(response) => {
|
||||
let raw = "";
|
||||
response.setEncoding("utf8");
|
||||
response.on("data", (chunk) => {
|
||||
raw += chunk;
|
||||
});
|
||||
response.on("end", () => {
|
||||
let bodyValue = null;
|
||||
if (raw.length > 0) {
|
||||
try {
|
||||
bodyValue = JSON.parse(raw);
|
||||
} catch (error) {
|
||||
reject(new Error(`invalid JSON response from ${urlString}: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
}
|
||||
resolve({
|
||||
statusCode: response.statusCode ?? 0,
|
||||
headers: response.headers,
|
||||
body: bodyValue,
|
||||
raw
|
||||
});
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
request.on("error", reject);
|
||||
request.setTimeout(timeoutMs, () => {
|
||||
request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`));
|
||||
});
|
||||
|
||||
if (payload) request.write(payload);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
function requestHead(urlString, { timeoutMs = 5000 } = {}) {
|
||||
const url = new URL(urlString);
|
||||
const client = url.protocol === "https:" ? httpsRequest : httpRequest;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = client(
|
||||
{
|
||||
protocol: url.protocol,
|
||||
hostname: url.hostname,
|
||||
port: url.port,
|
||||
path: `${url.pathname}${url.search}`,
|
||||
method: "HEAD"
|
||||
},
|
||||
(response) => {
|
||||
response.resume();
|
||||
response.on("end", () => {
|
||||
resolve({
|
||||
statusCode: response.statusCode ?? 0,
|
||||
headers: response.headers
|
||||
});
|
||||
});
|
||||
}
|
||||
);
|
||||
request.on("error", reject);
|
||||
request.setTimeout(timeoutMs, () => {
|
||||
request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`));
|
||||
});
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
function runKubectl(args, { input, timeoutMs = 5000 } = {}) {
|
||||
try {
|
||||
return {
|
||||
ok: true,
|
||||
command: `KUBECONFIG=${d601Kubeconfig} kubectl ${args.join(" ")}`,
|
||||
exitCode: 0,
|
||||
stdout: execFileSync("kubectl", args, {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, KUBECONFIG: d601Kubeconfig },
|
||||
input,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
timeout: timeoutMs
|
||||
})
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
command: `KUBECONFIG=${d601Kubeconfig} kubectl ${args.join(" ")}`,
|
||||
exitCode: typeof error.status === "number" ? error.status : 1,
|
||||
stdout: String(error.stdout ?? ""),
|
||||
stderr: String(error.stderr ?? ""),
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function parseProbeJson(probe) {
|
||||
if (!probe.ok || !probe.stdout.trim()) return null;
|
||||
try {
|
||||
return JSON.parse(probe.stdout);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function compactError(probe) {
|
||||
return (probe.stderr || probe.error || probe.stdout || "unknown error")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.slice(0, 400);
|
||||
}
|
||||
|
||||
function items(document) {
|
||||
if (!document) return [];
|
||||
return Array.isArray(document.items) ? document.items : [document];
|
||||
}
|
||||
|
||||
function named(document, name) {
|
||||
return items(document).find((item) => item?.metadata?.name === name) ?? null;
|
||||
}
|
||||
|
||||
function firstContainer(workload) {
|
||||
return workload?.spec?.template?.spec?.containers?.[0] ?? null;
|
||||
}
|
||||
|
||||
function envMap(container) {
|
||||
const values = {};
|
||||
const valueFrom = [];
|
||||
for (const entry of container?.env ?? []) {
|
||||
if (Object.hasOwn(entry, "value")) values[entry.name] = entry.value;
|
||||
else if (entry.valueFrom) valueFrom.push(entry.name);
|
||||
}
|
||||
return { values, valueFrom };
|
||||
}
|
||||
|
||||
function condition(workload, type) {
|
||||
return workload?.status?.conditions?.find((entry) => entry.type === type)?.status ?? "Unknown";
|
||||
}
|
||||
|
||||
function deploymentSummary(deployment) {
|
||||
const container = firstContainer(deployment);
|
||||
const replicas = deployment?.status?.replicas ?? deployment?.spec?.replicas ?? 0;
|
||||
const readyReplicas = deployment?.status?.readyReplicas ?? 0;
|
||||
return {
|
||||
name: deployment?.metadata?.name ?? null,
|
||||
image: container?.image ?? null,
|
||||
readyReplicas,
|
||||
replicas,
|
||||
available: condition(deployment, "Available"),
|
||||
ready: replicas > 0 && readyReplicas >= replicas && condition(deployment, "Available") === "True",
|
||||
env: envMap(container)
|
||||
};
|
||||
}
|
||||
|
||||
function workerTemplateSummary(job) {
|
||||
const container = firstContainer(job);
|
||||
return {
|
||||
name: job?.metadata?.name ?? null,
|
||||
suspend: job?.spec?.suspend ?? null,
|
||||
image: container?.image ?? null,
|
||||
restartPolicy: job?.spec?.template?.spec?.restartPolicy ?? null,
|
||||
env: envMap(container)
|
||||
};
|
||||
}
|
||||
|
||||
function serviceUrl(service, fallbackPort) {
|
||||
const clusterIP = service?.spec?.clusterIP;
|
||||
if (!clusterIP || clusterIP === "None") return null;
|
||||
return `http://${clusterIP}:${service?.spec?.ports?.[0]?.port ?? fallbackPort}`;
|
||||
}
|
||||
|
||||
function endpointCount(endpoint) {
|
||||
return (endpoint?.subsets ?? []).reduce((total, subset) => total + (subset.addresses?.length ?? 0), 0);
|
||||
}
|
||||
|
||||
function buildWorkerDryRunJob(templateJob, fixture, liveSkillsEnv) {
|
||||
const templateContainer = firstContainer(templateJob);
|
||||
const labels = {
|
||||
"app.kubernetes.io/name": AGENT_WORKER_SERVICE_ID,
|
||||
"hwlab.pikastech.local/environment": ENVIRONMENT_DEV,
|
||||
"hwlab.pikastech.local/profile": ENVIRONMENT_DEV,
|
||||
"hwlab.pikastech.local/service-id": AGENT_WORKER_SERVICE_ID,
|
||||
"hwlab.pikastech.local/smoke": "dev-m4-agent-loop"
|
||||
};
|
||||
const reservedEnv = new Set([
|
||||
"HWLAB_AGENT_SESSION_ID",
|
||||
"HWLAB_PROJECT_ID",
|
||||
"HWLAB_SKILL_COMMIT_ID",
|
||||
"HWLAB_SKILL_VERSION",
|
||||
"HWLAB_WORKSPACE_ROOT"
|
||||
]);
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "Job",
|
||||
metadata: {
|
||||
name: "hwlab-agent-worker-smoke-dryrun-20260522",
|
||||
namespace: devNamespace,
|
||||
labels
|
||||
},
|
||||
spec: {
|
||||
suspend: true,
|
||||
backoffLimit: 0,
|
||||
template: {
|
||||
metadata: { labels },
|
||||
spec: {
|
||||
restartPolicy: templateJob?.spec?.template?.spec?.restartPolicy ?? "Never",
|
||||
containers: [
|
||||
{
|
||||
name: templateContainer?.name ?? AGENT_WORKER_SERVICE_ID,
|
||||
image: templateContainer?.image ?? "unknown",
|
||||
env: [
|
||||
...(templateContainer?.env ?? []).filter((entry) => !reservedEnv.has(entry.name)),
|
||||
{ name: "HWLAB_AGENT_SESSION_ID", value: fixture.agentSessionId },
|
||||
{ name: "HWLAB_PROJECT_ID", value: fixture.projectId },
|
||||
{ name: "HWLAB_SKILL_COMMIT_ID", value: liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? "" },
|
||||
{ name: "HWLAB_SKILL_VERSION", value: liveSkillsEnv.HWLAB_SKILLS_VERSION ?? "" },
|
||||
{ name: "HWLAB_WORKSPACE_ROOT", value: `/workspace/${fixture.projectId}/${fixture.agentSessionId}` }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function k3sBlockers(report) {
|
||||
const blockers = [];
|
||||
const push = (type, scope, summary, classification, nextFix) => {
|
||||
blockers.push({ type, scope, status: "open", summary, classification, nextFix });
|
||||
};
|
||||
if (!report.clusterReadable) {
|
||||
push(
|
||||
"environment_blocker",
|
||||
"d601-k3s-readonly",
|
||||
"D601 native k3s hwlab-dev resources were not readable with /etc/rancher/k3s/k3s.yaml.",
|
||||
"k3s readonly",
|
||||
"Mount or authorize read-only hwlab-dev access through the D601 native k3s kubeconfig."
|
||||
);
|
||||
return blockers;
|
||||
}
|
||||
if (report.agentManager.health.status !== "ok") {
|
||||
push(
|
||||
"agent_blocker",
|
||||
"agent-mgr-health",
|
||||
`hwlab-agent-mgr /health/live is ${report.agentManager.health.status}.`,
|
||||
"agent runtime",
|
||||
"Inject explicit skill commit and version into agent-mgr health configuration, then redeploy DEV."
|
||||
);
|
||||
}
|
||||
if (report.skillsInjection.status !== "pass") {
|
||||
push(
|
||||
"agent_blocker",
|
||||
"skills-commit-version-injection",
|
||||
`DEV skills injection is incomplete: missing ${report.skillsInjection.missing.join(", ")}.`,
|
||||
"skills injection",
|
||||
"Add HWLAB_SKILLS_VERSION to hwlab-agent-skills and ensure scheduled worker jobs receive HWLAB_SKILL_COMMIT_ID plus HWLAB_SKILL_VERSION."
|
||||
);
|
||||
}
|
||||
if (report.workerJob.serverDryRun.status !== "pass") {
|
||||
push(
|
||||
"agent_blocker",
|
||||
"worker-job-admission",
|
||||
report.workerJob.serverDryRun.summary,
|
||||
"worker job",
|
||||
"Fix the hwlab-agent-worker-template image/env/job spec until server-side dry-run admission succeeds."
|
||||
);
|
||||
}
|
||||
return blockers;
|
||||
}
|
||||
|
||||
export async function collectPublicEntrypoints() {
|
||||
const apiHealth = await requestJson(`${DEV_ENDPOINT}/health`).catch((error) => ({
|
||||
statusCode: 0,
|
||||
body: null,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
}));
|
||||
const apiLive = await requestJson(`${DEV_ENDPOINT}/health/live`).catch((error) => ({
|
||||
statusCode: 0,
|
||||
body: null,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
}));
|
||||
const frontend = await requestHead(`${DEV_FRONTEND_ENDPOINT}/`).catch((error) => ({
|
||||
statusCode: 0,
|
||||
headers: {},
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
}));
|
||||
return {
|
||||
status: apiHealth.statusCode >= 200 &&
|
||||
apiHealth.statusCode < 300 &&
|
||||
apiLive.statusCode >= 200 &&
|
||||
apiLive.statusCode < 300 &&
|
||||
frontend.statusCode >= 200 &&
|
||||
frontend.statusCode < 400
|
||||
? "pass"
|
||||
: "blocked",
|
||||
commands: [
|
||||
`curl -fsS --max-time 10 ${DEV_ENDPOINT}/health`,
|
||||
`curl -fsS --max-time 10 ${DEV_ENDPOINT}/health/live`,
|
||||
`curl -fsS -I --max-time 10 ${DEV_FRONTEND_ENDPOINT}/`
|
||||
],
|
||||
api: {
|
||||
endpoint: DEV_ENDPOINT,
|
||||
health: {
|
||||
statusCode: apiHealth.statusCode,
|
||||
serviceId: apiHealth.body?.serviceId ?? null,
|
||||
environment: apiHealth.body?.environment ?? null,
|
||||
status: apiHealth.body?.status ?? null,
|
||||
error: apiHealth.error ?? null
|
||||
},
|
||||
live: {
|
||||
statusCode: apiLive.statusCode,
|
||||
serviceId: apiLive.body?.serviceId ?? null,
|
||||
environment: apiLive.body?.environment ?? null,
|
||||
status: apiLive.body?.status ?? null,
|
||||
db: apiLive.body?.db ? {
|
||||
status: apiLive.body.db.status ?? null,
|
||||
connected: apiLive.body.db.connected ?? null,
|
||||
ready: apiLive.body.db.ready ?? null,
|
||||
evidence: apiLive.body.db.evidence ?? null
|
||||
} : null,
|
||||
error: apiLive.error ?? null
|
||||
}
|
||||
},
|
||||
frontend: {
|
||||
endpoint: DEV_FRONTEND_ENDPOINT,
|
||||
statusCode: frontend.statusCode,
|
||||
contentType: frontend.headers?.["content-type"] ?? null,
|
||||
contentLength: frontend.headers?.["content-length"] ?? null,
|
||||
error: frontend.error ?? null
|
||||
},
|
||||
summary: `Public DEV API ${DEV_ENDPOINT} health=${apiHealth.statusCode} live=${apiLive.statusCode}; frontend ${DEV_FRONTEND_ENDPOINT}/=${frontend.statusCode}.`
|
||||
};
|
||||
}
|
||||
|
||||
export async function collectD601K3sNativeEvidence(fixture) {
|
||||
const namespaceProbe = runKubectl(["get", "namespace", devNamespace, "-o", "json"]);
|
||||
const deploymentsProbe = runKubectl([
|
||||
"-n",
|
||||
devNamespace,
|
||||
"get",
|
||||
"deploy",
|
||||
AGENT_MGR_SERVICE_ID,
|
||||
AGENT_SKILLS_SERVICE_ID,
|
||||
"-o",
|
||||
"json"
|
||||
]);
|
||||
const workerTemplateProbe = runKubectl([
|
||||
"-n",
|
||||
devNamespace,
|
||||
"get",
|
||||
"job",
|
||||
"hwlab-agent-worker-template",
|
||||
"-o",
|
||||
"json"
|
||||
]);
|
||||
const servicesProbe = runKubectl([
|
||||
"-n",
|
||||
devNamespace,
|
||||
"get",
|
||||
"svc",
|
||||
AGENT_MGR_SERVICE_ID,
|
||||
AGENT_SKILLS_SERVICE_ID,
|
||||
"-o",
|
||||
"json"
|
||||
]);
|
||||
const endpointsProbe = runKubectl([
|
||||
"-n",
|
||||
devNamespace,
|
||||
"get",
|
||||
"endpoints",
|
||||
AGENT_MGR_SERVICE_ID,
|
||||
AGENT_SKILLS_SERVICE_ID,
|
||||
"-o",
|
||||
"json"
|
||||
]);
|
||||
const canCreateJobs = runKubectl(["auth", "can-i", "create", "jobs", "-n", devNamespace]);
|
||||
const canDeleteJobs = runKubectl(["auth", "can-i", "delete", "jobs", "-n", devNamespace]);
|
||||
const canGetSecrets = runKubectl(["auth", "can-i", "get", "secrets", "-n", devNamespace]);
|
||||
|
||||
const deployments = parseProbeJson(deploymentsProbe);
|
||||
const services = parseProbeJson(servicesProbe);
|
||||
const endpoints = parseProbeJson(endpointsProbe);
|
||||
const workerTemplate = parseProbeJson(workerTemplateProbe);
|
||||
const agentMgrDeployment = named(deployments, AGENT_MGR_SERVICE_ID);
|
||||
const skillsDeployment = named(deployments, AGENT_SKILLS_SERVICE_ID);
|
||||
const agentMgrService = named(services, AGENT_MGR_SERVICE_ID);
|
||||
const skillsService = named(services, AGENT_SKILLS_SERVICE_ID);
|
||||
const agentMgrEndpoint = named(endpoints, AGENT_MGR_SERVICE_ID);
|
||||
const skillsEndpoint = named(endpoints, AGENT_SKILLS_SERVICE_ID);
|
||||
const agentMgrSummary = deploymentSummary(agentMgrDeployment);
|
||||
const skillsSummary = deploymentSummary(skillsDeployment);
|
||||
const workerSummary = workerTemplateSummary(workerTemplate);
|
||||
const liveSkillsEnv = skillsSummary.env.values;
|
||||
const agentMgrUrl = serviceUrl(agentMgrService, 7410);
|
||||
const skillsUrl = serviceUrl(skillsService, 7430);
|
||||
const agentMgrHealth = agentMgrUrl ? await requestJson(`${agentMgrUrl}/health/live`).catch((error) => ({
|
||||
statusCode: 0,
|
||||
body: null,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
})) : null;
|
||||
const skillsHealth = skillsUrl ? await requestJson(`${skillsUrl}/health/live`).catch((error) => ({
|
||||
statusCode: 0,
|
||||
body: null,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
})) : null;
|
||||
const workerDryRunManifest = workerTemplate ? buildWorkerDryRunJob(workerTemplate, fixture, liveSkillsEnv) : null;
|
||||
const workerDryRunProbe = workerDryRunManifest
|
||||
? runKubectl(["-n", devNamespace, "create", "-f", "-", "--dry-run=server", "-o", "json"], {
|
||||
input: `${JSON.stringify(workerDryRunManifest)}\n`
|
||||
})
|
||||
: { ok: false, stdout: "", stderr: "worker template was not readable" };
|
||||
const workerDryRun = parseProbeJson(workerDryRunProbe);
|
||||
const workerInjectedEnv = envMap(workerDryRunManifest?.spec?.template?.spec?.containers?.[0] ?? null).values;
|
||||
const missingSkills = [];
|
||||
if (!liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID) missingSkills.push("hwlab-agent-skills.HWLAB_SKILLS_COMMIT_ID");
|
||||
if (!liveSkillsEnv.HWLAB_SKILLS_VERSION) missingSkills.push("hwlab-agent-skills.HWLAB_SKILLS_VERSION");
|
||||
if (!workerInjectedEnv.HWLAB_SKILL_COMMIT_ID) missingSkills.push("worker-dry-run.HWLAB_SKILL_COMMIT_ID");
|
||||
if (!workerInjectedEnv.HWLAB_SKILL_VERSION) missingSkills.push("worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV");
|
||||
const clusterReadable = namespaceProbe.ok && deploymentsProbe.ok && workerTemplateProbe.ok && servicesProbe.ok;
|
||||
const evidence = [
|
||||
`k3s:namespace=${devNamespace}:read=${namespaceProbe.ok}`,
|
||||
`k3s:deploy/${AGENT_MGR_SERVICE_ID}:ready=${agentMgrSummary.readyReplicas}/${agentMgrSummary.replicas}:available=${agentMgrSummary.available}`,
|
||||
`k3s:deploy/${AGENT_SKILLS_SERVICE_ID}:ready=${skillsSummary.readyReplicas}/${skillsSummary.replicas}:available=${skillsSummary.available}`,
|
||||
`k3s:svc/${AGENT_MGR_SERVICE_ID}:endpoints=${endpointCount(agentMgrEndpoint)}`,
|
||||
`k3s:svc/${AGENT_SKILLS_SERVICE_ID}:endpoints=${endpointCount(skillsEndpoint)}`,
|
||||
`agent-mgr:/health/live:${agentMgrHealth?.body?.health?.status ?? agentMgrHealth?.body?.status ?? "unreachable"}`,
|
||||
`agent-skills:/health/live:${skillsHealth?.body?.status ?? "unreachable"}:${skillsHealth?.body?.revision ?? "unknown"}`,
|
||||
`worker-template:suspend=${workerSummary.suspend}:image=${workerSummary.image ?? "unknown"}`,
|
||||
`worker-job:server-dry-run=${workerDryRunProbe.ok}:persisted=false`,
|
||||
`skills-injection:commit=${liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? "missing"}:version=${liveSkillsEnv.HWLAB_SKILLS_VERSION ?? "missing"}`,
|
||||
`secret-rbac:get=${canGetSecrets.stdout.trim() || "unknown"}:secretResourcesRead=false`
|
||||
];
|
||||
const report = {
|
||||
status: "pass",
|
||||
mode: "dev-read-only-plus-server-dry-run",
|
||||
generatedAt: new Date().toISOString(),
|
||||
commands: [
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl get namespace ${devNamespace} -o json`,
|
||||
d601K3sReadonlyCommand,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get deploy ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get job hwlab-agent-worker-template -o json`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get svc ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get endpoints ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl auth can-i create jobs -n ${devNamespace}`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl auth can-i delete jobs -n ${devNamespace}`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl auth can-i get secrets -n ${devNamespace}`,
|
||||
workerServerDryRunCommand
|
||||
],
|
||||
safety: {
|
||||
namespace: devNamespace,
|
||||
kubeconfig: d601Kubeconfig,
|
||||
readOnly: true,
|
||||
prodTouched: false,
|
||||
secretResourcesRead: false,
|
||||
secretMaterialPrinted: false,
|
||||
servicesRestarted: false,
|
||||
workerJobPersisted: false,
|
||||
longRunningAgentStarted: false,
|
||||
note: "The smoke checks Secret RBAC only with kubectl auth can-i; it never reads Secret resources or kubeconfig contents."
|
||||
},
|
||||
clusterReadable,
|
||||
namespace: {
|
||||
name: devNamespace,
|
||||
readable: namespaceProbe.ok,
|
||||
error: namespaceProbe.ok ? null : compactError(namespaceProbe)
|
||||
},
|
||||
rbac: {
|
||||
createJobs: canCreateJobs.stdout.trim() === "yes",
|
||||
deleteJobs: canDeleteJobs.stdout.trim() === "yes",
|
||||
getSecrets: canGetSecrets.stdout.trim() === "yes",
|
||||
secretReadNotPerformed: true
|
||||
},
|
||||
agentManager: {
|
||||
deployment: agentMgrSummary,
|
||||
endpointCount: endpointCount(agentMgrEndpoint),
|
||||
health: {
|
||||
statusCode: agentMgrHealth?.statusCode ?? 0,
|
||||
status: agentMgrHealth?.body?.health?.status ?? agentMgrHealth?.body?.status ?? "unreachable",
|
||||
serviceId: agentMgrHealth?.body?.serviceId ?? null,
|
||||
missingSkills: agentMgrHealth?.body?.health?.skills?.missing ?? [],
|
||||
error: agentMgrHealth?.error ?? null
|
||||
}
|
||||
},
|
||||
agentSkills: {
|
||||
deployment: skillsSummary,
|
||||
endpointCount: endpointCount(skillsEndpoint),
|
||||
health: {
|
||||
statusCode: skillsHealth?.statusCode ?? 0,
|
||||
status: skillsHealth?.body?.status ?? "unreachable",
|
||||
serviceId: skillsHealth?.body?.serviceId ?? null,
|
||||
revision: skillsHealth?.body?.revision ?? null,
|
||||
error: skillsHealth?.error ?? null
|
||||
}
|
||||
},
|
||||
workerJob: {
|
||||
template: workerSummary,
|
||||
serverDryRun: {
|
||||
status: workerDryRunProbe.ok ? "pass" : "blocked",
|
||||
accepted: workerDryRunProbe.ok,
|
||||
command: workerServerDryRunCommand,
|
||||
persisted: false,
|
||||
generatedName: workerDryRunManifest?.metadata?.name ?? null,
|
||||
injectedEnvNames: Object.keys(workerInjectedEnv).sort(),
|
||||
dryRunKind: workerDryRun?.kind ?? null,
|
||||
dryRunSuspend: workerDryRun?.spec?.suspend ?? null,
|
||||
summary: workerDryRunProbe.ok
|
||||
? "Server-side admission accepted a suspended worker Job generated from the template with session/workspace/skills env injected; no Job was persisted."
|
||||
: compactError(workerDryRunProbe)
|
||||
}
|
||||
},
|
||||
skillsInjection: {
|
||||
status: missingSkills.length === 0 ? "pass" : "blocked",
|
||||
source: "k3s deployment env plus generated worker dry-run manifest",
|
||||
serviceCommitId: liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? null,
|
||||
serviceVersion: liveSkillsEnv.HWLAB_SKILLS_VERSION ?? null,
|
||||
workerDryRunCommitId: workerInjectedEnv.HWLAB_SKILL_COMMIT_ID ?? null,
|
||||
workerDryRunVersion: workerInjectedEnv.HWLAB_SKILL_VERSION ?? null,
|
||||
workerDryRunSource: "derived-from-live-dev-skills-env",
|
||||
missing: missingSkills
|
||||
},
|
||||
traceEvidenceCleanup: {
|
||||
status: "dry-run-only",
|
||||
summary: "Live worker execution was not started. Trace/evidence/cleanup closure is covered by the bounded local dry-run and remains explicitly non-M3 DEV-LIVE."
|
||||
},
|
||||
evidence
|
||||
};
|
||||
report.blockers = k3sBlockers(report);
|
||||
report.status = report.blockers.length === 0 ? "pass" : "blocked";
|
||||
report.minimumFixRecommendations = report.blockers.map((blocker) => blocker.nextFix).filter(Boolean);
|
||||
return report;
|
||||
}
|
||||
Reference in New Issue
Block a user