import { execFileSync } from "node:child_process"; import { request as httpRequest } from "node:http"; import { request as httpsRequest } from "node:https"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { AGENT_MGR_SERVICE_ID, AGENT_SKILLS_SERVICE_ID, AGENT_WORKER_SERVICE_ID } from "../../internal/agent/index.mjs"; import { RUNTIME_DURABLE_ADAPTER_MISSING, RUNTIME_DURABILITY_REQUIRED_EVIDENCE, RUNTIME_STORE_KIND_POSTGRES } from "../../internal/db/runtime-store.ts"; import { CLOUD_CORE_MIGRATION_ID } from "../../internal/db/schema.ts"; import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); export const d601Kubeconfig = "/etc/rancher/k3s/k3s.yaml"; export const devNamespace = "hwlab-dev"; export const d601K3sReadonlyCommand = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods,svc,deploy,job,cm -o wide"; export const workerServerDryRunCommand = "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json"; export function isDbLiveReady(body) { const db = body?.db; return Boolean(db?.ready && db?.connected && db?.liveConnected !== false && db?.liveDbEvidence); } export function isDbLiveBlocked(body) { const db = body?.db; if (!db || typeof db !== "object") { return false; } if (isDbLiveReady(body)) { return false; } return db.connected === false || db.ready === false || db.liveConnected === false || db.liveDbEvidence === false || ["blocked", "degraded", "failed"].includes(db.status); } export function summarizeDbBlocker(body) { const db = body?.db ?? {}; const missing = Array.isArray(db.missingEnv) && db.missingEnv.length > 0 ? ` missing ${db.missingEnv.join(", ")}` : ""; return `cloud-api /health/live reports DB ${db.status ?? "not ready"}; connected=${db.connected ?? "unknown"}; ready=${db.ready ?? "unknown"}; liveDbEvidence=${db.liveDbEvidence ?? "unknown"}.${missing}`; } export function runtimeDurabilityFromHealth(body) { const runtime = body?.runtime ?? {}; const durability = body?.readiness?.durability ?? {}; const migration = runtime.migration ?? {}; const migrationGate = runtime.gates?.migration ?? durability.gates?.migration ?? {}; const blocker = runtime.blocker ?? durability.blocker ?? (Array.isArray(body?.blockerCodes) ? body.blockerCodes.find((code) => String(code).startsWith("runtime_durable_adapter_")) : null) ?? RUNTIME_DURABLE_ADAPTER_MISSING; const durableRequested = Boolean( runtime.durableRequested || durability.durableRequested || runtime.adapter === RUNTIME_STORE_KIND_POSTGRES || durability.adapter === RUNTIME_STORE_KIND_POSTGRES ); const ready = Boolean( runtime.durable === true && runtime.ready === true && runtime.liveRuntimeEvidence === true && durability.ready !== false ); return { adapter: runtime.adapter ?? durability.adapter ?? "unknown", durable: Boolean(runtime.durable), durableRequested, ready, runtimeStatus: runtime.status ?? durability.runtimeStatus ?? "unknown", blocker, blockedLayer: runtime.durabilityContract?.blockedLayer ?? durability.blockedLayer ?? null, requiredEvidence: runtime.durabilityContract?.requiredEvidence ?? durability.requiredEvidence ?? RUNTIME_DURABILITY_REQUIRED_EVIDENCE, migration: { checked: migration.checked ?? migrationGate.checked ?? false, ready: migration.ready ?? migrationGate.ready ?? false, missing: migration.missing ?? (migration.ready === false ? true : null), requiredMigrationId: migration.requiredMigrationId ?? CLOUD_CORE_MIGRATION_ID, appliedMigrationId: migration.appliedMigrationId ?? null }, queryAttempted: Boolean(runtime.connection?.queryAttempted ?? durability.queryAttempted), queryResult: runtime.connection?.queryResult ?? durability.queryResult ?? "unknown" }; } export function isRuntimeDurableAdapterBlocked(body) { const runtime = runtimeDurabilityFromHealth(body); if (!runtime.durableRequested) { return false; } return !runtime.ready || ["blocked", "degraded", "failed"].includes(runtime.runtimeStatus) || String(runtime.blocker).startsWith("runtime_durable_adapter_"); } export function summarizeRuntimeDurableBlocker(body) { const runtime = runtimeDurabilityFromHealth(body); const migration = runtime.migration; return `cloud-api durable runtime adapter is blocked: ${runtime.blocker}; adapter=${runtime.adapter}; durable=${runtime.durable}; ready=${runtime.ready}; migration=${migration.requiredMigrationId} checked=${migration.checked} ready=${migration.ready} missing=${migration.missing ?? "unknown"}; queryResult=${runtime.queryResult}.`; } export function publicDbLiveSummary(body) { const db = body?.db ?? null; if (!db || typeof db !== "object") { return null; } return { status: db.status ?? null, connected: db.connected ?? null, liveConnected: db.liveConnected ?? null, ready: db.ready ?? null, configReady: db.configReady ?? null, connectionChecked: db.connectionChecked ?? null, connectionAttempted: db.connectionAttempted ?? null, connectionResult: db.connectionResult ?? null, endpointSource: db.endpointSource ?? db.connection?.endpointSource ?? null, liveDbEvidence: db.liveDbEvidence ?? null, evidence: db.evidence ?? null }; } export function publicRuntimeDurabilitySummary(body) { if (!body?.runtime && !body?.readiness?.durability) { return null; } const runtime = runtimeDurabilityFromHealth(body); return { adapter: runtime.adapter, durable: runtime.durable, durableRequested: runtime.durableRequested, ready: runtime.ready, status: runtime.runtimeStatus, blocker: runtime.blocker, blockedLayer: runtime.blockedLayer, requiredEvidence: runtime.requiredEvidence, liveRuntimeEvidence: Boolean(runtime.ready), migration: runtime.migration, queryAttempted: runtime.queryAttempted, queryResult: runtime.queryResult }; } export function classifyCloudApiLiveReadiness(body) { if (isDbLiveBlocked(body)) { return { blocked: true, blockerType: "runtime_blocker", blockerScope: "db-live", sourceIssue: "pikasTech/HWLAB#49", blockedClassification: "DB live", blockerSummary: summarizeDbBlocker(body), summary: "Blocked at DB live readiness before scheduling a DEV agent task.", evidenceSuffix: "db-blocked", dbLiveReady: false, runtimeDurableReady: false }; } if (isRuntimeDurableAdapterBlocked(body)) { const runtime = runtimeDurabilityFromHealth(body); return { blocked: true, blockerType: "runtime_blocker", blockerScope: "runtime-durable-adapter", sourceIssue: "pikasTech/HWLAB#164", blockedClassification: runtime.blocker, blockerSummary: summarizeRuntimeDurableBlocker(body), summary: "Blocked at runtime durable adapter readiness before scheduling a DEV agent task.", evidenceSuffix: `runtime-durable-adapter-blocked:${runtime.blocker}`, dbLiveReady: isDbLiveReady(body), runtimeDurableReady: false }; } return { blocked: false, blockerType: null, blockerScope: null, blockedClassification: "none", blockerSummary: "", summary: "Live DEV preflight observed DB live and durable runtime adapter readiness.", evidenceSuffix: null, dbLiveReady: isDbLiveReady(body), runtimeDurableReady: true }; } export function requestJson(urlString, { method = "GET", headers = {}, body, timeoutMs = 5000 } = {}) { const url = new URL(urlString); const client = url.protocol === "https:" ? httpsRequest : httpRequest; const payload = body === undefined ? null : Buffer.from(body); return new Promise((resolve, reject) => { const request = client( { protocol: url.protocol, hostname: url.hostname, port: url.port, path: `${url.pathname}${url.search}`, method, headers }, (response) => { let raw = ""; response.setEncoding("utf8"); response.on("data", (chunk) => { raw += chunk; }); response.on("end", () => { let bodyValue = null; if (raw.length > 0) { try { bodyValue = JSON.parse(raw); } catch (error) { reject(new Error(`invalid JSON response from ${urlString}: ${error.message}`)); return; } } resolve({ statusCode: response.statusCode ?? 0, headers: response.headers, body: bodyValue, raw }); }); } ); request.on("error", reject); request.setTimeout(timeoutMs, () => { request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`)); }); if (payload) request.write(payload); request.end(); }); } function requestHead(urlString, { timeoutMs = 5000 } = {}) { const url = new URL(urlString); const client = url.protocol === "https:" ? httpsRequest : httpRequest; return new Promise((resolve, reject) => { const request = client( { protocol: url.protocol, hostname: url.hostname, port: url.port, path: `${url.pathname}${url.search}`, method: "HEAD" }, (response) => { response.resume(); response.on("end", () => { resolve({ statusCode: response.statusCode ?? 0, headers: response.headers }); }); } ); request.on("error", reject); request.setTimeout(timeoutMs, () => { request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`)); }); request.end(); }); } function runKubectl(args, { input, timeoutMs = 5000 } = {}) { try { return { ok: true, command: `KUBECONFIG=${d601Kubeconfig} kubectl ${args.join(" ")}`, exitCode: 0, stdout: execFileSync("kubectl", args, { cwd: repoRoot, encoding: "utf8", env: { ...process.env, KUBECONFIG: d601Kubeconfig }, input, stdio: ["pipe", "pipe", "pipe"], timeout: timeoutMs }) }; } catch (error) { return { ok: false, command: `KUBECONFIG=${d601Kubeconfig} kubectl ${args.join(" ")}`, exitCode: typeof error.status === "number" ? error.status : 1, stdout: String(error.stdout ?? ""), stderr: String(error.stderr ?? ""), error: error instanceof Error ? error.message : String(error) }; } } function parseProbeJson(probe) { if (!probe.ok || !probe.stdout.trim()) return null; try { return JSON.parse(probe.stdout); } catch { return null; } } function compactError(probe) { return (probe.stderr || probe.error || probe.stdout || "unknown error") .replace(/\s+/g, " ") .trim() .slice(0, 400); } function items(document) { if (!document) return []; return Array.isArray(document.items) ? document.items : [document]; } function named(document, name) { return items(document).find((item) => item?.metadata?.name === name) ?? null; } function firstContainer(workload) { return workload?.spec?.template?.spec?.containers?.[0] ?? null; } function envMap(container) { const values = {}; const valueFrom = []; for (const entry of container?.env ?? []) { if (Object.hasOwn(entry, "value")) values[entry.name] = entry.value; else if (entry.valueFrom) valueFrom.push(entry.name); } return { values, valueFrom }; } function condition(workload, type) { return workload?.status?.conditions?.find((entry) => entry.type === type)?.status ?? "Unknown"; } function deploymentSummary(deployment) { const container = firstContainer(deployment); const replicas = deployment?.status?.replicas ?? deployment?.spec?.replicas ?? 0; const readyReplicas = deployment?.status?.readyReplicas ?? 0; return { name: deployment?.metadata?.name ?? null, image: container?.image ?? null, readyReplicas, replicas, available: condition(deployment, "Available"), ready: replicas > 0 && readyReplicas >= replicas && condition(deployment, "Available") === "True", env: envMap(container) }; } function workerTemplateSummary(job) { const container = firstContainer(job); return { name: job?.metadata?.name ?? null, suspend: job?.spec?.suspend ?? null, image: container?.image ?? null, restartPolicy: job?.spec?.template?.spec?.restartPolicy ?? null, env: envMap(container) }; } function serviceUrl(service, fallbackPort) { const clusterIP = service?.spec?.clusterIP; if (!clusterIP || clusterIP === "None") return null; return `http://${clusterIP}:${service?.spec?.ports?.[0]?.port ?? fallbackPort}`; } function endpointCount(endpoint) { return (endpoint?.subsets ?? []).reduce((total, subset) => total + (subset.addresses?.length ?? 0), 0); } function buildWorkerDryRunJob(templateJob, fixture, liveSkillsEnv) { const templateContainer = firstContainer(templateJob); const labels = { "app.kubernetes.io/name": AGENT_WORKER_SERVICE_ID, "hwlab.pikastech.local/environment": ENVIRONMENT_DEV, "hwlab.pikastech.local/profile": ENVIRONMENT_DEV, "hwlab.pikastech.local/service-id": AGENT_WORKER_SERVICE_ID, "hwlab.pikastech.local/smoke": "dev-m4-agent-loop" }; const reservedEnv = new Set([ "HWLAB_AGENT_SESSION_ID", "HWLAB_PROJECT_ID", "HWLAB_SKILL_COMMIT_ID", "HWLAB_SKILL_VERSION", "HWLAB_WORKSPACE_ROOT" ]); return { apiVersion: "batch/v1", kind: "Job", metadata: { name: "hwlab-agent-worker-smoke-dryrun-20260522", namespace: devNamespace, labels }, spec: { suspend: true, backoffLimit: 0, template: { metadata: { labels }, spec: { restartPolicy: templateJob?.spec?.template?.spec?.restartPolicy ?? "Never", containers: [ { name: templateContainer?.name ?? AGENT_WORKER_SERVICE_ID, image: templateContainer?.image ?? "unknown", env: [ ...(templateContainer?.env ?? []).filter((entry) => !reservedEnv.has(entry.name)), { name: "HWLAB_AGENT_SESSION_ID", value: fixture.agentSessionId }, { name: "HWLAB_PROJECT_ID", value: fixture.projectId }, { name: "HWLAB_SKILL_COMMIT_ID", value: liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? "" }, { name: "HWLAB_SKILL_VERSION", value: liveSkillsEnv.HWLAB_SKILLS_VERSION ?? "" }, { name: "HWLAB_WORKSPACE_ROOT", value: `/workspace/${fixture.projectId}/${fixture.agentSessionId}` } ] } ] } } } }; } function k3sBlockers(report) { const blockers = []; const push = (type, scope, summary, classification, nextFix) => { blockers.push({ type, scope, status: "open", summary, classification, nextFix }); }; if (!report.clusterReadable) { push( "environment_blocker", "d601-k3s-readonly", "D601 native k3s hwlab-dev resources were not readable with /etc/rancher/k3s/k3s.yaml.", "k3s readonly", "Mount or authorize read-only hwlab-dev access through the D601 native k3s kubeconfig." ); return blockers; } if (report.agentManager.health.status !== "ok") { push( "agent_blocker", "agent-mgr-health", `hwlab-agent-mgr /health/live is ${report.agentManager.health.status}.`, "agent runtime", "Inject explicit skill commit and version into agent-mgr health configuration, then redeploy DEV." ); } if (report.skillsInjection.status !== "pass") { push( "agent_blocker", "skills-commit-version-injection", `DEV skills injection is incomplete: missing ${report.skillsInjection.missing.join(", ")}.`, "skills injection", "Add HWLAB_SKILLS_VERSION to hwlab-agent-skills and ensure scheduled worker jobs receive HWLAB_SKILL_COMMIT_ID plus HWLAB_SKILL_VERSION." ); } if (report.workerJob.serverDryRun.status !== "pass") { push( "agent_blocker", "worker-job-admission", report.workerJob.serverDryRun.summary, "worker job", "Fix the hwlab-agent-worker-template image/env/job spec until server-side dry-run admission succeeds." ); } return blockers; } export async function collectPublicEntrypoints() { const apiHealth = await requestJson(`${DEV_ENDPOINT}/health`).catch((error) => ({ statusCode: 0, body: null, error: error instanceof Error ? error.message : String(error) })); const apiLive = await requestJson(`${DEV_ENDPOINT}/health/live`).catch((error) => ({ statusCode: 0, body: null, error: error instanceof Error ? error.message : String(error) })); const frontend = await requestHead(`${DEV_FRONTEND_ENDPOINT}/`).catch((error) => ({ statusCode: 0, headers: {}, error: error instanceof Error ? error.message : String(error) })); return { status: apiHealth.statusCode >= 200 && apiHealth.statusCode < 300 && apiLive.statusCode >= 200 && apiLive.statusCode < 300 && frontend.statusCode >= 200 && frontend.statusCode < 400 ? "pass" : "blocked", commands: [ `curl -fsS --max-time 10 ${DEV_ENDPOINT}/health`, `curl -fsS --max-time 10 ${DEV_ENDPOINT}/health/live`, `curl -fsS -I --max-time 10 ${DEV_FRONTEND_ENDPOINT}/` ], api: { endpoint: DEV_ENDPOINT, health: { statusCode: apiHealth.statusCode, serviceId: apiHealth.body?.serviceId ?? null, environment: apiHealth.body?.environment ?? null, status: apiHealth.body?.status ?? null, error: apiHealth.error ?? null }, live: { statusCode: apiLive.statusCode, serviceId: apiLive.body?.serviceId ?? null, environment: apiLive.body?.environment ?? null, status: apiLive.body?.status ?? null, db: publicDbLiveSummary(apiLive.body), runtime: publicRuntimeDurabilitySummary(apiLive.body), blockerCodes: Array.isArray(apiLive.body?.blockerCodes) ? apiLive.body.blockerCodes : [], error: apiLive.error ?? null } }, frontend: { endpoint: DEV_FRONTEND_ENDPOINT, statusCode: frontend.statusCode, contentType: frontend.headers?.["content-type"] ?? null, contentLength: frontend.headers?.["content-length"] ?? null, error: frontend.error ?? null }, summary: `Public DEV API ${DEV_ENDPOINT} health=${apiHealth.statusCode} live=${apiLive.statusCode}; frontend ${DEV_FRONTEND_ENDPOINT}/=${frontend.statusCode}.` }; } export async function collectD601K3sNativeEvidence(fixture) { const namespaceProbe = runKubectl(["get", "namespace", devNamespace, "-o", "json"]); const deploymentsProbe = runKubectl([ "-n", devNamespace, "get", "deploy", AGENT_MGR_SERVICE_ID, AGENT_SKILLS_SERVICE_ID, "-o", "json" ]); const workerTemplateProbe = runKubectl([ "-n", devNamespace, "get", "job", "hwlab-agent-worker-template", "-o", "json" ]); const servicesProbe = runKubectl([ "-n", devNamespace, "get", "svc", AGENT_MGR_SERVICE_ID, AGENT_SKILLS_SERVICE_ID, "-o", "json" ]); const endpointsProbe = runKubectl([ "-n", devNamespace, "get", "endpoints", AGENT_MGR_SERVICE_ID, AGENT_SKILLS_SERVICE_ID, "-o", "json" ]); const canCreateJobs = runKubectl(["auth", "can-i", "create", "jobs", "-n", devNamespace]); const canDeleteJobs = runKubectl(["auth", "can-i", "delete", "jobs", "-n", devNamespace]); const canGetSecrets = runKubectl(["auth", "can-i", "get", "secrets", "-n", devNamespace]); const deployments = parseProbeJson(deploymentsProbe); const services = parseProbeJson(servicesProbe); const endpoints = parseProbeJson(endpointsProbe); const workerTemplate = parseProbeJson(workerTemplateProbe); const agentMgrDeployment = named(deployments, AGENT_MGR_SERVICE_ID); const skillsDeployment = named(deployments, AGENT_SKILLS_SERVICE_ID); const agentMgrService = named(services, AGENT_MGR_SERVICE_ID); const skillsService = named(services, AGENT_SKILLS_SERVICE_ID); const agentMgrEndpoint = named(endpoints, AGENT_MGR_SERVICE_ID); const skillsEndpoint = named(endpoints, AGENT_SKILLS_SERVICE_ID); const agentMgrSummary = deploymentSummary(agentMgrDeployment); const skillsSummary = deploymentSummary(skillsDeployment); const workerSummary = workerTemplateSummary(workerTemplate); const liveSkillsEnv = skillsSummary.env.values; const agentMgrUrl = serviceUrl(agentMgrService, 7410); const skillsUrl = serviceUrl(skillsService, 7430); const agentMgrHealth = agentMgrUrl ? await requestJson(`${agentMgrUrl}/health/live`).catch((error) => ({ statusCode: 0, body: null, error: error instanceof Error ? error.message : String(error) })) : null; const skillsHealth = skillsUrl ? await requestJson(`${skillsUrl}/health/live`).catch((error) => ({ statusCode: 0, body: null, error: error instanceof Error ? error.message : String(error) })) : null; const workerDryRunManifest = workerTemplate ? buildWorkerDryRunJob(workerTemplate, fixture, liveSkillsEnv) : null; const workerDryRunProbe = workerDryRunManifest ? runKubectl(["-n", devNamespace, "create", "-f", "-", "--dry-run=server", "-o", "json"], { input: `${JSON.stringify(workerDryRunManifest)}\n` }) : { ok: false, stdout: "", stderr: "worker template was not readable" }; const workerDryRun = parseProbeJson(workerDryRunProbe); const workerInjectedEnv = envMap(workerDryRunManifest?.spec?.template?.spec?.containers?.[0] ?? null).values; const missingSkills = []; if (!liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID) missingSkills.push("hwlab-agent-skills.HWLAB_SKILLS_COMMIT_ID"); if (!liveSkillsEnv.HWLAB_SKILLS_VERSION) missingSkills.push("hwlab-agent-skills.HWLAB_SKILLS_VERSION"); if (!workerInjectedEnv.HWLAB_SKILL_COMMIT_ID) missingSkills.push("worker-dry-run.HWLAB_SKILL_COMMIT_ID"); if (!workerInjectedEnv.HWLAB_SKILL_VERSION) missingSkills.push("worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV"); const clusterReadable = namespaceProbe.ok && deploymentsProbe.ok && workerTemplateProbe.ok && servicesProbe.ok; const evidence = [ `k3s:namespace=${devNamespace}:read=${namespaceProbe.ok}`, `k3s:deploy/${AGENT_MGR_SERVICE_ID}:ready=${agentMgrSummary.readyReplicas}/${agentMgrSummary.replicas}:available=${agentMgrSummary.available}`, `k3s:deploy/${AGENT_SKILLS_SERVICE_ID}:ready=${skillsSummary.readyReplicas}/${skillsSummary.replicas}:available=${skillsSummary.available}`, `k3s:svc/${AGENT_MGR_SERVICE_ID}:endpoints=${endpointCount(agentMgrEndpoint)}`, `k3s:svc/${AGENT_SKILLS_SERVICE_ID}:endpoints=${endpointCount(skillsEndpoint)}`, `agent-mgr:/health/live:${agentMgrHealth?.body?.health?.status ?? agentMgrHealth?.body?.status ?? "unreachable"}`, `agent-skills:/health/live:${skillsHealth?.body?.status ?? "unreachable"}:${skillsHealth?.body?.revision ?? "unknown"}`, `worker-template:suspend=${workerSummary.suspend}:image=${workerSummary.image ?? "unknown"}`, `worker-job:server-dry-run=${workerDryRunProbe.ok}:persisted=false`, `skills-injection:commit=${liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? "missing"}:version=${liveSkillsEnv.HWLAB_SKILLS_VERSION ?? "missing"}`, `secret-rbac:get=${canGetSecrets.stdout.trim() || "unknown"}:secretResourcesRead=false` ]; const report = { status: "pass", mode: "dev-read-only-plus-server-dry-run", generatedAt: new Date().toISOString(), commands: [ `KUBECONFIG=${d601Kubeconfig} kubectl get namespace ${devNamespace} -o json`, d601K3sReadonlyCommand, `KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get deploy ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`, `KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get job hwlab-agent-worker-template -o json`, `KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get svc ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`, `KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get endpoints ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`, `KUBECONFIG=${d601Kubeconfig} kubectl auth can-i create jobs -n ${devNamespace}`, `KUBECONFIG=${d601Kubeconfig} kubectl auth can-i delete jobs -n ${devNamespace}`, `KUBECONFIG=${d601Kubeconfig} kubectl auth can-i get secrets -n ${devNamespace}`, workerServerDryRunCommand ], safety: { namespace: devNamespace, kubeconfig: d601Kubeconfig, readOnly: true, prodTouched: false, secretResourcesRead: false, secretMaterialPrinted: false, servicesRestarted: false, workerJobPersisted: false, longRunningAgentStarted: false, note: "The smoke checks Secret RBAC only with kubectl auth can-i; it never reads Secret resources or kubeconfig contents." }, clusterReadable, namespace: { name: devNamespace, readable: namespaceProbe.ok, error: namespaceProbe.ok ? null : compactError(namespaceProbe) }, rbac: { createJobs: canCreateJobs.stdout.trim() === "yes", deleteJobs: canDeleteJobs.stdout.trim() === "yes", getSecrets: canGetSecrets.stdout.trim() === "yes", secretReadNotPerformed: true }, agentManager: { deployment: agentMgrSummary, endpointCount: endpointCount(agentMgrEndpoint), health: { statusCode: agentMgrHealth?.statusCode ?? 0, status: agentMgrHealth?.body?.health?.status ?? agentMgrHealth?.body?.status ?? "unreachable", serviceId: agentMgrHealth?.body?.serviceId ?? null, missingSkills: agentMgrHealth?.body?.health?.skills?.missing ?? [], error: agentMgrHealth?.error ?? null } }, agentSkills: { deployment: skillsSummary, endpointCount: endpointCount(skillsEndpoint), health: { statusCode: skillsHealth?.statusCode ?? 0, status: skillsHealth?.body?.status ?? "unreachable", serviceId: skillsHealth?.body?.serviceId ?? null, revision: skillsHealth?.body?.revision ?? null, error: skillsHealth?.error ?? null } }, workerJob: { template: workerSummary, serverDryRun: { status: workerDryRunProbe.ok ? "pass" : "blocked", accepted: workerDryRunProbe.ok, command: workerServerDryRunCommand, persisted: false, generatedName: workerDryRunManifest?.metadata?.name ?? null, injectedEnvNames: Object.keys(workerInjectedEnv).sort(), dryRunKind: workerDryRun?.kind ?? null, dryRunSuspend: workerDryRun?.spec?.suspend ?? null, summary: workerDryRunProbe.ok ? "Server-side admission accepted a suspended worker Job generated from the template with session/workspace/skills env injected; no Job was persisted." : compactError(workerDryRunProbe) } }, skillsInjection: { status: missingSkills.length === 0 ? "pass" : "blocked", source: "k3s deployment env plus generated worker dry-run manifest", serviceCommitId: liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? null, serviceVersion: liveSkillsEnv.HWLAB_SKILLS_VERSION ?? null, workerDryRunCommitId: workerInjectedEnv.HWLAB_SKILL_COMMIT_ID ?? null, workerDryRunVersion: workerInjectedEnv.HWLAB_SKILL_VERSION ?? null, workerDryRunSource: "derived-from-live-dev-skills-env", missing: missingSkills }, traceEvidenceCleanup: { status: "dry-run-only", summary: "Live worker execution was not started. Trace/evidence/cleanup closure is covered by the bounded local dry-run and remains explicitly non-M3 DEV-LIVE." }, evidence }; report.blockers = k3sBlockers(report); report.status = report.blockers.length === 0 ? "pass" : "blocked"; report.minimumFixRecommendations = report.blockers.map((blocker) => blocker.nextFix).filter(Boolean); return report; }