From ae23972fd9dfc8e3e062c07e954133c86990b783 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Fri, 22 May 2026 20:45:23 +0800 Subject: [PATCH] test: refresh DEV M4 agent smoke evidence (#139) Commander review: direction is correct. This PR records DEV M4 agent-loop evidence as a bounded preflight, keeps M4 subordinate to the M3 trusted-loop mainline, and preserves blockers for DB live readiness and skill version injection. No PROD/service restart is included. --- docs/dev-m4-agent-loop.md | 18 + reports/dev-gate/dev-m4-agent-loop.json | 269 +++++++++- scripts/dev-m4-agent-loop-smoke.mjs | 160 +++--- scripts/src/dev-m4-agent-loop-smoke-lib.mjs | 567 ++++++++++++++++++++ 4 files changed, 935 insertions(+), 79 deletions(-) create mode 100644 scripts/src/dev-m4-agent-loop-smoke-lib.mjs diff --git a/docs/dev-m4-agent-loop.md b/docs/dev-m4-agent-loop.md index 68fbcdd5..c79d5e0e 100644 --- a/docs/dev-m4-agent-loop.md +++ b/docs/dev-m4-agent-loop.md @@ -46,6 +46,24 @@ The live path must only observe existing DEV state. It must not read tokens or secrets, must not try to run a real agent workload, and must stop if the M3 runtime preconditions are not ready. +On D601, the live preflight also records native k3s evidence with the explicit +kubeconfig and namespace: + +```sh +KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods,svc,deploy,job,cm -o wide +``` + +Worker createability is checked with server-side dry-run admission from the +existing `hwlab-agent-worker-template`. The generated Job stays suspended and is +not persisted: + +```sh +KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json +``` + +The smoke may check Secret RBAC with `kubectl auth can-i`, but it must not read +Secret resources, kubeconfig contents, or token values. + Any hardware assistance from M4 must go through the cloud API hardware surface and the patch-panel trusted path for `DO1 -> patch-panel -> DI1`. The agent must not connect directly to box or gateway simulators. diff --git a/reports/dev-gate/dev-m4-agent-loop.json b/reports/dev-gate/dev-m4-agent-loop.json index e8535642..52814ca0 100644 --- a/reports/dev-gate/dev-m4-agent-loop.json +++ b/reports/dev-gate/dev-m4-agent-loop.json @@ -4,7 +4,7 @@ "reportVersion": "v1", "issue": "pikasTech/HWLAB#37", "taskId": "dev-m4-agent-loop", - "commitId": "8e89409", + "commitId": "0614202", "acceptanceLevel": "dev_m4_agent_loop", "devOnly": true, "prodDisabled": true, @@ -28,8 +28,11 @@ "node --check scripts/validate-dev-gate-report.mjs", "node scripts/validate-dev-gate-report.mjs", "node --check scripts/dev-m4-agent-loop-smoke.mjs", + "node --check scripts/src/dev-m4-agent-loop-smoke-lib.mjs", "node scripts/dev-m4-agent-loop-smoke.mjs --dry-run", - "node scripts/dev-m4-agent-loop-smoke.mjs --live --confirm-dev --confirmed-non-production" + "node scripts/dev-m4-agent-loop-smoke.mjs --live --confirm-dev --confirmed-non-production", + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods,svc,deploy,job,cm -o wide", + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json" ], "localSmoke": { "status": "pass", @@ -122,11 +125,245 @@ ] } }, + "d601K3sNative": { + "status": "blocked", + "mode": "dev-read-only-plus-server-dry-run", + "generatedAt": "2026-05-22T12:12:47.156Z", + "commands": [ + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get namespace hwlab-dev -o json", + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods,svc,deploy,job,cm -o wide", + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get deploy hwlab-agent-mgr hwlab-agent-skills -o json", + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get job hwlab-agent-worker-template -o json", + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get svc hwlab-agent-mgr hwlab-agent-skills -o json", + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get endpoints hwlab-agent-mgr hwlab-agent-skills -o json", + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl auth can-i create jobs -n hwlab-dev", + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl auth can-i delete jobs -n hwlab-dev", + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl auth can-i get secrets -n hwlab-dev", + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json" + ], + "safety": { + "namespace": "hwlab-dev", + "kubeconfig": "/etc/rancher/k3s/k3s.yaml", + "readOnly": true, + "prodTouched": false, + "secretResourcesRead": false, + "secretMaterialPrinted": false, + "servicesRestarted": false, + "workerJobPersisted": false, + "longRunningAgentStarted": false, + "note": "The smoke checks Secret RBAC only with kubectl auth can-i; it never reads Secret resources or kubeconfig contents." + }, + "clusterReadable": true, + "namespace": { + "name": "hwlab-dev", + "readable": true, + "error": null + }, + "rbac": { + "createJobs": true, + "deleteJobs": true, + "getSecrets": true, + "secretReadNotPerformed": true + }, + "agentManager": { + "deployment": { + "name": "hwlab-agent-mgr", + "image": "127.0.0.1:5000/hwlab/hwlab-agent-mgr:cb35ada", + "readyReplicas": 1, + "replicas": 1, + "available": "True", + "ready": true, + "env": { + "values": { + "HWLAB_AGENT_SESSION_MODE": "control-plane" + }, + "valueFrom": [] + } + }, + "endpointCount": 1, + "health": { + "statusCode": 200, + "status": "degraded", + "serviceId": "hwlab-agent-mgr", + "missingSkills": [ + "manager.skillCommitId", + "manager.skillVersion" + ], + "error": null + } + }, + "agentSkills": { + "deployment": { + "name": "hwlab-agent-skills", + "image": "127.0.0.1:5000/hwlab/hwlab-agent-skills:cb35ada", + "readyReplicas": 1, + "replicas": 1, + "available": "True", + "ready": true, + "env": { + "values": { + "HWLAB_SKILLS_COMMIT_ID": "cb35ada" + }, + "valueFrom": [] + } + }, + "endpointCount": 1, + "health": { + "statusCode": 200, + "status": "ok", + "serviceId": "hwlab-agent-skills", + "revision": "cb35ada6860653b27269b0a57991184118cbf4b1", + "error": null + } + }, + "workerJob": { + "template": { + "name": "hwlab-agent-worker-template", + "suspend": true, + "image": "127.0.0.1:5000/hwlab/hwlab-agent-worker:cb35ada", + "restartPolicy": "Never", + "env": { + "values": { + "HWLAB_AGENT_SESSION_MODE": "session-scoped" + }, + "valueFrom": [] + } + }, + "serverDryRun": { + "status": "pass", + "accepted": true, + "command": "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json", + "persisted": false, + "generatedName": "hwlab-agent-worker-smoke-dryrun-20260522", + "injectedEnvNames": [ + "HWLAB_AGENT_SESSION_ID", + "HWLAB_AGENT_SESSION_MODE", + "HWLAB_PROJECT_ID", + "HWLAB_SKILL_COMMIT_ID", + "HWLAB_SKILL_VERSION", + "HWLAB_WORKSPACE_ROOT" + ], + "dryRunKind": "Job", + "dryRunSuspend": true, + "summary": "Server-side admission accepted a suspended worker Job generated from the template with session/workspace/skills env injected; no Job was persisted." + } + }, + "skillsInjection": { + "status": "blocked", + "source": "k3s deployment env plus generated worker dry-run manifest", + "serviceCommitId": "cb35ada", + "serviceVersion": null, + "workerDryRunCommitId": "cb35ada", + "workerDryRunVersion": "", + "workerDryRunSource": "derived-from-live-dev-skills-env", + "missing": [ + "hwlab-agent-skills.HWLAB_SKILLS_VERSION", + "worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV" + ] + }, + "traceEvidenceCleanup": { + "status": "dry-run-only", + "summary": "Live worker execution was not started. Trace/evidence/cleanup closure is covered by the bounded local dry-run and remains explicitly non-M3 DEV-LIVE." + }, + "evidence": [ + "k3s:namespace=hwlab-dev:read=true", + "k3s:deploy/hwlab-agent-mgr:ready=1/1:available=True", + "k3s:deploy/hwlab-agent-skills:ready=1/1:available=True", + "k3s:svc/hwlab-agent-mgr:endpoints=1", + "k3s:svc/hwlab-agent-skills:endpoints=1", + "agent-mgr:/health/live:degraded", + "agent-skills:/health/live:ok:cb35ada6860653b27269b0a57991184118cbf4b1", + "worker-template:suspend=true:image=127.0.0.1:5000/hwlab/hwlab-agent-worker:cb35ada", + "worker-job:server-dry-run=true:persisted=false", + "skills-injection:commit=cb35ada:version=missing", + "secret-rbac:get=yes:secretResourcesRead=false" + ], + "blockers": [ + { + "type": "agent_blocker", + "scope": "agent-mgr-health", + "status": "open", + "summary": "hwlab-agent-mgr /health/live is degraded.", + "classification": "agent runtime", + "nextFix": "Inject explicit skill commit and version into agent-mgr health configuration, then redeploy DEV." + }, + { + "type": "agent_blocker", + "scope": "skills-commit-version-injection", + "status": "open", + "summary": "DEV skills injection is incomplete: missing hwlab-agent-skills.HWLAB_SKILLS_VERSION, worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV.", + "classification": "skills injection", + "nextFix": "Add HWLAB_SKILLS_VERSION to hwlab-agent-skills and ensure scheduled worker jobs receive HWLAB_SKILL_COMMIT_ID plus HWLAB_SKILL_VERSION." + } + ], + "minimumFixRecommendations": [ + "Inject explicit skill commit and version into agent-mgr health configuration, then redeploy DEV.", + "Add HWLAB_SKILLS_VERSION to hwlab-agent-skills and ensure scheduled worker jobs receive HWLAB_SKILL_COMMIT_ID plus HWLAB_SKILL_VERSION." + ] + }, + "publicEntrypoints": { + "status": "pass", + "commands": [ + "curl -fsS --max-time 10 http://74.48.78.17:16667/health", + "curl -fsS --max-time 10 http://74.48.78.17:16667/health/live", + "curl -fsS -I --max-time 10 http://74.48.78.17:16666/" + ], + "api": { + "endpoint": "http://74.48.78.17:16667", + "health": { + "statusCode": 200, + "serviceId": "hwlab-edge-proxy", + "environment": "dev", + "status": "ok", + "error": null + }, + "live": { + "statusCode": 200, + "serviceId": "hwlab-cloud-api", + "environment": "dev", + "status": "degraded", + "db": { + "status": "degraded", + "connected": false, + "ready": false, + "evidence": "env_presence_only_no_live_db" + }, + "error": null + } + }, + "frontend": { + "endpoint": "http://74.48.78.17:16666", + "statusCode": 200, + "contentType": "text/html; charset=utf-8", + "contentLength": "11180", + "error": null + }, + "summary": "Public DEV API http://74.48.78.17:16667 health=200 live=200; frontend http://74.48.78.17:16666/=200." + }, + "safetyEvidence": { + "devOnly": true, + "namespace": "hwlab-dev", + "kubeconfig": "/etc/rancher/k3s/k3s.yaml", + "prodTouched": false, + "prodNamespaceTouched": false, + "secretMaterialRead": false, + "secretResourcesRead": false, + "servicesRestarted": false, + "longRunningAgentStarted": false, + "workerJobPersisted": false, + "notes": [ + "kubectl commands were scoped to hwlab-dev and /etc/rancher/k3s/k3s.yaml.", + "Secret RBAC was checked with kubectl auth can-i only; no Secret resource or secret value was read.", + "Worker createability used server-side dry-run with suspend=true and did not persist a Job." + ] + }, "devPreconditions": { "status": "blocked", "classification": "DB live", "requirements": [ "DEV route remains frozen at http://74.48.78.17:16667", + "DEV browser route remains frozen at http://74.48.78.17:16666", + "D601 native k3s access uses KUBECONFIG=/etc/rancher/k3s/k3s.yaml and namespace hwlab-dev", "No secret reads, real deployment, or long-running agent task is attempted", "Agent manager, worker, and skills are reachable only through the cloud API boundary", "M4 hardware assistance must go through cloud-api/hardware API and patch-panel; direct box/gateway access is forbidden" @@ -140,6 +377,22 @@ "status": "open", "summary": "cloud-api /health/live reports DB degraded; connected=false; ready=false.", "classification": "DB live" + }, + { + "status": "open", + "type": "agent_blocker", + "scope": "agent-mgr-health", + "summary": "hwlab-agent-mgr /health/live is degraded.", + "classification": "agent runtime", + "nextFix": "Inject explicit skill commit and version into agent-mgr health configuration, then redeploy DEV." + }, + { + "status": "open", + "type": "agent_blocker", + "scope": "skills-commit-version-injection", + "summary": "DEV skills injection is incomplete: missing hwlab-agent-skills.HWLAB_SKILLS_VERSION, worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV.", + "classification": "skills injection", + "nextFix": "Add HWLAB_SKILLS_VERSION to hwlab-agent-skills and ensure scheduled worker jobs receive HWLAB_SKILL_COMMIT_ID plus HWLAB_SKILL_VERSION." } ], "livePreflight": { @@ -149,6 +402,18 @@ "node scripts/dev-m4-agent-loop-smoke.mjs --live --confirm-dev --confirmed-non-production" ], "evidence": [ + "k3s:namespace=hwlab-dev:read=true", + "k3s:deploy/hwlab-agent-mgr:ready=1/1:available=True", + "k3s:deploy/hwlab-agent-skills:ready=1/1:available=True", + "k3s:svc/hwlab-agent-mgr:endpoints=1", + "k3s:svc/hwlab-agent-skills:endpoints=1", + "agent-mgr:/health/live:degraded", + "agent-skills:/health/live:ok:cb35ada6860653b27269b0a57991184118cbf4b1", + "worker-template:suspend=true:image=127.0.0.1:5000/hwlab/hwlab-agent-worker:cb35ada", + "worker-job:server-dry-run=true:persisted=false", + "skills-injection:commit=cb35ada:version=missing", + "secret-rbac:get=yes:secretResourcesRead=false", + "Public DEV API http://74.48.78.17:16667 health=200 live=200; frontend http://74.48.78.17:16666/=200.", "health:hwlab-edge-proxy:dev:ok", "live:hwlab-cloud-api:degraded:db-blocked" ], diff --git a/scripts/dev-m4-agent-loop-smoke.mjs b/scripts/dev-m4-agent-loop-smoke.mjs index 2a86e06e..dec3a225 100644 --- a/scripts/dev-m4-agent-loop-smoke.mjs +++ b/scripts/dev-m4-agent-loop-smoke.mjs @@ -1,8 +1,6 @@ #!/usr/bin/env node import assert from "node:assert/strict"; import { execFileSync } from "node:child_process"; -import { request as httpRequest } from "node:http"; -import { request as httpsRequest } from "node:https"; import { mkdtemp, readFile, writeFile, mkdir, rm } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; @@ -21,7 +19,16 @@ import { runLocalWorkerDryRun } from "../internal/agent/runtime.mjs"; import { activeReportLifecycle } from "../internal/dev-report-lifecycle.mjs"; -import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs"; +import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs"; +import { + collectD601K3sNativeEvidence, + collectPublicEntrypoints, + d601K3sReadonlyCommand, + d601Kubeconfig, + devNamespace, + requestJson, + workerServerDryRunCommand +} from "./src/dev-m4-agent-loop-smoke-lib.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const fixedNow = () => "2026-05-22T00:00:00.000Z"; @@ -67,77 +74,35 @@ function currentOriginMainCommitId() { }).trim(); } -function requestJson(urlString, { method = "GET", headers = {}, body, timeoutMs = 5000 } = {}) { - const url = new URL(urlString); - const client = url.protocol === "https:" ? httpsRequest : httpRequest; - const payload = body === undefined ? null : Buffer.from(body); - - return new Promise((resolve, reject) => { - const request = client( - { - protocol: url.protocol, - hostname: url.hostname, - port: url.port, - path: `${url.pathname}${url.search}`, - method, - headers: { - ...headers - } - }, - (response) => { - let raw = ""; - response.setEncoding("utf8"); - response.on("data", (chunk) => { - raw += chunk; - }); - response.on("end", () => { - let parsed = null; - if (raw.length > 0) { - try { - parsed = JSON.parse(raw); - } catch (error) { - reject(new Error(`invalid JSON response from ${urlString}: ${error.message}`)); - return; - } - } - resolve({ - statusCode: response.statusCode ?? 0, - headers: response.headers, - body: parsed, - raw - }); - }); - } - ); - - request.on("error", reject); - request.setTimeout(timeoutMs, () => { - request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`)); - }); - - if (payload) { - request.write(payload); - } - request.end(); - }); -} - function statusForResult(result) { - return result.blocked ? "blocked" : "pass"; + return result.blocked || (result.additionalBlockers?.length ?? 0) > 0 ? "blocked" : "pass"; } function blockerForResult(result) { - return result.blocked - ? [ - { - type: result.blockerType, - scope: result.blockerScope ?? "devPreconditions", - status: "open", - summary: result.blockerSummary, - classification: result.blockedClassification ?? "other" - } - ] - : []; + const blockers = []; + if (result.blocked && result.blockerType) { + blockers.push({ + type: result.blockerType, + scope: result.blockerScope ?? "devPreconditions", + status: "open", + summary: result.blockerSummary, + classification: result.blockedClassification ?? "other" + }); + } + for (const blocker of result.additionalBlockers ?? []) { + blockers.push({ status: "open", ...blocker }); + } + const seen = new Set(); + return blockers.filter((blocker) => { + const key = `${blocker.type}:${blocker.scope}`; + if (seen.has(key)) return false; + seen.add(key); + return true; + }); +} + +function classificationForResult(result) { + return result.blockedClassification ?? result.additionalBlockers?.[0]?.classification ?? "none"; } function buildDryRunEvidence({ status = "not_run", details = [] } = {}) { @@ -243,7 +208,12 @@ function buildDependencyState(result) { }; } -function buildReport(fixture, liveEvidence, result, { dryRun = buildDryRunEvidence() } = {}) { +function buildReport( + fixture, + liveEvidence, + result, + { dryRun = buildDryRunEvidence(), d601K3s = null, publicEntrypoints = null } = {} +) { const sourceContract = { status: "pass", documents: requiredDocs, @@ -254,9 +224,13 @@ function buildReport(fixture, liveEvidence, result, { dryRun = buildDryRunEviden "node --check scripts/validate-dev-gate-report.mjs", "node scripts/validate-dev-gate-report.mjs", "node --check scripts/dev-m4-agent-loop-smoke.mjs", + "node --check scripts/src/dev-m4-agent-loop-smoke-lib.mjs", dryRunCommand, - preflightCommand + preflightCommand, + d601K3sReadonlyCommand, + workerServerDryRunCommand ]; + const blockers = blockerForResult(result); return { $schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json", @@ -283,21 +257,42 @@ function buildReport(fixture, liveEvidence, result, { dryRun = buildDryRunEviden dryRun, componentEvidence: buildComponentEvidence(fixture, dryRun, result), dependencyState: buildDependencyState(result), + d601K3sNative: d601K3s, + publicEntrypoints, + safetyEvidence: { + devOnly: true, + namespace: devNamespace, + kubeconfig: d601Kubeconfig, + prodTouched: false, + prodNamespaceTouched: false, + secretMaterialRead: false, + secretResourcesRead: false, + servicesRestarted: false, + longRunningAgentStarted: false, + workerJobPersisted: false, + notes: [ + "kubectl commands were scoped to hwlab-dev and /etc/rancher/k3s/k3s.yaml.", + "Secret RBAC was checked with kubectl auth can-i only; no Secret resource or secret value was read.", + "Worker createability used server-side dry-run with suspend=true and did not persist a Job." + ] + }, devPreconditions: { status: statusForResult(result), - classification: result.blockedClassification ?? "none", + classification: classificationForResult(result), requirements: [ `DEV route remains frozen at ${DEV_ENDPOINT}`, + `DEV browser route remains frozen at ${DEV_FRONTEND_ENDPOINT}`, + `D601 native k3s access uses KUBECONFIG=${d601Kubeconfig} and namespace ${devNamespace}`, "No secret reads, real deployment, or long-running agent task is attempted", "Agent manager, worker, and skills are reachable only through the cloud API boundary", "M4 hardware assistance must go through cloud-api/hardware API and patch-panel; direct box/gateway access is forbidden" ], summary: result.summary }, - blockers: blockerForResult(result), + blockers, livePreflight: { - status: result.blocked ? "blocked" : "pass", - classification: result.blockedClassification ?? "none", + status: blockers.length > 0 ? "blocked" : "pass", + classification: classificationForResult(result), commands: [preflightCommand], evidence: liveEvidence.length ? liveEvidence : ["No live DEV observation was recorded."], summary: result.summary @@ -379,6 +374,12 @@ async function runDryRun(fixture) { async function runLivePreflight(fixture, dryRun) { const liveEvidence = []; + const [d601K3s, publicEntrypoints] = await Promise.all([ + collectD601K3sNativeEvidence(fixture), + collectPublicEntrypoints() + ]); + liveEvidence.push(...d601K3s.evidence); + liveEvidence.push(publicEntrypoints.summary); let result = { blocked: true, @@ -529,7 +530,12 @@ async function runLivePreflight(fixture, dryRun) { }; } - const report = buildReport(fixture, liveEvidence, result, { dryRun }); + result.additionalBlockers = d601K3s.blockers; + if (!result.blocked && result.additionalBlockers.length > 0) { + result.summary = "D601 native k3s agent evidence is blocked even though the public preflight path did not produce an earlier blocker."; + } + + const report = buildReport(fixture, liveEvidence, result, { dryRun, d601K3s, publicEntrypoints }); await writeJSON(reportPath, report); process.stdout.write(JSON.stringify({ diff --git a/scripts/src/dev-m4-agent-loop-smoke-lib.mjs b/scripts/src/dev-m4-agent-loop-smoke-lib.mjs new file mode 100644 index 00000000..ad9fedb9 --- /dev/null +++ b/scripts/src/dev-m4-agent-loop-smoke-lib.mjs @@ -0,0 +1,567 @@ +import { execFileSync } from "node:child_process"; +import { request as httpRequest } from "node:http"; +import { request as httpsRequest } from "node:https"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { + AGENT_MGR_SERVICE_ID, + AGENT_SKILLS_SERVICE_ID, + AGENT_WORKER_SERVICE_ID +} from "../../internal/agent/index.mjs"; +import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); + +export const d601Kubeconfig = "/etc/rancher/k3s/k3s.yaml"; +export const devNamespace = "hwlab-dev"; +export const d601K3sReadonlyCommand = + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods,svc,deploy,job,cm -o wide"; +export const workerServerDryRunCommand = + "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json"; + +export function requestJson(urlString, { method = "GET", headers = {}, body, timeoutMs = 5000 } = {}) { + const url = new URL(urlString); + const client = url.protocol === "https:" ? httpsRequest : httpRequest; + const payload = body === undefined ? null : Buffer.from(body); + + return new Promise((resolve, reject) => { + const request = client( + { + protocol: url.protocol, + hostname: url.hostname, + port: url.port, + path: `${url.pathname}${url.search}`, + method, + headers + }, + (response) => { + let raw = ""; + response.setEncoding("utf8"); + response.on("data", (chunk) => { + raw += chunk; + }); + response.on("end", () => { + let bodyValue = null; + if (raw.length > 0) { + try { + bodyValue = JSON.parse(raw); + } catch (error) { + reject(new Error(`invalid JSON response from ${urlString}: ${error.message}`)); + return; + } + } + resolve({ + statusCode: response.statusCode ?? 0, + headers: response.headers, + body: bodyValue, + raw + }); + }); + } + ); + + request.on("error", reject); + request.setTimeout(timeoutMs, () => { + request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`)); + }); + + if (payload) request.write(payload); + request.end(); + }); +} + +function requestHead(urlString, { timeoutMs = 5000 } = {}) { + const url = new URL(urlString); + const client = url.protocol === "https:" ? httpsRequest : httpRequest; + + return new Promise((resolve, reject) => { + const request = client( + { + protocol: url.protocol, + hostname: url.hostname, + port: url.port, + path: `${url.pathname}${url.search}`, + method: "HEAD" + }, + (response) => { + response.resume(); + response.on("end", () => { + resolve({ + statusCode: response.statusCode ?? 0, + headers: response.headers + }); + }); + } + ); + request.on("error", reject); + request.setTimeout(timeoutMs, () => { + request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`)); + }); + request.end(); + }); +} + +function runKubectl(args, { input, timeoutMs = 5000 } = {}) { + try { + return { + ok: true, + command: `KUBECONFIG=${d601Kubeconfig} kubectl ${args.join(" ")}`, + exitCode: 0, + stdout: execFileSync("kubectl", args, { + cwd: repoRoot, + encoding: "utf8", + env: { ...process.env, KUBECONFIG: d601Kubeconfig }, + input, + stdio: ["pipe", "pipe", "pipe"], + timeout: timeoutMs + }) + }; + } catch (error) { + return { + ok: false, + command: `KUBECONFIG=${d601Kubeconfig} kubectl ${args.join(" ")}`, + exitCode: typeof error.status === "number" ? error.status : 1, + stdout: String(error.stdout ?? ""), + stderr: String(error.stderr ?? ""), + error: error instanceof Error ? error.message : String(error) + }; + } +} + +function parseProbeJson(probe) { + if (!probe.ok || !probe.stdout.trim()) return null; + try { + return JSON.parse(probe.stdout); + } catch { + return null; + } +} + +function compactError(probe) { + return (probe.stderr || probe.error || probe.stdout || "unknown error") + .replace(/\s+/g, " ") + .trim() + .slice(0, 400); +} + +function items(document) { + if (!document) return []; + return Array.isArray(document.items) ? document.items : [document]; +} + +function named(document, name) { + return items(document).find((item) => item?.metadata?.name === name) ?? null; +} + +function firstContainer(workload) { + return workload?.spec?.template?.spec?.containers?.[0] ?? null; +} + +function envMap(container) { + const values = {}; + const valueFrom = []; + for (const entry of container?.env ?? []) { + if (Object.hasOwn(entry, "value")) values[entry.name] = entry.value; + else if (entry.valueFrom) valueFrom.push(entry.name); + } + return { values, valueFrom }; +} + +function condition(workload, type) { + return workload?.status?.conditions?.find((entry) => entry.type === type)?.status ?? "Unknown"; +} + +function deploymentSummary(deployment) { + const container = firstContainer(deployment); + const replicas = deployment?.status?.replicas ?? deployment?.spec?.replicas ?? 0; + const readyReplicas = deployment?.status?.readyReplicas ?? 0; + return { + name: deployment?.metadata?.name ?? null, + image: container?.image ?? null, + readyReplicas, + replicas, + available: condition(deployment, "Available"), + ready: replicas > 0 && readyReplicas >= replicas && condition(deployment, "Available") === "True", + env: envMap(container) + }; +} + +function workerTemplateSummary(job) { + const container = firstContainer(job); + return { + name: job?.metadata?.name ?? null, + suspend: job?.spec?.suspend ?? null, + image: container?.image ?? null, + restartPolicy: job?.spec?.template?.spec?.restartPolicy ?? null, + env: envMap(container) + }; +} + +function serviceUrl(service, fallbackPort) { + const clusterIP = service?.spec?.clusterIP; + if (!clusterIP || clusterIP === "None") return null; + return `http://${clusterIP}:${service?.spec?.ports?.[0]?.port ?? fallbackPort}`; +} + +function endpointCount(endpoint) { + return (endpoint?.subsets ?? []).reduce((total, subset) => total + (subset.addresses?.length ?? 0), 0); +} + +function buildWorkerDryRunJob(templateJob, fixture, liveSkillsEnv) { + const templateContainer = firstContainer(templateJob); + const labels = { + "app.kubernetes.io/name": AGENT_WORKER_SERVICE_ID, + "hwlab.pikastech.local/environment": ENVIRONMENT_DEV, + "hwlab.pikastech.local/profile": ENVIRONMENT_DEV, + "hwlab.pikastech.local/service-id": AGENT_WORKER_SERVICE_ID, + "hwlab.pikastech.local/smoke": "dev-m4-agent-loop" + }; + const reservedEnv = new Set([ + "HWLAB_AGENT_SESSION_ID", + "HWLAB_PROJECT_ID", + "HWLAB_SKILL_COMMIT_ID", + "HWLAB_SKILL_VERSION", + "HWLAB_WORKSPACE_ROOT" + ]); + return { + apiVersion: "batch/v1", + kind: "Job", + metadata: { + name: "hwlab-agent-worker-smoke-dryrun-20260522", + namespace: devNamespace, + labels + }, + spec: { + suspend: true, + backoffLimit: 0, + template: { + metadata: { labels }, + spec: { + restartPolicy: templateJob?.spec?.template?.spec?.restartPolicy ?? "Never", + containers: [ + { + name: templateContainer?.name ?? AGENT_WORKER_SERVICE_ID, + image: templateContainer?.image ?? "unknown", + env: [ + ...(templateContainer?.env ?? []).filter((entry) => !reservedEnv.has(entry.name)), + { name: "HWLAB_AGENT_SESSION_ID", value: fixture.agentSessionId }, + { name: "HWLAB_PROJECT_ID", value: fixture.projectId }, + { name: "HWLAB_SKILL_COMMIT_ID", value: liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? "" }, + { name: "HWLAB_SKILL_VERSION", value: liveSkillsEnv.HWLAB_SKILLS_VERSION ?? "" }, + { name: "HWLAB_WORKSPACE_ROOT", value: `/workspace/${fixture.projectId}/${fixture.agentSessionId}` } + ] + } + ] + } + } + } + }; +} + +function k3sBlockers(report) { + const blockers = []; + const push = (type, scope, summary, classification, nextFix) => { + blockers.push({ type, scope, status: "open", summary, classification, nextFix }); + }; + if (!report.clusterReadable) { + push( + "environment_blocker", + "d601-k3s-readonly", + "D601 native k3s hwlab-dev resources were not readable with /etc/rancher/k3s/k3s.yaml.", + "k3s readonly", + "Mount or authorize read-only hwlab-dev access through the D601 native k3s kubeconfig." + ); + return blockers; + } + if (report.agentManager.health.status !== "ok") { + push( + "agent_blocker", + "agent-mgr-health", + `hwlab-agent-mgr /health/live is ${report.agentManager.health.status}.`, + "agent runtime", + "Inject explicit skill commit and version into agent-mgr health configuration, then redeploy DEV." + ); + } + if (report.skillsInjection.status !== "pass") { + push( + "agent_blocker", + "skills-commit-version-injection", + `DEV skills injection is incomplete: missing ${report.skillsInjection.missing.join(", ")}.`, + "skills injection", + "Add HWLAB_SKILLS_VERSION to hwlab-agent-skills and ensure scheduled worker jobs receive HWLAB_SKILL_COMMIT_ID plus HWLAB_SKILL_VERSION." + ); + } + if (report.workerJob.serverDryRun.status !== "pass") { + push( + "agent_blocker", + "worker-job-admission", + report.workerJob.serverDryRun.summary, + "worker job", + "Fix the hwlab-agent-worker-template image/env/job spec until server-side dry-run admission succeeds." + ); + } + return blockers; +} + +export async function collectPublicEntrypoints() { + const apiHealth = await requestJson(`${DEV_ENDPOINT}/health`).catch((error) => ({ + statusCode: 0, + body: null, + error: error instanceof Error ? error.message : String(error) + })); + const apiLive = await requestJson(`${DEV_ENDPOINT}/health/live`).catch((error) => ({ + statusCode: 0, + body: null, + error: error instanceof Error ? error.message : String(error) + })); + const frontend = await requestHead(`${DEV_FRONTEND_ENDPOINT}/`).catch((error) => ({ + statusCode: 0, + headers: {}, + error: error instanceof Error ? error.message : String(error) + })); + return { + status: apiHealth.statusCode >= 200 && + apiHealth.statusCode < 300 && + apiLive.statusCode >= 200 && + apiLive.statusCode < 300 && + frontend.statusCode >= 200 && + frontend.statusCode < 400 + ? "pass" + : "blocked", + commands: [ + `curl -fsS --max-time 10 ${DEV_ENDPOINT}/health`, + `curl -fsS --max-time 10 ${DEV_ENDPOINT}/health/live`, + `curl -fsS -I --max-time 10 ${DEV_FRONTEND_ENDPOINT}/` + ], + api: { + endpoint: DEV_ENDPOINT, + health: { + statusCode: apiHealth.statusCode, + serviceId: apiHealth.body?.serviceId ?? null, + environment: apiHealth.body?.environment ?? null, + status: apiHealth.body?.status ?? null, + error: apiHealth.error ?? null + }, + live: { + statusCode: apiLive.statusCode, + serviceId: apiLive.body?.serviceId ?? null, + environment: apiLive.body?.environment ?? null, + status: apiLive.body?.status ?? null, + db: apiLive.body?.db ? { + status: apiLive.body.db.status ?? null, + connected: apiLive.body.db.connected ?? null, + ready: apiLive.body.db.ready ?? null, + evidence: apiLive.body.db.evidence ?? null + } : null, + error: apiLive.error ?? null + } + }, + frontend: { + endpoint: DEV_FRONTEND_ENDPOINT, + statusCode: frontend.statusCode, + contentType: frontend.headers?.["content-type"] ?? null, + contentLength: frontend.headers?.["content-length"] ?? null, + error: frontend.error ?? null + }, + summary: `Public DEV API ${DEV_ENDPOINT} health=${apiHealth.statusCode} live=${apiLive.statusCode}; frontend ${DEV_FRONTEND_ENDPOINT}/=${frontend.statusCode}.` + }; +} + +export async function collectD601K3sNativeEvidence(fixture) { + const namespaceProbe = runKubectl(["get", "namespace", devNamespace, "-o", "json"]); + const deploymentsProbe = runKubectl([ + "-n", + devNamespace, + "get", + "deploy", + AGENT_MGR_SERVICE_ID, + AGENT_SKILLS_SERVICE_ID, + "-o", + "json" + ]); + const workerTemplateProbe = runKubectl([ + "-n", + devNamespace, + "get", + "job", + "hwlab-agent-worker-template", + "-o", + "json" + ]); + const servicesProbe = runKubectl([ + "-n", + devNamespace, + "get", + "svc", + AGENT_MGR_SERVICE_ID, + AGENT_SKILLS_SERVICE_ID, + "-o", + "json" + ]); + const endpointsProbe = runKubectl([ + "-n", + devNamespace, + "get", + "endpoints", + AGENT_MGR_SERVICE_ID, + AGENT_SKILLS_SERVICE_ID, + "-o", + "json" + ]); + const canCreateJobs = runKubectl(["auth", "can-i", "create", "jobs", "-n", devNamespace]); + const canDeleteJobs = runKubectl(["auth", "can-i", "delete", "jobs", "-n", devNamespace]); + const canGetSecrets = runKubectl(["auth", "can-i", "get", "secrets", "-n", devNamespace]); + + const deployments = parseProbeJson(deploymentsProbe); + const services = parseProbeJson(servicesProbe); + const endpoints = parseProbeJson(endpointsProbe); + const workerTemplate = parseProbeJson(workerTemplateProbe); + const agentMgrDeployment = named(deployments, AGENT_MGR_SERVICE_ID); + const skillsDeployment = named(deployments, AGENT_SKILLS_SERVICE_ID); + const agentMgrService = named(services, AGENT_MGR_SERVICE_ID); + const skillsService = named(services, AGENT_SKILLS_SERVICE_ID); + const agentMgrEndpoint = named(endpoints, AGENT_MGR_SERVICE_ID); + const skillsEndpoint = named(endpoints, AGENT_SKILLS_SERVICE_ID); + const agentMgrSummary = deploymentSummary(agentMgrDeployment); + const skillsSummary = deploymentSummary(skillsDeployment); + const workerSummary = workerTemplateSummary(workerTemplate); + const liveSkillsEnv = skillsSummary.env.values; + const agentMgrUrl = serviceUrl(agentMgrService, 7410); + const skillsUrl = serviceUrl(skillsService, 7430); + const agentMgrHealth = agentMgrUrl ? await requestJson(`${agentMgrUrl}/health/live`).catch((error) => ({ + statusCode: 0, + body: null, + error: error instanceof Error ? error.message : String(error) + })) : null; + const skillsHealth = skillsUrl ? await requestJson(`${skillsUrl}/health/live`).catch((error) => ({ + statusCode: 0, + body: null, + error: error instanceof Error ? error.message : String(error) + })) : null; + const workerDryRunManifest = workerTemplate ? buildWorkerDryRunJob(workerTemplate, fixture, liveSkillsEnv) : null; + const workerDryRunProbe = workerDryRunManifest + ? runKubectl(["-n", devNamespace, "create", "-f", "-", "--dry-run=server", "-o", "json"], { + input: `${JSON.stringify(workerDryRunManifest)}\n` + }) + : { ok: false, stdout: "", stderr: "worker template was not readable" }; + const workerDryRun = parseProbeJson(workerDryRunProbe); + const workerInjectedEnv = envMap(workerDryRunManifest?.spec?.template?.spec?.containers?.[0] ?? null).values; + const missingSkills = []; + if (!liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID) missingSkills.push("hwlab-agent-skills.HWLAB_SKILLS_COMMIT_ID"); + if (!liveSkillsEnv.HWLAB_SKILLS_VERSION) missingSkills.push("hwlab-agent-skills.HWLAB_SKILLS_VERSION"); + if (!workerInjectedEnv.HWLAB_SKILL_COMMIT_ID) missingSkills.push("worker-dry-run.HWLAB_SKILL_COMMIT_ID"); + if (!workerInjectedEnv.HWLAB_SKILL_VERSION) missingSkills.push("worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV"); + const clusterReadable = namespaceProbe.ok && deploymentsProbe.ok && workerTemplateProbe.ok && servicesProbe.ok; + const evidence = [ + `k3s:namespace=${devNamespace}:read=${namespaceProbe.ok}`, + `k3s:deploy/${AGENT_MGR_SERVICE_ID}:ready=${agentMgrSummary.readyReplicas}/${agentMgrSummary.replicas}:available=${agentMgrSummary.available}`, + `k3s:deploy/${AGENT_SKILLS_SERVICE_ID}:ready=${skillsSummary.readyReplicas}/${skillsSummary.replicas}:available=${skillsSummary.available}`, + `k3s:svc/${AGENT_MGR_SERVICE_ID}:endpoints=${endpointCount(agentMgrEndpoint)}`, + `k3s:svc/${AGENT_SKILLS_SERVICE_ID}:endpoints=${endpointCount(skillsEndpoint)}`, + `agent-mgr:/health/live:${agentMgrHealth?.body?.health?.status ?? agentMgrHealth?.body?.status ?? "unreachable"}`, + `agent-skills:/health/live:${skillsHealth?.body?.status ?? "unreachable"}:${skillsHealth?.body?.revision ?? "unknown"}`, + `worker-template:suspend=${workerSummary.suspend}:image=${workerSummary.image ?? "unknown"}`, + `worker-job:server-dry-run=${workerDryRunProbe.ok}:persisted=false`, + `skills-injection:commit=${liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? "missing"}:version=${liveSkillsEnv.HWLAB_SKILLS_VERSION ?? "missing"}`, + `secret-rbac:get=${canGetSecrets.stdout.trim() || "unknown"}:secretResourcesRead=false` + ]; + const report = { + status: "pass", + mode: "dev-read-only-plus-server-dry-run", + generatedAt: new Date().toISOString(), + commands: [ + `KUBECONFIG=${d601Kubeconfig} kubectl get namespace ${devNamespace} -o json`, + d601K3sReadonlyCommand, + `KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get deploy ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`, + `KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get job hwlab-agent-worker-template -o json`, + `KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get svc ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`, + `KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get endpoints ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`, + `KUBECONFIG=${d601Kubeconfig} kubectl auth can-i create jobs -n ${devNamespace}`, + `KUBECONFIG=${d601Kubeconfig} kubectl auth can-i delete jobs -n ${devNamespace}`, + `KUBECONFIG=${d601Kubeconfig} kubectl auth can-i get secrets -n ${devNamespace}`, + workerServerDryRunCommand + ], + safety: { + namespace: devNamespace, + kubeconfig: d601Kubeconfig, + readOnly: true, + prodTouched: false, + secretResourcesRead: false, + secretMaterialPrinted: false, + servicesRestarted: false, + workerJobPersisted: false, + longRunningAgentStarted: false, + note: "The smoke checks Secret RBAC only with kubectl auth can-i; it never reads Secret resources or kubeconfig contents." + }, + clusterReadable, + namespace: { + name: devNamespace, + readable: namespaceProbe.ok, + error: namespaceProbe.ok ? null : compactError(namespaceProbe) + }, + rbac: { + createJobs: canCreateJobs.stdout.trim() === "yes", + deleteJobs: canDeleteJobs.stdout.trim() === "yes", + getSecrets: canGetSecrets.stdout.trim() === "yes", + secretReadNotPerformed: true + }, + agentManager: { + deployment: agentMgrSummary, + endpointCount: endpointCount(agentMgrEndpoint), + health: { + statusCode: agentMgrHealth?.statusCode ?? 0, + status: agentMgrHealth?.body?.health?.status ?? agentMgrHealth?.body?.status ?? "unreachable", + serviceId: agentMgrHealth?.body?.serviceId ?? null, + missingSkills: agentMgrHealth?.body?.health?.skills?.missing ?? [], + error: agentMgrHealth?.error ?? null + } + }, + agentSkills: { + deployment: skillsSummary, + endpointCount: endpointCount(skillsEndpoint), + health: { + statusCode: skillsHealth?.statusCode ?? 0, + status: skillsHealth?.body?.status ?? "unreachable", + serviceId: skillsHealth?.body?.serviceId ?? null, + revision: skillsHealth?.body?.revision ?? null, + error: skillsHealth?.error ?? null + } + }, + workerJob: { + template: workerSummary, + serverDryRun: { + status: workerDryRunProbe.ok ? "pass" : "blocked", + accepted: workerDryRunProbe.ok, + command: workerServerDryRunCommand, + persisted: false, + generatedName: workerDryRunManifest?.metadata?.name ?? null, + injectedEnvNames: Object.keys(workerInjectedEnv).sort(), + dryRunKind: workerDryRun?.kind ?? null, + dryRunSuspend: workerDryRun?.spec?.suspend ?? null, + summary: workerDryRunProbe.ok + ? "Server-side admission accepted a suspended worker Job generated from the template with session/workspace/skills env injected; no Job was persisted." + : compactError(workerDryRunProbe) + } + }, + skillsInjection: { + status: missingSkills.length === 0 ? "pass" : "blocked", + source: "k3s deployment env plus generated worker dry-run manifest", + serviceCommitId: liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? null, + serviceVersion: liveSkillsEnv.HWLAB_SKILLS_VERSION ?? null, + workerDryRunCommitId: workerInjectedEnv.HWLAB_SKILL_COMMIT_ID ?? null, + workerDryRunVersion: workerInjectedEnv.HWLAB_SKILL_VERSION ?? null, + workerDryRunSource: "derived-from-live-dev-skills-env", + missing: missingSkills + }, + traceEvidenceCleanup: { + status: "dry-run-only", + summary: "Live worker execution was not started. Trace/evidence/cleanup closure is covered by the bounded local dry-run and remains explicitly non-M3 DEV-LIVE." + }, + evidence + }; + report.blockers = k3sBlockers(report); + report.status = report.blockers.length === 0 ? "pass" : "blocked"; + report.minimumFixRecommendations = report.blockers.map((blocker) => blocker.nextFix).filter(Boolean); + return report; +}