test: refresh DEV M4 agent smoke evidence (#139)
Commander review: direction is correct. This PR records DEV M4 agent-loop evidence as a bounded preflight, keeps M4 subordinate to the M3 trusted-loop mainline, and preserves blockers for DB live readiness and skill version injection. No PROD/service restart is included.
This commit is contained in:
Lyon
@@ -46,6 +46,24 @@ The live path must only observe existing DEV state. It must not read tokens or
|
||||
secrets, must not try to run a real agent workload, and must stop if the M3
|
||||
runtime preconditions are not ready.
|
||||
|
||||
On D601, the live preflight also records native k3s evidence with the explicit
|
||||
kubeconfig and namespace:
|
||||
|
||||
```sh
|
||||
KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods,svc,deploy,job,cm -o wide
|
||||
```
|
||||
|
||||
Worker createability is checked with server-side dry-run admission from the
|
||||
existing `hwlab-agent-worker-template`. The generated Job stays suspended and is
|
||||
not persisted:
|
||||
|
||||
```sh
|
||||
KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json
|
||||
```
|
||||
|
||||
The smoke may check Secret RBAC with `kubectl auth can-i`, but it must not read
|
||||
Secret resources, kubeconfig contents, or token values.
|
||||
|
||||
Any hardware assistance from M4 must go through the cloud API hardware surface
|
||||
and the patch-panel trusted path for `DO1 -> patch-panel -> DI1`. The agent
|
||||
must not connect directly to box or gateway simulators.
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
"reportVersion": "v1",
|
||||
"issue": "pikasTech/HWLAB#37",
|
||||
"taskId": "dev-m4-agent-loop",
|
||||
"commitId": "8e89409",
|
||||
"commitId": "0614202",
|
||||
"acceptanceLevel": "dev_m4_agent_loop",
|
||||
"devOnly": true,
|
||||
"prodDisabled": true,
|
||||
@@ -28,8 +28,11 @@
|
||||
"node --check scripts/validate-dev-gate-report.mjs",
|
||||
"node scripts/validate-dev-gate-report.mjs",
|
||||
"node --check scripts/dev-m4-agent-loop-smoke.mjs",
|
||||
"node --check scripts/src/dev-m4-agent-loop-smoke-lib.mjs",
|
||||
"node scripts/dev-m4-agent-loop-smoke.mjs --dry-run",
|
||||
"node scripts/dev-m4-agent-loop-smoke.mjs --live --confirm-dev --confirmed-non-production"
|
||||
"node scripts/dev-m4-agent-loop-smoke.mjs --live --confirm-dev --confirmed-non-production",
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods,svc,deploy,job,cm -o wide",
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json"
|
||||
],
|
||||
"localSmoke": {
|
||||
"status": "pass",
|
||||
@@ -122,11 +125,245 @@
|
||||
]
|
||||
}
|
||||
},
|
||||
"d601K3sNative": {
|
||||
"status": "blocked",
|
||||
"mode": "dev-read-only-plus-server-dry-run",
|
||||
"generatedAt": "2026-05-22T12:12:47.156Z",
|
||||
"commands": [
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl get namespace hwlab-dev -o json",
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods,svc,deploy,job,cm -o wide",
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get deploy hwlab-agent-mgr hwlab-agent-skills -o json",
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get job hwlab-agent-worker-template -o json",
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get svc hwlab-agent-mgr hwlab-agent-skills -o json",
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get endpoints hwlab-agent-mgr hwlab-agent-skills -o json",
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl auth can-i create jobs -n hwlab-dev",
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl auth can-i delete jobs -n hwlab-dev",
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl auth can-i get secrets -n hwlab-dev",
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json"
|
||||
],
|
||||
"safety": {
|
||||
"namespace": "hwlab-dev",
|
||||
"kubeconfig": "/etc/rancher/k3s/k3s.yaml",
|
||||
"readOnly": true,
|
||||
"prodTouched": false,
|
||||
"secretResourcesRead": false,
|
||||
"secretMaterialPrinted": false,
|
||||
"servicesRestarted": false,
|
||||
"workerJobPersisted": false,
|
||||
"longRunningAgentStarted": false,
|
||||
"note": "The smoke checks Secret RBAC only with kubectl auth can-i; it never reads Secret resources or kubeconfig contents."
|
||||
},
|
||||
"clusterReadable": true,
|
||||
"namespace": {
|
||||
"name": "hwlab-dev",
|
||||
"readable": true,
|
||||
"error": null
|
||||
},
|
||||
"rbac": {
|
||||
"createJobs": true,
|
||||
"deleteJobs": true,
|
||||
"getSecrets": true,
|
||||
"secretReadNotPerformed": true
|
||||
},
|
||||
"agentManager": {
|
||||
"deployment": {
|
||||
"name": "hwlab-agent-mgr",
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-agent-mgr:cb35ada",
|
||||
"readyReplicas": 1,
|
||||
"replicas": 1,
|
||||
"available": "True",
|
||||
"ready": true,
|
||||
"env": {
|
||||
"values": {
|
||||
"HWLAB_AGENT_SESSION_MODE": "control-plane"
|
||||
},
|
||||
"valueFrom": []
|
||||
}
|
||||
},
|
||||
"endpointCount": 1,
|
||||
"health": {
|
||||
"statusCode": 200,
|
||||
"status": "degraded",
|
||||
"serviceId": "hwlab-agent-mgr",
|
||||
"missingSkills": [
|
||||
"manager.skillCommitId",
|
||||
"manager.skillVersion"
|
||||
],
|
||||
"error": null
|
||||
}
|
||||
},
|
||||
"agentSkills": {
|
||||
"deployment": {
|
||||
"name": "hwlab-agent-skills",
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-agent-skills:cb35ada",
|
||||
"readyReplicas": 1,
|
||||
"replicas": 1,
|
||||
"available": "True",
|
||||
"ready": true,
|
||||
"env": {
|
||||
"values": {
|
||||
"HWLAB_SKILLS_COMMIT_ID": "cb35ada"
|
||||
},
|
||||
"valueFrom": []
|
||||
}
|
||||
},
|
||||
"endpointCount": 1,
|
||||
"health": {
|
||||
"statusCode": 200,
|
||||
"status": "ok",
|
||||
"serviceId": "hwlab-agent-skills",
|
||||
"revision": "cb35ada6860653b27269b0a57991184118cbf4b1",
|
||||
"error": null
|
||||
}
|
||||
},
|
||||
"workerJob": {
|
||||
"template": {
|
||||
"name": "hwlab-agent-worker-template",
|
||||
"suspend": true,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-agent-worker:cb35ada",
|
||||
"restartPolicy": "Never",
|
||||
"env": {
|
||||
"values": {
|
||||
"HWLAB_AGENT_SESSION_MODE": "session-scoped"
|
||||
},
|
||||
"valueFrom": []
|
||||
}
|
||||
},
|
||||
"serverDryRun": {
|
||||
"status": "pass",
|
||||
"accepted": true,
|
||||
"command": "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json",
|
||||
"persisted": false,
|
||||
"generatedName": "hwlab-agent-worker-smoke-dryrun-20260522",
|
||||
"injectedEnvNames": [
|
||||
"HWLAB_AGENT_SESSION_ID",
|
||||
"HWLAB_AGENT_SESSION_MODE",
|
||||
"HWLAB_PROJECT_ID",
|
||||
"HWLAB_SKILL_COMMIT_ID",
|
||||
"HWLAB_SKILL_VERSION",
|
||||
"HWLAB_WORKSPACE_ROOT"
|
||||
],
|
||||
"dryRunKind": "Job",
|
||||
"dryRunSuspend": true,
|
||||
"summary": "Server-side admission accepted a suspended worker Job generated from the template with session/workspace/skills env injected; no Job was persisted."
|
||||
}
|
||||
},
|
||||
"skillsInjection": {
|
||||
"status": "blocked",
|
||||
"source": "k3s deployment env plus generated worker dry-run manifest",
|
||||
"serviceCommitId": "cb35ada",
|
||||
"serviceVersion": null,
|
||||
"workerDryRunCommitId": "cb35ada",
|
||||
"workerDryRunVersion": "",
|
||||
"workerDryRunSource": "derived-from-live-dev-skills-env",
|
||||
"missing": [
|
||||
"hwlab-agent-skills.HWLAB_SKILLS_VERSION",
|
||||
"worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV"
|
||||
]
|
||||
},
|
||||
"traceEvidenceCleanup": {
|
||||
"status": "dry-run-only",
|
||||
"summary": "Live worker execution was not started. Trace/evidence/cleanup closure is covered by the bounded local dry-run and remains explicitly non-M3 DEV-LIVE."
|
||||
},
|
||||
"evidence": [
|
||||
"k3s:namespace=hwlab-dev:read=true",
|
||||
"k3s:deploy/hwlab-agent-mgr:ready=1/1:available=True",
|
||||
"k3s:deploy/hwlab-agent-skills:ready=1/1:available=True",
|
||||
"k3s:svc/hwlab-agent-mgr:endpoints=1",
|
||||
"k3s:svc/hwlab-agent-skills:endpoints=1",
|
||||
"agent-mgr:/health/live:degraded",
|
||||
"agent-skills:/health/live:ok:cb35ada6860653b27269b0a57991184118cbf4b1",
|
||||
"worker-template:suspend=true:image=127.0.0.1:5000/hwlab/hwlab-agent-worker:cb35ada",
|
||||
"worker-job:server-dry-run=true:persisted=false",
|
||||
"skills-injection:commit=cb35ada:version=missing",
|
||||
"secret-rbac:get=yes:secretResourcesRead=false"
|
||||
],
|
||||
"blockers": [
|
||||
{
|
||||
"type": "agent_blocker",
|
||||
"scope": "agent-mgr-health",
|
||||
"status": "open",
|
||||
"summary": "hwlab-agent-mgr /health/live is degraded.",
|
||||
"classification": "agent runtime",
|
||||
"nextFix": "Inject explicit skill commit and version into agent-mgr health configuration, then redeploy DEV."
|
||||
},
|
||||
{
|
||||
"type": "agent_blocker",
|
||||
"scope": "skills-commit-version-injection",
|
||||
"status": "open",
|
||||
"summary": "DEV skills injection is incomplete: missing hwlab-agent-skills.HWLAB_SKILLS_VERSION, worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV.",
|
||||
"classification": "skills injection",
|
||||
"nextFix": "Add HWLAB_SKILLS_VERSION to hwlab-agent-skills and ensure scheduled worker jobs receive HWLAB_SKILL_COMMIT_ID plus HWLAB_SKILL_VERSION."
|
||||
}
|
||||
],
|
||||
"minimumFixRecommendations": [
|
||||
"Inject explicit skill commit and version into agent-mgr health configuration, then redeploy DEV.",
|
||||
"Add HWLAB_SKILLS_VERSION to hwlab-agent-skills and ensure scheduled worker jobs receive HWLAB_SKILL_COMMIT_ID plus HWLAB_SKILL_VERSION."
|
||||
]
|
||||
},
|
||||
"publicEntrypoints": {
|
||||
"status": "pass",
|
||||
"commands": [
|
||||
"curl -fsS --max-time 10 http://74.48.78.17:16667/health",
|
||||
"curl -fsS --max-time 10 http://74.48.78.17:16667/health/live",
|
||||
"curl -fsS -I --max-time 10 http://74.48.78.17:16666/"
|
||||
],
|
||||
"api": {
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"health": {
|
||||
"statusCode": 200,
|
||||
"serviceId": "hwlab-edge-proxy",
|
||||
"environment": "dev",
|
||||
"status": "ok",
|
||||
"error": null
|
||||
},
|
||||
"live": {
|
||||
"statusCode": 200,
|
||||
"serviceId": "hwlab-cloud-api",
|
||||
"environment": "dev",
|
||||
"status": "degraded",
|
||||
"db": {
|
||||
"status": "degraded",
|
||||
"connected": false,
|
||||
"ready": false,
|
||||
"evidence": "env_presence_only_no_live_db"
|
||||
},
|
||||
"error": null
|
||||
}
|
||||
},
|
||||
"frontend": {
|
||||
"endpoint": "http://74.48.78.17:16666",
|
||||
"statusCode": 200,
|
||||
"contentType": "text/html; charset=utf-8",
|
||||
"contentLength": "11180",
|
||||
"error": null
|
||||
},
|
||||
"summary": "Public DEV API http://74.48.78.17:16667 health=200 live=200; frontend http://74.48.78.17:16666/=200."
|
||||
},
|
||||
"safetyEvidence": {
|
||||
"devOnly": true,
|
||||
"namespace": "hwlab-dev",
|
||||
"kubeconfig": "/etc/rancher/k3s/k3s.yaml",
|
||||
"prodTouched": false,
|
||||
"prodNamespaceTouched": false,
|
||||
"secretMaterialRead": false,
|
||||
"secretResourcesRead": false,
|
||||
"servicesRestarted": false,
|
||||
"longRunningAgentStarted": false,
|
||||
"workerJobPersisted": false,
|
||||
"notes": [
|
||||
"kubectl commands were scoped to hwlab-dev and /etc/rancher/k3s/k3s.yaml.",
|
||||
"Secret RBAC was checked with kubectl auth can-i only; no Secret resource or secret value was read.",
|
||||
"Worker createability used server-side dry-run with suspend=true and did not persist a Job."
|
||||
]
|
||||
},
|
||||
"devPreconditions": {
|
||||
"status": "blocked",
|
||||
"classification": "DB live",
|
||||
"requirements": [
|
||||
"DEV route remains frozen at http://74.48.78.17:16667",
|
||||
"DEV browser route remains frozen at http://74.48.78.17:16666",
|
||||
"D601 native k3s access uses KUBECONFIG=/etc/rancher/k3s/k3s.yaml and namespace hwlab-dev",
|
||||
"No secret reads, real deployment, or long-running agent task is attempted",
|
||||
"Agent manager, worker, and skills are reachable only through the cloud API boundary",
|
||||
"M4 hardware assistance must go through cloud-api/hardware API and patch-panel; direct box/gateway access is forbidden"
|
||||
@@ -140,6 +377,22 @@
|
||||
"status": "open",
|
||||
"summary": "cloud-api /health/live reports DB degraded; connected=false; ready=false.",
|
||||
"classification": "DB live"
|
||||
},
|
||||
{
|
||||
"status": "open",
|
||||
"type": "agent_blocker",
|
||||
"scope": "agent-mgr-health",
|
||||
"summary": "hwlab-agent-mgr /health/live is degraded.",
|
||||
"classification": "agent runtime",
|
||||
"nextFix": "Inject explicit skill commit and version into agent-mgr health configuration, then redeploy DEV."
|
||||
},
|
||||
{
|
||||
"status": "open",
|
||||
"type": "agent_blocker",
|
||||
"scope": "skills-commit-version-injection",
|
||||
"summary": "DEV skills injection is incomplete: missing hwlab-agent-skills.HWLAB_SKILLS_VERSION, worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV.",
|
||||
"classification": "skills injection",
|
||||
"nextFix": "Add HWLAB_SKILLS_VERSION to hwlab-agent-skills and ensure scheduled worker jobs receive HWLAB_SKILL_COMMIT_ID plus HWLAB_SKILL_VERSION."
|
||||
}
|
||||
],
|
||||
"livePreflight": {
|
||||
@@ -149,6 +402,18 @@
|
||||
"node scripts/dev-m4-agent-loop-smoke.mjs --live --confirm-dev --confirmed-non-production"
|
||||
],
|
||||
"evidence": [
|
||||
"k3s:namespace=hwlab-dev:read=true",
|
||||
"k3s:deploy/hwlab-agent-mgr:ready=1/1:available=True",
|
||||
"k3s:deploy/hwlab-agent-skills:ready=1/1:available=True",
|
||||
"k3s:svc/hwlab-agent-mgr:endpoints=1",
|
||||
"k3s:svc/hwlab-agent-skills:endpoints=1",
|
||||
"agent-mgr:/health/live:degraded",
|
||||
"agent-skills:/health/live:ok:cb35ada6860653b27269b0a57991184118cbf4b1",
|
||||
"worker-template:suspend=true:image=127.0.0.1:5000/hwlab/hwlab-agent-worker:cb35ada",
|
||||
"worker-job:server-dry-run=true:persisted=false",
|
||||
"skills-injection:commit=cb35ada:version=missing",
|
||||
"secret-rbac:get=yes:secretResourcesRead=false",
|
||||
"Public DEV API http://74.48.78.17:16667 health=200 live=200; frontend http://74.48.78.17:16666/=200.",
|
||||
"health:hwlab-edge-proxy:dev:ok",
|
||||
"live:hwlab-cloud-api:degraded:db-blocked"
|
||||
],
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
#!/usr/bin/env node
|
||||
import assert from "node:assert/strict";
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { request as httpRequest } from "node:http";
|
||||
import { request as httpsRequest } from "node:https";
|
||||
import { mkdtemp, readFile, writeFile, mkdir, rm } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
@@ -21,7 +19,16 @@ import {
|
||||
runLocalWorkerDryRun
|
||||
} from "../internal/agent/runtime.mjs";
|
||||
import { activeReportLifecycle } from "../internal/dev-report-lifecycle.mjs";
|
||||
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs";
|
||||
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs";
|
||||
import {
|
||||
collectD601K3sNativeEvidence,
|
||||
collectPublicEntrypoints,
|
||||
d601K3sReadonlyCommand,
|
||||
d601Kubeconfig,
|
||||
devNamespace,
|
||||
requestJson,
|
||||
workerServerDryRunCommand
|
||||
} from "./src/dev-m4-agent-loop-smoke-lib.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
||||
const fixedNow = () => "2026-05-22T00:00:00.000Z";
|
||||
@@ -67,77 +74,35 @@ function currentOriginMainCommitId() {
|
||||
}).trim();
|
||||
}
|
||||
|
||||
function requestJson(urlString, { method = "GET", headers = {}, body, timeoutMs = 5000 } = {}) {
|
||||
const url = new URL(urlString);
|
||||
const client = url.protocol === "https:" ? httpsRequest : httpRequest;
|
||||
const payload = body === undefined ? null : Buffer.from(body);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = client(
|
||||
{
|
||||
protocol: url.protocol,
|
||||
hostname: url.hostname,
|
||||
port: url.port,
|
||||
path: `${url.pathname}${url.search}`,
|
||||
method,
|
||||
headers: {
|
||||
...headers
|
||||
}
|
||||
},
|
||||
(response) => {
|
||||
let raw = "";
|
||||
response.setEncoding("utf8");
|
||||
response.on("data", (chunk) => {
|
||||
raw += chunk;
|
||||
});
|
||||
response.on("end", () => {
|
||||
let parsed = null;
|
||||
if (raw.length > 0) {
|
||||
try {
|
||||
parsed = JSON.parse(raw);
|
||||
} catch (error) {
|
||||
reject(new Error(`invalid JSON response from ${urlString}: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
}
|
||||
resolve({
|
||||
statusCode: response.statusCode ?? 0,
|
||||
headers: response.headers,
|
||||
body: parsed,
|
||||
raw
|
||||
});
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
request.on("error", reject);
|
||||
request.setTimeout(timeoutMs, () => {
|
||||
request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`));
|
||||
});
|
||||
|
||||
if (payload) {
|
||||
request.write(payload);
|
||||
}
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
function statusForResult(result) {
|
||||
return result.blocked ? "blocked" : "pass";
|
||||
return result.blocked || (result.additionalBlockers?.length ?? 0) > 0 ? "blocked" : "pass";
|
||||
}
|
||||
|
||||
function blockerForResult(result) {
|
||||
return result.blocked
|
||||
? [
|
||||
{
|
||||
type: result.blockerType,
|
||||
scope: result.blockerScope ?? "devPreconditions",
|
||||
status: "open",
|
||||
summary: result.blockerSummary,
|
||||
classification: result.blockedClassification ?? "other"
|
||||
}
|
||||
]
|
||||
: [];
|
||||
const blockers = [];
|
||||
if (result.blocked && result.blockerType) {
|
||||
blockers.push({
|
||||
type: result.blockerType,
|
||||
scope: result.blockerScope ?? "devPreconditions",
|
||||
status: "open",
|
||||
summary: result.blockerSummary,
|
||||
classification: result.blockedClassification ?? "other"
|
||||
});
|
||||
}
|
||||
for (const blocker of result.additionalBlockers ?? []) {
|
||||
blockers.push({ status: "open", ...blocker });
|
||||
}
|
||||
const seen = new Set();
|
||||
return blockers.filter((blocker) => {
|
||||
const key = `${blocker.type}:${blocker.scope}`;
|
||||
if (seen.has(key)) return false;
|
||||
seen.add(key);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
|
||||
function classificationForResult(result) {
|
||||
return result.blockedClassification ?? result.additionalBlockers?.[0]?.classification ?? "none";
|
||||
}
|
||||
|
||||
function buildDryRunEvidence({ status = "not_run", details = [] } = {}) {
|
||||
@@ -243,7 +208,12 @@ function buildDependencyState(result) {
|
||||
};
|
||||
}
|
||||
|
||||
function buildReport(fixture, liveEvidence, result, { dryRun = buildDryRunEvidence() } = {}) {
|
||||
function buildReport(
|
||||
fixture,
|
||||
liveEvidence,
|
||||
result,
|
||||
{ dryRun = buildDryRunEvidence(), d601K3s = null, publicEntrypoints = null } = {}
|
||||
) {
|
||||
const sourceContract = {
|
||||
status: "pass",
|
||||
documents: requiredDocs,
|
||||
@@ -254,9 +224,13 @@ function buildReport(fixture, liveEvidence, result, { dryRun = buildDryRunEviden
|
||||
"node --check scripts/validate-dev-gate-report.mjs",
|
||||
"node scripts/validate-dev-gate-report.mjs",
|
||||
"node --check scripts/dev-m4-agent-loop-smoke.mjs",
|
||||
"node --check scripts/src/dev-m4-agent-loop-smoke-lib.mjs",
|
||||
dryRunCommand,
|
||||
preflightCommand
|
||||
preflightCommand,
|
||||
d601K3sReadonlyCommand,
|
||||
workerServerDryRunCommand
|
||||
];
|
||||
const blockers = blockerForResult(result);
|
||||
|
||||
return {
|
||||
$schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json",
|
||||
@@ -283,21 +257,42 @@ function buildReport(fixture, liveEvidence, result, { dryRun = buildDryRunEviden
|
||||
dryRun,
|
||||
componentEvidence: buildComponentEvidence(fixture, dryRun, result),
|
||||
dependencyState: buildDependencyState(result),
|
||||
d601K3sNative: d601K3s,
|
||||
publicEntrypoints,
|
||||
safetyEvidence: {
|
||||
devOnly: true,
|
||||
namespace: devNamespace,
|
||||
kubeconfig: d601Kubeconfig,
|
||||
prodTouched: false,
|
||||
prodNamespaceTouched: false,
|
||||
secretMaterialRead: false,
|
||||
secretResourcesRead: false,
|
||||
servicesRestarted: false,
|
||||
longRunningAgentStarted: false,
|
||||
workerJobPersisted: false,
|
||||
notes: [
|
||||
"kubectl commands were scoped to hwlab-dev and /etc/rancher/k3s/k3s.yaml.",
|
||||
"Secret RBAC was checked with kubectl auth can-i only; no Secret resource or secret value was read.",
|
||||
"Worker createability used server-side dry-run with suspend=true and did not persist a Job."
|
||||
]
|
||||
},
|
||||
devPreconditions: {
|
||||
status: statusForResult(result),
|
||||
classification: result.blockedClassification ?? "none",
|
||||
classification: classificationForResult(result),
|
||||
requirements: [
|
||||
`DEV route remains frozen at ${DEV_ENDPOINT}`,
|
||||
`DEV browser route remains frozen at ${DEV_FRONTEND_ENDPOINT}`,
|
||||
`D601 native k3s access uses KUBECONFIG=${d601Kubeconfig} and namespace ${devNamespace}`,
|
||||
"No secret reads, real deployment, or long-running agent task is attempted",
|
||||
"Agent manager, worker, and skills are reachable only through the cloud API boundary",
|
||||
"M4 hardware assistance must go through cloud-api/hardware API and patch-panel; direct box/gateway access is forbidden"
|
||||
],
|
||||
summary: result.summary
|
||||
},
|
||||
blockers: blockerForResult(result),
|
||||
blockers,
|
||||
livePreflight: {
|
||||
status: result.blocked ? "blocked" : "pass",
|
||||
classification: result.blockedClassification ?? "none",
|
||||
status: blockers.length > 0 ? "blocked" : "pass",
|
||||
classification: classificationForResult(result),
|
||||
commands: [preflightCommand],
|
||||
evidence: liveEvidence.length ? liveEvidence : ["No live DEV observation was recorded."],
|
||||
summary: result.summary
|
||||
@@ -379,6 +374,12 @@ async function runDryRun(fixture) {
|
||||
|
||||
async function runLivePreflight(fixture, dryRun) {
|
||||
const liveEvidence = [];
|
||||
const [d601K3s, publicEntrypoints] = await Promise.all([
|
||||
collectD601K3sNativeEvidence(fixture),
|
||||
collectPublicEntrypoints()
|
||||
]);
|
||||
liveEvidence.push(...d601K3s.evidence);
|
||||
liveEvidence.push(publicEntrypoints.summary);
|
||||
|
||||
let result = {
|
||||
blocked: true,
|
||||
@@ -529,7 +530,12 @@ async function runLivePreflight(fixture, dryRun) {
|
||||
};
|
||||
}
|
||||
|
||||
const report = buildReport(fixture, liveEvidence, result, { dryRun });
|
||||
result.additionalBlockers = d601K3s.blockers;
|
||||
if (!result.blocked && result.additionalBlockers.length > 0) {
|
||||
result.summary = "D601 native k3s agent evidence is blocked even though the public preflight path did not produce an earlier blocker.";
|
||||
}
|
||||
|
||||
const report = buildReport(fixture, liveEvidence, result, { dryRun, d601K3s, publicEntrypoints });
|
||||
await writeJSON(reportPath, report);
|
||||
|
||||
process.stdout.write(JSON.stringify({
|
||||
|
||||
@@ -0,0 +1,567 @@
|
||||
import { execFileSync } from "node:child_process";
|
||||
import { request as httpRequest } from "node:http";
|
||||
import { request as httpsRequest } from "node:https";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import {
|
||||
AGENT_MGR_SERVICE_ID,
|
||||
AGENT_SKILLS_SERVICE_ID,
|
||||
AGENT_WORKER_SERVICE_ID
|
||||
} from "../../internal/agent/index.mjs";
|
||||
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
|
||||
export const d601Kubeconfig = "/etc/rancher/k3s/k3s.yaml";
|
||||
export const devNamespace = "hwlab-dev";
|
||||
export const d601K3sReadonlyCommand =
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods,svc,deploy,job,cm -o wide";
|
||||
export const workerServerDryRunCommand =
|
||||
"KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev create -f - --dry-run=server -o json";
|
||||
|
||||
export function requestJson(urlString, { method = "GET", headers = {}, body, timeoutMs = 5000 } = {}) {
|
||||
const url = new URL(urlString);
|
||||
const client = url.protocol === "https:" ? httpsRequest : httpRequest;
|
||||
const payload = body === undefined ? null : Buffer.from(body);
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = client(
|
||||
{
|
||||
protocol: url.protocol,
|
||||
hostname: url.hostname,
|
||||
port: url.port,
|
||||
path: `${url.pathname}${url.search}`,
|
||||
method,
|
||||
headers
|
||||
},
|
||||
(response) => {
|
||||
let raw = "";
|
||||
response.setEncoding("utf8");
|
||||
response.on("data", (chunk) => {
|
||||
raw += chunk;
|
||||
});
|
||||
response.on("end", () => {
|
||||
let bodyValue = null;
|
||||
if (raw.length > 0) {
|
||||
try {
|
||||
bodyValue = JSON.parse(raw);
|
||||
} catch (error) {
|
||||
reject(new Error(`invalid JSON response from ${urlString}: ${error.message}`));
|
||||
return;
|
||||
}
|
||||
}
|
||||
resolve({
|
||||
statusCode: response.statusCode ?? 0,
|
||||
headers: response.headers,
|
||||
body: bodyValue,
|
||||
raw
|
||||
});
|
||||
});
|
||||
}
|
||||
);
|
||||
|
||||
request.on("error", reject);
|
||||
request.setTimeout(timeoutMs, () => {
|
||||
request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`));
|
||||
});
|
||||
|
||||
if (payload) request.write(payload);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
function requestHead(urlString, { timeoutMs = 5000 } = {}) {
|
||||
const url = new URL(urlString);
|
||||
const client = url.protocol === "https:" ? httpsRequest : httpRequest;
|
||||
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = client(
|
||||
{
|
||||
protocol: url.protocol,
|
||||
hostname: url.hostname,
|
||||
port: url.port,
|
||||
path: `${url.pathname}${url.search}`,
|
||||
method: "HEAD"
|
||||
},
|
||||
(response) => {
|
||||
response.resume();
|
||||
response.on("end", () => {
|
||||
resolve({
|
||||
statusCode: response.statusCode ?? 0,
|
||||
headers: response.headers
|
||||
});
|
||||
});
|
||||
}
|
||||
);
|
||||
request.on("error", reject);
|
||||
request.setTimeout(timeoutMs, () => {
|
||||
request.destroy(new Error(`request to ${urlString} timed out after ${timeoutMs}ms`));
|
||||
});
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
function runKubectl(args, { input, timeoutMs = 5000 } = {}) {
|
||||
try {
|
||||
return {
|
||||
ok: true,
|
||||
command: `KUBECONFIG=${d601Kubeconfig} kubectl ${args.join(" ")}`,
|
||||
exitCode: 0,
|
||||
stdout: execFileSync("kubectl", args, {
|
||||
cwd: repoRoot,
|
||||
encoding: "utf8",
|
||||
env: { ...process.env, KUBECONFIG: d601Kubeconfig },
|
||||
input,
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
timeout: timeoutMs
|
||||
})
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
ok: false,
|
||||
command: `KUBECONFIG=${d601Kubeconfig} kubectl ${args.join(" ")}`,
|
||||
exitCode: typeof error.status === "number" ? error.status : 1,
|
||||
stdout: String(error.stdout ?? ""),
|
||||
stderr: String(error.stderr ?? ""),
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function parseProbeJson(probe) {
|
||||
if (!probe.ok || !probe.stdout.trim()) return null;
|
||||
try {
|
||||
return JSON.parse(probe.stdout);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function compactError(probe) {
|
||||
return (probe.stderr || probe.error || probe.stdout || "unknown error")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim()
|
||||
.slice(0, 400);
|
||||
}
|
||||
|
||||
function items(document) {
|
||||
if (!document) return [];
|
||||
return Array.isArray(document.items) ? document.items : [document];
|
||||
}
|
||||
|
||||
function named(document, name) {
|
||||
return items(document).find((item) => item?.metadata?.name === name) ?? null;
|
||||
}
|
||||
|
||||
function firstContainer(workload) {
|
||||
return workload?.spec?.template?.spec?.containers?.[0] ?? null;
|
||||
}
|
||||
|
||||
function envMap(container) {
|
||||
const values = {};
|
||||
const valueFrom = [];
|
||||
for (const entry of container?.env ?? []) {
|
||||
if (Object.hasOwn(entry, "value")) values[entry.name] = entry.value;
|
||||
else if (entry.valueFrom) valueFrom.push(entry.name);
|
||||
}
|
||||
return { values, valueFrom };
|
||||
}
|
||||
|
||||
function condition(workload, type) {
|
||||
return workload?.status?.conditions?.find((entry) => entry.type === type)?.status ?? "Unknown";
|
||||
}
|
||||
|
||||
function deploymentSummary(deployment) {
|
||||
const container = firstContainer(deployment);
|
||||
const replicas = deployment?.status?.replicas ?? deployment?.spec?.replicas ?? 0;
|
||||
const readyReplicas = deployment?.status?.readyReplicas ?? 0;
|
||||
return {
|
||||
name: deployment?.metadata?.name ?? null,
|
||||
image: container?.image ?? null,
|
||||
readyReplicas,
|
||||
replicas,
|
||||
available: condition(deployment, "Available"),
|
||||
ready: replicas > 0 && readyReplicas >= replicas && condition(deployment, "Available") === "True",
|
||||
env: envMap(container)
|
||||
};
|
||||
}
|
||||
|
||||
function workerTemplateSummary(job) {
|
||||
const container = firstContainer(job);
|
||||
return {
|
||||
name: job?.metadata?.name ?? null,
|
||||
suspend: job?.spec?.suspend ?? null,
|
||||
image: container?.image ?? null,
|
||||
restartPolicy: job?.spec?.template?.spec?.restartPolicy ?? null,
|
||||
env: envMap(container)
|
||||
};
|
||||
}
|
||||
|
||||
function serviceUrl(service, fallbackPort) {
|
||||
const clusterIP = service?.spec?.clusterIP;
|
||||
if (!clusterIP || clusterIP === "None") return null;
|
||||
return `http://${clusterIP}:${service?.spec?.ports?.[0]?.port ?? fallbackPort}`;
|
||||
}
|
||||
|
||||
function endpointCount(endpoint) {
|
||||
return (endpoint?.subsets ?? []).reduce((total, subset) => total + (subset.addresses?.length ?? 0), 0);
|
||||
}
|
||||
|
||||
function buildWorkerDryRunJob(templateJob, fixture, liveSkillsEnv) {
|
||||
const templateContainer = firstContainer(templateJob);
|
||||
const labels = {
|
||||
"app.kubernetes.io/name": AGENT_WORKER_SERVICE_ID,
|
||||
"hwlab.pikastech.local/environment": ENVIRONMENT_DEV,
|
||||
"hwlab.pikastech.local/profile": ENVIRONMENT_DEV,
|
||||
"hwlab.pikastech.local/service-id": AGENT_WORKER_SERVICE_ID,
|
||||
"hwlab.pikastech.local/smoke": "dev-m4-agent-loop"
|
||||
};
|
||||
const reservedEnv = new Set([
|
||||
"HWLAB_AGENT_SESSION_ID",
|
||||
"HWLAB_PROJECT_ID",
|
||||
"HWLAB_SKILL_COMMIT_ID",
|
||||
"HWLAB_SKILL_VERSION",
|
||||
"HWLAB_WORKSPACE_ROOT"
|
||||
]);
|
||||
return {
|
||||
apiVersion: "batch/v1",
|
||||
kind: "Job",
|
||||
metadata: {
|
||||
name: "hwlab-agent-worker-smoke-dryrun-20260522",
|
||||
namespace: devNamespace,
|
||||
labels
|
||||
},
|
||||
spec: {
|
||||
suspend: true,
|
||||
backoffLimit: 0,
|
||||
template: {
|
||||
metadata: { labels },
|
||||
spec: {
|
||||
restartPolicy: templateJob?.spec?.template?.spec?.restartPolicy ?? "Never",
|
||||
containers: [
|
||||
{
|
||||
name: templateContainer?.name ?? AGENT_WORKER_SERVICE_ID,
|
||||
image: templateContainer?.image ?? "unknown",
|
||||
env: [
|
||||
...(templateContainer?.env ?? []).filter((entry) => !reservedEnv.has(entry.name)),
|
||||
{ name: "HWLAB_AGENT_SESSION_ID", value: fixture.agentSessionId },
|
||||
{ name: "HWLAB_PROJECT_ID", value: fixture.projectId },
|
||||
{ name: "HWLAB_SKILL_COMMIT_ID", value: liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? "" },
|
||||
{ name: "HWLAB_SKILL_VERSION", value: liveSkillsEnv.HWLAB_SKILLS_VERSION ?? "" },
|
||||
{ name: "HWLAB_WORKSPACE_ROOT", value: `/workspace/${fixture.projectId}/${fixture.agentSessionId}` }
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function k3sBlockers(report) {
|
||||
const blockers = [];
|
||||
const push = (type, scope, summary, classification, nextFix) => {
|
||||
blockers.push({ type, scope, status: "open", summary, classification, nextFix });
|
||||
};
|
||||
if (!report.clusterReadable) {
|
||||
push(
|
||||
"environment_blocker",
|
||||
"d601-k3s-readonly",
|
||||
"D601 native k3s hwlab-dev resources were not readable with /etc/rancher/k3s/k3s.yaml.",
|
||||
"k3s readonly",
|
||||
"Mount or authorize read-only hwlab-dev access through the D601 native k3s kubeconfig."
|
||||
);
|
||||
return blockers;
|
||||
}
|
||||
if (report.agentManager.health.status !== "ok") {
|
||||
push(
|
||||
"agent_blocker",
|
||||
"agent-mgr-health",
|
||||
`hwlab-agent-mgr /health/live is ${report.agentManager.health.status}.`,
|
||||
"agent runtime",
|
||||
"Inject explicit skill commit and version into agent-mgr health configuration, then redeploy DEV."
|
||||
);
|
||||
}
|
||||
if (report.skillsInjection.status !== "pass") {
|
||||
push(
|
||||
"agent_blocker",
|
||||
"skills-commit-version-injection",
|
||||
`DEV skills injection is incomplete: missing ${report.skillsInjection.missing.join(", ")}.`,
|
||||
"skills injection",
|
||||
"Add HWLAB_SKILLS_VERSION to hwlab-agent-skills and ensure scheduled worker jobs receive HWLAB_SKILL_COMMIT_ID plus HWLAB_SKILL_VERSION."
|
||||
);
|
||||
}
|
||||
if (report.workerJob.serverDryRun.status !== "pass") {
|
||||
push(
|
||||
"agent_blocker",
|
||||
"worker-job-admission",
|
||||
report.workerJob.serverDryRun.summary,
|
||||
"worker job",
|
||||
"Fix the hwlab-agent-worker-template image/env/job spec until server-side dry-run admission succeeds."
|
||||
);
|
||||
}
|
||||
return blockers;
|
||||
}
|
||||
|
||||
export async function collectPublicEntrypoints() {
|
||||
const apiHealth = await requestJson(`${DEV_ENDPOINT}/health`).catch((error) => ({
|
||||
statusCode: 0,
|
||||
body: null,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
}));
|
||||
const apiLive = await requestJson(`${DEV_ENDPOINT}/health/live`).catch((error) => ({
|
||||
statusCode: 0,
|
||||
body: null,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
}));
|
||||
const frontend = await requestHead(`${DEV_FRONTEND_ENDPOINT}/`).catch((error) => ({
|
||||
statusCode: 0,
|
||||
headers: {},
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
}));
|
||||
return {
|
||||
status: apiHealth.statusCode >= 200 &&
|
||||
apiHealth.statusCode < 300 &&
|
||||
apiLive.statusCode >= 200 &&
|
||||
apiLive.statusCode < 300 &&
|
||||
frontend.statusCode >= 200 &&
|
||||
frontend.statusCode < 400
|
||||
? "pass"
|
||||
: "blocked",
|
||||
commands: [
|
||||
`curl -fsS --max-time 10 ${DEV_ENDPOINT}/health`,
|
||||
`curl -fsS --max-time 10 ${DEV_ENDPOINT}/health/live`,
|
||||
`curl -fsS -I --max-time 10 ${DEV_FRONTEND_ENDPOINT}/`
|
||||
],
|
||||
api: {
|
||||
endpoint: DEV_ENDPOINT,
|
||||
health: {
|
||||
statusCode: apiHealth.statusCode,
|
||||
serviceId: apiHealth.body?.serviceId ?? null,
|
||||
environment: apiHealth.body?.environment ?? null,
|
||||
status: apiHealth.body?.status ?? null,
|
||||
error: apiHealth.error ?? null
|
||||
},
|
||||
live: {
|
||||
statusCode: apiLive.statusCode,
|
||||
serviceId: apiLive.body?.serviceId ?? null,
|
||||
environment: apiLive.body?.environment ?? null,
|
||||
status: apiLive.body?.status ?? null,
|
||||
db: apiLive.body?.db ? {
|
||||
status: apiLive.body.db.status ?? null,
|
||||
connected: apiLive.body.db.connected ?? null,
|
||||
ready: apiLive.body.db.ready ?? null,
|
||||
evidence: apiLive.body.db.evidence ?? null
|
||||
} : null,
|
||||
error: apiLive.error ?? null
|
||||
}
|
||||
},
|
||||
frontend: {
|
||||
endpoint: DEV_FRONTEND_ENDPOINT,
|
||||
statusCode: frontend.statusCode,
|
||||
contentType: frontend.headers?.["content-type"] ?? null,
|
||||
contentLength: frontend.headers?.["content-length"] ?? null,
|
||||
error: frontend.error ?? null
|
||||
},
|
||||
summary: `Public DEV API ${DEV_ENDPOINT} health=${apiHealth.statusCode} live=${apiLive.statusCode}; frontend ${DEV_FRONTEND_ENDPOINT}/=${frontend.statusCode}.`
|
||||
};
|
||||
}
|
||||
|
||||
export async function collectD601K3sNativeEvidence(fixture) {
|
||||
const namespaceProbe = runKubectl(["get", "namespace", devNamespace, "-o", "json"]);
|
||||
const deploymentsProbe = runKubectl([
|
||||
"-n",
|
||||
devNamespace,
|
||||
"get",
|
||||
"deploy",
|
||||
AGENT_MGR_SERVICE_ID,
|
||||
AGENT_SKILLS_SERVICE_ID,
|
||||
"-o",
|
||||
"json"
|
||||
]);
|
||||
const workerTemplateProbe = runKubectl([
|
||||
"-n",
|
||||
devNamespace,
|
||||
"get",
|
||||
"job",
|
||||
"hwlab-agent-worker-template",
|
||||
"-o",
|
||||
"json"
|
||||
]);
|
||||
const servicesProbe = runKubectl([
|
||||
"-n",
|
||||
devNamespace,
|
||||
"get",
|
||||
"svc",
|
||||
AGENT_MGR_SERVICE_ID,
|
||||
AGENT_SKILLS_SERVICE_ID,
|
||||
"-o",
|
||||
"json"
|
||||
]);
|
||||
const endpointsProbe = runKubectl([
|
||||
"-n",
|
||||
devNamespace,
|
||||
"get",
|
||||
"endpoints",
|
||||
AGENT_MGR_SERVICE_ID,
|
||||
AGENT_SKILLS_SERVICE_ID,
|
||||
"-o",
|
||||
"json"
|
||||
]);
|
||||
const canCreateJobs = runKubectl(["auth", "can-i", "create", "jobs", "-n", devNamespace]);
|
||||
const canDeleteJobs = runKubectl(["auth", "can-i", "delete", "jobs", "-n", devNamespace]);
|
||||
const canGetSecrets = runKubectl(["auth", "can-i", "get", "secrets", "-n", devNamespace]);
|
||||
|
||||
const deployments = parseProbeJson(deploymentsProbe);
|
||||
const services = parseProbeJson(servicesProbe);
|
||||
const endpoints = parseProbeJson(endpointsProbe);
|
||||
const workerTemplate = parseProbeJson(workerTemplateProbe);
|
||||
const agentMgrDeployment = named(deployments, AGENT_MGR_SERVICE_ID);
|
||||
const skillsDeployment = named(deployments, AGENT_SKILLS_SERVICE_ID);
|
||||
const agentMgrService = named(services, AGENT_MGR_SERVICE_ID);
|
||||
const skillsService = named(services, AGENT_SKILLS_SERVICE_ID);
|
||||
const agentMgrEndpoint = named(endpoints, AGENT_MGR_SERVICE_ID);
|
||||
const skillsEndpoint = named(endpoints, AGENT_SKILLS_SERVICE_ID);
|
||||
const agentMgrSummary = deploymentSummary(agentMgrDeployment);
|
||||
const skillsSummary = deploymentSummary(skillsDeployment);
|
||||
const workerSummary = workerTemplateSummary(workerTemplate);
|
||||
const liveSkillsEnv = skillsSummary.env.values;
|
||||
const agentMgrUrl = serviceUrl(agentMgrService, 7410);
|
||||
const skillsUrl = serviceUrl(skillsService, 7430);
|
||||
const agentMgrHealth = agentMgrUrl ? await requestJson(`${agentMgrUrl}/health/live`).catch((error) => ({
|
||||
statusCode: 0,
|
||||
body: null,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
})) : null;
|
||||
const skillsHealth = skillsUrl ? await requestJson(`${skillsUrl}/health/live`).catch((error) => ({
|
||||
statusCode: 0,
|
||||
body: null,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
})) : null;
|
||||
const workerDryRunManifest = workerTemplate ? buildWorkerDryRunJob(workerTemplate, fixture, liveSkillsEnv) : null;
|
||||
const workerDryRunProbe = workerDryRunManifest
|
||||
? runKubectl(["-n", devNamespace, "create", "-f", "-", "--dry-run=server", "-o", "json"], {
|
||||
input: `${JSON.stringify(workerDryRunManifest)}\n`
|
||||
})
|
||||
: { ok: false, stdout: "", stderr: "worker template was not readable" };
|
||||
const workerDryRun = parseProbeJson(workerDryRunProbe);
|
||||
const workerInjectedEnv = envMap(workerDryRunManifest?.spec?.template?.spec?.containers?.[0] ?? null).values;
|
||||
const missingSkills = [];
|
||||
if (!liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID) missingSkills.push("hwlab-agent-skills.HWLAB_SKILLS_COMMIT_ID");
|
||||
if (!liveSkillsEnv.HWLAB_SKILLS_VERSION) missingSkills.push("hwlab-agent-skills.HWLAB_SKILLS_VERSION");
|
||||
if (!workerInjectedEnv.HWLAB_SKILL_COMMIT_ID) missingSkills.push("worker-dry-run.HWLAB_SKILL_COMMIT_ID");
|
||||
if (!workerInjectedEnv.HWLAB_SKILL_VERSION) missingSkills.push("worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV");
|
||||
const clusterReadable = namespaceProbe.ok && deploymentsProbe.ok && workerTemplateProbe.ok && servicesProbe.ok;
|
||||
const evidence = [
|
||||
`k3s:namespace=${devNamespace}:read=${namespaceProbe.ok}`,
|
||||
`k3s:deploy/${AGENT_MGR_SERVICE_ID}:ready=${agentMgrSummary.readyReplicas}/${agentMgrSummary.replicas}:available=${agentMgrSummary.available}`,
|
||||
`k3s:deploy/${AGENT_SKILLS_SERVICE_ID}:ready=${skillsSummary.readyReplicas}/${skillsSummary.replicas}:available=${skillsSummary.available}`,
|
||||
`k3s:svc/${AGENT_MGR_SERVICE_ID}:endpoints=${endpointCount(agentMgrEndpoint)}`,
|
||||
`k3s:svc/${AGENT_SKILLS_SERVICE_ID}:endpoints=${endpointCount(skillsEndpoint)}`,
|
||||
`agent-mgr:/health/live:${agentMgrHealth?.body?.health?.status ?? agentMgrHealth?.body?.status ?? "unreachable"}`,
|
||||
`agent-skills:/health/live:${skillsHealth?.body?.status ?? "unreachable"}:${skillsHealth?.body?.revision ?? "unknown"}`,
|
||||
`worker-template:suspend=${workerSummary.suspend}:image=${workerSummary.image ?? "unknown"}`,
|
||||
`worker-job:server-dry-run=${workerDryRunProbe.ok}:persisted=false`,
|
||||
`skills-injection:commit=${liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? "missing"}:version=${liveSkillsEnv.HWLAB_SKILLS_VERSION ?? "missing"}`,
|
||||
`secret-rbac:get=${canGetSecrets.stdout.trim() || "unknown"}:secretResourcesRead=false`
|
||||
];
|
||||
const report = {
|
||||
status: "pass",
|
||||
mode: "dev-read-only-plus-server-dry-run",
|
||||
generatedAt: new Date().toISOString(),
|
||||
commands: [
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl get namespace ${devNamespace} -o json`,
|
||||
d601K3sReadonlyCommand,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get deploy ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get job hwlab-agent-worker-template -o json`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get svc ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl -n ${devNamespace} get endpoints ${AGENT_MGR_SERVICE_ID} ${AGENT_SKILLS_SERVICE_ID} -o json`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl auth can-i create jobs -n ${devNamespace}`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl auth can-i delete jobs -n ${devNamespace}`,
|
||||
`KUBECONFIG=${d601Kubeconfig} kubectl auth can-i get secrets -n ${devNamespace}`,
|
||||
workerServerDryRunCommand
|
||||
],
|
||||
safety: {
|
||||
namespace: devNamespace,
|
||||
kubeconfig: d601Kubeconfig,
|
||||
readOnly: true,
|
||||
prodTouched: false,
|
||||
secretResourcesRead: false,
|
||||
secretMaterialPrinted: false,
|
||||
servicesRestarted: false,
|
||||
workerJobPersisted: false,
|
||||
longRunningAgentStarted: false,
|
||||
note: "The smoke checks Secret RBAC only with kubectl auth can-i; it never reads Secret resources or kubeconfig contents."
|
||||
},
|
||||
clusterReadable,
|
||||
namespace: {
|
||||
name: devNamespace,
|
||||
readable: namespaceProbe.ok,
|
||||
error: namespaceProbe.ok ? null : compactError(namespaceProbe)
|
||||
},
|
||||
rbac: {
|
||||
createJobs: canCreateJobs.stdout.trim() === "yes",
|
||||
deleteJobs: canDeleteJobs.stdout.trim() === "yes",
|
||||
getSecrets: canGetSecrets.stdout.trim() === "yes",
|
||||
secretReadNotPerformed: true
|
||||
},
|
||||
agentManager: {
|
||||
deployment: agentMgrSummary,
|
||||
endpointCount: endpointCount(agentMgrEndpoint),
|
||||
health: {
|
||||
statusCode: agentMgrHealth?.statusCode ?? 0,
|
||||
status: agentMgrHealth?.body?.health?.status ?? agentMgrHealth?.body?.status ?? "unreachable",
|
||||
serviceId: agentMgrHealth?.body?.serviceId ?? null,
|
||||
missingSkills: agentMgrHealth?.body?.health?.skills?.missing ?? [],
|
||||
error: agentMgrHealth?.error ?? null
|
||||
}
|
||||
},
|
||||
agentSkills: {
|
||||
deployment: skillsSummary,
|
||||
endpointCount: endpointCount(skillsEndpoint),
|
||||
health: {
|
||||
statusCode: skillsHealth?.statusCode ?? 0,
|
||||
status: skillsHealth?.body?.status ?? "unreachable",
|
||||
serviceId: skillsHealth?.body?.serviceId ?? null,
|
||||
revision: skillsHealth?.body?.revision ?? null,
|
||||
error: skillsHealth?.error ?? null
|
||||
}
|
||||
},
|
||||
workerJob: {
|
||||
template: workerSummary,
|
||||
serverDryRun: {
|
||||
status: workerDryRunProbe.ok ? "pass" : "blocked",
|
||||
accepted: workerDryRunProbe.ok,
|
||||
command: workerServerDryRunCommand,
|
||||
persisted: false,
|
||||
generatedName: workerDryRunManifest?.metadata?.name ?? null,
|
||||
injectedEnvNames: Object.keys(workerInjectedEnv).sort(),
|
||||
dryRunKind: workerDryRun?.kind ?? null,
|
||||
dryRunSuspend: workerDryRun?.spec?.suspend ?? null,
|
||||
summary: workerDryRunProbe.ok
|
||||
? "Server-side admission accepted a suspended worker Job generated from the template with session/workspace/skills env injected; no Job was persisted."
|
||||
: compactError(workerDryRunProbe)
|
||||
}
|
||||
},
|
||||
skillsInjection: {
|
||||
status: missingSkills.length === 0 ? "pass" : "blocked",
|
||||
source: "k3s deployment env plus generated worker dry-run manifest",
|
||||
serviceCommitId: liveSkillsEnv.HWLAB_SKILLS_COMMIT_ID ?? null,
|
||||
serviceVersion: liveSkillsEnv.HWLAB_SKILLS_VERSION ?? null,
|
||||
workerDryRunCommitId: workerInjectedEnv.HWLAB_SKILL_COMMIT_ID ?? null,
|
||||
workerDryRunVersion: workerInjectedEnv.HWLAB_SKILL_VERSION ?? null,
|
||||
workerDryRunSource: "derived-from-live-dev-skills-env",
|
||||
missing: missingSkills
|
||||
},
|
||||
traceEvidenceCleanup: {
|
||||
status: "dry-run-only",
|
||||
summary: "Live worker execution was not started. Trace/evidence/cleanup closure is covered by the bounded local dry-run and remains explicitly non-M3 DEV-LIVE."
|
||||
},
|
||||
evidence
|
||||
};
|
||||
report.blockers = k3sBlockers(report);
|
||||
report.status = report.blockers.length === 0 ? "pass" : "blocked";
|
||||
report.minimumFixRecommendations = report.blockers.map((blocker) => blocker.nextFix).filter(Boolean);
|
||||
return report;
|
||||
}
|
||||
Reference in New Issue
Block a user