From 85da2dcb1917e8d8e96682111ea606a18d571ba7 Mon Sep 17 00:00:00 2001
From: Lyon <88232613+pikasTech@users.noreply.github.com>
Date: Fri, 22 May 2026 20:44:30 +0800
Subject: [PATCH] fix: separate runner kubeconfig gap from D601 status (#144)
Commander review: direction is correct. This PR separates runner kubeconfig/read-only observability gaps from D601 public endpoint/k3s availability and keeps M3 blocked until real operation/trace/audit/evidence are observed. It does not touch PROD or restart services.
---
reports/d601-k3s-readonly-observability.json | 92 +++++++-
reports/dev-gate/dev-m3-hardware-loop.json | 78 ++++---
.../dev-gate/dev-m5-gate-aggregator-v2.json | 187 ++++++++++++-----
reports/dev-gate/dev-m5-gate-aggregator-v2.md | 18 +-
scripts/dev-m3-hardware-loop-smoke.mjs | 84 +++++++-
.../src/d601-k3s-readonly-observability.mjs | 196 +++++++++++++++++-
.../src/dev-evidence-blocker-aggregator.mjs | 107 +++++++---
scripts/validate-dev-gate-report.mjs | 48 +++++
8 files changed, 669 insertions(+), 141 deletions(-)
diff --git a/reports/d601-k3s-readonly-observability.json b/reports/d601-k3s-readonly-observability.json
index 9756c263..3b3acd8e 100644
--- a/reports/d601-k3s-readonly-observability.json
+++ b/reports/d601-k3s-readonly-observability.json
@@ -12,7 +12,7 @@
"pikasTech/HWLAB#39",
"pikasTech/HWLAB#42"
],
- "generatedAt": "2026-05-22T07:21:35.915Z",
+ "generatedAt": "2026-05-22T12:34:14.403Z",
"mode": "read-only",
"devOnly": true,
"prodDisabled": true,
@@ -48,7 +48,24 @@
"node --check scripts/src/d601-k3s-readonly-observability.mjs",
"node scripts/d601-k3s-readonly-observability.mjs"
],
- "conclusion": "ready",
+ "conclusion": "blocked",
+ "runnerKubeconfigReadable": false,
+ "runnerKubeconfigProbeOk": true,
+ "runnerKubeconfigProbeExitCode": 0,
+ "runnerKubeconfigProbeStderr": "empty",
+ "d601PublicEndpointsReachable": true,
+ "d601K3sUnavailable": false,
+ "statusInference": {
+ "runnerKubeconfigReadable": false,
+ "runnerKubeconfigProbeOk": true,
+ "runnerKubeconfigProbeExitCode": 0,
+ "runnerKubeconfigProbeStderr": "empty",
+ "d601PublicEndpointsReachable": true,
+ "d601K3sUnavailable": false,
+ "conclusion": "Runner kubeconfig readability is an observability signal only; it is not used to infer D601 k3s or public DEV endpoint availability.",
+ "blockerClassification": "runner_permission_mount_gap",
+ "sourceIssue": "pikasTech/HWLAB#46"
+ },
"environment": {
"binaries": {
"kubectl": {
@@ -118,6 +135,22 @@
"error": "ENOENT"
}
],
+ "runnerKubeconfig": {
+ "path": "/etc/rancher/k3s/k3s.yaml",
+ "mode": "read-only",
+ "resource": "pods",
+ "command": "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods -o name",
+ "runnerKubeconfigReadable": false,
+ "runnerKubeconfigProbeOk": true,
+ "runnerKubeconfigProbeExitCode": 0,
+ "runnerKubeconfigProbeStderr": "empty",
+ "stdoutSummary": "13 pod name(s) observed",
+ "classification": "runner_permission_mount_gap",
+ "sourceIssue": "pikasTech/HWLAB#46",
+ "secretValuesRead": false,
+ "secretResourcesRead": false,
+ "note": "This probe forces KUBECONFIG=/etc/rancher/k3s/k3s.yaml for a read-only kubectl get pods command and records only exit code plus redacted stderr/stdout summary. A successful kubectl exit code does not make the kubeconfig file readable; kubectl may use an alternate in-cluster fallback."
+ },
"sshBridge": {
"sshBinaryAvailable": true,
"env": [
@@ -172,6 +205,41 @@
"note": "SSH bridge discovery checks only binary, environment variable presence, and file metadata; it does not read SSH config or private keys."
}
},
+ "publicEndpoints": {
+ "activeFrontendEndpoint": "http://74.48.78.17:16666",
+ "activeApiEndpoint": "http://74.48.78.17:16667",
+ "d601PublicEndpointsReachable": true,
+ "probes": [
+ {
+ "id": "frontend-16666",
+ "url": "http://74.48.78.17:16666/",
+ "reached": true,
+ "ok": true,
+ "status": 200,
+ "bodySummary": " "
+ },
+ {
+ "id": "api-health-16667",
+ "url": "http://74.48.78.17:16667/health",
+ "reached": true,
+ "ok": true,
+ "status": 200,
+ "serviceId": "hwlab-edge-proxy",
+ "environment": "dev",
+ "bodySummary": "json serviceId=hwlab-edge-proxy status=ok"
+ },
+ {
+ "id": "api-live-16667",
+ "url": "http://74.48.78.17:16667/health/live",
+ "reached": true,
+ "ok": true,
+ "status": 200,
+ "serviceId": "hwlab-cloud-api",
+ "environment": "dev",
+ "bodySummary": "json serviceId=hwlab-cloud-api status=degraded"
+ }
+ ]
+ },
"cluster": {
"namespace": "hwlab-dev",
"attemptedExecutors": [
@@ -285,7 +353,7 @@
"nodeName": "d601"
},
{
- "name": "hwlab-cloud-web-cd9655b98-gxtmc",
+ "name": "hwlab-cloud-web-5b7487b5c4-fzvgr",
"phase": "Running",
"readyContainers": 1,
"totalContainers": 1,
@@ -623,10 +691,20 @@
}
]
},
- "blockers": [],
+ "blockers": [
+ {
+ "status": "open",
+ "type": "observability_blocker",
+ "scope": "runner-kubeconfig-readonly-gap",
+ "classification": "runner_permission_mount_gap",
+ "sourceIssue": "pikasTech/HWLAB#46",
+ "summary": "The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately.",
+ "nextTask": "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report."
+ }
+ ],
"runnerImageFollowUp": {
- "needed": false,
- "suggestedUniDeskIssue": null
+ "needed": true,
+ "suggestedUniDeskIssue": "Code Queue runner image should include kubectl and a documented read-only hwlab-dev kubeconfig mount/permission path; keep Secret resources and token material inaccessible to reports."
},
- "notes": "This report uses command availability checks, file metadata checks, kubectl/k3s kubectl read-only get probes, and auth can-i checks only. It does not print kubeconfig contents, token material, Secret resources, ConfigMap values, SSH config contents, or private keys."
+ "notes": "This report uses command availability checks, file metadata checks, public DEV endpoint GET probes, kubectl/k3s kubectl read-only get probes, and auth can-i checks only. It does not print kubeconfig contents, token material, Secret resources, ConfigMap values, SSH config contents, or private keys. A failed runner /etc/rancher/k3s/k3s.yaml probe is classified as a runner permission/mount or read-only observability gap and is not evidence that D601 k3s is globally unavailable."
}
diff --git a/reports/dev-gate/dev-m3-hardware-loop.json b/reports/dev-gate/dev-m3-hardware-loop.json
index 27ec8f93..3372552c 100644
--- a/reports/dev-gate/dev-m3-hardware-loop.json
+++ b/reports/dev-gate/dev-m3-hardware-loop.json
@@ -4,10 +4,18 @@
"reportVersion": "v1",
"issue": "pikasTech/HWLAB#38",
"taskId": "dev-m3-hardware-loop",
- "commitId": "86d769525caf",
+ "commitId": "bf47a95a97a9",
"acceptanceLevel": "dev_m3_hardware_loop",
"devOnly": true,
"prodDisabled": true,
+ "reportLifecycle": {
+ "version": "v1",
+ "state": "active",
+ "activeEndpoint": "http://74.48.78.17:16667",
+ "activeBrowserEndpoint": "http://74.48.78.17:16666",
+ "deprecatedEndpoint": null,
+ "summary": "Current M3 DEV hardware trusted-loop report; DEV-LIVE requires res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, audit, and evidence ids."
+ },
"sourceContract": {
"status": "pass",
"documents": [
@@ -27,6 +35,7 @@
],
"runtimeTarget": {
"endpoint": "http://74.48.78.17:16667",
+ "frontendEndpoint": "http://74.48.78.17:16666",
"namespace": "hwlab-dev",
"environment": "dev",
"requiredBoxSimulators": 2,
@@ -67,16 +76,16 @@
"status": "pass",
"summary": "DEV ingress returned HWLAB identity at /health/live.",
"evidence": [
- "{\"url\":\"http://74.48.78.17:16667/health/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T04:30:50.824Z\",\"observedAt\":\"2026-05-22T04:30:51.649Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"status\":\"degraded\",\"service\":{\"id\":\"hwlab-cloud-api\",\"role\":\"cloud-api\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T04:30:51.463Z\",\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"configured_without_live_connection\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"redacted\":true}],\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"valuesRedacted\":true,\"liveDbEvidence\":false},\"evidence\":\"env_presence_only_no_live_db\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}",
- "{\"url\":\"http://74.48.78.17:16667/health\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T04:30:51.650Z\",\"observedAt\":\"2026-05-22T04:30:52.025Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-edge-proxy\",\"environment\":\"dev\",\"status\":\"ok\",\"service\":{\"id\":\"hwlab-edge-proxy\",\"role\":\"public-dev-ingress\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada6860653b27269b0a57991184118cbf4b1\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"ghcr.io/pikastech/hwlab-edge-proxy:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T04:30:51.840Z\",\"details\":{\"upstream\":\"http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667\",\"mode\":\"dev-edge-proxy\"}}}",
- "{\"url\":\"http://74.48.78.17:16667/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T04:30:52.025Z\",\"observedAt\":\"2026-05-22T04:30:52.401Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"status\":\"live\"}}",
- "{\"url\":\"http://74.48.78.17:16667/v1\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T04:30:52.401Z\",\"observedAt\":\"2026-05-22T04:30:53.001Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"adapter\":\"rest\",\"status\":\"degraded\",\"rpcBridge\":\"POST /v1/rpc/{method}\",\"methods\":[\"system.health\",\"cloud.adapter.describe\",\"gateway.session.register\",\"box.resource.register\",\"box.capability.report\",\"hardware.operation.request\",\"hardware.invoke.shell\",\"audit.event.write\",\"audit.event.query\",\"evidence.record.write\",\"evidence.record.query\"],\"auditFields\":[\"requestId\",\"actor\",\"source\",\"operation\",\"target\",\"result\",\"timestamp\"],\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"configured_without_live_connection\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"redacted\":true}],\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"valuesRedacted\":true,\"liveDbEvidence\":false},\"evidence\":\"env_presence_only_no_live_db\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}"
+ "{\"url\":\"http://74.48.78.17:16667/health/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:42.977Z\",\"observedAt\":\"2026-05-22T12:36:43.545Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"status\":\"degraded\",\"service\":{\"id\":\"hwlab-cloud-api\",\"role\":\"cloud-api\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T12:36:43.343Z\",\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"configured_without_live_connection\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"redacted\":true}],\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"valuesRedacted\":true,\"liveDbEvidence\":false},\"evidence\":\"env_presence_only_no_live_db\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}",
+ "{\"url\":\"http://74.48.78.17:16667/health\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:43.545Z\",\"observedAt\":\"2026-05-22T12:36:44.191Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-edge-proxy\",\"environment\":\"dev\",\"status\":\"ok\",\"service\":{\"id\":\"hwlab-edge-proxy\",\"role\":\"public-dev-ingress\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada6860653b27269b0a57991184118cbf4b1\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"ghcr.io/pikastech/hwlab-edge-proxy:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T12:36:43.989Z\",\"details\":{\"upstream\":\"http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667\",\"mode\":\"dev-edge-proxy\"}}}",
+ "{\"url\":\"http://74.48.78.17:16667/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:44.192Z\",\"observedAt\":\"2026-05-22T12:36:44.570Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"status\":\"live\"}}",
+ "{\"url\":\"http://74.48.78.17:16667/v1\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:44.570Z\",\"observedAt\":\"2026-05-22T12:36:45.204Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"adapter\":\"rest\",\"status\":\"degraded\",\"rpcBridge\":\"POST /v1/rpc/{method}\",\"methods\":[\"system.health\",\"cloud.adapter.describe\",\"gateway.session.register\",\"box.resource.register\",\"box.capability.report\",\"hardware.operation.request\",\"hardware.invoke.shell\",\"audit.event.write\",\"audit.event.query\",\"evidence.record.write\",\"evidence.record.query\"],\"auditFields\":[\"requestId\",\"actor\",\"source\",\"operation\",\"target\",\"result\",\"timestamp\"],\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"configured_without_live_connection\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"redacted\":true}],\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"valuesRedacted\":true,\"liveDbEvidence\":false},\"evidence\":\"env_presence_only_no_live_db\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}"
]
},
{
"id": "two-box-simu-online",
"status": "not_run",
- "summary": "A live DEV M3 runtime contract check failed before all checks passed.",
+ "summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -84,7 +93,7 @@
{
"id": "two-gateway-simu-online",
"status": "not_run",
- "summary": "A live DEV M3 runtime contract check failed before all checks passed.",
+ "summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -92,7 +101,7 @@
{
"id": "patch-panel-healthy",
"status": "not_run",
- "summary": "A live DEV M3 runtime contract check failed before all checks passed.",
+ "summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -100,7 +109,7 @@
{
"id": "wiring-do1-di1-applied",
"status": "not_run",
- "summary": "A live DEV M3 runtime contract check failed before all checks passed.",
+ "summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -108,7 +117,7 @@
{
"id": "direct-call-do-write-di-read",
"status": "not_run",
- "summary": "A live DEV M3 runtime contract check failed before all checks passed.",
+ "summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -116,7 +125,7 @@
{
"id": "audit-evidence-traceable",
"status": "not_run",
- "summary": "A live DEV M3 runtime contract check failed before all checks passed.",
+ "summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -170,9 +179,31 @@
"id": "kubectl-hwlab-dev",
"status": "observed",
"command": "kubectl get deploy,po,svc -n hwlab-dev -o wide",
- "summary": "NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR deployment.apps/hwlab-agent-mgr 1/1 1 1 4h38m hwlab-agent-mgr 127.0.0.1:5000/hwlab/hwlab-agent-mgr:cb35ada app.kubernetes.io/name=hwlab-agent-mgr,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-agent-skills 1/1 1 1 66m hwlab-agent-skills 127.0.0.1:5000/hwlab/hwlab-agent-skills:cb35ada app.kubernetes.io/name=hwlab-agent-skills,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-box-simu 2/2 2 2 4h38m hwlab-box-simu 127.0.0.1:5000/hwlab/hwlab-box-simu:cb35ada app.kubernetes.io/name=hwlab-box-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-api 1/1 1 1 4h38m hwlab-cloud-api 127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada app.kubernetes.io/name=hwlab-cloud-api,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-web 1/1 1 1 4h38m hwlab-cloud-web 127.0.0.1:5000/hwlab/hwlab-cloud-web:cb35ada app.kubernetes.io/name=hwlab-cloud-web,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-edge-proxy 1/1 1 1 4h38m hwlab-edge-proxy 127.0.0.1:5000/hwlab/hwlab-edge-proxy:cb35ada app.kubernetes.io/name=hwlab-edge-proxy,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-frpc 1/1 1 1 4h33m frpc 127.0.0.1:5000/hwlab/frpc:v0.68.1 app.kubernetes.io/name=hwlab-frpc deployment.apps/hwlab-gateway 0/0 0 0 4h38m hwlab-gateway 127.0.0.1:5000/hwlab/hwlab-gateway:cb35ada app.kubernetes.io/name=hwlab-gateway,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-gateway-simu 2/2 2 2 4h38m hwlab-gateway-simu 127.0.0.1:5000/hwlab/hwlab-gateway-simu:cb35ada app.kubernetes.io/name=hwlab-gateway-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-patch-panel 1/1 1 1 4h38m hwlab-patch-panel 127.0.0.1:5000/hwlab/hwlab-patch-panel:cb35ada app.kubernetes.io/name=hwlab-patch-panel,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/prof"
+ "summary": "NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR deployment.apps/hwlab-agent-mgr 1/1 1 1 12h hwlab-agent-mgr 127.0.0.1:5000/hwlab/hwlab-agent-mgr:cb35ada app.kubernetes.io/name=hwlab-agent-mgr,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-agent-skills 1/1 1 1 9h hwlab-agent-skills 127.0.0.1:5000/hwlab/hwlab-agent-skills:cb35ada app.kubernetes.io/name=hwlab-agent-skills,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-box-simu 2/2 2 2 12h hwlab-box-simu 127.0.0.1:5000/hwlab/hwlab-box-simu:cb35ada app.kubernetes.io/name=hwlab-box-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-api 1/1 1 1 12h hwlab-cloud-api 127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada app.kubernetes.io/name=hwlab-cloud-api,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-web 1/1 1 1 12h hwlab-cloud-web 127.0.0.1:5000/hwlab/hwlab-cloud-web:bf47a95 app.kubernetes.io/name=hwlab-cloud-web,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-edge-proxy 1/1 1 1 12h hwlab-edge-proxy 127.0.0.1:5000/hwlab/hwlab-edge-proxy:cb35ada app.kubernetes.io/name=hwlab-edge-proxy,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-frpc 1/1 1 1 12h frpc 127.0.0.1:5000/hwlab/frpc:v0.68.1 app.kubernetes.io/name=hwlab-frpc deployment.apps/hwlab-gateway 0/0 0 0 12h hwlab-gateway 127.0.0.1:5000/hwlab/hwlab-gateway:cb35ada app.kubernetes.io/name=hwlab-gateway,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-gateway-simu 2/2 2 2 12h hwlab-gateway-simu 127.0.0.1:5000/hwlab/hwlab-gateway-simu:cb35ada app.kubernetes.io/name=hwlab-gateway-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-patch-panel 1/1 1 1 12h hwlab-patch-panel 127.0.0.1:5000/hwlab/hwlab-patch-panel:cb35ada app.kubernetes.io/name=hwlab-patch-panel,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.app"
+ },
+ {
+ "id": "runner-k3s-kubeconfig-readonly",
+ "status": "observed",
+ "command": "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods -o name",
+ "summary": "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate. runnerKubeconfigReadable=false; exitCode=0; stderr=empty."
}
],
+ "d601Observability": {
+ "runnerKubeconfigReadable": false,
+ "runnerKubeconfigProbeOk": true,
+ "runnerKubeconfigProbeExitCode": 0,
+ "runnerKubeconfigProbeStderr": "empty",
+ "d601PublicEndpointsReachable": true,
+ "d601K3sUnavailable": false,
+ "classification": "runner_permission_mount_gap",
+ "sourceIssue": "pikasTech/HWLAB#46",
+ "command": "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods -o name",
+ "stdoutSummary": "13 pod name(s) observed",
+ "summary": "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate.",
+ "inferenceRule": "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately.",
+ "secretValuesRead": false,
+ "secretResourcesRead": false
+ },
"liveOperation": {
"status": "not_run",
"operationId": "not_observed",
@@ -183,25 +214,18 @@
},
"blockers": [
{
- "type": "runtime_blocker",
- "scope": "m3-hardware-loop-runtime",
+ "type": "observability_blocker",
+ "scope": "m3-service-discovery",
"status": "open",
- "classification": "hardware loop",
- "summary": "patch-panel active DO1 -> DI1 connection missing"
+ "classification": "runner permission/mount gap",
+ "sourceIssue": "pikasTech/HWLAB#46",
+ "summary": "DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline."
}
],
"summary": {
"status": "blocked",
- "classification": "hardware loop",
- "observedAt": "2026-05-22T04:30:50.820Z",
- "result": "Live DEV M3 runtime was reachable but did not satisfy the M3 hardware-loop contract."
- },
- "reportLifecycle": {
- "version": "v1",
- "state": "active",
- "activeEndpoint": "http://74.48.78.17:16667",
- "activeBrowserEndpoint": "http://74.48.78.17:16666",
- "deprecatedEndpoint": null,
- "summary": "Current M3 DEV hardware trusted-loop report; DEV-LIVE requires res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, audit, and evidence ids."
+ "classification": "runner permission/mount gap",
+ "observedAt": "2026-05-22T12:36:42.972Z",
+ "result": "DEV ingress was reachable, but live M3 simulator and patch-panel targets were not discoverable."
}
}
diff --git a/reports/dev-gate/dev-m5-gate-aggregator-v2.json b/reports/dev-gate/dev-m5-gate-aggregator-v2.json
index dda7c15d..477a504f 100644
--- a/reports/dev-gate/dev-m5-gate-aggregator-v2.json
+++ b/reports/dev-gate/dev-m5-gate-aggregator-v2.json
@@ -20,8 +20,8 @@
"pikasTech/HWLAB#46",
"pikasTech/HWLAB#64"
],
- "generatedAt": "2026-05-22T11:50:42.733Z",
- "generatedFromCommit": "f933a81c83a5",
+ "generatedAt": "2026-05-22T12:36:45.366Z",
+ "generatedFromCommit": "bf47a95a97a9",
"environment": "dev",
"endpoint": "http://74.48.78.17:16667",
"frontendEndpoint": "http://74.48.78.17:16666",
@@ -108,7 +108,7 @@
"taskId": "dev-m3-hardware-loop",
"lifecycleState": "active",
"status": "blocked",
- "commitId": "86d769525caf"
+ "commitId": "bf47a95a97a9"
},
"devM4Agent": {
"path": "reports/dev-gate/dev-m4-agent-loop.json",
@@ -131,7 +131,7 @@
"issue": "pikasTech/HWLAB#46",
"taskId": "d601-k3s-readonly-observability",
"lifecycleState": "active",
- "status": "ready",
+ "status": "blocked",
"commitId": "unknown"
}
},
@@ -192,9 +192,9 @@
},
{
"id": "d601-k3s-observability",
- "status": "pass",
+ "status": "blocked",
"evidenceLevel": "DEV-LIVE",
- "summary": "D601 hwlab-dev cluster was readable."
+ "summary": "D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false."
},
{
"id": "dev-edge-frp-16667",
@@ -265,11 +265,25 @@
],
"nextRequired": "Provide live DB connection evidence through redacted health output; route reachability alone is insufficient."
},
+ "d601RunnerObservability": {
+ "label": "D601 runner observability",
+ "status": "blocked",
+ "evidenceLevel": "DEV-LIVE",
+ "summary": "D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false.",
+ "evidence": [
+ "runnerKubeconfigReadable=false",
+ "runnerKubeconfigProbeExitCode=0",
+ "runnerKubeconfigProbeStderr=empty",
+ "d601PublicEndpointsReachable=true",
+ "d601K3sUnavailable=false"
+ ],
+ "nextRequired": "Treat #46 runner kubeconfig/readonly gaps separately from D601 service health; rerun read-only observability after the mount or permission path is repaired."
+ },
"m3HardwareTrustedLoop": {
"label": "M3 hardware trusted loop",
"status": "blocked",
"evidenceLevel": "BLOCKED",
- "summary": "No live DEV operation has run yet. Blocker: patch-panel active DO1 -> DI1 connection missing.",
+ "summary": "No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline..",
"evidence": [
"operationId=not_observed",
"traceId=not_observed",
@@ -357,15 +371,18 @@
"milestone": "M3",
"status": "blocked",
"currentLevel": "BLOCKED",
- "blockerClass": "hardware-loop-runtime",
- "dependency": "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.",
+ "blockerClass": "runner-readonly-observability-gap",
+ "dependency": "Runner read-only service discovery must be repaired before the real DEV trusted loop can be observed; this does not mean D601/k3s is globally unavailable.",
"evidence": [
"operationId=not_observed",
"traceId=not_observed",
"auditId=not_observed",
- "evidenceId=not_observed"
+ "evidenceId=not_observed",
+ "runnerKubeconfigReadable=false",
+ "d601PublicEndpointsReachable=true",
+ "d601K3sUnavailable=false"
],
- "nextRequired": "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.",
+ "nextRequired": "Repair the #46 runner readonly kubeconfig/service-discovery gap, then run the bounded DEV M3 live smoke only when direct simulator and patch-panel targets are discoverable.",
"nonPromotionReason": "Public frontend, route, and artifact evidence do not prove the required hardware loop."
},
{
@@ -424,7 +441,7 @@
"highestVisibleLevel": "DEV-LIVE",
"liveEvidence": "pass",
"evidenceCount": 7,
- "blockerCount": 5,
+ "blockerCount": 7,
"summary": "deploy/runtime readiness is blocked before live DEV; highest visible level is DEV-LIVE; status is blocked."
},
{
@@ -433,7 +450,7 @@
"highestVisibleLevel": "LOCAL",
"liveEvidence": "missing_or_blocked",
"evidenceCount": 3,
- "blockerCount": 2,
+ "blockerCount": 3,
"summary": "hardware loop has source/local shape but no live operation; highest visible level is LOCAL; status is blocked."
},
{
@@ -451,7 +468,7 @@
"highestVisibleLevel": "DRY-RUN",
"liveEvidence": "missing_or_blocked",
"evidenceCount": 2,
- "blockerCount": 9,
+ "blockerCount": 8,
"summary": "dry-run is green but live MVP gate is blocked; highest visible level is DRY-RUN; status is blocked."
}
],
@@ -651,8 +668,8 @@
"reportPath": "reports/d601-k3s-readonly-observability.json",
"commitId": "unknown",
"lifecycleState": "active",
- "level": "BLOCKED",
- "status": "ready",
+ "level": "DEV-LIVE",
+ "status": "blocked",
"category": "d601-observability",
"commands": [
"node --check scripts/d601-k3s-readonly-observability.mjs",
@@ -660,18 +677,20 @@
"node scripts/d601-k3s-readonly-observability.mjs"
],
"evidence": [
- "kubectl=available",
- "k3s=available",
- "clusterReadable=yes"
+ "runnerKubeconfigReadable=false",
+ "runnerKubeconfigProbeExitCode=0",
+ "runnerKubeconfigProbeStderr=empty",
+ "d601PublicEndpointsReachable=true",
+ "d601K3sUnavailable=false"
],
- "summary": "D601 k3s observability is blocked; cluster state has not been read."
+ "summary": "D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false."
},
{
"milestone": "M3",
"issue": "pikasTech/HWLAB#38",
"taskId": "dev-m3-hardware-loop",
"reportPath": "reports/dev-gate/dev-m3-hardware-loop.json",
- "commitId": "86d769525caf",
+ "commitId": "bf47a95a97a9",
"lifecycleState": "active",
"level": "SOURCE",
"status": "manifest-ready",
@@ -710,7 +729,7 @@
"issue": "pikasTech/HWLAB#38",
"taskId": "dev-m3-hardware-loop",
"reportPath": "reports/dev-gate/dev-m3-hardware-loop.json",
- "commitId": "86d769525caf",
+ "commitId": "bf47a95a97a9",
"lifecycleState": "active",
"level": "BLOCKED",
"status": "not_run",
@@ -724,7 +743,7 @@
"auditId=not_observed",
"evidenceId=not_observed"
],
- "summary": "No live DEV operation has run yet. Blocker: patch-panel active DO1 -> DI1 connection missing."
+ "summary": "No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.."
},
{
"milestone": "M4",
@@ -970,19 +989,19 @@
"category": "edge-frp-contract",
"reportPath": "reports/dev-gate/dev-edge-health.json",
"summary": "Committed edge report proves read-only public HTTP on :16667 /health and /health/live; remaining edge report blocker is DB readiness, not route/frp reachability."
- }
- ],
- "BLOCKED": [
+ },
{
"milestone": "M2",
"issue": "pikasTech/HWLAB#46",
"taskId": "d601-k3s-readonly-observability",
"lifecycleState": "active",
- "status": "ready",
+ "status": "blocked",
"category": "d601-observability",
"reportPath": "reports/d601-k3s-readonly-observability.json",
- "summary": "D601 k3s observability is blocked; cluster state has not been read."
- },
+ "summary": "D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false."
+ }
+ ],
+ "BLOCKED": [
{
"milestone": "M3",
"issue": "pikasTech/HWLAB#38",
@@ -991,7 +1010,7 @@
"status": "not_run",
"category": "hardware-loop-live",
"reportPath": "reports/dev-gate/dev-m3-hardware-loop.json",
- "summary": "No live DEV operation has run yet. Blocker: patch-panel active DO1 -> DI1 connection missing."
+ "summary": "No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.."
},
{
"milestone": "M4",
@@ -1033,6 +1052,25 @@
"summary": "reports/dev-gate/dev-artifacts.json does not prove all required HWLAB service artifacts for origin/main 8e89409; current status is published with 13/13 required services published and source states resolved: 13 source-present, 0 intentionally-disabled.",
"nextTask": "Complete DEV artifact publishing for every enabled required HWLAB service at the artifact source commit, or prove the target commit has no artifact build input changes; keep disabled services marked not_published with reasons."
},
+ {
+ "priority": "P1",
+ "order": 4,
+ "type": "observability_blocker",
+ "scope": "runner-kubeconfig-readonly-gap",
+ "sourceIssue": "pikasTech/HWLAB#46",
+ "source": "reports/d601-k3s-readonly-observability.json",
+ "summary": "The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately.",
+ "nextTask": "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report."
+ },
+ {
+ "priority": "P1",
+ "order": 4,
+ "type": "observability_blocker",
+ "scope": "m3-service-discovery",
+ "sourceIssue": "pikasTech/HWLAB#38",
+ "source": "reports/dev-gate/dev-m3-hardware-loop.json",
+ "summary": "DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline."
+ },
{
"priority": "P1",
"order": 5,
@@ -1080,15 +1118,6 @@
"source": "reports/dev-gate/dev-mvp-gate-report.json",
"summary": "cloud-api /health/live reports DB degraded; db.ready=false, db.connected=false, and live DB evidence is not established."
},
- {
- "priority": "P0",
- "order": 6,
- "type": "runtime_blocker",
- "scope": "m3-hardware-loop-runtime",
- "sourceIssue": "pikasTech/HWLAB#38",
- "source": "reports/dev-gate/dev-m3-hardware-loop.json",
- "summary": "patch-panel active DO1 -> DI1 connection missing"
- },
{
"priority": "P0",
"order": 6,
@@ -1137,6 +1166,47 @@
],
"rationale": "The gate cannot promote deploy or runtime observations without immutable image provenance and digests."
},
+ {
+ "id": "d601-k3s-readonly-observability:runner-kubeconfig-readonly-gap",
+ "priority": "P1",
+ "type": "observability_blocker",
+ "scope": "runner-kubeconfig-readonly-gap",
+ "status": "open",
+ "source": "reports/d601-k3s-readonly-observability.json",
+ "sourceIssue": "pikasTech/HWLAB#46",
+ "summary": "The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately.",
+ "nextTask": "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report.",
+ "unblockOrder": 4,
+ "unblocks": [
+ "pikasTech/HWLAB#34",
+ "pikasTech/HWLAB#33",
+ "pikasTech/HWLAB#36",
+ "pikasTech/HWLAB#38",
+ "pikasTech/HWLAB#46",
+ "pikasTech/HWLAB#64"
+ ],
+ "rationale": "Runner read-only observability must be repaired without treating the runner gap as proof that D601 k3s or public DEV endpoints are unavailable."
+ },
+ {
+ "id": "dev-m3-hardware-loop:m3-service-discovery",
+ "priority": "P1",
+ "type": "observability_blocker",
+ "scope": "m3-service-discovery",
+ "status": "open",
+ "source": "reports/dev-gate/dev-m3-hardware-loop.json",
+ "sourceIssue": "pikasTech/HWLAB#38",
+ "summary": "DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.",
+ "unblockOrder": 4,
+ "unblocks": [
+ "pikasTech/HWLAB#34",
+ "pikasTech/HWLAB#33",
+ "pikasTech/HWLAB#36",
+ "pikasTech/HWLAB#38",
+ "pikasTech/HWLAB#46",
+ "pikasTech/HWLAB#64"
+ ],
+ "rationale": "Runner read-only observability must be repaired without treating the runner gap as proof that D601 k3s or public DEV endpoints are unavailable."
+ },
{
"id": "dev-edge-health:cloud-api-db",
"priority": "P1",
@@ -1222,23 +1292,6 @@
],
"rationale": "M4 and M5 cannot claim live agent or MVP evidence until cloud-api /health/live proves DB readiness with redacted live evidence."
},
- {
- "id": "dev-m3-hardware-loop:m3-hardware-loop-runtime",
- "priority": "P0",
- "type": "runtime_blocker",
- "scope": "m3-hardware-loop-runtime",
- "status": "open",
- "source": "reports/dev-gate/dev-m3-hardware-loop.json",
- "sourceIssue": "pikasTech/HWLAB#38",
- "summary": "patch-panel active DO1 -> DI1 connection missing",
- "unblockOrder": 6,
- "unblocks": [
- "pikasTech/HWLAB#38",
- "pikasTech/HWLAB#39",
- "pikasTech/HWLAB#64"
- ],
- "rationale": "M3 remains blocked until the real DEV hardware trusted loop proves DO1 -> patch-panel -> DI1 with operation, trace, audit, and evidence identifiers."
- },
{
"id": "dev-mvp-gate-report:m3-hardware-loop-runtime",
"priority": "P0",
@@ -1278,6 +1331,26 @@
},
{
"order": 2,
+ "blockerOrder": 4,
+ "priority": "P1",
+ "scopes": [
+ "runner-kubeconfig-readonly-gap",
+ "m3-service-discovery"
+ ],
+ "sourceIssues": [
+ "pikasTech/HWLAB#33",
+ "pikasTech/HWLAB#34",
+ "pikasTech/HWLAB#36",
+ "pikasTech/HWLAB#38",
+ "pikasTech/HWLAB#46",
+ "pikasTech/HWLAB#64"
+ ],
+ "rationale": "Runner read-only observability must be repaired without treating the runner gap as proof that D601 k3s or public DEV endpoints are unavailable.",
+ "action": "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report.",
+ "evidenceRequired": "Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write."
+ },
+ {
+ "order": 3,
"blockerOrder": 5,
"priority": "P1",
"scopes": [
@@ -1297,7 +1370,7 @@
"evidenceRequired": "Cloud API health/live output showing DB env ready and redacted secret references, without secret material."
},
{
- "order": 3,
+ "order": 4,
"blockerOrder": 6,
"priority": "P0",
"scopes": [
diff --git a/reports/dev-gate/dev-m5-gate-aggregator-v2.md b/reports/dev-gate/dev-m5-gate-aggregator-v2.md
index b24f26f1..bcebd7c3 100644
--- a/reports/dev-gate/dev-m5-gate-aggregator-v2.md
+++ b/reports/dev-gate/dev-m5-gate-aggregator-v2.md
@@ -1,7 +1,7 @@
# HWLAB M5 DEV Gate Aggregator v2
Status: blocked
-Generated from: `f933a81c83a5`
+Generated from: `bf47a95a97a9`
Scope: DEV only, report-only
Active frontend: `http://74.48.78.17:16666/`
Active API/live: `http://74.48.78.17:16667/health/live`
@@ -22,7 +22,8 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be
| Frontend DEV revision | pass | DEV-LIVE | http://74.48.78.17:16666/ serves the accepted Cloud Workbench frontend revision 1e8805664970839b72be40c34636b08f6d18b131; this is browser/frontend evidence only. | Keep frontend revision proof separate from DB live readiness, M3 hardware-loop evidence, M4 agent-loop evidence, and M5 acceptance. |
| EDGE/ROUTE live | pass | DEV-LIVE | http://74.48.78.17:16666/, http://74.48.78.17:16667/health, and http://74.48.78.17:16667/health/live returned accepted HWLAB DEV responses in the active M2 read-only smoke. | Keep this separated from DB readiness, M3/M4 loop evidence, and M5 acceptance. |
| DB live/degraded | blocked | BLOCKED | cloud-api DB status=degraded; configReady=true; ready=false; connected=false; liveDbEvidence=false. | Provide live DB connection evidence through redacted health output; route reachability alone is insufficient. |
-| M3 hardware trusted loop | blocked | BLOCKED | No live DEV operation has run yet. Blocker: patch-panel active DO1 -> DI1 connection missing. | Only a real DEV res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 observation with operation/trace/audit/evidence can clear M3. |
+| D601 runner observability | blocked | DEV-LIVE | D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false. | Treat #46 runner kubeconfig/readonly gaps separately from D601 service health; rerun read-only observability after the mount or permission path is repaired. |
+| M3 hardware trusted loop | blocked | BLOCKED | No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.. | Only a real DEV res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 observation with operation/trace/audit/evidence can clear M3. |
| M4 agent loop | blocked | BLOCKED | Blocked at DB live readiness before scheduling a DEV agent task. | Do not schedule or claim the agent loop as live until DB live and required runtime/evidence preconditions pass. |
| artifact/desired-state source | blocked | BLOCKED | artifact targetCovered=false; artifactSource=7e29522b65c8; target=8e89409dda5d; desiredApplyMode=dry-run; mutationAttempted=false. | Refresh artifact/source coverage for current origin/main and keep desired-state apply separate from read-only route proof. |
@@ -55,7 +56,7 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be
| m0-source-contract | pass | SOURCE | M0 contract checks are source-level evidence only. |
| m1-local-smoke | pass | LOCAL | M1 local smoke is not a live DEV substitute. |
| artifact-publish-digests | blocked | BLOCKED | artifactState=published, ciPublished=true, registryVerified=true, sha256=13, not_published=0, targetCovered=false |
-| d601-k3s-observability | pass | DEV-LIVE | D601 hwlab-dev cluster was readable. |
+| d601-k3s-observability | blocked | DEV-LIVE | D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false. |
| dev-edge-frp-16667 | pass | DEV-LIVE | Committed edge report proves read-only public HTTP on :16667 /health and /health/live; this is route evidence, not DB/M3/M4/M5 acceptance. |
| cloud-api-db-ready | blocked | BLOCKED | cloud-api DB status=degraded; ready=false; connected=false; liveDbEvidence=false. |
| m3-hardware-trusted-loop | blocked | BLOCKED | M3 trusted loop is blocked until res_boxsimu_1:DO1 -> patch-panel -> res_boxsimu_2:DI1 is proven with operation/trace/audit/evidence. |
@@ -66,7 +67,7 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be
| Milestone | Status | Current level | Blocker class | Dependency | Required next proof |
| --- | --- | --- | --- | --- | --- |
-| M3 | blocked | BLOCKED | hardware-loop-runtime | Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1. | Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs. |
+| M3 | blocked | BLOCKED | runner-readonly-observability-gap | Runner read-only service discovery must be repaired before the real DEV trusted loop can be observed; this does not mean D601/k3s is globally unavailable. | Repair the #46 runner readonly kubeconfig/service-discovery gap, then run the bounded DEV M3 live smoke only when direct simulator and patch-panel targets are discoverable. |
| M4 | blocked | BLOCKED | db-live-readiness | Cloud API /health/live must report DB ready=true, connected=true, and liveDbEvidence=true before live agent scheduling/evidence closure. | Repair DB live readiness and rerun the M4 live preflight without scheduling a DEV agent task before preconditions pass. |
| M5 | blocked | BLOCKED | composite-db-m3-m4-live | M5 needs DB live readiness, M3 trusted-loop DEV evidence, M4 live preflight/evidence closure, and current source/artifact coverage. | After DB/M3/M4 blockers are cleared, run only the bounded DEV MVP live gate command with explicit DEV/non-PROD confirmations. |
@@ -76,21 +77,24 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be
| --- | ---: | --- | --- | --- |
| P1 | 3 | contract_blocker | artifact-source-commit | source commit origin/main 8e89409 is not covered by artifact source 7e29522; target changes since the artifact source touch artifact build inputs. |
| P1 | 3 | runtime_blocker | dev-artifact-publish | reports/dev-gate/dev-artifacts.json does not prove all required HWLAB service artifacts for origin/main 8e89409; current status is published with 13/13 required services published and source states resolved: 13 source-present, 0 intentionally-disabled. |
+| P1 | 4 | observability_blocker | runner-kubeconfig-readonly-gap | The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately. |
+| P1 | 4 | observability_blocker | m3-service-discovery | DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline. |
| P1 | 5 | runtime_blocker | cloud-api-db | cloud-api DB env is injected, but runtime health has not attempted a live DB connection |
| P1 | 5 | network_blocker | cloud-api-db | cloud-api DB env is injected, but runtime health has not attempted a live DB connection |
| P1 | 5 | runtime_blocker | cloud-api-db-health-gate | cloud-api DB runtime env is not ready; missing HWLAB_CLOUD_DB_URL, HWLAB_CLOUD_DB_SSL_MODE. |
| P1 | 5 | runtime_blocker | db-live | cloud-api /health/live reports DB degraded; connected=false; ready=false. |
| P1 | 5 | runtime_blocker | db-live | cloud-api /health/live reports DB degraded; db.ready=false, db.connected=false, and live DB evidence is not established. |
-| P0 | 6 | runtime_blocker | m3-hardware-loop-runtime | patch-panel active DO1 -> DI1 connection missing |
| P0 | 6 | runtime_blocker | m3-hardware-loop-runtime | Live M3 smoke reached DEV simulators, but patch-panel active DO1 -> DI1 connection is missing. |
## Next Unblock Order
1. Refresh without fake digests using `node scripts/refresh-artifact-catalog.mjs --target-ref origin/main --blocked`, or after a successful publish run `node scripts/refresh-artifact-catalog.mjs --target-ref origin/main --publish-report reports/dev-gate/dev-artifacts.json`.
Evidence required: Artifact publish report with ciPublished=true, registryVerified=true, and sha256 digest for each frozen service ID.
-2. Deploy cloud-api DB runtime readiness probe and/or repair DEV DB connectivity, then rerun the read-only health smoke without reading or printing the DB secret value.
+2. Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report.
+ Evidence required: Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write.
+3. Deploy cloud-api DB runtime readiness probe and/or repair DEV DB connectivity, then rerun the read-only health smoke without reading or printing the DB secret value.
Evidence required: Cloud API health/live output showing DB env ready and redacted secret references, without secret material.
-3. Prove the real DEV M3 trusted loop res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, trace, audit, and evidence identifiers.
+4. Prove the real DEV M3 trusted loop res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, trace, audit, and evidence identifiers.
Evidence required: Operation, trace, audit, and evidence IDs from the real DEV DO1 -> patch-panel -> DI1 trusted loop.
## Validation
diff --git a/scripts/dev-m3-hardware-loop-smoke.mjs b/scripts/dev-m3-hardware-loop-smoke.mjs
index 71ede2e3..cb38f7dc 100644
--- a/scripts/dev-m3-hardware-loop-smoke.mjs
+++ b/scripts/dev-m3-hardware-loop-smoke.mjs
@@ -2,6 +2,7 @@
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { randomUUID } from "node:crypto";
+import { accessSync, constants as fsConstants } from "node:fs";
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
import { request as httpRequest } from "node:http";
import { request as httpsRequest } from "node:https";
@@ -9,7 +10,7 @@ import path from "node:path";
import { fileURLToPath } from "node:url";
import { activeReportLifecycle } from "../internal/dev-report-lifecycle.mjs";
-import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs";
+import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs";
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const defaultReportPath = "reports/dev-gate/dev-m3-hardware-loop.json";
@@ -19,6 +20,7 @@ const deployWorkloadsPath = "deploy/k8s/base/workloads.yaml";
const requiredManifestCardinalityCommand = "node scripts/validate-dev-m3-cardinality.mjs";
const requiredM3BoxIds = Object.freeze(["boxsimu_1", "boxsimu_2"]);
const requiredM3BoxResources = Object.freeze(["res_boxsimu_1", "res_boxsimu_2"]);
+const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
const serviceTargets = Object.freeze([
{
@@ -156,12 +158,57 @@ function commandResult(command, args, { timeoutMs = 5000, maxChars = 3000 } = {}
}
}
+function runnerKubeconfigCommandText(args) {
+ return [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args].join(" ");
+}
+
function commandExists(command) {
const result = commandResult("bash", ["-lc", `command -v ${command}`], { timeoutMs: 2000 });
return result.exitCode === 0 && result.stdout.trim().length > 0;
}
-function collectLocalRuntimeObservations() {
+function fileReadable(filePath) {
+ try {
+ accessSync(filePath, fsConstants.R_OK);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+function collectD601RunnerObservability(ingress) {
+ const args = ["-n", namespace, "get", "pods", "-o", "name"];
+ const probe = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], {
+ timeoutMs: 5000,
+ maxChars: 2000
+ });
+ const stderr = oneLine(probe.stderr || "") || "empty";
+ const stdoutLines = String(probe.stdout ?? "")
+ .split("\n")
+ .map((line) => line.trim())
+ .filter(Boolean);
+ const d601PublicEndpointsReachable = ingress.probes.some((item) => item.ok && isHwlabDevIdentity(item));
+ const runnerKubeconfigReadable = fileReadable(runnerK3sKubeconfigPath);
+
+ return {
+ runnerKubeconfigReadable,
+ runnerKubeconfigProbeOk: probe.exitCode === 0,
+ runnerKubeconfigProbeExitCode: probe.exitCode,
+ runnerKubeconfigProbeStderr: stderr,
+ d601PublicEndpointsReachable,
+ d601K3sUnavailable: false,
+ classification: "runner_permission_mount_gap",
+ sourceIssue: "pikasTech/HWLAB#46",
+ command: runnerKubeconfigCommandText(args),
+ stdoutSummary: probe.exitCode === 0 ? `${stdoutLines.length} pod name(s) observed` : undefined,
+ summary: "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate.",
+ inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately.",
+ secretValuesRead: false,
+ secretResourcesRead: false
+ };
+}
+
+function collectLocalRuntimeObservations(d601Observability) {
const observations = [];
if (commandExists("docker")) {
@@ -206,6 +253,13 @@ function collectLocalRuntimeObservations() {
});
}
+ observations.push({
+ id: "runner-k3s-kubeconfig-readonly",
+ status: d601Observability.runnerKubeconfigProbeOk ? "observed" : "blocked",
+ command: d601Observability.command,
+ summary: `${d601Observability.summary} runnerKubeconfigReadable=${d601Observability.runnerKubeconfigReadable}; exitCode=${d601Observability.runnerKubeconfigProbeExitCode}; stderr=${d601Observability.runnerKubeconfigProbeStderr}.`
+ });
+
return observations;
}
@@ -587,6 +641,7 @@ function baseReport({ commitId, observedAt }) {
],
runtimeTarget: {
endpoint: DEV_ENDPOINT,
+ frontendEndpoint: DEV_FRONTEND_ENDPOINT,
namespace,
environment: ENVIRONMENT_DEV,
requiredBoxSimulators: 2,
@@ -623,6 +678,17 @@ function baseReport({ commitId, observedAt }) {
],
readOnlySupplementalEvidence: [],
localRuntimeObservations: [],
+ d601Observability: {
+ runnerKubeconfigReadable: false,
+ runnerKubeconfigProbeOk: false,
+ runnerKubeconfigProbeExitCode: null,
+ runnerKubeconfigProbeStderr: "not_observed",
+ d601PublicEndpointsReachable: false,
+ d601K3sUnavailable: false,
+ classification: "not_observed",
+ sourceIssue: "pikasTech/HWLAB#46",
+ inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately."
+ },
liveOperation: {
status: "not_run",
operationId: "not_observed",
@@ -683,7 +749,8 @@ async function main() {
report.readOnlySupplementalEvidence = await collectReadOnlySupplementalEvidence();
const ingress = await observeDevIngress();
- report.localRuntimeObservations = collectLocalRuntimeObservations();
+ report.d601Observability = collectD601RunnerObservability(ingress);
+ report.localRuntimeObservations = collectLocalRuntimeObservations(report.d601Observability);
report.liveChecks.push({
id: "dev-ingress-health",
status: ingress.accepted ? "pass" : "blocked",
@@ -727,21 +794,22 @@ async function main() {
if (missing.length > 0) {
addNotRunM3Checks(report, "Stopped before M3 direct checks because service target URLs were not provided.");
report.blockers.push({
- type: "runtime_blocker",
+ type: "observability_blocker",
scope: "m3-service-discovery",
status: "open",
- classification: "hardware loop",
- summary: `DEV ingress is reachable, but M3 smoke lacks direct DEV service URLs: ${missing.join(", ")}.`
+ classification: "runner permission/mount gap",
+ sourceIssue: "pikasTech/HWLAB#46",
+ summary: `DEV ingress is reachable on frozen :16667, but this runner cannot use ${runnerK3sKubeconfigPath} as readable kubeconfig to discover direct M3 service URLs (${missing.join(", ")}); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.`
});
report.summary = {
status: "blocked",
- classification: "hardware loop",
+ classification: "runner permission/mount gap",
observedAt,
result: "DEV ingress was reachable, but live M3 simulator and patch-panel targets were not discoverable."
};
await writeReport(report, reportPath);
console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`);
- console.log("[dev-m3-smoke] blocker=runtime_blocker scope=m3-service-discovery");
+ console.log("[dev-m3-smoke] blocker=observability_blocker scope=m3-service-discovery");
return;
}
diff --git a/scripts/src/d601-k3s-readonly-observability.mjs b/scripts/src/d601-k3s-readonly-observability.mjs
index 13a2af35..02c0b3d8 100644
--- a/scripts/src/d601-k3s-readonly-observability.mjs
+++ b/scripts/src/d601-k3s-readonly-observability.mjs
@@ -7,6 +7,8 @@ import path from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
+import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT } from "../../internal/protocol/index.mjs";
+
const execFileAsync = promisify(execFile);
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const namespace = "hwlab-dev";
@@ -26,6 +28,12 @@ const kubeconfigCandidatePaths = [
"/etc/rancher/k3s/k3s.yaml",
"/var/lib/rancher/k3s/server/cred/admin.kubeconfig"
];
+const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
+const publicEndpointProbeTargets = [
+ { id: "frontend-16666", url: `${DEV_FRONTEND_ENDPOINT}/` },
+ { id: "api-health-16667", url: `${DEV_ENDPOINT}/health` },
+ { id: "api-live-16667", url: `${DEV_ENDPOINT}/health/live` }
+];
const sshEnvVars = [
"HWLAB_D601_SSH_HOST",
"HWLAB_D601_SSH_TARGET",
@@ -106,6 +114,10 @@ function commandText(command, args) {
return [command, ...args].join(" ");
}
+function runnerKubeconfigCommandText(args) {
+ return [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args].join(" ");
+}
+
async function run(command, args = [], options = {}) {
try {
const result = await execFileAsync(command, args, {
@@ -176,6 +188,89 @@ async function fileProbe(filePath) {
}
}
+async function collectRunnerKubeconfigProbe(timeoutMs) {
+ const metadata = await fileProbe(runnerK3sKubeconfigPath);
+ const args = ["-n", namespace, "get", "pods", "-o", "name"];
+ const probe = await run("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], {
+ timeoutMs
+ });
+ const stdoutLines = probe.stdout
+ ? probe.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
+ : [];
+
+ return {
+ path: runnerK3sKubeconfigPath,
+ mode: "read-only",
+ resource: "pods",
+ command: runnerKubeconfigCommandText(args),
+ runnerKubeconfigReadable: metadata.readable === true,
+ runnerKubeconfigProbeOk: probe.ok,
+ runnerKubeconfigProbeExitCode: probe.exitCode,
+ runnerKubeconfigProbeStderr: oneLine(probe.stderr || probe.error || "") || "empty",
+ stdoutSummary: probe.ok ? `${stdoutLines.length} pod name(s) observed` : undefined,
+ classification: metadata.readable
+ ? probe.ok
+ ? "readable"
+ : "readonly_observability_gap"
+ : "runner_permission_mount_gap",
+ sourceIssue: issue,
+ secretValuesRead: false,
+ secretResourcesRead: false,
+ note: "This probe forces KUBECONFIG=/etc/rancher/k3s/k3s.yaml for a read-only kubectl get pods command and records only exit code plus redacted stderr/stdout summary. A successful kubectl exit code does not make the kubeconfig file readable; kubectl may use an alternate in-cluster fallback."
+ };
+}
+
+async function probePublicEndpoint(target, timeoutMs) {
+ const controller = new AbortController();
+ const timeout = setTimeout(() => controller.abort(), timeoutMs);
+ try {
+ const response = await fetch(target.url, {
+ method: "GET",
+ signal: controller.signal,
+ redirect: "manual"
+ });
+ const body = await response.text();
+ let json = null;
+ try {
+ json = body ? JSON.parse(body) : null;
+ } catch {}
+ return {
+ id: target.id,
+ url: target.url,
+ reached: true,
+ ok: response.status >= 200 && response.status < 400,
+ status: response.status,
+ serviceId: json?.serviceId,
+ environment: json?.environment,
+ bodySummary: json
+ ? `json serviceId=${json.serviceId ?? "unknown"} status=${json.status ?? "unknown"}`
+ : oneLine(body.slice(0, 160))
+ };
+ } catch (error) {
+ return {
+ id: target.id,
+ url: target.url,
+ reached: false,
+ ok: false,
+ error: error instanceof Error ? error.message : String(error)
+ };
+ } finally {
+ clearTimeout(timeout);
+ }
+}
+
+async function collectPublicEndpoints(timeoutMs) {
+ const probes = await Promise.all(
+ publicEndpointProbeTargets.map((target) => probePublicEndpoint(target, timeoutMs))
+ );
+ return {
+ activeFrontendEndpoint: DEV_FRONTEND_ENDPOINT,
+ activeApiEndpoint: DEV_ENDPOINT,
+ d601PublicEndpointsReachable: probes.every((probe) => probe.ok === true),
+ probes
+ };
+}
+
async function collectBinary(name, versionArgs) {
const executablePath = await findExecutable(name);
const result = {
@@ -391,6 +486,51 @@ async function collectClusterObservability(binaries, timeoutMs) {
};
}
+function probeLooksLikeClusterServerUnavailable(probe) {
+ const text = oneLine(`${probe?.stderr ?? ""} ${probe?.error ?? ""}`).toLowerCase();
+ return [
+ "connection refused",
+ "no route to host",
+ "i/o timeout",
+ "context deadline exceeded",
+ "unable to connect to the server",
+ "the server is currently unable to handle the request"
+ ].some((needle) => text.includes(needle));
+}
+
+function inferD601K3sUnavailable({ cluster, publicEndpoints }) {
+ if (cluster.readable === true || publicEndpoints.d601PublicEndpointsReachable === true) {
+ return false;
+ }
+
+ return (cluster.executors ?? []).some((executor) =>
+ Object.values(executor.probes ?? {}).some((probe) => {
+ if (probe && typeof probe === "object" && !Array.isArray(probe) && Object.hasOwn(probe, "ok")) {
+ return probeLooksLikeClusterServerUnavailable(probe);
+ }
+ return Object.values(probe ?? {}).some((nested) => probeLooksLikeClusterServerUnavailable(nested));
+ })
+ );
+}
+
+function d601StatusInference({ runnerKubeconfig, publicEndpoints, d601K3sUnavailable }) {
+ return {
+ runnerKubeconfigReadable: runnerKubeconfig.runnerKubeconfigReadable,
+ runnerKubeconfigProbeOk: runnerKubeconfig.runnerKubeconfigProbeOk,
+ runnerKubeconfigProbeExitCode: runnerKubeconfig.runnerKubeconfigProbeExitCode,
+ runnerKubeconfigProbeStderr: runnerKubeconfig.runnerKubeconfigProbeStderr,
+ d601PublicEndpointsReachable: publicEndpoints.d601PublicEndpointsReachable,
+ d601K3sUnavailable,
+ conclusion: d601K3sUnavailable
+ ? "Direct read-only probes indicate D601 k3s may be unavailable."
+ : "Runner kubeconfig readability is an observability signal only; it is not used to infer D601 k3s or public DEV endpoint availability.",
+ blockerClassification: runnerKubeconfig.runnerKubeconfigReadable
+ ? "none"
+ : "runner_permission_mount_gap",
+ sourceIssue: issue
+ };
+}
+
function addBlocker(blockers, blocker) {
assert.ok(blockerTypes.has(blocker.type), `unknown blocker type ${blocker.type}`);
const key = `${blocker.type}:${blocker.scope}`;
@@ -399,7 +539,7 @@ function addBlocker(blockers, blocker) {
}
}
-function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {
+function buildBlockers({ binaries, kubeconfig, runnerKubeconfig, sshBridge, cluster }) {
const blockers = [];
const hasDirectClient = binaries.kubectl.available || binaries.k3s.available;
if (!hasDirectClient) {
@@ -412,12 +552,24 @@ function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {
}
if (!kubeconfig.some((item) => item.readable)) {
addBlocker(blockers, {
- type: "environment_blocker",
- scope: "d601-kubeconfig-path",
- summary: "No readable KUBECONFIG/default k3s kubeconfig path was found by metadata checks.",
+ type: "observability_blocker",
+ scope: "runner-kubeconfig-readonly-gap",
+ classification: "runner_permission_mount_gap",
+ sourceIssue: issue,
+ summary: "No readable KUBECONFIG/default k3s kubeconfig path was found by metadata checks; this is a runner permission/mount gap and does not prove D601 k3s is unavailable.",
nextTask: "Mount a read-only kubeconfig for hwlab-dev, or document the approved k3s local kubeconfig path without exposing token material."
});
}
+ if (!runnerKubeconfig.runnerKubeconfigReadable) {
+ addBlocker(blockers, {
+ type: "observability_blocker",
+ scope: "runner-kubeconfig-readonly-gap",
+ classification: "runner_permission_mount_gap",
+ sourceIssue: issue,
+ summary: "The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately.",
+ nextTask: "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report."
+ });
+ }
if (!sshBridge.configured && !cluster.readable) {
addBlocker(blockers, {
type: "environment_blocker",
@@ -431,7 +583,9 @@ function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {
addBlocker(blockers, {
type: "observability_blocker",
scope: "hwlab-dev-readonly-rbac",
- summary: `Read-only cluster probes did not return pods, services, and ConfigMaps for ${namespace}; attempted executors: ${attempted}.`,
+ classification: "readonly_observability_gap",
+ sourceIssue: issue,
+ summary: `Read-only runner probes did not return pods, services, and ConfigMaps for ${namespace}; attempted executors: ${attempted}. This is an observability gap unless independent evidence proves D601 k3s unavailable.`,
nextTask: "Grant or mount read-only access that can get namespace, pods, services, and ConfigMaps in hwlab-dev, then rerun this preflight."
});
}
@@ -440,7 +594,7 @@ function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {
function runnerImageFollowUp(blockers) {
const scopes = new Set(blockers.map((blocker) => blocker.scope));
- const needed = scopes.has("d601-k3s-client-binary") || scopes.has("d601-kubeconfig-path");
+ const needed = scopes.has("d601-k3s-client-binary") || scopes.has("runner-kubeconfig-readonly-gap");
return {
needed,
suggestedUniDeskIssue: needed
@@ -457,13 +611,21 @@ async function writeReport(report, reportPath) {
async function buildReport(args) {
const binaries = await collectBinaries();
- const [kubeconfig, sshBridge, cluster] = await Promise.all([
+ const [kubeconfig, runnerKubeconfig, sshBridge, cluster, publicEndpoints] = await Promise.all([
collectKubeconfigPaths(),
+ collectRunnerKubeconfigProbe(args.timeoutMs),
collectSshBridge(binaries),
- collectClusterObservability(binaries, args.timeoutMs)
+ collectClusterObservability(binaries, args.timeoutMs),
+ collectPublicEndpoints(args.timeoutMs)
]);
- const blockers = buildBlockers({ binaries, kubeconfig, sshBridge, cluster });
+ const d601K3sUnavailable = inferD601K3sUnavailable({ cluster, publicEndpoints });
+ const blockers = buildBlockers({ binaries, kubeconfig, runnerKubeconfig, sshBridge, cluster });
const conclusion = cluster.readable && blockers.length === 0 ? "ready" : "blocked";
+ const statusInference = d601StatusInference({
+ runnerKubeconfig,
+ publicEndpoints,
+ d601K3sUnavailable
+ });
return {
$schema: "https://hwlab.pikastech.local/schemas/d601-k3s-readonly-observability-report.schema.json",
$id: "https://hwlab.pikastech.local/reports/d601-k3s-readonly-observability.json",
@@ -497,15 +659,24 @@ async function buildReport(args) {
"node scripts/d601-k3s-readonly-observability.mjs"
],
conclusion,
+ runnerKubeconfigReadable: statusInference.runnerKubeconfigReadable,
+ runnerKubeconfigProbeOk: statusInference.runnerKubeconfigProbeOk,
+ runnerKubeconfigProbeExitCode: statusInference.runnerKubeconfigProbeExitCode,
+ runnerKubeconfigProbeStderr: statusInference.runnerKubeconfigProbeStderr,
+ d601PublicEndpointsReachable: statusInference.d601PublicEndpointsReachable,
+ d601K3sUnavailable: statusInference.d601K3sUnavailable,
+ statusInference,
environment: {
binaries,
kubeconfig,
+ runnerKubeconfig,
sshBridge
},
+ publicEndpoints,
cluster,
blockers,
runnerImageFollowUp: runnerImageFollowUp(blockers),
- notes: "This report uses command availability checks, file metadata checks, kubectl/k3s kubectl read-only get probes, and auth can-i checks only. It does not print kubeconfig contents, token material, Secret resources, ConfigMap values, SSH config contents, or private keys."
+ notes: "This report uses command availability checks, file metadata checks, public DEV endpoint GET probes, kubectl/k3s kubectl read-only get probes, and auth can-i checks only. It does not print kubeconfig contents, token material, Secret resources, ConfigMap values, SSH config contents, or private keys. A failed runner /etc/rancher/k3s/k3s.yaml probe is classified as a runner permission/mount or read-only observability gap and is not evidence that D601 k3s is globally unavailable."
};
}
@@ -517,6 +688,11 @@ function printSummary(args, report) {
namespace: report.namespace,
report: args.writeReport ? args.reportPath : null,
readable: report.cluster.readable,
+ runnerKubeconfigReadable: report.runnerKubeconfigReadable,
+ runnerKubeconfigProbeOk: report.runnerKubeconfigProbeOk,
+ runnerKubeconfigProbeExitCode: report.runnerKubeconfigProbeExitCode,
+ d601PublicEndpointsReachable: report.d601PublicEndpointsReachable,
+ d601K3sUnavailable: report.d601K3sUnavailable,
attemptedExecutors: report.cluster.attemptedExecutors,
blockers: report.blockers.map((blocker) => ({
type: blocker.type,
diff --git a/scripts/src/dev-evidence-blocker-aggregator.mjs b/scripts/src/dev-evidence-blocker-aggregator.mjs
index a7bf116b..9c4e8c78 100644
--- a/scripts/src/dev-evidence-blocker-aggregator.mjs
+++ b/scripts/src/dev-evidence-blocker-aggregator.mjs
@@ -263,13 +263,19 @@ function applyPriority(blocker) {
};
}
- if (blocker.scope === "kubectl" || blocker.scope === "d601-k3s" || blocker.scope.startsWith("d601-")) {
+ if (
+ blocker.scope === "kubectl" ||
+ blocker.scope === "d601-k3s" ||
+ blocker.scope === "runner-kubeconfig-readonly-gap" ||
+ blocker.scope === "m3-service-discovery" ||
+ blocker.scope.startsWith("d601-")
+ ) {
return {
...blocker,
priority: "P1",
unblockOrder: 4,
- unblocks: [issue(34), issue(33), issue(36), issue(38)],
- rationale: "D601 hwlab-dev cluster state must be observable read-only before any live apply or M3/M4/M5 acceptance."
+ unblocks: [issue(34), issue(33), issue(36), issue(38), issue(46), issue(64)],
+ rationale: "Runner read-only observability must be repaired without treating the runner gap as proof that D601 k3s or public DEV endpoints are unavailable."
};
}
@@ -279,7 +285,7 @@ function applyPriority(blocker) {
priority: "P1",
unblockOrder: 4,
unblocks: [issue(34), issue(33), issue(36), issue(38), issue(39)],
- rationale: "D601 has a client path, but read-only hwlab-dev pods/services/configmaps must be observable before live acceptance evidence can be trusted."
+ rationale: "D601 has a client path, but read-only hwlab-dev pods/services/configmaps must be observable before live acceptance evidence can be trusted; this is not evidence that D601 is globally offline."
};
}
@@ -462,17 +468,13 @@ function collectM2Evidence(reports) {
}),
reportEvidence({
milestone: "M2",
- level: "BLOCKED",
+ level: d601.cluster?.readable === true || d601.d601PublicEndpointsReachable === true ? "DEV-LIVE" : "BLOCKED",
report: d601,
status: d601.conclusion,
category: "d601-observability",
- evidence: [
- `kubectl=${d601.environment?.binaries?.kubectl?.available === true ? "available" : "missing"}`,
- `k3s=${d601.environment?.binaries?.k3s?.available === true ? "available" : "missing"}`,
- `clusterReadable=${d601.cluster?.readable === true ? "yes" : "no"}`
- ],
+ evidence: d601ObservationSummary(d601),
commands: d601.validationCommands,
- summary: "D601 k3s observability is blocked; cluster state has not been read."
+ summary: d601ObservabilitySummary(d601)
})
];
}
@@ -522,7 +524,10 @@ function collectM3Evidence(reports) {
function m3TrustedLoopSummary(m3Report) {
const operationSummary = m3Report.liveOperation?.summary ?? "No live M3 hardware operation was observed.";
- const blockerSummary = m3Report.blockers?.find((blocker) => blocker.scope === "m3-hardware-loop-runtime")?.summary;
+ const blockerSummary = m3Report.blockers?.find((blocker) =>
+ blocker.scope === "m3-hardware-loop-runtime" ||
+ blocker.scope === "m3-service-discovery"
+ )?.summary;
if (statusIsPass(m3Report.liveOperation?.status) || !blockerSummary) {
return operationSummary;
}
@@ -658,6 +663,32 @@ function publicEndpointEvidence(m2Report) {
});
}
+function d601ObservationSummary(d601) {
+ return [
+ `runnerKubeconfigReadable=${d601.runnerKubeconfigReadable === true}`,
+ `runnerKubeconfigProbeExitCode=${d601.runnerKubeconfigProbeExitCode ?? "unknown"}`,
+ `runnerKubeconfigProbeStderr=${d601.runnerKubeconfigProbeStderr || "empty"}`,
+ `d601PublicEndpointsReachable=${d601.d601PublicEndpointsReachable === true}`,
+ `d601K3sUnavailable=${d601.d601K3sUnavailable === true}`
+ ];
+}
+
+function d601ObservabilitySummary(d601) {
+ if (d601.d601PublicEndpointsReachable === true && d601.runnerKubeconfigReadable === false) {
+ const clusterNote = d601.cluster?.readable === true
+ ? " Alternate read-only cluster probes are readable."
+ : "";
+ return `D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline.${clusterNote} ${d601ObservationSummary(d601).join(", ")}.`;
+ }
+ if (d601.cluster?.readable === true) {
+ return `D601 hwlab-dev cluster is readable from this runner; ${d601ObservationSummary(d601).join(", ")}.`;
+ }
+ if (d601.d601K3sUnavailable === true) {
+ return `Direct read-only probes indicate D601 k3s may be unavailable; ${d601ObservationSummary(d601).join(", ")}.`;
+ }
+ return `D601 read-only observability is blocked without proving D601 unavailable; ${d601ObservationSummary(d601).join(", ")}.`;
+}
+
function isStaleLegacyIngressBlocker(blocker, sourceReport, reports) {
const text = `${blocker.scope ?? ""} ${blocker.summary ?? ""}`;
const staleLegacyPort = deprecatedLegacyPublicEndpoints.some((endpoint) => text.includes(endpoint));
@@ -792,8 +823,6 @@ function buildDoD(reports, milestones, blockers) {
const preflight = reports.devPreflight;
const artifactIdentity = preflight.artifactIdentity;
const d601 = reports.d601Observability;
- const d601HasClient = d601.environment?.binaries?.kubectl?.available === true ||
- d601.environment?.binaries?.k3s?.available === true;
const edgeLive = hasCurrentLiveEdgeEvidence(reports.devEdgeHealth);
const artifactCurrent = artifactIdentity.publishVerified === true &&
artifactIdentity.targetCoverage?.covered !== false &&
@@ -827,13 +856,9 @@ function buildDoD(reports, milestones, blockers) {
},
{
id: "d601-k3s-observability",
- status: d601.cluster?.readable ? "pass" : "blocked",
- evidenceLevel: d601.cluster?.readable ? "DEV-LIVE" : "BLOCKED",
- summary: d601.cluster?.readable
- ? "D601 hwlab-dev cluster was readable."
- : d601HasClient
- ? "D601 kubectl/k3s clients are present, but read-only hwlab-dev pods/services/configmaps probes are still blocked."
- : "D601 runner lacks kubectl/k3s/kubeconfig observability for hwlab-dev."
+ status: d601.runnerKubeconfigReadable === false ? "blocked" : d601.cluster?.readable ? "pass" : "blocked",
+ evidenceLevel: d601.cluster?.readable || d601.d601PublicEndpointsReachable ? "DEV-LIVE" : "BLOCKED",
+ summary: d601ObservabilitySummary(d601)
},
{
id: "dev-edge-frp-16667",
@@ -928,6 +953,21 @@ function buildCurrentDevLayering(reports, blockers) {
evidence: reports.devM5Gate.devPreconditions?.evidence?.filter((line) => line.includes("/health/live") || line.includes("DB")) ?? [],
nextRequired: "Provide live DB connection evidence through redacted health output; route reachability alone is insufficient."
},
+ d601RunnerObservability: {
+ label: "D601 runner observability",
+ status: reports.d601Observability.runnerKubeconfigReadable === false
+ ? "blocked"
+ : reports.d601Observability.cluster?.readable === true
+ ? "pass"
+ : "blocked",
+ evidenceLevel: reports.d601Observability.cluster?.readable === true ||
+ reports.d601Observability.d601PublicEndpointsReachable === true
+ ? "DEV-LIVE"
+ : "BLOCKED",
+ summary: d601ObservabilitySummary(reports.d601Observability),
+ evidence: d601ObservationSummary(reports.d601Observability),
+ nextRequired: "Treat #46 runner kubeconfig/readonly gaps separately from D601 service health; rerun read-only observability after the mount or permission path is repaired."
+ },
m3HardwareTrustedLoop: {
label: "M3 hardware trusted loop",
status: m3Live ? "pass" : "blocked",
@@ -968,21 +1008,36 @@ function buildMilestoneBlockerClassification(reports) {
const cloudDb = cloudApiDbStatus(reports);
const m3Live = statusIsPass(reports.devM3Hardware.liveOperation?.status);
const m4Live = statusIsPass(reports.devM4Agent.livePreflight?.status);
+ const m3ServiceDiscoveryBlocked = reports.devM3Hardware.blockers?.some((blocker) =>
+ blocker.scope === "m3-service-discovery"
+ ) === true;
+ const m3BlockerClass = m3Live
+ ? "cleared"
+ : m3ServiceDiscoveryBlocked
+ ? "runner-readonly-observability-gap"
+ : "hardware-loop-runtime";
return [
{
milestone: "M3",
status: m3Live ? "pass" : "blocked",
currentLevel: m3Live ? "DEV-LIVE" : "BLOCKED",
- blockerClass: m3Live ? "cleared" : "hardware-loop-runtime",
- dependency: "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.",
+ blockerClass: m3BlockerClass,
+ dependency: m3ServiceDiscoveryBlocked
+ ? "Runner read-only service discovery must be repaired before the real DEV trusted loop can be observed; this does not mean D601/k3s is globally unavailable."
+ : "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.",
evidence: [
`operationId=${reports.devM3Hardware.liveOperation?.operationId ?? "not_observed"}`,
`traceId=${reports.devM3Hardware.liveOperation?.traceId ?? "not_observed"}`,
`auditId=${reports.devM3Hardware.liveOperation?.auditId ?? "not_observed"}`,
- `evidenceId=${reports.devM3Hardware.liveOperation?.evidenceId ?? "not_observed"}`
+ `evidenceId=${reports.devM3Hardware.liveOperation?.evidenceId ?? "not_observed"}`,
+ `runnerKubeconfigReadable=${reports.devM3Hardware.d601Observability?.runnerKubeconfigReadable === true}`,
+ `d601PublicEndpointsReachable=${reports.devM3Hardware.d601Observability?.d601PublicEndpointsReachable === true}`,
+ `d601K3sUnavailable=${reports.devM3Hardware.d601Observability?.d601K3sUnavailable === true}`
],
- nextRequired: "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.",
+ nextRequired: m3ServiceDiscoveryBlocked
+ ? "Repair the #46 runner readonly kubeconfig/service-discovery gap, then run the bounded DEV M3 live smoke only when direct simulator and patch-panel targets are discoverable."
+ : "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.",
nonPromotionReason: "Public frontend, route, and artifact evidence do not prove the required hardware loop."
},
{
@@ -1071,6 +1126,7 @@ function fallbackAction(scope) {
if (scope.includes("cloud-api-db")) return "Configure DEV cloud-api DB env readiness and rerun health/preflight without exposing secrets.";
if (scope === "db-live") return "Repair DEV cloud-api DB live readiness, then rerun the read-only health and M4 preflight reports without exposing secret values.";
if (scope === "m3-hardware-loop-runtime") return "Prove the real DEV M3 trusted loop res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, trace, audit, and evidence identifiers.";
+ if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery") return "Repair the #46 runner kubeconfig mount/permission or document the approved alternate read-only KUBECONFIG/service-discovery path; do not classify this as D601 global offline.";
if (scope.includes("kubectl") || scope.includes("k3s")) return "Provide read-only kubectl/kubeconfig observability for hwlab-dev.";
return "Resolve the blocker and attach source/local/dry-run/DEV-live evidence at the correct level.";
}
@@ -1083,6 +1139,7 @@ function evidenceRequiredFor(scope) {
if (scope.includes("cloud-api-db")) return "Cloud API health/live output showing DB env ready and redacted secret references, without secret material.";
if (scope === "db-live") return "Cloud API /health/live output with ready=true, connected=true, liveDbEvidence=true, and redacted secret references.";
if (scope === "m3-hardware-loop-runtime") return "Operation, trace, audit, and evidence IDs from the real DEV DO1 -> patch-panel -> DI1 trusted loop.";
+ if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery") return "Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write.";
if (scope.includes("edge") || scope.includes("ingress") || scope.includes("frp")) return "Read-only DEV route observation for :16667/frp/edge/router with HWLAB service identity and artifact identity.";
return "A committed report with the exact evidence level and command used.";
}
diff --git a/scripts/validate-dev-gate-report.mjs b/scripts/validate-dev-gate-report.mjs
index ef2d0772..de61ac04 100644
--- a/scripts/validate-dev-gate-report.mjs
+++ b/scripts/validate-dev-gate-report.mjs
@@ -1271,6 +1271,13 @@ async function validateDevM3Report(report, label) {
assertObject(report.runtimeTarget, `${label}.runtimeTarget`);
assert.equal(report.runtimeTarget.endpoint, "http://74.48.78.17:16667", `${label}.runtimeTarget.endpoint`);
+ if (Object.hasOwn(report.runtimeTarget, "frontendEndpoint")) {
+ assert.equal(
+ report.runtimeTarget.frontendEndpoint,
+ "http://74.48.78.17:16666",
+ `${label}.runtimeTarget.frontendEndpoint`
+ );
+ }
assert.equal(report.runtimeTarget.namespace, "hwlab-dev", `${label}.runtimeTarget.namespace`);
assert.equal(report.runtimeTarget.environment, "dev", `${label}.runtimeTarget.environment`);
assert.equal(report.runtimeTarget.requiredBoxSimulators, 2, `${label}.runtimeTarget.requiredBoxSimulators`);
@@ -1322,6 +1329,47 @@ async function validateDevM3Report(report, label) {
}
}
+ if (Object.hasOwn(report, "d601Observability")) {
+ assertObject(report.d601Observability, `${label}.d601Observability`);
+ for (const field of [
+ "runnerKubeconfigReadable",
+ "runnerKubeconfigProbeExitCode",
+ "runnerKubeconfigProbeStderr",
+ "d601PublicEndpointsReachable",
+ "d601K3sUnavailable",
+ "classification",
+ "sourceIssue",
+ "inferenceRule"
+ ]) {
+ assert.ok(Object.hasOwn(report.d601Observability, field), `${label}.d601Observability missing ${field}`);
+ }
+ assertBoolean(report.d601Observability.runnerKubeconfigReadable, `${label}.d601Observability.runnerKubeconfigReadable`);
+ if (Object.hasOwn(report.d601Observability, "runnerKubeconfigProbeOk")) {
+ assertBoolean(report.d601Observability.runnerKubeconfigProbeOk, `${label}.d601Observability.runnerKubeconfigProbeOk`);
+ }
+ assert.ok(
+ Number.isInteger(report.d601Observability.runnerKubeconfigProbeExitCode) ||
+ report.d601Observability.runnerKubeconfigProbeExitCode === null,
+ `${label}.d601Observability.runnerKubeconfigProbeExitCode`
+ );
+ assertString(report.d601Observability.runnerKubeconfigProbeStderr, `${label}.d601Observability.runnerKubeconfigProbeStderr`);
+ assertBoolean(report.d601Observability.d601PublicEndpointsReachable, `${label}.d601Observability.d601PublicEndpointsReachable`);
+ assertBoolean(report.d601Observability.d601K3sUnavailable, `${label}.d601Observability.d601K3sUnavailable`);
+ if (
+ report.d601Observability.runnerKubeconfigReadable === false &&
+ report.d601Observability.d601PublicEndpointsReachable === true
+ ) {
+ assert.equal(
+ report.d601Observability.d601K3sUnavailable,
+ false,
+ `${label}.d601Observability runner gap must not imply D601 k3s unavailable`
+ );
+ }
+ assertString(report.d601Observability.classification, `${label}.d601Observability.classification`);
+ assertString(report.d601Observability.sourceIssue, `${label}.d601Observability.sourceIssue`);
+ assertString(report.d601Observability.inferenceRule, `${label}.d601Observability.inferenceRule`);
+ }
+
if (Object.hasOwn(report, "readOnlySupplementalEvidence")) {
assertArray(report.readOnlySupplementalEvidence, `${label}.readOnlySupplementalEvidence`);
for (const [index, evidence] of report.readOnlySupplementalEvidence.entries()) {