mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: separate runner kubeconfig gap from D601 status (#144)

Browse Source 
Commander review: direction is correct. This PR separates runner kubeconfig/read-only observability gaps from D601 public endpoint/k3s availability and keeps M3 blocked until real operation/trace/audit/evidence are observed. It does not touch PROD or restart services.
This commit is contained in:
Lyon
2026-05-22 20:44:30 +08:00
committed by GitHub
parent bf47a95a97
commit 85da2dcb19
8 changed files with 669 additions and 141 deletions
Download Patch File Download Diff File Expand all files Collapse all files
reports/d601-k3s-readonly-observability.json
+85 -7
View File
@@ -12,7 +12,7 @@
"pikasTech/HWLAB#39",
"pikasTech/HWLAB#42"
],
"generatedAt": "2026-05-22T07:21:35.915Z",
"generatedAt": "2026-05-22T12:34:14.403Z",
"mode": "read-only",
"devOnly": true,
"prodDisabled": true,
@@ -48,7 +48,24 @@
"node --check scripts/src/d601-k3s-readonly-observability.mjs",
"node scripts/d601-k3s-readonly-observability.mjs"
],
"conclusion": "ready",
"conclusion": "blocked",
"runnerKubeconfigReadable": false,
"runnerKubeconfigProbeOk": true,
"runnerKubeconfigProbeExitCode": 0,
"runnerKubeconfigProbeStderr": "empty",
"d601PublicEndpointsReachable": true,
"d601K3sUnavailable": false,
"statusInference": {
"runnerKubeconfigReadable": false,
"runnerKubeconfigProbeOk": true,
"runnerKubeconfigProbeExitCode": 0,
"runnerKubeconfigProbeStderr": "empty",
"d601PublicEndpointsReachable": true,
"d601K3sUnavailable": false,
"conclusion": "Runner kubeconfig readability is an observability signal only; it is not used to infer D601 k3s or public DEV endpoint availability.",
"blockerClassification": "runner_permission_mount_gap",
"sourceIssue": "pikasTech/HWLAB#46"
},
"environment": {
"binaries": {
"kubectl": {
@@ -118,6 +135,22 @@
"error": "ENOENT"
}
],
"runnerKubeconfig": {
"path": "/etc/rancher/k3s/k3s.yaml",
"mode": "read-only",
"resource": "pods",
"command": "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods -o name",
"runnerKubeconfigReadable": false,
"runnerKubeconfigProbeOk": true,
"runnerKubeconfigProbeExitCode": 0,
"runnerKubeconfigProbeStderr": "empty",
"stdoutSummary": "13 pod name(s) observed",
"classification": "runner_permission_mount_gap",
"sourceIssue": "pikasTech/HWLAB#46",
"secretValuesRead": false,
"secretResourcesRead": false,
"note": "This probe forces KUBECONFIG=/etc/rancher/k3s/k3s.yaml for a read-only kubectl get pods command and records only exit code plus redacted stderr/stdout summary. A successful kubectl exit code does not make the kubeconfig file readable; kubectl may use an alternate in-cluster fallback."
},
"sshBridge": {
"sshBinaryAvailable": true,
"env": [
@@ -172,6 +205,41 @@
"note": "SSH bridge discovery checks only binary, environment variable presence, and file metadata; it does not read SSH config or private keys."
}
},
"publicEndpoints": {
"activeFrontendEndpoint": "http://74.48.78.17:16666",
"activeApiEndpoint": "http://74.48.78.17:16667",
"d601PublicEndpointsReachable": true,
"probes": [
{
"id": "frontend-16666",
"url": "http://74.48.78.17:16666/",
"reached": true,
"ok": true,
"status": 200,
"bodySummary": "<!doctype html> <html lang=\"zh-CN\"> <head> <meta charset=\"utf-8\" /> <meta name=\"viewport\" content=\"width=device-width, initial-scale=1\" /> <title>"
},
{
"id": "api-health-16667",
"url": "http://74.48.78.17:16667/health",
"reached": true,
"ok": true,
"status": 200,
"serviceId": "hwlab-edge-proxy",
"environment": "dev",
"bodySummary": "json serviceId=hwlab-edge-proxy status=ok"
},
{
"id": "api-live-16667",
"url": "http://74.48.78.17:16667/health/live",
"reached": true,
"ok": true,
"status": 200,
"serviceId": "hwlab-cloud-api",
"environment": "dev",
"bodySummary": "json serviceId=hwlab-cloud-api status=degraded"
}
]
},
"cluster": {
"namespace": "hwlab-dev",
"attemptedExecutors": [
@@ -285,7 +353,7 @@
"nodeName": "d601"
},
{
"name": "hwlab-cloud-web-cd9655b98-gxtmc",
"name": "hwlab-cloud-web-5b7487b5c4-fzvgr",
"phase": "Running",
"readyContainers": 1,
"totalContainers": 1,
@@ -623,10 +691,20 @@
}
]
},
"blockers": [],
"blockers": [
{
"status": "open",
"type": "observability_blocker",
"scope": "runner-kubeconfig-readonly-gap",
"classification": "runner_permission_mount_gap",
"sourceIssue": "pikasTech/HWLAB#46",
"summary": "The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately.",
"nextTask": "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report."
}
],
"runnerImageFollowUp": {
"needed": false,
"suggestedUniDeskIssue": null
"needed": true,
"suggestedUniDeskIssue": "Code Queue runner image should include kubectl and a documented read-only hwlab-dev kubeconfig mount/permission path; keep Secret resources and token material inaccessible to reports."
},
"notes": "This report uses command availability checks, file metadata checks, kubectl/k3s kubectl read-only get probes, and auth can-i checks only. It does not print kubeconfig contents, token material, Secret resources, ConfigMap values, SSH config contents, or private keys."
"notes": "This report uses command availability checks, file metadata checks, public DEV endpoint GET probes, kubectl/k3s kubectl read-only get probes, and auth can-i checks only. It does not print kubeconfig contents, token material, Secret resources, ConfigMap values, SSH config contents, or private keys. A failed runner /etc/rancher/k3s/k3s.yaml probe is classified as a runner permission/mount or read-only observability gap and is not evidence that D601 k3s is globally unavailable."
}
reports/dev-gate/dev-m3-hardware-loop.json
+51 -27
View File
@@ -4,10 +4,18 @@
"reportVersion": "v1",
"issue": "pikasTech/HWLAB#38",
"taskId": "dev-m3-hardware-loop",
"commitId": "86d769525caf",
"commitId": "bf47a95a97a9",
"acceptanceLevel": "dev_m3_hardware_loop",
"devOnly": true,
"prodDisabled": true,
"reportLifecycle": {
"version": "v1",
"state": "active",
"activeEndpoint": "http://74.48.78.17:16667",
"activeBrowserEndpoint": "http://74.48.78.17:16666",
"deprecatedEndpoint": null,
"summary": "Current M3 DEV hardware trusted-loop report; DEV-LIVE requires res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, audit, and evidence ids."
},
"sourceContract": {
"status": "pass",
"documents": [
@@ -27,6 +35,7 @@
],
"runtimeTarget": {
"endpoint": "http://74.48.78.17:16667",
"frontendEndpoint": "http://74.48.78.17:16666",
"namespace": "hwlab-dev",
"environment": "dev",
"requiredBoxSimulators": 2,
@@ -67,16 +76,16 @@
"status": "pass",
"summary": "DEV ingress returned HWLAB identity at /health/live.",
"evidence": [
"{\"url\":\"http://74.48.78.17:16667/health/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T04:30:50.824Z\",\"observedAt\":\"2026-05-22T04:30:51.649Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"status\":\"degraded\",\"service\":{\"id\":\"hwlab-cloud-api\",\"role\":\"cloud-api\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T04:30:51.463Z\",\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"configured_without_live_connection\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"redacted\":true}],\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"valuesRedacted\":true,\"liveDbEvidence\":false},\"evidence\":\"env_presence_only_no_live_db\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}",
"{\"url\":\"http://74.48.78.17:16667/health\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T04:30:51.650Z\",\"observedAt\":\"2026-05-22T04:30:52.025Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-edge-proxy\",\"environment\":\"dev\",\"status\":\"ok\",\"service\":{\"id\":\"hwlab-edge-proxy\",\"role\":\"public-dev-ingress\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada6860653b27269b0a57991184118cbf4b1\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"ghcr.io/pikastech/hwlab-edge-proxy:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T04:30:51.840Z\",\"details\":{\"upstream\":\"http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667\",\"mode\":\"dev-edge-proxy\"}}}",
"{\"url\":\"http://74.48.78.17:16667/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T04:30:52.025Z\",\"observedAt\":\"2026-05-22T04:30:52.401Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"status\":\"live\"}}",
"{\"url\":\"http://74.48.78.17:16667/v1\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T04:30:52.401Z\",\"observedAt\":\"2026-05-22T04:30:53.001Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"adapter\":\"rest\",\"status\":\"degraded\",\"rpcBridge\":\"POST /v1/rpc/{method}\",\"methods\":[\"system.health\",\"cloud.adapter.describe\",\"gateway.session.register\",\"box.resource.register\",\"box.capability.report\",\"hardware.operation.request\",\"hardware.invoke.shell\",\"audit.event.write\",\"audit.event.query\",\"evidence.record.write\",\"evidence.record.query\"],\"auditFields\":[\"requestId\",\"actor\",\"source\",\"operation\",\"target\",\"result\",\"timestamp\"],\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"configured_without_live_connection\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"redacted\":true}],\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"valuesRedacted\":true,\"liveDbEvidence\":false},\"evidence\":\"env_presence_only_no_live_db\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}"
"{\"url\":\"http://74.48.78.17:16667/health/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:42.977Z\",\"observedAt\":\"2026-05-22T12:36:43.545Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"status\":\"degraded\",\"service\":{\"id\":\"hwlab-cloud-api\",\"role\":\"cloud-api\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T12:36:43.343Z\",\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"configured_without_live_connection\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"redacted\":true}],\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"valuesRedacted\":true,\"liveDbEvidence\":false},\"evidence\":\"env_presence_only_no_live_db\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}",
"{\"url\":\"http://74.48.78.17:16667/health\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:43.545Z\",\"observedAt\":\"2026-05-22T12:36:44.191Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-edge-proxy\",\"environment\":\"dev\",\"status\":\"ok\",\"service\":{\"id\":\"hwlab-edge-proxy\",\"role\":\"public-dev-ingress\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada6860653b27269b0a57991184118cbf4b1\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"ghcr.io/pikastech/hwlab-edge-proxy:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T12:36:43.989Z\",\"details\":{\"upstream\":\"http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667\",\"mode\":\"dev-edge-proxy\"}}}",
"{\"url\":\"http://74.48.78.17:16667/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:44.192Z\",\"observedAt\":\"2026-05-22T12:36:44.570Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"status\":\"live\"}}",
"{\"url\":\"http://74.48.78.17:16667/v1\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:44.570Z\",\"observedAt\":\"2026-05-22T12:36:45.204Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"adapter\":\"rest\",\"status\":\"degraded\",\"rpcBridge\":\"POST /v1/rpc/{method}\",\"methods\":[\"system.health\",\"cloud.adapter.describe\",\"gateway.session.register\",\"box.resource.register\",\"box.capability.report\",\"hardware.operation.request\",\"hardware.invoke.shell\",\"audit.event.write\",\"audit.event.query\",\"evidence.record.write\",\"evidence.record.query\"],\"auditFields\":[\"requestId\",\"actor\",\"source\",\"operation\",\"target\",\"result\",\"timestamp\"],\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"configured_without_live_connection\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"redacted\":true}],\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"valuesRedacted\":true,\"liveDbEvidence\":false},\"evidence\":\"env_presence_only_no_live_db\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}"
]
},
{
"id": "two-box-simu-online",
"status": "not_run",
"summary": "A live DEV M3 runtime contract check failed before all checks passed.",
"summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -84,7 +93,7 @@
{
"id": "two-gateway-simu-online",
"status": "not_run",
"summary": "A live DEV M3 runtime contract check failed before all checks passed.",
"summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -92,7 +101,7 @@
{
"id": "patch-panel-healthy",
"status": "not_run",
"summary": "A live DEV M3 runtime contract check failed before all checks passed.",
"summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -100,7 +109,7 @@
{
"id": "wiring-do1-di1-applied",
"status": "not_run",
"summary": "A live DEV M3 runtime contract check failed before all checks passed.",
"summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -108,7 +117,7 @@
{
"id": "direct-call-do-write-di-read",
"status": "not_run",
"summary": "A live DEV M3 runtime contract check failed before all checks passed.",
"summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -116,7 +125,7 @@
{
"id": "audit-evidence-traceable",
"status": "not_run",
"summary": "A live DEV M3 runtime contract check failed before all checks passed.",
"summary": "Stopped before M3 direct checks because service target URLs were not provided.",
"evidence": [
"No live DEV evidence collected for this check."
]
@@ -170,9 +179,31 @@
"id": "kubectl-hwlab-dev",
"status": "observed",
"command": "kubectl get deploy,po,svc -n hwlab-dev -o wide",
"summary": "NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR deployment.apps/hwlab-agent-mgr 1/1 1 1 4h38m hwlab-agent-mgr 127.0.0.1:5000/hwlab/hwlab-agent-mgr:cb35ada app.kubernetes.io/name=hwlab-agent-mgr,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-agent-skills 1/1 1 1 66m hwlab-agent-skills 127.0.0.1:5000/hwlab/hwlab-agent-skills:cb35ada app.kubernetes.io/name=hwlab-agent-skills,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-box-simu 2/2 2 2 4h38m hwlab-box-simu 127.0.0.1:5000/hwlab/hwlab-box-simu:cb35ada app.kubernetes.io/name=hwlab-box-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-api 1/1 1 1 4h38m hwlab-cloud-api 127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada app.kubernetes.io/name=hwlab-cloud-api,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-web 1/1 1 1 4h38m hwlab-cloud-web 127.0.0.1:5000/hwlab/hwlab-cloud-web:cb35ada app.kubernetes.io/name=hwlab-cloud-web,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-edge-proxy 1/1 1 1 4h38m hwlab-edge-proxy 127.0.0.1:5000/hwlab/hwlab-edge-proxy:cb35ada app.kubernetes.io/name=hwlab-edge-proxy,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-frpc 1/1 1 1 4h33m frpc 127.0.0.1:5000/hwlab/frpc:v0.68.1 app.kubernetes.io/name=hwlab-frpc deployment.apps/hwlab-gateway 0/0 0 0 4h38m hwlab-gateway 127.0.0.1:5000/hwlab/hwlab-gateway:cb35ada app.kubernetes.io/name=hwlab-gateway,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-gateway-simu 2/2 2 2 4h38m hwlab-gateway-simu 127.0.0.1:5000/hwlab/hwlab-gateway-simu:cb35ada app.kubernetes.io/name=hwlab-gateway-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-patch-panel 1/1 1 1 4h38m hwlab-patch-panel 127.0.0.1:5000/hwlab/hwlab-patch-panel:cb35ada app.kubernetes.io/name=hwlab-patch-panel,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/prof"
"summary": "NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR deployment.apps/hwlab-agent-mgr 1/1 1 1 12h hwlab-agent-mgr 127.0.0.1:5000/hwlab/hwlab-agent-mgr:cb35ada app.kubernetes.io/name=hwlab-agent-mgr,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-agent-skills 1/1 1 1 9h hwlab-agent-skills 127.0.0.1:5000/hwlab/hwlab-agent-skills:cb35ada app.kubernetes.io/name=hwlab-agent-skills,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-box-simu 2/2 2 2 12h hwlab-box-simu 127.0.0.1:5000/hwlab/hwlab-box-simu:cb35ada app.kubernetes.io/name=hwlab-box-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-api 1/1 1 1 12h hwlab-cloud-api 127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada app.kubernetes.io/name=hwlab-cloud-api,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-web 1/1 1 1 12h hwlab-cloud-web 127.0.0.1:5000/hwlab/hwlab-cloud-web:bf47a95 app.kubernetes.io/name=hwlab-cloud-web,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-edge-proxy 1/1 1 1 12h hwlab-edge-proxy 127.0.0.1:5000/hwlab/hwlab-edge-proxy:cb35ada app.kubernetes.io/name=hwlab-edge-proxy,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-frpc 1/1 1 1 12h frpc 127.0.0.1:5000/hwlab/frpc:v0.68.1 app.kubernetes.io/name=hwlab-frpc deployment.apps/hwlab-gateway 0/0 0 0 12h hwlab-gateway 127.0.0.1:5000/hwlab/hwlab-gateway:cb35ada app.kubernetes.io/name=hwlab-gateway,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-gateway-simu 2/2 2 2 12h hwlab-gateway-simu 127.0.0.1:5000/hwlab/hwlab-gateway-simu:cb35ada app.kubernetes.io/name=hwlab-gateway-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-patch-panel 1/1 1 1 12h hwlab-patch-panel 127.0.0.1:5000/hwlab/hwlab-patch-panel:cb35ada app.kubernetes.io/name=hwlab-patch-panel,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.app"
},
{
"id": "runner-k3s-kubeconfig-readonly",
"status": "observed",
"command": "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods -o name",
"summary": "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate. runnerKubeconfigReadable=false; exitCode=0; stderr=empty."
}
],
"d601Observability": {
"runnerKubeconfigReadable": false,
"runnerKubeconfigProbeOk": true,
"runnerKubeconfigProbeExitCode": 0,
"runnerKubeconfigProbeStderr": "empty",
"d601PublicEndpointsReachable": true,
"d601K3sUnavailable": false,
"classification": "runner_permission_mount_gap",
"sourceIssue": "pikasTech/HWLAB#46",
"command": "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods -o name",
"stdoutSummary": "13 pod name(s) observed",
"summary": "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate.",
"inferenceRule": "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately.",
"secretValuesRead": false,
"secretResourcesRead": false
},
"liveOperation": {
"status": "not_run",
"operationId": "not_observed",
@@ -183,25 +214,18 @@
},
"blockers": [
{
"type": "runtime_blocker",
"scope": "m3-hardware-loop-runtime",
"type": "observability_blocker",
"scope": "m3-service-discovery",
"status": "open",
"classification": "hardware loop",
"summary": "patch-panel active DO1 -> DI1 connection missing"
"classification": "runner permission/mount gap",
"sourceIssue": "pikasTech/HWLAB#46",
"summary": "DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline."
}
],
"summary": {
"status": "blocked",
"classification": "hardware loop",
"observedAt": "2026-05-22T04:30:50.820Z",
"result": "Live DEV M3 runtime was reachable but did not satisfy the M3 hardware-loop contract."
},
"reportLifecycle": {
"version": "v1",
"state": "active",
"activeEndpoint": "http://74.48.78.17:16667",
"activeBrowserEndpoint": "http://74.48.78.17:16666",
"deprecatedEndpoint": null,
"summary": "Current M3 DEV hardware trusted-loop report; DEV-LIVE requires res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, audit, and evidence ids."
"classification": "runner permission/mount gap",
"observedAt": "2026-05-22T12:36:42.972Z",
"result": "DEV ingress was reachable, but live M3 simulator and patch-panel targets were not discoverable."
}
}
reports/dev-gate/dev-m5-gate-aggregator-v2.json
+130 -57
View File
@@ -20,8 +20,8 @@
"pikasTech/HWLAB#46",
"pikasTech/HWLAB#64"
],
"generatedAt": "2026-05-22T11:50:42.733Z",
"generatedFromCommit": "f933a81c83a5",
"generatedAt": "2026-05-22T12:36:45.366Z",
"generatedFromCommit": "bf47a95a97a9",
"environment": "dev",
"endpoint": "http://74.48.78.17:16667",
"frontendEndpoint": "http://74.48.78.17:16666",
@@ -108,7 +108,7 @@
"taskId": "dev-m3-hardware-loop",
"lifecycleState": "active",
"status": "blocked",
"commitId": "86d769525caf"
"commitId": "bf47a95a97a9"
},
"devM4Agent": {
"path": "reports/dev-gate/dev-m4-agent-loop.json",
@@ -131,7 +131,7 @@
"issue": "pikasTech/HWLAB#46",
"taskId": "d601-k3s-readonly-observability",
"lifecycleState": "active",
"status": "ready",
"status": "blocked",
"commitId": "unknown"
}
},
@@ -192,9 +192,9 @@
},
{
"id": "d601-k3s-observability",
"status": "pass",
"status": "blocked",
"evidenceLevel": "DEV-LIVE",
"summary": "D601 hwlab-dev cluster was readable."
"summary": "D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false."
},
{
"id": "dev-edge-frp-16667",
@@ -265,11 +265,25 @@
],
"nextRequired": "Provide live DB connection evidence through redacted health output; route reachability alone is insufficient."
},
"d601RunnerObservability": {
"label": "D601 runner observability",
"status": "blocked",
"evidenceLevel": "DEV-LIVE",
"summary": "D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false.",
"evidence": [
"runnerKubeconfigReadable=false",
"runnerKubeconfigProbeExitCode=0",
"runnerKubeconfigProbeStderr=empty",
"d601PublicEndpointsReachable=true",
"d601K3sUnavailable=false"
],
"nextRequired": "Treat #46 runner kubeconfig/readonly gaps separately from D601 service health; rerun read-only observability after the mount or permission path is repaired."
},
"m3HardwareTrustedLoop": {
"label": "M3 hardware trusted loop",
"status": "blocked",
"evidenceLevel": "BLOCKED",
"summary": "No live DEV operation has run yet. Blocker: patch-panel active DO1 -> DI1 connection missing.",
"summary": "No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline..",
"evidence": [
"operationId=not_observed",
"traceId=not_observed",
@@ -357,15 +371,18 @@
"milestone": "M3",
"status": "blocked",
"currentLevel": "BLOCKED",
"blockerClass": "hardware-loop-runtime",
"dependency": "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.",
"blockerClass": "runner-readonly-observability-gap",
"dependency": "Runner read-only service discovery must be repaired before the real DEV trusted loop can be observed; this does not mean D601/k3s is globally unavailable.",
"evidence": [
"operationId=not_observed",
"traceId=not_observed",
"auditId=not_observed",
"evidenceId=not_observed"
"evidenceId=not_observed",
"runnerKubeconfigReadable=false",
"d601PublicEndpointsReachable=true",
"d601K3sUnavailable=false"
],
"nextRequired": "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.",
"nextRequired": "Repair the #46 runner readonly kubeconfig/service-discovery gap, then run the bounded DEV M3 live smoke only when direct simulator and patch-panel targets are discoverable.",
"nonPromotionReason": "Public frontend, route, and artifact evidence do not prove the required hardware loop."
},
{
@@ -424,7 +441,7 @@
"highestVisibleLevel": "DEV-LIVE",
"liveEvidence": "pass",
"evidenceCount": 7,
"blockerCount": 5,
"blockerCount": 7,
"summary": "deploy/runtime readiness is blocked before live DEV; highest visible level is DEV-LIVE; status is blocked."
},
{
@@ -433,7 +450,7 @@
"highestVisibleLevel": "LOCAL",
"liveEvidence": "missing_or_blocked",
"evidenceCount": 3,
"blockerCount": 2,
"blockerCount": 3,
"summary": "hardware loop has source/local shape but no live operation; highest visible level is LOCAL; status is blocked."
},
{
@@ -451,7 +468,7 @@
"highestVisibleLevel": "DRY-RUN",
"liveEvidence": "missing_or_blocked",
"evidenceCount": 2,
"blockerCount": 9,
"blockerCount": 8,
"summary": "dry-run is green but live MVP gate is blocked; highest visible level is DRY-RUN; status is blocked."
}
],
@@ -651,8 +668,8 @@
"reportPath": "reports/d601-k3s-readonly-observability.json",
"commitId": "unknown",
"lifecycleState": "active",
"level": "BLOCKED",
"status": "ready",
"level": "DEV-LIVE",
"status": "blocked",
"category": "d601-observability",
"commands": [
"node --check scripts/d601-k3s-readonly-observability.mjs",
@@ -660,18 +677,20 @@
"node scripts/d601-k3s-readonly-observability.mjs"
],
"evidence": [
"kubectl=available",
"k3s=available",
"clusterReadable=yes"
"runnerKubeconfigReadable=false",
"runnerKubeconfigProbeExitCode=0",
"runnerKubeconfigProbeStderr=empty",
"d601PublicEndpointsReachable=true",
"d601K3sUnavailable=false"
],
"summary": "D601 k3s observability is blocked; cluster state has not been read."
"summary": "D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false."
},
{
"milestone": "M3",
"issue": "pikasTech/HWLAB#38",
"taskId": "dev-m3-hardware-loop",
"reportPath": "reports/dev-gate/dev-m3-hardware-loop.json",
"commitId": "86d769525caf",
"commitId": "bf47a95a97a9",
"lifecycleState": "active",
"level": "SOURCE",
"status": "manifest-ready",
@@ -710,7 +729,7 @@
"issue": "pikasTech/HWLAB#38",
"taskId": "dev-m3-hardware-loop",
"reportPath": "reports/dev-gate/dev-m3-hardware-loop.json",
"commitId": "86d769525caf",
"commitId": "bf47a95a97a9",
"lifecycleState": "active",
"level": "BLOCKED",
"status": "not_run",
@@ -724,7 +743,7 @@
"auditId=not_observed",
"evidenceId=not_observed"
],
"summary": "No live DEV operation has run yet. Blocker: patch-panel active DO1 -> DI1 connection missing."
"summary": "No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.."
},
{
"milestone": "M4",
@@ -970,19 +989,19 @@
"category": "edge-frp-contract",
"reportPath": "reports/dev-gate/dev-edge-health.json",
"summary": "Committed edge report proves read-only public HTTP on :16667 /health and /health/live; remaining edge report blocker is DB readiness, not route/frp reachability."
}
],
"BLOCKED": [
},
{
"milestone": "M2",
"issue": "pikasTech/HWLAB#46",
"taskId": "d601-k3s-readonly-observability",
"lifecycleState": "active",
"status": "ready",
"status": "blocked",
"category": "d601-observability",
"reportPath": "reports/d601-k3s-readonly-observability.json",
"summary": "D601 k3s observability is blocked; cluster state has not been read."
},
"summary": "D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false."
}
],
"BLOCKED": [
{
"milestone": "M3",
"issue": "pikasTech/HWLAB#38",
@@ -991,7 +1010,7 @@
"status": "not_run",
"category": "hardware-loop-live",
"reportPath": "reports/dev-gate/dev-m3-hardware-loop.json",
"summary": "No live DEV operation has run yet. Blocker: patch-panel active DO1 -> DI1 connection missing."
"summary": "No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.."
},
{
"milestone": "M4",
@@ -1033,6 +1052,25 @@
"summary": "reports/dev-gate/dev-artifacts.json does not prove all required HWLAB service artifacts for origin/main 8e89409; current status is published with 13/13 required services published and source states resolved: 13 source-present, 0 intentionally-disabled.",
"nextTask": "Complete DEV artifact publishing for every enabled required HWLAB service at the artifact source commit, or prove the target commit has no artifact build input changes; keep disabled services marked not_published with reasons."
},
{
"priority": "P1",
"order": 4,
"type": "observability_blocker",
"scope": "runner-kubeconfig-readonly-gap",
"sourceIssue": "pikasTech/HWLAB#46",
"source": "reports/d601-k3s-readonly-observability.json",
"summary": "The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately.",
"nextTask": "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report."
},
{
"priority": "P1",
"order": 4,
"type": "observability_blocker",
"scope": "m3-service-discovery",
"sourceIssue": "pikasTech/HWLAB#38",
"source": "reports/dev-gate/dev-m3-hardware-loop.json",
"summary": "DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline."
},
{
"priority": "P1",
"order": 5,
@@ -1080,15 +1118,6 @@
"source": "reports/dev-gate/dev-mvp-gate-report.json",
"summary": "cloud-api /health/live reports DB degraded; db.ready=false, db.connected=false, and live DB evidence is not established."
},
{
"priority": "P0",
"order": 6,
"type": "runtime_blocker",
"scope": "m3-hardware-loop-runtime",
"sourceIssue": "pikasTech/HWLAB#38",
"source": "reports/dev-gate/dev-m3-hardware-loop.json",
"summary": "patch-panel active DO1 -> DI1 connection missing"
},
{
"priority": "P0",
"order": 6,
@@ -1137,6 +1166,47 @@
],
"rationale": "The gate cannot promote deploy or runtime observations without immutable image provenance and digests."
},
{
"id": "d601-k3s-readonly-observability:runner-kubeconfig-readonly-gap",
"priority": "P1",
"type": "observability_blocker",
"scope": "runner-kubeconfig-readonly-gap",
"status": "open",
"source": "reports/d601-k3s-readonly-observability.json",
"sourceIssue": "pikasTech/HWLAB#46",
"summary": "The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately.",
"nextTask": "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report.",
"unblockOrder": 4,
"unblocks": [
"pikasTech/HWLAB#34",
"pikasTech/HWLAB#33",
"pikasTech/HWLAB#36",
"pikasTech/HWLAB#38",
"pikasTech/HWLAB#46",
"pikasTech/HWLAB#64"
],
"rationale": "Runner read-only observability must be repaired without treating the runner gap as proof that D601 k3s or public DEV endpoints are unavailable."
},
{
"id": "dev-m3-hardware-loop:m3-service-discovery",
"priority": "P1",
"type": "observability_blocker",
"scope": "m3-service-discovery",
"status": "open",
"source": "reports/dev-gate/dev-m3-hardware-loop.json",
"sourceIssue": "pikasTech/HWLAB#38",
"summary": "DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.",
"unblockOrder": 4,
"unblocks": [
"pikasTech/HWLAB#34",
"pikasTech/HWLAB#33",
"pikasTech/HWLAB#36",
"pikasTech/HWLAB#38",
"pikasTech/HWLAB#46",
"pikasTech/HWLAB#64"
],
"rationale": "Runner read-only observability must be repaired without treating the runner gap as proof that D601 k3s or public DEV endpoints are unavailable."
},
{
"id": "dev-edge-health:cloud-api-db",
"priority": "P1",
@@ -1222,23 +1292,6 @@
],
"rationale": "M4 and M5 cannot claim live agent or MVP evidence until cloud-api /health/live proves DB readiness with redacted live evidence."
},
{
"id": "dev-m3-hardware-loop:m3-hardware-loop-runtime",
"priority": "P0",
"type": "runtime_blocker",
"scope": "m3-hardware-loop-runtime",
"status": "open",
"source": "reports/dev-gate/dev-m3-hardware-loop.json",
"sourceIssue": "pikasTech/HWLAB#38",
"summary": "patch-panel active DO1 -> DI1 connection missing",
"unblockOrder": 6,
"unblocks": [
"pikasTech/HWLAB#38",
"pikasTech/HWLAB#39",
"pikasTech/HWLAB#64"
],
"rationale": "M3 remains blocked until the real DEV hardware trusted loop proves DO1 -> patch-panel -> DI1 with operation, trace, audit, and evidence identifiers."
},
{
"id": "dev-mvp-gate-report:m3-hardware-loop-runtime",
"priority": "P0",
@@ -1278,6 +1331,26 @@
},
{
"order": 2,
"blockerOrder": 4,
"priority": "P1",
"scopes": [
"runner-kubeconfig-readonly-gap",
"m3-service-discovery"
],
"sourceIssues": [
"pikasTech/HWLAB#33",
"pikasTech/HWLAB#34",
"pikasTech/HWLAB#36",
"pikasTech/HWLAB#38",
"pikasTech/HWLAB#46",
"pikasTech/HWLAB#64"
],
"rationale": "Runner read-only observability must be repaired without treating the runner gap as proof that D601 k3s or public DEV endpoints are unavailable.",
"action": "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report.",
"evidenceRequired": "Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write."
},
{
"order": 3,
"blockerOrder": 5,
"priority": "P1",
"scopes": [
@@ -1297,7 +1370,7 @@
"evidenceRequired": "Cloud API health/live output showing DB env ready and redacted secret references, without secret material."
},
{
"order": 3,
"order": 4,
"blockerOrder": 6,
"priority": "P0",
"scopes": [
reports/dev-gate/dev-m5-gate-aggregator-v2.md
+11 -7
View File
@@ -1,7 +1,7 @@
# HWLAB M5 DEV Gate Aggregator v2
Status: blocked
Generated from: `f933a81c83a5`
Generated from: `bf47a95a97a9`
Scope: DEV only, report-only
Active frontend: `http://74.48.78.17:16666/`
Active API/live: `http://74.48.78.17:16667/health/live`
@@ -22,7 +22,8 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be
| Frontend DEV revision | pass | DEV-LIVE | http://74.48.78.17:16666/ serves the accepted Cloud Workbench frontend revision 1e8805664970839b72be40c34636b08f6d18b131; this is browser/frontend evidence only. | Keep frontend revision proof separate from DB live readiness, M3 hardware-loop evidence, M4 agent-loop evidence, and M5 acceptance. |
| EDGE/ROUTE live | pass | DEV-LIVE | http://74.48.78.17:16666/, http://74.48.78.17:16667/health, and http://74.48.78.17:16667/health/live returned accepted HWLAB DEV responses in the active M2 read-only smoke. | Keep this separated from DB readiness, M3/M4 loop evidence, and M5 acceptance. |
| DB live/degraded | blocked | BLOCKED | cloud-api DB status=degraded; configReady=true; ready=false; connected=false; liveDbEvidence=false. | Provide live DB connection evidence through redacted health output; route reachability alone is insufficient. |
| M3 hardware trusted loop | blocked | BLOCKED | No live DEV operation has run yet. Blocker: patch-panel active DO1 -> DI1 connection missing. | Only a real DEV res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 observation with operation/trace/audit/evidence can clear M3. |
| D601 runner observability | blocked | DEV-LIVE | D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false. | Treat #46 runner kubeconfig/readonly gaps separately from D601 service health; rerun read-only observability after the mount or permission path is repaired. |
| M3 hardware trusted loop | blocked | BLOCKED | No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.. | Only a real DEV res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 observation with operation/trace/audit/evidence can clear M3. |
| M4 agent loop | blocked | BLOCKED | Blocked at DB live readiness before scheduling a DEV agent task. | Do not schedule or claim the agent loop as live until DB live and required runtime/evidence preconditions pass. |
| artifact/desired-state source | blocked | BLOCKED | artifact targetCovered=false; artifactSource=7e29522b65c8; target=8e89409dda5d; desiredApplyMode=dry-run; mutationAttempted=false. | Refresh artifact/source coverage for current origin/main and keep desired-state apply separate from read-only route proof. |
@@ -55,7 +56,7 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be
| m0-source-contract | pass | SOURCE | M0 contract checks are source-level evidence only. |
| m1-local-smoke | pass | LOCAL | M1 local smoke is not a live DEV substitute. |
| artifact-publish-digests | blocked | BLOCKED | artifactState=published, ciPublished=true, registryVerified=true, sha256=13, not_published=0, targetCovered=false |
| d601-k3s-observability | pass | DEV-LIVE | D601 hwlab-dev cluster was readable. |
| d601-k3s-observability | blocked | DEV-LIVE | D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false. |
| dev-edge-frp-16667 | pass | DEV-LIVE | Committed edge report proves read-only public HTTP on :16667 /health and /health/live; this is route evidence, not DB/M3/M4/M5 acceptance. |
| cloud-api-db-ready | blocked | BLOCKED | cloud-api DB status=degraded; ready=false; connected=false; liveDbEvidence=false. |
| m3-hardware-trusted-loop | blocked | BLOCKED | M3 trusted loop is blocked until res_boxsimu_1:DO1 -> patch-panel -> res_boxsimu_2:DI1 is proven with operation/trace/audit/evidence. |
@@ -66,7 +67,7 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be
| Milestone | Status | Current level | Blocker class | Dependency | Required next proof |
| --- | --- | --- | --- | --- | --- |
| M3 | blocked | BLOCKED | hardware-loop-runtime | Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1. | Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs. |
| M3 | blocked | BLOCKED | runner-readonly-observability-gap | Runner read-only service discovery must be repaired before the real DEV trusted loop can be observed; this does not mean D601/k3s is globally unavailable. | Repair the #46 runner readonly kubeconfig/service-discovery gap, then run the bounded DEV M3 live smoke only when direct simulator and patch-panel targets are discoverable. |
| M4 | blocked | BLOCKED | db-live-readiness | Cloud API /health/live must report DB ready=true, connected=true, and liveDbEvidence=true before live agent scheduling/evidence closure. | Repair DB live readiness and rerun the M4 live preflight without scheduling a DEV agent task before preconditions pass. |
| M5 | blocked | BLOCKED | composite-db-m3-m4-live | M5 needs DB live readiness, M3 trusted-loop DEV evidence, M4 live preflight/evidence closure, and current source/artifact coverage. | After DB/M3/M4 blockers are cleared, run only the bounded DEV MVP live gate command with explicit DEV/non-PROD confirmations. |
@@ -76,21 +77,24 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be
| --- | ---: | --- | --- | --- |
| P1 | 3 | contract_blocker | artifact-source-commit | source commit origin/main 8e89409 is not covered by artifact source 7e29522; target changes since the artifact source touch artifact build inputs. |
| P1 | 3 | runtime_blocker | dev-artifact-publish | reports/dev-gate/dev-artifacts.json does not prove all required HWLAB service artifacts for origin/main 8e89409; current status is published with 13/13 required services published and source states resolved: 13 source-present, 0 intentionally-disabled. |
| P1 | 4 | observability_blocker | runner-kubeconfig-readonly-gap | The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately. |
| P1 | 4 | observability_blocker | m3-service-discovery | DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline. |
| P1 | 5 | runtime_blocker | cloud-api-db | cloud-api DB env is injected, but runtime health has not attempted a live DB connection |
| P1 | 5 | network_blocker | cloud-api-db | cloud-api DB env is injected, but runtime health has not attempted a live DB connection |
| P1 | 5 | runtime_blocker | cloud-api-db-health-gate | cloud-api DB runtime env is not ready; missing HWLAB_CLOUD_DB_URL, HWLAB_CLOUD_DB_SSL_MODE. |
| P1 | 5 | runtime_blocker | db-live | cloud-api /health/live reports DB degraded; connected=false; ready=false. |
| P1 | 5 | runtime_blocker | db-live | cloud-api /health/live reports DB degraded; db.ready=false, db.connected=false, and live DB evidence is not established. |
| P0 | 6 | runtime_blocker | m3-hardware-loop-runtime | patch-panel active DO1 -> DI1 connection missing |
| P0 | 6 | runtime_blocker | m3-hardware-loop-runtime | Live M3 smoke reached DEV simulators, but patch-panel active DO1 -> DI1 connection is missing. |
## Next Unblock Order
1. Refresh without fake digests using `node scripts/refresh-artifact-catalog.mjs --target-ref origin/main --blocked`, or after a successful publish run `node scripts/refresh-artifact-catalog.mjs --target-ref origin/main --publish-report reports/dev-gate/dev-artifacts.json`.
Evidence required: Artifact publish report with ciPublished=true, registryVerified=true, and sha256 digest for each frozen service ID.
2. Deploy cloud-api DB runtime readiness probe and/or repair DEV DB connectivity, then rerun the read-only health smoke without reading or printing the DB secret value.
2. Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report.
Evidence required: Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write.
3. Deploy cloud-api DB runtime readiness probe and/or repair DEV DB connectivity, then rerun the read-only health smoke without reading or printing the DB secret value.
Evidence required: Cloud API health/live output showing DB env ready and redacted secret references, without secret material.
3. Prove the real DEV M3 trusted loop res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, trace, audit, and evidence identifiers.
4. Prove the real DEV M3 trusted loop res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, trace, audit, and evidence identifiers.
Evidence required: Operation, trace, audit, and evidence IDs from the real DEV DO1 -> patch-panel -> DI1 trusted loop.
## Validation
scripts/dev-m3-hardware-loop-smoke.mjs
+76 -8
View File
@@ -2,6 +2,7 @@
import assert from "node:assert/strict";
import { execFileSync } from "node:child_process";
import { randomUUID } from "node:crypto";
import { accessSync, constants as fsConstants } from "node:fs";
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
import { request as httpRequest } from "node:http";
import { request as httpsRequest } from "node:https";
@@ -9,7 +10,7 @@ import path from "node:path";
import { fileURLToPath } from "node:url";
import { activeReportLifecycle } from "../internal/dev-report-lifecycle.mjs";
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs";
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT, ENVIRONMENT_DEV } from "../internal/protocol/index.mjs";
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const defaultReportPath = "reports/dev-gate/dev-m3-hardware-loop.json";
@@ -19,6 +20,7 @@ const deployWorkloadsPath = "deploy/k8s/base/workloads.yaml";
const requiredManifestCardinalityCommand = "node scripts/validate-dev-m3-cardinality.mjs";
const requiredM3BoxIds = Object.freeze(["boxsimu_1", "boxsimu_2"]);
const requiredM3BoxResources = Object.freeze(["res_boxsimu_1", "res_boxsimu_2"]);
const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
const serviceTargets = Object.freeze([
{
@@ -156,12 +158,57 @@ function commandResult(command, args, { timeoutMs = 5000, maxChars = 3000 } = {}
}
}
function runnerKubeconfigCommandText(args) {
return [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args].join(" ");
}
function commandExists(command) {
const result = commandResult("bash", ["-lc", `command -v ${command}`], { timeoutMs: 2000 });
return result.exitCode === 0 && result.stdout.trim().length > 0;
}
function collectLocalRuntimeObservations() {
function fileReadable(filePath) {
try {
accessSync(filePath, fsConstants.R_OK);
return true;
} catch {
return false;
}
}
function collectD601RunnerObservability(ingress) {
const args = ["-n", namespace, "get", "pods", "-o", "name"];
const probe = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], {
timeoutMs: 5000,
maxChars: 2000
});
const stderr = oneLine(probe.stderr || "") || "empty";
const stdoutLines = String(probe.stdout ?? "")
.split("\n")
.map((line) => line.trim())
.filter(Boolean);
const d601PublicEndpointsReachable = ingress.probes.some((item) => item.ok && isHwlabDevIdentity(item));
const runnerKubeconfigReadable = fileReadable(runnerK3sKubeconfigPath);
return {
runnerKubeconfigReadable,
runnerKubeconfigProbeOk: probe.exitCode === 0,
runnerKubeconfigProbeExitCode: probe.exitCode,
runnerKubeconfigProbeStderr: stderr,
d601PublicEndpointsReachable,
d601K3sUnavailable: false,
classification: "runner_permission_mount_gap",
sourceIssue: "pikasTech/HWLAB#46",
command: runnerKubeconfigCommandText(args),
stdoutSummary: probe.exitCode === 0 ? `${stdoutLines.length} pod name(s) observed` : undefined,
summary: "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate.",
inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately.",
secretValuesRead: false,
secretResourcesRead: false
};
}
function collectLocalRuntimeObservations(d601Observability) {
const observations = [];
if (commandExists("docker")) {
@@ -206,6 +253,13 @@ function collectLocalRuntimeObservations() {
});
}
observations.push({
id: "runner-k3s-kubeconfig-readonly",
status: d601Observability.runnerKubeconfigProbeOk ? "observed" : "blocked",
command: d601Observability.command,
summary: `${d601Observability.summary} runnerKubeconfigReadable=${d601Observability.runnerKubeconfigReadable}; exitCode=${d601Observability.runnerKubeconfigProbeExitCode}; stderr=${d601Observability.runnerKubeconfigProbeStderr}.`
});
return observations;
}
@@ -587,6 +641,7 @@ function baseReport({ commitId, observedAt }) {
],
runtimeTarget: {
endpoint: DEV_ENDPOINT,
frontendEndpoint: DEV_FRONTEND_ENDPOINT,
namespace,
environment: ENVIRONMENT_DEV,
requiredBoxSimulators: 2,
@@ -623,6 +678,17 @@ function baseReport({ commitId, observedAt }) {
],
readOnlySupplementalEvidence: [],
localRuntimeObservations: [],
d601Observability: {
runnerKubeconfigReadable: false,
runnerKubeconfigProbeOk: false,
runnerKubeconfigProbeExitCode: null,
runnerKubeconfigProbeStderr: "not_observed",
d601PublicEndpointsReachable: false,
d601K3sUnavailable: false,
classification: "not_observed",
sourceIssue: "pikasTech/HWLAB#46",
inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately."
},
liveOperation: {
status: "not_run",
operationId: "not_observed",
@@ -683,7 +749,8 @@ async function main() {
report.readOnlySupplementalEvidence = await collectReadOnlySupplementalEvidence();
const ingress = await observeDevIngress();
report.localRuntimeObservations = collectLocalRuntimeObservations();
report.d601Observability = collectD601RunnerObservability(ingress);
report.localRuntimeObservations = collectLocalRuntimeObservations(report.d601Observability);
report.liveChecks.push({
id: "dev-ingress-health",
status: ingress.accepted ? "pass" : "blocked",
@@ -727,21 +794,22 @@ async function main() {
if (missing.length > 0) {
addNotRunM3Checks(report, "Stopped before M3 direct checks because service target URLs were not provided.");
report.blockers.push({
type: "runtime_blocker",
type: "observability_blocker",
scope: "m3-service-discovery",
status: "open",
classification: "hardware loop",
summary: `DEV ingress is reachable, but M3 smoke lacks direct DEV service URLs: ${missing.join(", ")}.`
classification: "runner permission/mount gap",
sourceIssue: "pikasTech/HWLAB#46",
summary: `DEV ingress is reachable on frozen :16667, but this runner cannot use ${runnerK3sKubeconfigPath} as readable kubeconfig to discover direct M3 service URLs (${missing.join(", ")}); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.`
});
report.summary = {
status: "blocked",
classification: "hardware loop",
classification: "runner permission/mount gap",
observedAt,
result: "DEV ingress was reachable, but live M3 simulator and patch-panel targets were not discoverable."
};
await writeReport(report, reportPath);
console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`);
console.log("[dev-m3-smoke] blocker=runtime_blocker scope=m3-service-discovery");
console.log("[dev-m3-smoke] blocker=observability_blocker scope=m3-service-discovery");
return;
}
scripts/src/d601-k3s-readonly-observability.mjs
+186 -10
View File
@@ -7,6 +7,8 @@ import path from "node:path";
import { fileURLToPath } from "node:url";
import { promisify } from "node:util";
import { DEV_ENDPOINT, DEV_FRONTEND_ENDPOINT } from "../../internal/protocol/index.mjs";
const execFileAsync = promisify(execFile);
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
const namespace = "hwlab-dev";
@@ -26,6 +28,12 @@ const kubeconfigCandidatePaths = [
"/etc/rancher/k3s/k3s.yaml",
"/var/lib/rancher/k3s/server/cred/admin.kubeconfig"
];
const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
const publicEndpointProbeTargets = [
{ id: "frontend-16666", url: `${DEV_FRONTEND_ENDPOINT}/` },
{ id: "api-health-16667", url: `${DEV_ENDPOINT}/health` },
{ id: "api-live-16667", url: `${DEV_ENDPOINT}/health/live` }
];
const sshEnvVars = [
"HWLAB_D601_SSH_HOST",
"HWLAB_D601_SSH_TARGET",
@@ -106,6 +114,10 @@ function commandText(command, args) {
return [command, ...args].join(" ");
}
function runnerKubeconfigCommandText(args) {
return [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args].join(" ");
}
async function run(command, args = [], options = {}) {
try {
const result = await execFileAsync(command, args, {
@@ -176,6 +188,89 @@ async function fileProbe(filePath) {
}
}
async function collectRunnerKubeconfigProbe(timeoutMs) {
const metadata = await fileProbe(runnerK3sKubeconfigPath);
const args = ["-n", namespace, "get", "pods", "-o", "name"];
const probe = await run("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], {
timeoutMs
});
const stdoutLines = probe.stdout
? probe.stdout.split("\n").map((line) => line.trim()).filter(Boolean)
: [];
return {
path: runnerK3sKubeconfigPath,
mode: "read-only",
resource: "pods",
command: runnerKubeconfigCommandText(args),
runnerKubeconfigReadable: metadata.readable === true,
runnerKubeconfigProbeOk: probe.ok,
runnerKubeconfigProbeExitCode: probe.exitCode,
runnerKubeconfigProbeStderr: oneLine(probe.stderr || probe.error || "") || "empty",
stdoutSummary: probe.ok ? `${stdoutLines.length} pod name(s) observed` : undefined,
classification: metadata.readable
? probe.ok
? "readable"
: "readonly_observability_gap"
: "runner_permission_mount_gap",
sourceIssue: issue,
secretValuesRead: false,
secretResourcesRead: false,
note: "This probe forces KUBECONFIG=/etc/rancher/k3s/k3s.yaml for a read-only kubectl get pods command and records only exit code plus redacted stderr/stdout summary. A successful kubectl exit code does not make the kubeconfig file readable; kubectl may use an alternate in-cluster fallback."
};
}
async function probePublicEndpoint(target, timeoutMs) {
const controller = new AbortController();
const timeout = setTimeout(() => controller.abort(), timeoutMs);
try {
const response = await fetch(target.url, {
method: "GET",
signal: controller.signal,
redirect: "manual"
});
const body = await response.text();
let json = null;
try {
json = body ? JSON.parse(body) : null;
} catch {}
return {
id: target.id,
url: target.url,
reached: true,
ok: response.status >= 200 && response.status < 400,
status: response.status,
serviceId: json?.serviceId,
environment: json?.environment,
bodySummary: json
? `json serviceId=${json.serviceId ?? "unknown"} status=${json.status ?? "unknown"}`
: oneLine(body.slice(0, 160))
};
} catch (error) {
return {
id: target.id,
url: target.url,
reached: false,
ok: false,
error: error instanceof Error ? error.message : String(error)
};
} finally {
clearTimeout(timeout);
}
}
async function collectPublicEndpoints(timeoutMs) {
const probes = await Promise.all(
publicEndpointProbeTargets.map((target) => probePublicEndpoint(target, timeoutMs))
);
return {
activeFrontendEndpoint: DEV_FRONTEND_ENDPOINT,
activeApiEndpoint: DEV_ENDPOINT,
d601PublicEndpointsReachable: probes.every((probe) => probe.ok === true),
probes
};
}
async function collectBinary(name, versionArgs) {
const executablePath = await findExecutable(name);
const result = {
@@ -391,6 +486,51 @@ async function collectClusterObservability(binaries, timeoutMs) {
};
}
function probeLooksLikeClusterServerUnavailable(probe) {
const text = oneLine(`${probe?.stderr ?? ""} ${probe?.error ?? ""}`).toLowerCase();
return [
"connection refused",
"no route to host",
"i/o timeout",
"context deadline exceeded",
"unable to connect to the server",
"the server is currently unable to handle the request"
].some((needle) => text.includes(needle));
}
function inferD601K3sUnavailable({ cluster, publicEndpoints }) {
if (cluster.readable === true || publicEndpoints.d601PublicEndpointsReachable === true) {
return false;
}
return (cluster.executors ?? []).some((executor) =>
Object.values(executor.probes ?? {}).some((probe) => {
if (probe && typeof probe === "object" && !Array.isArray(probe) && Object.hasOwn(probe, "ok")) {
return probeLooksLikeClusterServerUnavailable(probe);
}
return Object.values(probe ?? {}).some((nested) => probeLooksLikeClusterServerUnavailable(nested));
})
);
}
function d601StatusInference({ runnerKubeconfig, publicEndpoints, d601K3sUnavailable }) {
return {
runnerKubeconfigReadable: runnerKubeconfig.runnerKubeconfigReadable,
runnerKubeconfigProbeOk: runnerKubeconfig.runnerKubeconfigProbeOk,
runnerKubeconfigProbeExitCode: runnerKubeconfig.runnerKubeconfigProbeExitCode,
runnerKubeconfigProbeStderr: runnerKubeconfig.runnerKubeconfigProbeStderr,
d601PublicEndpointsReachable: publicEndpoints.d601PublicEndpointsReachable,
d601K3sUnavailable,
conclusion: d601K3sUnavailable
? "Direct read-only probes indicate D601 k3s may be unavailable."
: "Runner kubeconfig readability is an observability signal only; it is not used to infer D601 k3s or public DEV endpoint availability.",
blockerClassification: runnerKubeconfig.runnerKubeconfigReadable
? "none"
: "runner_permission_mount_gap",
sourceIssue: issue
};
}
function addBlocker(blockers, blocker) {
assert.ok(blockerTypes.has(blocker.type), `unknown blocker type ${blocker.type}`);
const key = `${blocker.type}:${blocker.scope}`;
@@ -399,7 +539,7 @@ function addBlocker(blockers, blocker) {
}
}
function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {
function buildBlockers({ binaries, kubeconfig, runnerKubeconfig, sshBridge, cluster }) {
const blockers = [];
const hasDirectClient = binaries.kubectl.available || binaries.k3s.available;
if (!hasDirectClient) {
@@ -412,12 +552,24 @@ function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {
}
if (!kubeconfig.some((item) => item.readable)) {
addBlocker(blockers, {
type: "environment_blocker",
scope: "d601-kubeconfig-path",
summary: "No readable KUBECONFIG/default k3s kubeconfig path was found by metadata checks.",
type: "observability_blocker",
scope: "runner-kubeconfig-readonly-gap",
classification: "runner_permission_mount_gap",
sourceIssue: issue,
summary: "No readable KUBECONFIG/default k3s kubeconfig path was found by metadata checks; this is a runner permission/mount gap and does not prove D601 k3s is unavailable.",
nextTask: "Mount a read-only kubeconfig for hwlab-dev, or document the approved k3s local kubeconfig path without exposing token material."
});
}
if (!runnerKubeconfig.runnerKubeconfigReadable) {
addBlocker(blockers, {
type: "observability_blocker",
scope: "runner-kubeconfig-readonly-gap",
classification: "runner_permission_mount_gap",
sourceIssue: issue,
summary: "The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately.",
nextTask: "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report."
});
}
if (!sshBridge.configured && !cluster.readable) {
addBlocker(blockers, {
type: "environment_blocker",
@@ -431,7 +583,9 @@ function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {
addBlocker(blockers, {
type: "observability_blocker",
scope: "hwlab-dev-readonly-rbac",
summary: `Read-only cluster probes did not return pods, services, and ConfigMaps for ${namespace}; attempted executors: ${attempted}.`,
classification: "readonly_observability_gap",
sourceIssue: issue,
summary: `Read-only runner probes did not return pods, services, and ConfigMaps for ${namespace}; attempted executors: ${attempted}. This is an observability gap unless independent evidence proves D601 k3s unavailable.`,
nextTask: "Grant or mount read-only access that can get namespace, pods, services, and ConfigMaps in hwlab-dev, then rerun this preflight."
});
}
@@ -440,7 +594,7 @@ function buildBlockers({ binaries, kubeconfig, sshBridge, cluster }) {
function runnerImageFollowUp(blockers) {
const scopes = new Set(blockers.map((blocker) => blocker.scope));
const needed = scopes.has("d601-k3s-client-binary") || scopes.has("d601-kubeconfig-path");
const needed = scopes.has("d601-k3s-client-binary") || scopes.has("runner-kubeconfig-readonly-gap");
return {
needed,
suggestedUniDeskIssue: needed
@@ -457,13 +611,21 @@ async function writeReport(report, reportPath) {
async function buildReport(args) {
const binaries = await collectBinaries();
const [kubeconfig, sshBridge, cluster] = await Promise.all([
const [kubeconfig, runnerKubeconfig, sshBridge, cluster, publicEndpoints] = await Promise.all([
collectKubeconfigPaths(),
collectRunnerKubeconfigProbe(args.timeoutMs),
collectSshBridge(binaries),
collectClusterObservability(binaries, args.timeoutMs)
collectClusterObservability(binaries, args.timeoutMs),
collectPublicEndpoints(args.timeoutMs)
]);
const blockers = buildBlockers({ binaries, kubeconfig, sshBridge, cluster });
const d601K3sUnavailable = inferD601K3sUnavailable({ cluster, publicEndpoints });
const blockers = buildBlockers({ binaries, kubeconfig, runnerKubeconfig, sshBridge, cluster });
const conclusion = cluster.readable && blockers.length === 0 ? "ready" : "blocked";
const statusInference = d601StatusInference({
runnerKubeconfig,
publicEndpoints,
d601K3sUnavailable
});
return {
$schema: "https://hwlab.pikastech.local/schemas/d601-k3s-readonly-observability-report.schema.json",
$id: "https://hwlab.pikastech.local/reports/d601-k3s-readonly-observability.json",
@@ -497,15 +659,24 @@ async function buildReport(args) {
"node scripts/d601-k3s-readonly-observability.mjs"
],
conclusion,
runnerKubeconfigReadable: statusInference.runnerKubeconfigReadable,
runnerKubeconfigProbeOk: statusInference.runnerKubeconfigProbeOk,
runnerKubeconfigProbeExitCode: statusInference.runnerKubeconfigProbeExitCode,
runnerKubeconfigProbeStderr: statusInference.runnerKubeconfigProbeStderr,
d601PublicEndpointsReachable: statusInference.d601PublicEndpointsReachable,
d601K3sUnavailable: statusInference.d601K3sUnavailable,
statusInference,
environment: {
binaries,
kubeconfig,
runnerKubeconfig,
sshBridge
},
publicEndpoints,
cluster,
blockers,
runnerImageFollowUp: runnerImageFollowUp(blockers),
notes: "This report uses command availability checks, file metadata checks, kubectl/k3s kubectl read-only get probes, and auth can-i checks only. It does not print kubeconfig contents, token material, Secret resources, ConfigMap values, SSH config contents, or private keys."
notes: "This report uses command availability checks, file metadata checks, public DEV endpoint GET probes, kubectl/k3s kubectl read-only get probes, and auth can-i checks only. It does not print kubeconfig contents, token material, Secret resources, ConfigMap values, SSH config contents, or private keys. A failed runner /etc/rancher/k3s/k3s.yaml probe is classified as a runner permission/mount or read-only observability gap and is not evidence that D601 k3s is globally unavailable."
};
}
@@ -517,6 +688,11 @@ function printSummary(args, report) {
namespace: report.namespace,
report: args.writeReport ? args.reportPath : null,
readable: report.cluster.readable,
runnerKubeconfigReadable: report.runnerKubeconfigReadable,
runnerKubeconfigProbeOk: report.runnerKubeconfigProbeOk,
runnerKubeconfigProbeExitCode: report.runnerKubeconfigProbeExitCode,
d601PublicEndpointsReachable: report.d601PublicEndpointsReachable,
d601K3sUnavailable: report.d601K3sUnavailable,
attemptedExecutors: report.cluster.attemptedExecutors,
blockers: report.blockers.map((blocker) => ({
type: blocker.type,
scripts/src/dev-evidence-blocker-aggregator.mjs
+82 -25
View File
@@ -263,13 +263,19 @@ function applyPriority(blocker) {
};
}
if (blocker.scope === "kubectl" || blocker.scope === "d601-k3s" || blocker.scope.startsWith("d601-")) {
if (
blocker.scope === "kubectl" ||
blocker.scope === "d601-k3s" ||
blocker.scope === "runner-kubeconfig-readonly-gap" ||
blocker.scope === "m3-service-discovery" ||
blocker.scope.startsWith("d601-")
) {
return {
...blocker,
priority: "P1",
unblockOrder: 4,
unblocks: [issue(34), issue(33), issue(36), issue(38)],
rationale: "D601 hwlab-dev cluster state must be observable read-only before any live apply or M3/M4/M5 acceptance."
unblocks: [issue(34), issue(33), issue(36), issue(38), issue(46), issue(64)],
rationale: "Runner read-only observability must be repaired without treating the runner gap as proof that D601 k3s or public DEV endpoints are unavailable."
};
}
@@ -279,7 +285,7 @@ function applyPriority(blocker) {
priority: "P1",
unblockOrder: 4,
unblocks: [issue(34), issue(33), issue(36), issue(38), issue(39)],
rationale: "D601 has a client path, but read-only hwlab-dev pods/services/configmaps must be observable before live acceptance evidence can be trusted."
rationale: "D601 has a client path, but read-only hwlab-dev pods/services/configmaps must be observable before live acceptance evidence can be trusted; this is not evidence that D601 is globally offline."
};
}
@@ -462,17 +468,13 @@ function collectM2Evidence(reports) {
}),
reportEvidence({
milestone: "M2",
level: "BLOCKED",
level: d601.cluster?.readable === true || d601.d601PublicEndpointsReachable === true ? "DEV-LIVE" : "BLOCKED",
report: d601,
status: d601.conclusion,
category: "d601-observability",
evidence: [
`kubectl=${d601.environment?.binaries?.kubectl?.available === true ? "available" : "missing"}`,
`k3s=${d601.environment?.binaries?.k3s?.available === true ? "available" : "missing"}`,
`clusterReadable=${d601.cluster?.readable === true ? "yes" : "no"}`
],
evidence: d601ObservationSummary(d601),
commands: d601.validationCommands,
summary: "D601 k3s observability is blocked; cluster state has not been read."
summary: d601ObservabilitySummary(d601)
})
];
}
@@ -522,7 +524,10 @@ function collectM3Evidence(reports) {
function m3TrustedLoopSummary(m3Report) {
const operationSummary = m3Report.liveOperation?.summary ?? "No live M3 hardware operation was observed.";
const blockerSummary = m3Report.blockers?.find((blocker) => blocker.scope === "m3-hardware-loop-runtime")?.summary;
const blockerSummary = m3Report.blockers?.find((blocker) =>
blocker.scope === "m3-hardware-loop-runtime" ||
blocker.scope === "m3-service-discovery"
)?.summary;
if (statusIsPass(m3Report.liveOperation?.status) || !blockerSummary) {
return operationSummary;
}
@@ -658,6 +663,32 @@ function publicEndpointEvidence(m2Report) {
});
}
function d601ObservationSummary(d601) {
return [
`runnerKubeconfigReadable=${d601.runnerKubeconfigReadable === true}`,
`runnerKubeconfigProbeExitCode=${d601.runnerKubeconfigProbeExitCode ?? "unknown"}`,
`runnerKubeconfigProbeStderr=${d601.runnerKubeconfigProbeStderr || "empty"}`,
`d601PublicEndpointsReachable=${d601.d601PublicEndpointsReachable === true}`,
`d601K3sUnavailable=${d601.d601K3sUnavailable === true}`
];
}
function d601ObservabilitySummary(d601) {
if (d601.d601PublicEndpointsReachable === true && d601.runnerKubeconfigReadable === false) {
const clusterNote = d601.cluster?.readable === true
? " Alternate read-only cluster probes are readable."
: "";
return `D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline.${clusterNote} ${d601ObservationSummary(d601).join(", ")}.`;
}
if (d601.cluster?.readable === true) {
return `D601 hwlab-dev cluster is readable from this runner; ${d601ObservationSummary(d601).join(", ")}.`;
}
if (d601.d601K3sUnavailable === true) {
return `Direct read-only probes indicate D601 k3s may be unavailable; ${d601ObservationSummary(d601).join(", ")}.`;
}
return `D601 read-only observability is blocked without proving D601 unavailable; ${d601ObservationSummary(d601).join(", ")}.`;
}
function isStaleLegacyIngressBlocker(blocker, sourceReport, reports) {
const text = `${blocker.scope ?? ""} ${blocker.summary ?? ""}`;
const staleLegacyPort = deprecatedLegacyPublicEndpoints.some((endpoint) => text.includes(endpoint));
@@ -792,8 +823,6 @@ function buildDoD(reports, milestones, blockers) {
const preflight = reports.devPreflight;
const artifactIdentity = preflight.artifactIdentity;
const d601 = reports.d601Observability;
const d601HasClient = d601.environment?.binaries?.kubectl?.available === true ||
d601.environment?.binaries?.k3s?.available === true;
const edgeLive = hasCurrentLiveEdgeEvidence(reports.devEdgeHealth);
const artifactCurrent = artifactIdentity.publishVerified === true &&
artifactIdentity.targetCoverage?.covered !== false &&
@@ -827,13 +856,9 @@ function buildDoD(reports, milestones, blockers) {
},
{
id: "d601-k3s-observability",
status: d601.cluster?.readable ? "pass" : "blocked",
evidenceLevel: d601.cluster?.readable ? "DEV-LIVE" : "BLOCKED",
summary: d601.cluster?.readable
? "D601 hwlab-dev cluster was readable."
: d601HasClient
? "D601 kubectl/k3s clients are present, but read-only hwlab-dev pods/services/configmaps probes are still blocked."
: "D601 runner lacks kubectl/k3s/kubeconfig observability for hwlab-dev."
status: d601.runnerKubeconfigReadable === false ? "blocked" : d601.cluster?.readable ? "pass" : "blocked",
evidenceLevel: d601.cluster?.readable || d601.d601PublicEndpointsReachable ? "DEV-LIVE" : "BLOCKED",
summary: d601ObservabilitySummary(d601)
},
{
id: "dev-edge-frp-16667",
@@ -928,6 +953,21 @@ function buildCurrentDevLayering(reports, blockers) {
evidence: reports.devM5Gate.devPreconditions?.evidence?.filter((line) => line.includes("/health/live") || line.includes("DB")) ?? [],
nextRequired: "Provide live DB connection evidence through redacted health output; route reachability alone is insufficient."
},
d601RunnerObservability: {
label: "D601 runner observability",
status: reports.d601Observability.runnerKubeconfigReadable === false
? "blocked"
: reports.d601Observability.cluster?.readable === true
? "pass"
: "blocked",
evidenceLevel: reports.d601Observability.cluster?.readable === true ||
reports.d601Observability.d601PublicEndpointsReachable === true
? "DEV-LIVE"
: "BLOCKED",
summary: d601ObservabilitySummary(reports.d601Observability),
evidence: d601ObservationSummary(reports.d601Observability),
nextRequired: "Treat #46 runner kubeconfig/readonly gaps separately from D601 service health; rerun read-only observability after the mount or permission path is repaired."
},
m3HardwareTrustedLoop: {
label: "M3 hardware trusted loop",
status: m3Live ? "pass" : "blocked",
@@ -968,21 +1008,36 @@ function buildMilestoneBlockerClassification(reports) {
const cloudDb = cloudApiDbStatus(reports);
const m3Live = statusIsPass(reports.devM3Hardware.liveOperation?.status);
const m4Live = statusIsPass(reports.devM4Agent.livePreflight?.status);
const m3ServiceDiscoveryBlocked = reports.devM3Hardware.blockers?.some((blocker) =>
blocker.scope === "m3-service-discovery"
) === true;
const m3BlockerClass = m3Live
? "cleared"
: m3ServiceDiscoveryBlocked
? "runner-readonly-observability-gap"
: "hardware-loop-runtime";
return [
{
milestone: "M3",
status: m3Live ? "pass" : "blocked",
currentLevel: m3Live ? "DEV-LIVE" : "BLOCKED",
blockerClass: m3Live ? "cleared" : "hardware-loop-runtime",
dependency: "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.",
blockerClass: m3BlockerClass,
dependency: m3ServiceDiscoveryBlocked
? "Runner read-only service discovery must be repaired before the real DEV trusted loop can be observed; this does not mean D601/k3s is globally unavailable."
: "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.",
evidence: [
`operationId=${reports.devM3Hardware.liveOperation?.operationId ?? "not_observed"}`,
`traceId=${reports.devM3Hardware.liveOperation?.traceId ?? "not_observed"}`,
`auditId=${reports.devM3Hardware.liveOperation?.auditId ?? "not_observed"}`,
`evidenceId=${reports.devM3Hardware.liveOperation?.evidenceId ?? "not_observed"}`
`evidenceId=${reports.devM3Hardware.liveOperation?.evidenceId ?? "not_observed"}`,
`runnerKubeconfigReadable=${reports.devM3Hardware.d601Observability?.runnerKubeconfigReadable === true}`,
`d601PublicEndpointsReachable=${reports.devM3Hardware.d601Observability?.d601PublicEndpointsReachable === true}`,
`d601K3sUnavailable=${reports.devM3Hardware.d601Observability?.d601K3sUnavailable === true}`
],
nextRequired: "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.",
nextRequired: m3ServiceDiscoveryBlocked
? "Repair the #46 runner readonly kubeconfig/service-discovery gap, then run the bounded DEV M3 live smoke only when direct simulator and patch-panel targets are discoverable."
: "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.",
nonPromotionReason: "Public frontend, route, and artifact evidence do not prove the required hardware loop."
},
{
@@ -1071,6 +1126,7 @@ function fallbackAction(scope) {
if (scope.includes("cloud-api-db")) return "Configure DEV cloud-api DB env readiness and rerun health/preflight without exposing secrets.";
if (scope === "db-live") return "Repair DEV cloud-api DB live readiness, then rerun the read-only health and M4 preflight reports without exposing secret values.";
if (scope === "m3-hardware-loop-runtime") return "Prove the real DEV M3 trusted loop res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, trace, audit, and evidence identifiers.";
if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery") return "Repair the #46 runner kubeconfig mount/permission or document the approved alternate read-only KUBECONFIG/service-discovery path; do not classify this as D601 global offline.";
if (scope.includes("kubectl") || scope.includes("k3s")) return "Provide read-only kubectl/kubeconfig observability for hwlab-dev.";
return "Resolve the blocker and attach source/local/dry-run/DEV-live evidence at the correct level.";
}
@@ -1083,6 +1139,7 @@ function evidenceRequiredFor(scope) {
if (scope.includes("cloud-api-db")) return "Cloud API health/live output showing DB env ready and redacted secret references, without secret material.";
if (scope === "db-live") return "Cloud API /health/live output with ready=true, connected=true, liveDbEvidence=true, and redacted secret references.";
if (scope === "m3-hardware-loop-runtime") return "Operation, trace, audit, and evidence IDs from the real DEV DO1 -> patch-panel -> DI1 trusted loop.";
if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery") return "Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write.";
if (scope.includes("edge") || scope.includes("ingress") || scope.includes("frp")) return "Read-only DEV route observation for :16667/frp/edge/router with HWLAB service identity and artifact identity.";
return "A committed report with the exact evidence level and command used.";
}
scripts/validate-dev-gate-report.mjs
+48
View File
@@ -1271,6 +1271,13 @@ async function validateDevM3Report(report, label) {
assertObject(report.runtimeTarget, `${label}.runtimeTarget`);
assert.equal(report.runtimeTarget.endpoint, "http://74.48.78.17:16667", `${label}.runtimeTarget.endpoint`);
if (Object.hasOwn(report.runtimeTarget, "frontendEndpoint")) {
assert.equal(
report.runtimeTarget.frontendEndpoint,
"http://74.48.78.17:16666",
`${label}.runtimeTarget.frontendEndpoint`
);
}
assert.equal(report.runtimeTarget.namespace, "hwlab-dev", `${label}.runtimeTarget.namespace`);
assert.equal(report.runtimeTarget.environment, "dev", `${label}.runtimeTarget.environment`);
assert.equal(report.runtimeTarget.requiredBoxSimulators, 2, `${label}.runtimeTarget.requiredBoxSimulators`);
@@ -1322,6 +1329,47 @@ async function validateDevM3Report(report, label) {
}
}
if (Object.hasOwn(report, "d601Observability")) {
assertObject(report.d601Observability, `${label}.d601Observability`);
for (const field of [
"runnerKubeconfigReadable",
"runnerKubeconfigProbeExitCode",
"runnerKubeconfigProbeStderr",
"d601PublicEndpointsReachable",
"d601K3sUnavailable",
"classification",
"sourceIssue",
"inferenceRule"
]) {
assert.ok(Object.hasOwn(report.d601Observability, field), `${label}.d601Observability missing ${field}`);
}
assertBoolean(report.d601Observability.runnerKubeconfigReadable, `${label}.d601Observability.runnerKubeconfigReadable`);
if (Object.hasOwn(report.d601Observability, "runnerKubeconfigProbeOk")) {
assertBoolean(report.d601Observability.runnerKubeconfigProbeOk, `${label}.d601Observability.runnerKubeconfigProbeOk`);
}
assert.ok(
Number.isInteger(report.d601Observability.runnerKubeconfigProbeExitCode) ||
report.d601Observability.runnerKubeconfigProbeExitCode === null,
`${label}.d601Observability.runnerKubeconfigProbeExitCode`
);
assertString(report.d601Observability.runnerKubeconfigProbeStderr, `${label}.d601Observability.runnerKubeconfigProbeStderr`);
assertBoolean(report.d601Observability.d601PublicEndpointsReachable, `${label}.d601Observability.d601PublicEndpointsReachable`);
assertBoolean(report.d601Observability.d601K3sUnavailable, `${label}.d601Observability.d601K3sUnavailable`);
if (
report.d601Observability.runnerKubeconfigReadable === false &&
report.d601Observability.d601PublicEndpointsReachable === true
) {
assert.equal(
report.d601Observability.d601K3sUnavailable,
false,
`${label}.d601Observability runner gap must not imply D601 k3s unavailable`
);
}
assertString(report.d601Observability.classification, `${label}.d601Observability.classification`);
assertString(report.d601Observability.sourceIssue, `${label}.d601Observability.sourceIssue`);
assertString(report.d601Observability.inferenceRule, `${label}.d601Observability.inferenceRule`);
}
if (Object.hasOwn(report, "readOnlySupplementalEvidence")) {
assertArray(report.readOnlySupplementalEvidence, `${label}.readOnlySupplementalEvidence`);
for (const [index, evidence] of report.readOnlySupplementalEvidence.entries()) {