diff --git a/reports/dev-gate/dev-m3-hardware-loop.json b/reports/dev-gate/dev-m3-hardware-loop.json index 3372552c..ce13d93a 100644 --- a/reports/dev-gate/dev-m3-hardware-loop.json +++ b/reports/dev-gate/dev-m3-hardware-loop.json @@ -4,7 +4,7 @@ "reportVersion": "v1", "issue": "pikasTech/HWLAB#38", "taskId": "dev-m3-hardware-loop", - "commitId": "bf47a95a97a9", + "commitId": "c7de4745f491", "acceptanceLevel": "dev_m3_hardware_loop", "devOnly": true, "prodDisabled": true, @@ -76,58 +76,67 @@ "status": "pass", "summary": "DEV ingress returned HWLAB identity at /health/live.", "evidence": [ - "{\"url\":\"http://74.48.78.17:16667/health/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:42.977Z\",\"observedAt\":\"2026-05-22T12:36:43.545Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"status\":\"degraded\",\"service\":{\"id\":\"hwlab-cloud-api\",\"role\":\"cloud-api\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T12:36:43.343Z\",\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"configured_without_live_connection\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"redacted\":true}],\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"valuesRedacted\":true,\"liveDbEvidence\":false},\"evidence\":\"env_presence_only_no_live_db\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}", - "{\"url\":\"http://74.48.78.17:16667/health\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:43.545Z\",\"observedAt\":\"2026-05-22T12:36:44.191Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-edge-proxy\",\"environment\":\"dev\",\"status\":\"ok\",\"service\":{\"id\":\"hwlab-edge-proxy\",\"role\":\"public-dev-ingress\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada6860653b27269b0a57991184118cbf4b1\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"ghcr.io/pikastech/hwlab-edge-proxy:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T12:36:43.989Z\",\"details\":{\"upstream\":\"http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667\",\"mode\":\"dev-edge-proxy\"}}}", - "{\"url\":\"http://74.48.78.17:16667/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:44.192Z\",\"observedAt\":\"2026-05-22T12:36:44.570Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"status\":\"live\"}}", - "{\"url\":\"http://74.48.78.17:16667/v1\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T12:36:44.570Z\",\"observedAt\":\"2026-05-22T12:36:45.204Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"adapter\":\"rest\",\"status\":\"degraded\",\"rpcBridge\":\"POST /v1/rpc/{method}\",\"methods\":[\"system.health\",\"cloud.adapter.describe\",\"gateway.session.register\",\"box.resource.register\",\"box.capability.report\",\"hardware.operation.request\",\"hardware.invoke.shell\",\"audit.event.write\",\"audit.event.query\",\"evidence.record.write\",\"evidence.record.query\"],\"auditFields\":[\"requestId\",\"actor\",\"source\",\"operation\",\"target\",\"result\",\"timestamp\"],\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"configured_without_live_connection\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"redacted\":true}],\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"valuesRedacted\":true,\"liveDbEvidence\":false},\"evidence\":\"env_presence_only_no_live_db\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}" + "{\"url\":\"http://74.48.78.17:16667/health/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T13:44:36.427Z\",\"observedAt\":\"2026-05-22T13:44:37.010Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"status\":\"degraded\",\"service\":{\"id\":\"hwlab-cloud-api\",\"role\":\"cloud-api\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"c7de474\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"127.0.0.1:5000/hwlab/hwlab-cloud-api:c7de474\",\"tag\":\"c7de474\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T13:44:36.806Z\",\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"liveConnected\":false,\"liveDbEvidence\":false,\"connectionChecked\":true,\"connectionAttempted\":true,\"connectionResult\":\"dns_error\",\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"live_connection_blocked\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"envInjected\":true,\"secretPresent\":\"not_observed_by_runtime\",\"secretKeyPresent\":\"not_observed_by_runtime\",\"redacted\":true}],\"connection\":{\"attempted\":true,\"networkAttempted\":true,\"probeType\":\"tcp-connect\",\"endpointRedacted\":true,\"valueRedacted\":true,\"timeoutMs\":1200,\"durationMs\":1,\"missingEnv\":[],\"result\":\"dns_error\",\"classification\":\"dns_resolution_failed\",\"errorCode\":\"ENOTFOUND\"},\"redaction\":{\"valuesRedacted\":true,\"endpointRedacted\":true,\"valueRedacted\":true,\"secretMaterialRead\":false,\"secretRefsOnly\":true},\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"secretMaterialRead\":false,\"valuesRedacted\":true,\"endpointRedacted\":true,\"liveDbEvidence\":false,\"liveDbConnectedEvidence\":false},\"blocker\":\"DB endpoint DNS resolution failed from the cloud-api runtime\",\"evidence\":\"live_db_tcp_connection_blocked\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}", + "{\"url\":\"http://74.48.78.17:16667/health\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T13:44:37.011Z\",\"observedAt\":\"2026-05-22T13:44:37.943Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-edge-proxy\",\"environment\":\"dev\",\"status\":\"ok\",\"service\":{\"id\":\"hwlab-edge-proxy\",\"role\":\"public-dev-ingress\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada6860653b27269b0a57991184118cbf4b1\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"ghcr.io/pikastech/hwlab-edge-proxy:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T13:44:37.190Z\",\"details\":{\"upstream\":\"http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667\",\"mode\":\"dev-edge-proxy\"}}}", + "{\"url\":\"http://74.48.78.17:16667/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T13:44:37.943Z\",\"observedAt\":\"2026-05-22T13:44:38.325Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"status\":\"live\"}}", + "{\"url\":\"http://74.48.78.17:16667/v1\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T13:44:38.325Z\",\"observedAt\":\"2026-05-22T13:44:38.710Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-api\",\"adapter\":\"rest\",\"status\":\"degraded\",\"rpcBridge\":\"POST /v1/rpc/{method}\",\"codeAgent\":{\"endpoint\":\"POST /v1/agent/chat\",\"status\":\"available\",\"schema\":[\"conversationId\",\"sessionId\",\"messageId\",\"status\",\"createdAt\",\"updatedAt\",\"traceId\",\"provider\",\"model\",\"backend\",\"error.message\"]},\"methods\":[\"system.health\",\"cloud.adapter.describe\",\"gateway.session.register\",\"box.resource.register\",\"box.capability.report\",\"hardware.operation.request\",\"hardware.invoke.shell\",\"audit.event.write\",\"audit.event.query\",\"evidence.record.write\",\"evidence.record.query\"],\"auditFields\":[\"requestId\",\"actor\",\"source\",\"operation\",\"target\",\"result\",\"timestamp\"],\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"liveConnected\":false,\"liveDbEvidence\":false,\"connectionChecked\":true,\"connectionAttempted\":true,\"connectionResult\":\"dns_error\",\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"live_connection_blocked\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"envInjected\":true,\"secretPresent\":\"not_observed_by_runtime\",\"secretKeyPresent\":\"not_observed_by_runtime\",\"redacted\":true}],\"connection\":{\"attempted\":true,\"networkAttempted\":true,\"probeType\":\"tcp-connect\",\"endpointRedacted\":true,\"valueRedacted\":true,\"timeoutMs\":1200,\"durationMs\":2,\"missingEnv\":[],\"result\":\"dns_error\",\"classification\":\"dns_resolution_failed\",\"errorCode\":\"ENOTFOUND\"},\"redaction\":{\"valuesRedacted\":true,\"endpointRedacted\":true,\"valueRedacted\":true,\"secretMaterialRead\":false,\"secretRefsOnly\":true},\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"secretMaterialRead\":false,\"valuesRedacted\":true,\"endpointRedacted\":true,\"liveDbEvidence\":false,\"liveDbConnectedEvidence\":false},\"blocker\":\"DB endpoint DNS resolution failed from the cloud-api runtime\",\"evidence\":\"live_db_tcp_connection_blocked\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}}}", + "{\"url\":\"http://74.48.78.17:16666/health/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T13:44:38.710Z\",\"observedAt\":\"2026-05-22T13:44:39.934Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-web\",\"environment\":\"dev\",\"status\":\"ok\",\"artifactKind\":\"cloud-web\",\"revision\":\"c7de474\"}}", + "{\"url\":\"http://74.48.78.17:16666/health\",\"method\":\"GET\",\"startedAt\":\"2026-05-22T13:44:39.934Z\",\"observedAt\":\"2026-05-22T13:44:40.314Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"serviceId\":\"hwlab-cloud-web\",\"environment\":\"dev\",\"status\":\"ok\",\"artifactKind\":\"cloud-web\",\"revision\":\"c7de474\"}}", + "{\"url\":\"http://74.48.78.17:16667/json-rpc\",\"method\":\"POST\",\"startedAt\":\"2026-05-22T13:44:40.314Z\",\"observedAt\":\"2026-05-22T13:44:40.709Z\",\"reached\":true,\"ok\":true,\"status\":200,\"statusText\":\"OK\",\"json\":{\"jsonrpc\":\"2.0\",\"id\":\"req_dev_m3_a6ad2ece-8c10-4938-bdbb-717ae63948bb\",\"result\":{\"status\":\"degraded\",\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"liveConnected\":false,\"liveDbEvidence\":false,\"connectionChecked\":true,\"connectionAttempted\":true,\"connectionResult\":\"dns_error\",\"configReady\":true,\"ready\":false,\"status\":\"degraded\",\"mode\":\"live_connection_blocked\",\"fields\":[{\"name\":\"HWLAB_CLOUD_DB_URL\",\"present\":true,\"redacted\":true,\"source\":\"k8s-secret-ref\",\"required\":true},{\"name\":\"HWLAB_CLOUD_DB_SSL_MODE\",\"present\":true,\"redacted\":false,\"source\":\"runtime-env\",\"required\":true}],\"missingEnv\":[],\"secretRefs\":[{\"env\":\"HWLAB_CLOUD_DB_URL\",\"secretName\":\"hwlab-cloud-api-dev-db\",\"secretKey\":\"database-url\",\"present\":true,\"envInjected\":true,\"secretPresent\":\"not_observed_by_runtime\",\"secretKeyPresent\":\"not_observed_by_runtime\",\"redacted\":true}],\"connection\":{\"attempted\":true,\"networkAttempted\":true,\"probeType\":\"tcp-connect\",\"endpointRedacted\":true,\"valueRedacted\":true,\"timeoutMs\":1200,\"durationMs\":1,\"missingEnv\":[],\"result\":\"dns_error\",\"classification\":\"dns_resolution_failed\",\"errorCode\":\"ENOTFOUND\"},\"redaction\":{\"valuesRedacted\":true,\"endpointRedacted\":true,\"valueRedacted\":true,\"secretMaterialRead\":false,\"secretRefsOnly\":true},\"safety\":{\"devOnly\":true,\"prodAllowed\":false,\"secretsRead\":false,\"secretMaterialRead\":false,\"valuesRedacted\":true,\"endpointRedacted\":true,\"liveDbEvidence\":false,\"liveDbConnectedEvidence\":false},\"blocker\":\"DB endpoint DNS resolution failed from the cloud-api runtime\",\"evidence\":\"live_db_tcp_connection_blocked\"},\"runtime\":{\"adapter\":\"memory\",\"durable\":false,\"status\":\"degraded\",\"reason\":\"live DB persistence is not connected; L1 runtime writes are process-local only\",\"counts\":{\"gatewaySessions\":0,\"boxResources\":0,\"boxCapabilities\":0,\"hardwareOperations\":0,\"auditEvents\":0,\"evidenceRecords\":0}}},\"meta\":{\"traceId\":\"trc_dev_m3_2393bb6a-6df3-4ad1-8794-b45530869d22\",\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"actorId\":\"system_code_queue_d601\"}}}" ] }, { "id": "two-box-simu-online", - "status": "not_run", - "summary": "Stopped before M3 direct checks because service target URLs were not provided.", + "status": "blocked", + "blockerClass": "runtime_blocker", + "summary": "Read-only direct target probes did not prove two distinct live DEV box-simu resources res_boxsimu_1 and res_boxsimu_2.", "evidence": [ - "No live DEV evidence collected for this check." + "{\"source\":\"kubernetes-endpointslices\",\"counts\":{\"boxSimu\":2,\"gatewaySimu\":2,\"patchPanel\":1,\"distinctBoxResources\":1,\"distinctGatewayIdentities\":1},\"selected\":{\"boxSimu1\":{\"id\":\"box-simu-1-candidate-1\",\"serviceId\":\"hwlab-box-simu\",\"baseUrl\":\"http://10.42.0.200:7201\",\"targetRef\":\"hwlab-box-simu-55688cbcc7-pftxc\",\"ok\":true,\"statusCode\":200,\"healthCode\":200,\"boxId\":\"boxsimu_1\",\"resourceId\":\"res_boxsimu_1\",\"gatewayId\":null,\"gatewaySessionId\":\"gws_mvp_simu\",\"patchPanelState\":null},\"boxSimu2\":null,\"gateways\":[{\"id\":\"gateway-simu-1-candidate-1\",\"serviceId\":\"hwlab-gateway-simu\",\"baseUrl\":\"http://10.42.0.204:7101\",\"targetRef\":\"hwlab-gateway-simu-699fd486b-phcgp\",\"ok\":true,\"statusCode\":200,\"healthCode\":200,\"boxId\":null,\"resourceId\":null,\"gatewayId\":\"gateway-a\",\"gatewaySessionId\":\"gws_gateway-a\",\"patchPanelState\":null}],\"patchPanel\":{\"id\":\"patch-panel-candidate-1\",\"serviceId\":\"hwlab-patch-panel\",\"baseUrl\":\"http://10.42.0.205:7301\",\"targetRef\":\"hwlab-patch-panel-757f9f44d5-7np8r\",\"ok\":true,\"statusCode\":200,\"healthCode\":200,\"boxId\":null,\"resourceId\":null,\"gatewayId\":null,\"gatewaySessionId\":\"gws_gwsimu_east\",\"patchPanelState\":\"active\"}}}" ] }, { "id": "two-gateway-simu-online", - "status": "not_run", - "summary": "Stopped before M3 direct checks because service target URLs were not provided.", + "status": "blocked", + "blockerClass": "runtime_blocker", + "summary": "Read-only direct target probes did not prove two distinct live DEV gateway-simu identities.", "evidence": [ - "No live DEV evidence collected for this check." + "{\"source\":\"kubernetes-endpointslices\",\"counts\":{\"boxSimu\":2,\"gatewaySimu\":2,\"patchPanel\":1,\"distinctBoxResources\":1,\"distinctGatewayIdentities\":1},\"selected\":{\"boxSimu1\":{\"id\":\"box-simu-1-candidate-1\",\"serviceId\":\"hwlab-box-simu\",\"baseUrl\":\"http://10.42.0.200:7201\",\"targetRef\":\"hwlab-box-simu-55688cbcc7-pftxc\",\"ok\":true,\"statusCode\":200,\"healthCode\":200,\"boxId\":\"boxsimu_1\",\"resourceId\":\"res_boxsimu_1\",\"gatewayId\":null,\"gatewaySessionId\":\"gws_mvp_simu\",\"patchPanelState\":null},\"boxSimu2\":null,\"gateways\":[{\"id\":\"gateway-simu-1-candidate-1\",\"serviceId\":\"hwlab-gateway-simu\",\"baseUrl\":\"http://10.42.0.204:7101\",\"targetRef\":\"hwlab-gateway-simu-699fd486b-phcgp\",\"ok\":true,\"statusCode\":200,\"healthCode\":200,\"boxId\":null,\"resourceId\":null,\"gatewayId\":\"gateway-a\",\"gatewaySessionId\":\"gws_gateway-a\",\"patchPanelState\":null}],\"patchPanel\":{\"id\":\"patch-panel-candidate-1\",\"serviceId\":\"hwlab-patch-panel\",\"baseUrl\":\"http://10.42.0.205:7301\",\"targetRef\":\"hwlab-patch-panel-757f9f44d5-7np8r\",\"ok\":true,\"statusCode\":200,\"healthCode\":200,\"boxId\":null,\"resourceId\":null,\"gatewayId\":null,\"gatewaySessionId\":\"gws_gwsimu_east\",\"patchPanelState\":\"active\"}}}" ] }, { "id": "patch-panel-healthy", - "status": "not_run", - "summary": "Stopped before M3 direct checks because service target URLs were not provided.", + "status": "pass", + "summary": "DEV patch-panel direct target returned live status.", "evidence": [ - "No live DEV evidence collected for this check." + "{\"id\":\"patch-panel-candidate-1\",\"serviceId\":\"hwlab-patch-panel\",\"baseUrl\":\"http://10.42.0.205:7301\",\"targetRef\":\"hwlab-patch-panel-757f9f44d5-7np8r\",\"ok\":true,\"statusCode\":200,\"healthCode\":200,\"boxId\":null,\"resourceId\":null,\"gatewayId\":null,\"gatewaySessionId\":\"gws_gwsimu_east\",\"patchPanelState\":\"active\"}" ] }, { "id": "wiring-do1-di1-applied", - "status": "not_run", - "summary": "Stopped before M3 direct checks because service target URLs were not provided.", + "status": "blocked", + "blockerClass": "runtime_blocker", + "summary": "DEV patch-panel wiring is active but does not contain the required res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 route.", "evidence": [ - "No live DEV evidence collected for this check." + "{\"url\":\"http://10.42.0.205:7301/wiring\",\"ok\":true,\"requiredConnection\":{\"fromResourceId\":\"res_boxsimu_1\",\"fromPort\":\"DO1\",\"toResourceId\":\"res_boxsimu_2\",\"toPort\":\"DI1\"},\"activeObserved\":[\"res_boxsim_alpha:uart0->res_boxsim_beta:uart0\",\"res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0\"],\"configuredObserved\":[\"res_boxsim_alpha:uart0->res_boxsim_beta:uart0\",\"res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0\"],\"hasRequiredActiveConnection\":false,\"hasRequiredConfiguredConnection\":false,\"configSource\":\"internal:mvp-topology\",\"syncableConnectionCount\":0}" ] }, { "id": "direct-call-do-write-di-read", "status": "not_run", - "summary": "Stopped before M3 direct checks because service target URLs were not provided.", + "summary": "Not attempted because read-only DEV M3 preconditions were blocked; no box-simu loopback or SOURCE/DRY-RUN substitute was used.", "evidence": [ - "No live DEV evidence collected for this check." + "No live DEV write was sent." ] }, { "id": "audit-evidence-traceable", "status": "not_run", - "summary": "Stopped before M3 direct checks because service target URLs were not provided.", + "summary": "Not attempted because the patch-panel-bound live operation was not safe to trigger.", "evidence": [ - "No live DEV evidence collected for this check." + "operationId=not_observed", + "traceId=not_observed", + "auditId=not_observed", + "evidenceId=not_observed" ] } ], @@ -179,13 +188,13 @@ "id": "kubectl-hwlab-dev", "status": "observed", "command": "kubectl get deploy,po,svc -n hwlab-dev -o wide", - "summary": "NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR deployment.apps/hwlab-agent-mgr 1/1 1 1 12h hwlab-agent-mgr 127.0.0.1:5000/hwlab/hwlab-agent-mgr:cb35ada app.kubernetes.io/name=hwlab-agent-mgr,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-agent-skills 1/1 1 1 9h hwlab-agent-skills 127.0.0.1:5000/hwlab/hwlab-agent-skills:cb35ada app.kubernetes.io/name=hwlab-agent-skills,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-box-simu 2/2 2 2 12h hwlab-box-simu 127.0.0.1:5000/hwlab/hwlab-box-simu:cb35ada app.kubernetes.io/name=hwlab-box-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-api 1/1 1 1 12h hwlab-cloud-api 127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada app.kubernetes.io/name=hwlab-cloud-api,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-web 1/1 1 1 12h hwlab-cloud-web 127.0.0.1:5000/hwlab/hwlab-cloud-web:bf47a95 app.kubernetes.io/name=hwlab-cloud-web,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-edge-proxy 1/1 1 1 12h hwlab-edge-proxy 127.0.0.1:5000/hwlab/hwlab-edge-proxy:cb35ada app.kubernetes.io/name=hwlab-edge-proxy,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-frpc 1/1 1 1 12h frpc 127.0.0.1:5000/hwlab/frpc:v0.68.1 app.kubernetes.io/name=hwlab-frpc deployment.apps/hwlab-gateway 0/0 0 0 12h hwlab-gateway 127.0.0.1:5000/hwlab/hwlab-gateway:cb35ada app.kubernetes.io/name=hwlab-gateway,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-gateway-simu 2/2 2 2 12h hwlab-gateway-simu 127.0.0.1:5000/hwlab/hwlab-gateway-simu:cb35ada app.kubernetes.io/name=hwlab-gateway-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-patch-panel 1/1 1 1 12h hwlab-patch-panel 127.0.0.1:5000/hwlab/hwlab-patch-panel:cb35ada app.kubernetes.io/name=hwlab-patch-panel,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.app" + "summary": "NAME READY UP-TO-DATE AVAILABLE AGE CONTAINERS IMAGES SELECTOR deployment.apps/hwlab-agent-mgr 1/1 1 1 13h hwlab-agent-mgr 127.0.0.1:5000/hwlab/hwlab-agent-mgr:cb35ada app.kubernetes.io/name=hwlab-agent-mgr,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-agent-skills 1/1 1 1 10h hwlab-agent-skills 127.0.0.1:5000/hwlab/hwlab-agent-skills:cb35ada app.kubernetes.io/name=hwlab-agent-skills,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-box-simu 2/2 2 2 13h hwlab-box-simu 127.0.0.1:5000/hwlab/hwlab-box-simu:cb35ada app.kubernetes.io/name=hwlab-box-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-api 1/1 1 1 13h hwlab-cloud-api 127.0.0.1:5000/hwlab/hwlab-cloud-api:c7de474 app.kubernetes.io/name=hwlab-cloud-api,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-cloud-web 1/1 1 1 13h hwlab-cloud-web 127.0.0.1:5000/hwlab/hwlab-cloud-web:c7de474 app.kubernetes.io/name=hwlab-cloud-web,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-edge-proxy 1/1 1 1 13h hwlab-edge-proxy 127.0.0.1:5000/hwlab/hwlab-edge-proxy:cb35ada app.kubernetes.io/name=hwlab-edge-proxy,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-frpc 1/1 1 1 13h frpc 127.0.0.1:5000/hwlab/frpc:v0.68.1 app.kubernetes.io/name=hwlab-frpc deployment.apps/hwlab-gateway 0/0 0 0 13h hwlab-gateway 127.0.0.1:5000/hwlab/hwlab-gateway:cb35ada app.kubernetes.io/name=hwlab-gateway,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-gateway-simu 2/2 2 2 13h hwlab-gateway-simu 127.0.0.1:5000/hwlab/hwlab-gateway-simu:cb35ada app.kubernetes.io/name=hwlab-gateway-simu,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.apps/hwlab-patch-panel 1/1 1 1 13h hwlab-patch-panel 127.0.0.1:5000/hwlab/hwlab-patch-panel:cb35ada app.kubernetes.io/name=hwlab-patch-panel,app.kubernetes.io/part-of=hwlab,hwlab.pikastech.local/environment=dev,hwlab.pikastech.local/profile=dev deployment.app" }, { "id": "runner-k3s-kubeconfig-readonly", "status": "observed", "command": "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods -o name", - "summary": "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate. runnerKubeconfigReadable=false; exitCode=0; stderr=empty." + "summary": "Runner /etc/rancher/k3s/k3s.yaml is not readable through fs access, but read-only kubectl observation succeeds; file readability, kubectl reachability, and service discovery are recorded separately. runnerKubeconfigReadable=false; exitCode=0; stderr=empty." } ], "d601Observability": { @@ -195,11 +204,11 @@ "runnerKubeconfigProbeStderr": "empty", "d601PublicEndpointsReachable": true, "d601K3sUnavailable": false, - "classification": "runner_permission_mount_gap", + "classification": "read_only_kubectl_available_kubeconfig_file_unreadable", "sourceIssue": "pikasTech/HWLAB#46", "command": "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get pods -o name", "stdoutSummary": "13 pod name(s) observed", - "summary": "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate.", + "summary": "Runner /etc/rancher/k3s/k3s.yaml is not readable through fs access, but read-only kubectl observation succeeds; file readability, kubectl reachability, and service discovery are recorded separately.", "inferenceRule": "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately.", "secretValuesRead": false, "secretResourcesRead": false @@ -210,22 +219,257 @@ "traceId": "not_observed", "auditId": "not_observed", "evidenceId": "not_observed", - "summary": "No live DEV operation has run yet." + "summary": "No live DEV operation was attempted because read-only direct target checks did not prove the required patch-panel-owned M3 route." }, "blockers": [ { - "type": "observability_blocker", - "scope": "m3-service-discovery", + "type": "runtime_blocker", + "scope": "m3-box-simu-identity", "status": "open", - "classification": "runner permission/mount gap", - "sourceIssue": "pikasTech/HWLAB#46", - "summary": "DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline." + "classification": "direct_target_identity_gap", + "sourceIssue": "pikasTech/HWLAB#64", + "summary": "DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=res_boxsimu_1." + }, + { + "type": "runtime_blocker", + "scope": "m3-gateway-simu-identity", + "status": "open", + "classification": "direct_target_identity_gap", + "sourceIssue": "pikasTech/HWLAB#64", + "summary": "DEV exposes two gateway-simu endpoints, but live gateway identities are not distinct; observed identities=gateway-a:gws_gateway-a." + }, + { + "type": "runtime_blocker", + "scope": "m3-patch-panel-wiring", + "status": "open", + "classification": "direct_target_m3_wiring_missing", + "sourceIssue": "pikasTech/HWLAB#64", + "summary": "DEV patch-panel is callable, but live wiring does not contain res_boxsimu_1:DO1 -> res_boxsimu_2:DI1; active=res_boxsim_alpha:uart0->res_boxsim_beta:uart0, res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0; configured=res_boxsim_alpha:uart0->res_boxsim_beta:uart0, res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0." } ], "summary": { "status": "blocked", - "classification": "runner permission/mount gap", - "observedAt": "2026-05-22T12:36:42.972Z", - "result": "DEV ingress was reachable, but live M3 simulator and patch-panel targets were not discoverable." + "classification": "direct_target_identity_gap", + "observedAt": "2026-05-22T13:44:36.422Z", + "result": "DEV ingress and direct targets were reachable, but M3 DEV-LIVE was not triggered because required identities or patch-panel wiring were missing." + }, + "directTargetDiscovery": { + "environmentMissing": [ + "HWLAB_DEV_BOX_SIMU_1_URL", + "HWLAB_DEV_BOX_SIMU_2_URL", + "HWLAB_DEV_GATEWAY_SIMU_1_URL", + "HWLAB_DEV_GATEWAY_SIMU_2_URL", + "HWLAB_DEV_PATCH_PANEL_URL" + ], + "kubernetes": { + "status": "observed", + "source": "kubernetes-endpointslices", + "command": "KUBECONFIG=/etc/rancher/k3s/k3s.yaml kubectl -n hwlab-dev get endpointslices.discovery.k8s.io -o json", + "exitCode": 0, + "services": { + "hwlab-box-simu": { + "endpointSlice": "hwlab-box-simu-qssf6", + "port": 7201, + "count": 2, + "endpoints": [ + { + "url": "http://10.42.0.200:7201", + "address": "10.42.0.200", + "port": 7201, + "targetRef": "hwlab-box-simu-55688cbcc7-pftxc" + }, + { + "url": "http://10.42.0.208:7201", + "address": "10.42.0.208", + "port": 7201, + "targetRef": "hwlab-box-simu-55688cbcc7-sq5fx" + } + ] + }, + "hwlab-gateway-simu": { + "endpointSlice": "hwlab-gateway-simu-dmm4m", + "port": 7101, + "count": 2, + "endpoints": [ + { + "url": "http://10.42.0.204:7101", + "address": "10.42.0.204", + "port": 7101, + "targetRef": "hwlab-gateway-simu-699fd486b-phcgp" + }, + { + "url": "http://10.42.0.209:7101", + "address": "10.42.0.209", + "port": 7101, + "targetRef": "hwlab-gateway-simu-699fd486b-j9pk7" + } + ] + }, + "hwlab-patch-panel": { + "endpointSlice": "hwlab-patch-panel-nl2px", + "port": 7301, + "count": 1, + "endpoints": [ + { + "url": "http://10.42.0.205:7301", + "address": "10.42.0.205", + "port": 7301, + "targetRef": "hwlab-patch-panel-757f9f44d5-7np8r" + } + ] + } + }, + "summary": "Observed endpoint slices for hwlab-box-simu, hwlab-gateway-simu, hwlab-patch-panel." + }, + "source": "kubernetes-endpointslices", + "counts": { + "boxSimu": 2, + "gatewaySimu": 2, + "patchPanel": 1, + "distinctBoxResources": 1, + "distinctGatewayIdentities": 1 + }, + "selected": { + "boxSimu1": { + "id": "box-simu-1-candidate-1", + "serviceId": "hwlab-box-simu", + "baseUrl": "http://10.42.0.200:7201", + "targetRef": "hwlab-box-simu-55688cbcc7-pftxc", + "ok": true, + "statusCode": 200, + "healthCode": 200, + "boxId": "boxsimu_1", + "resourceId": "res_boxsimu_1", + "gatewayId": null, + "gatewaySessionId": "gws_mvp_simu", + "patchPanelState": null + }, + "boxSimu2": null, + "gateways": [ + { + "id": "gateway-simu-1-candidate-1", + "serviceId": "hwlab-gateway-simu", + "baseUrl": "http://10.42.0.204:7101", + "targetRef": "hwlab-gateway-simu-699fd486b-phcgp", + "ok": true, + "statusCode": 200, + "healthCode": 200, + "boxId": null, + "resourceId": null, + "gatewayId": "gateway-a", + "gatewaySessionId": "gws_gateway-a", + "patchPanelState": null + } + ], + "patchPanel": { + "id": "patch-panel-candidate-1", + "serviceId": "hwlab-patch-panel", + "baseUrl": "http://10.42.0.205:7301", + "targetRef": "hwlab-patch-panel-757f9f44d5-7np8r", + "ok": true, + "statusCode": 200, + "healthCode": 200, + "boxId": null, + "resourceId": null, + "gatewayId": null, + "gatewaySessionId": "gws_gwsimu_east", + "patchPanelState": "active" + } + }, + "candidates": [ + { + "id": "box-simu-1-candidate-1", + "serviceId": "hwlab-box-simu", + "baseUrl": "http://10.42.0.200:7201", + "targetRef": "hwlab-box-simu-55688cbcc7-pftxc", + "ok": true, + "statusCode": 200, + "healthCode": 200, + "boxId": "boxsimu_1", + "resourceId": "res_boxsimu_1", + "gatewayId": null, + "gatewaySessionId": "gws_mvp_simu", + "patchPanelState": null + }, + { + "id": "box-simu-1-candidate-2", + "serviceId": "hwlab-box-simu", + "baseUrl": "http://10.42.0.208:7201", + "targetRef": "hwlab-box-simu-55688cbcc7-sq5fx", + "ok": true, + "statusCode": 200, + "healthCode": 200, + "boxId": "boxsimu_1", + "resourceId": "res_boxsimu_1", + "gatewayId": null, + "gatewaySessionId": "gws_mvp_simu", + "patchPanelState": null + }, + { + "id": "gateway-simu-1-candidate-1", + "serviceId": "hwlab-gateway-simu", + "baseUrl": "http://10.42.0.204:7101", + "targetRef": "hwlab-gateway-simu-699fd486b-phcgp", + "ok": true, + "statusCode": 200, + "healthCode": 200, + "boxId": null, + "resourceId": null, + "gatewayId": "gateway-a", + "gatewaySessionId": "gws_gateway-a", + "patchPanelState": null + }, + { + "id": "gateway-simu-1-candidate-2", + "serviceId": "hwlab-gateway-simu", + "baseUrl": "http://10.42.0.209:7101", + "targetRef": "hwlab-gateway-simu-699fd486b-j9pk7", + "ok": true, + "statusCode": 200, + "healthCode": 200, + "boxId": null, + "resourceId": null, + "gatewayId": "gateway-a", + "gatewaySessionId": "gws_gateway-a", + "patchPanelState": null + }, + { + "id": "patch-panel-candidate-1", + "serviceId": "hwlab-patch-panel", + "baseUrl": "http://10.42.0.205:7301", + "targetRef": "hwlab-patch-panel-757f9f44d5-7np8r", + "ok": true, + "statusCode": 200, + "healthCode": 200, + "boxId": null, + "resourceId": null, + "gatewayId": null, + "gatewaySessionId": "gws_gwsimu_east", + "patchPanelState": "active" + } + ], + "patchPanelWiring": { + "url": "http://10.42.0.205:7301/wiring", + "ok": true, + "requiredConnection": { + "fromResourceId": "res_boxsimu_1", + "fromPort": "DO1", + "toResourceId": "res_boxsimu_2", + "toPort": "DI1" + }, + "activeObserved": [ + "res_boxsim_alpha:uart0->res_boxsim_beta:uart0", + "res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0" + ], + "configuredObserved": [ + "res_boxsim_alpha:uart0->res_boxsim_beta:uart0", + "res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0" + ], + "hasRequiredActiveConnection": false, + "hasRequiredConfiguredConnection": false, + "configSource": "internal:mvp-topology", + "syncableConnectionCount": 0 + }, + "summary": "Direct DEV M3 targets were probed read-only, but required identities or patch-panel M3 wiring were not satisfied." } } diff --git a/reports/dev-gate/dev-m5-gate-aggregator-v2.json b/reports/dev-gate/dev-m5-gate-aggregator-v2.json index 477a504f..da93bfd5 100644 --- a/reports/dev-gate/dev-m5-gate-aggregator-v2.json +++ b/reports/dev-gate/dev-m5-gate-aggregator-v2.json @@ -20,8 +20,8 @@ "pikasTech/HWLAB#46", "pikasTech/HWLAB#64" ], - "generatedAt": "2026-05-22T12:36:45.366Z", - "generatedFromCommit": "bf47a95a97a9", + "generatedAt": "2026-05-22T13:44:40.940Z", + "generatedFromCommit": "c7de4745f491", "environment": "dev", "endpoint": "http://74.48.78.17:16667", "frontendEndpoint": "http://74.48.78.17:16666", @@ -108,7 +108,7 @@ "taskId": "dev-m3-hardware-loop", "lifecycleState": "active", "status": "blocked", - "commitId": "bf47a95a97a9" + "commitId": "c7de4745f491" }, "devM4Agent": { "path": "reports/dev-gate/dev-m4-agent-loop.json", @@ -116,7 +116,7 @@ "taskId": "dev-m4-agent-loop", "lifecycleState": "active", "status": "blocked", - "commitId": "8e89409" + "commitId": "0614202" }, "devM5Gate": { "path": "reports/dev-gate/dev-mvp-gate-report.json", @@ -283,7 +283,7 @@ "label": "M3 hardware trusted loop", "status": "blocked", "evidenceLevel": "BLOCKED", - "summary": "No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline..", + "summary": "No live DEV operation was attempted because read-only direct target checks did not prove the required patch-panel-owned M3 route. Blocker: DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=res_boxsimu_1.", "evidence": [ "operationId=not_observed", "traceId=not_observed", @@ -298,6 +298,18 @@ "evidenceLevel": "BLOCKED", "summary": "Blocked at DB live readiness before scheduling a DEV agent task.", "evidence": [ + "k3s:namespace=hwlab-dev:read=true", + "k3s:deploy/hwlab-agent-mgr:ready=1/1:available=True", + "k3s:deploy/hwlab-agent-skills:ready=1/1:available=True", + "k3s:svc/hwlab-agent-mgr:endpoints=1", + "k3s:svc/hwlab-agent-skills:endpoints=1", + "agent-mgr:/health/live:degraded", + "agent-skills:/health/live:ok:cb35ada6860653b27269b0a57991184118cbf4b1", + "worker-template:suspend=true:image=127.0.0.1:5000/hwlab/hwlab-agent-worker:cb35ada", + "worker-job:server-dry-run=true:persisted=false", + "skills-injection:commit=cb35ada:version=missing", + "secret-rbac:get=yes:secretResourcesRead=false", + "Public DEV API http://74.48.78.17:16667 health=200 live=200; frontend http://74.48.78.17:16666/=200.", "health:hwlab-edge-proxy:dev:ok", "live:hwlab-cloud-api:degraded:db-blocked" ], @@ -371,8 +383,8 @@ "milestone": "M3", "status": "blocked", "currentLevel": "BLOCKED", - "blockerClass": "runner-readonly-observability-gap", - "dependency": "Runner read-only service discovery must be repaired before the real DEV trusted loop can be observed; this does not mean D601/k3s is globally unavailable.", + "blockerClass": "direct-target-identity-gap", + "dependency": "Two distinct live box-simu resources and two distinct gateway-simu identities are required before M3 can run.", "evidence": [ "operationId=not_observed", "traceId=not_observed", @@ -380,9 +392,12 @@ "evidenceId=not_observed", "runnerKubeconfigReadable=false", "d601PublicEndpointsReachable=true", - "d601K3sUnavailable=false" + "d601K3sUnavailable=false", + "directTargetSource=kubernetes-endpointslices", + "directTargetCounts={\"boxSimu\":2,\"gatewaySimu\":2,\"patchPanel\":1,\"distinctBoxResources\":1,\"distinctGatewayIdentities\":1}", + "patchPanelM3Wiring=false" ], - "nextRequired": "Repair the #46 runner readonly kubeconfig/service-discovery gap, then run the bounded DEV M3 live smoke only when direct simulator and patch-panel targets are discoverable.", + "nextRequired": "Fix DEV simulator instance identity so two box-simu pods report res_boxsimu_1/res_boxsimu_2 and two gateway-simu pods report distinct gateway identities.", "nonPromotionReason": "Public frontend, route, and artifact evidence do not prove the required hardware loop." }, { @@ -441,7 +456,7 @@ "highestVisibleLevel": "DEV-LIVE", "liveEvidence": "pass", "evidenceCount": 7, - "blockerCount": 7, + "blockerCount": 6, "summary": "deploy/runtime readiness is blocked before live DEV; highest visible level is DEV-LIVE; status is blocked." }, { @@ -450,7 +465,7 @@ "highestVisibleLevel": "LOCAL", "liveEvidence": "missing_or_blocked", "evidenceCount": 3, - "blockerCount": 3, + "blockerCount": 5, "summary": "hardware loop has source/local shape but no live operation; highest visible level is LOCAL; status is blocked." }, { @@ -468,7 +483,7 @@ "highestVisibleLevel": "DRY-RUN", "liveEvidence": "missing_or_blocked", "evidenceCount": 2, - "blockerCount": 8, + "blockerCount": 13, "summary": "dry-run is green but live MVP gate is blocked; highest visible level is DRY-RUN; status is blocked." } ], @@ -690,7 +705,7 @@ "issue": "pikasTech/HWLAB#38", "taskId": "dev-m3-hardware-loop", "reportPath": "reports/dev-gate/dev-m3-hardware-loop.json", - "commitId": "bf47a95a97a9", + "commitId": "c7de4745f491", "lifecycleState": "active", "level": "SOURCE", "status": "manifest-ready", @@ -729,7 +744,7 @@ "issue": "pikasTech/HWLAB#38", "taskId": "dev-m3-hardware-loop", "reportPath": "reports/dev-gate/dev-m3-hardware-loop.json", - "commitId": "bf47a95a97a9", + "commitId": "c7de4745f491", "lifecycleState": "active", "level": "BLOCKED", "status": "not_run", @@ -743,14 +758,14 @@ "auditId=not_observed", "evidenceId=not_observed" ], - "summary": "No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.." + "summary": "No live DEV operation was attempted because read-only direct target checks did not prove the required patch-panel-owned M3 route. Blocker: DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=res_boxsimu_1." }, { "milestone": "M4", "issue": "pikasTech/HWLAB#37", "taskId": "dev-m4-agent-loop", "reportPath": "reports/dev-gate/dev-m4-agent-loop.json", - "commitId": "8e89409", + "commitId": "0614202", "lifecycleState": "active", "level": "LOCAL", "status": "pass", @@ -769,7 +784,7 @@ "issue": "pikasTech/HWLAB#37", "taskId": "dev-m4-agent-loop", "reportPath": "reports/dev-gate/dev-m4-agent-loop.json", - "commitId": "8e89409", + "commitId": "0614202", "lifecycleState": "active", "level": "DRY-RUN", "status": "pass", @@ -793,7 +808,7 @@ "issue": "pikasTech/HWLAB#37", "taskId": "dev-m4-agent-loop", "reportPath": "reports/dev-gate/dev-m4-agent-loop.json", - "commitId": "8e89409", + "commitId": "0614202", "lifecycleState": "active", "level": "BLOCKED", "status": "blocked", @@ -802,6 +817,18 @@ "node scripts/dev-m4-agent-loop-smoke.mjs --live --confirm-dev --confirmed-non-production" ], "evidence": [ + "k3s:namespace=hwlab-dev:read=true", + "k3s:deploy/hwlab-agent-mgr:ready=1/1:available=True", + "k3s:deploy/hwlab-agent-skills:ready=1/1:available=True", + "k3s:svc/hwlab-agent-mgr:endpoints=1", + "k3s:svc/hwlab-agent-skills:endpoints=1", + "agent-mgr:/health/live:degraded", + "agent-skills:/health/live:ok:cb35ada6860653b27269b0a57991184118cbf4b1", + "worker-template:suspend=true:image=127.0.0.1:5000/hwlab/hwlab-agent-worker:cb35ada", + "worker-job:server-dry-run=true:persisted=false", + "skills-injection:commit=cb35ada:version=missing", + "secret-rbac:get=yes:secretResourcesRead=false", + "Public DEV API http://74.48.78.17:16667 health=200 live=200; frontend http://74.48.78.17:16666/=200.", "health:hwlab-edge-proxy:dev:ok", "live:hwlab-cloud-api:degraded:db-blocked" ], @@ -1010,7 +1037,7 @@ "status": "not_run", "category": "hardware-loop-live", "reportPath": "reports/dev-gate/dev-m3-hardware-loop.json", - "summary": "No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.." + "summary": "No live DEV operation was attempted because read-only direct target checks did not prove the required patch-panel-owned M3 route. Blocker: DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=res_boxsimu_1." }, { "milestone": "M4", @@ -1062,15 +1089,6 @@ "summary": "The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately.", "nextTask": "Provide the approved read-only runner kubeconfig mount or document the intended alternate KUBECONFIG path, then rerun the read-only report." }, - { - "priority": "P1", - "order": 4, - "type": "observability_blocker", - "scope": "m3-service-discovery", - "sourceIssue": "pikasTech/HWLAB#38", - "source": "reports/dev-gate/dev-m3-hardware-loop.json", - "summary": "DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline." - }, { "priority": "P1", "order": 5, @@ -1118,6 +1136,33 @@ "source": "reports/dev-gate/dev-mvp-gate-report.json", "summary": "cloud-api /health/live reports DB degraded; db.ready=false, db.connected=false, and live DB evidence is not established." }, + { + "priority": "P0", + "order": 6, + "type": "runtime_blocker", + "scope": "m3-box-simu-identity", + "sourceIssue": "pikasTech/HWLAB#38", + "source": "reports/dev-gate/dev-m3-hardware-loop.json", + "summary": "DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=res_boxsimu_1." + }, + { + "priority": "P0", + "order": 6, + "type": "runtime_blocker", + "scope": "m3-gateway-simu-identity", + "sourceIssue": "pikasTech/HWLAB#38", + "source": "reports/dev-gate/dev-m3-hardware-loop.json", + "summary": "DEV exposes two gateway-simu endpoints, but live gateway identities are not distinct; observed identities=gateway-a:gws_gateway-a." + }, + { + "priority": "P0", + "order": 6, + "type": "runtime_blocker", + "scope": "m3-patch-panel-wiring", + "sourceIssue": "pikasTech/HWLAB#38", + "source": "reports/dev-gate/dev-m3-hardware-loop.json", + "summary": "DEV patch-panel is callable, but live wiring does not contain res_boxsimu_1:DO1 -> res_boxsimu_2:DI1; active=res_boxsim_alpha:uart0->res_boxsim_beta:uart0, res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0; configured=res_boxsim_alpha:uart0->res_boxsim_beta:uart0, res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0." + }, { "priority": "P0", "order": 6, @@ -1126,6 +1171,24 @@ "sourceIssue": "pikasTech/HWLAB#39", "source": "reports/dev-gate/dev-mvp-gate-report.json", "summary": "Live M3 smoke reached DEV simulators, but patch-panel active DO1 -> DI1 connection is missing." + }, + { + "priority": "P3", + "order": 99, + "type": "agent_blocker", + "scope": "agent-mgr-health", + "sourceIssue": "pikasTech/HWLAB#37", + "source": "reports/dev-gate/dev-m4-agent-loop.json", + "summary": "hwlab-agent-mgr /health/live is degraded." + }, + { + "priority": "P3", + "order": 99, + "type": "agent_blocker", + "scope": "skills-commit-version-injection", + "sourceIssue": "pikasTech/HWLAB#37", + "source": "reports/dev-gate/dev-m4-agent-loop.json", + "summary": "DEV skills injection is incomplete: missing hwlab-agent-skills.HWLAB_SKILLS_VERSION, worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV." } ] }, @@ -1187,26 +1250,6 @@ ], "rationale": "Runner read-only observability must be repaired without treating the runner gap as proof that D601 k3s or public DEV endpoints are unavailable." }, - { - "id": "dev-m3-hardware-loop:m3-service-discovery", - "priority": "P1", - "type": "observability_blocker", - "scope": "m3-service-discovery", - "status": "open", - "source": "reports/dev-gate/dev-m3-hardware-loop.json", - "sourceIssue": "pikasTech/HWLAB#38", - "summary": "DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.", - "unblockOrder": 4, - "unblocks": [ - "pikasTech/HWLAB#34", - "pikasTech/HWLAB#33", - "pikasTech/HWLAB#36", - "pikasTech/HWLAB#38", - "pikasTech/HWLAB#46", - "pikasTech/HWLAB#64" - ], - "rationale": "Runner read-only observability must be repaired without treating the runner gap as proof that D601 k3s or public DEV endpoints are unavailable." - }, { "id": "dev-edge-health:cloud-api-db", "priority": "P1", @@ -1292,6 +1335,57 @@ ], "rationale": "M4 and M5 cannot claim live agent or MVP evidence until cloud-api /health/live proves DB readiness with redacted live evidence." }, + { + "id": "dev-m3-hardware-loop:m3-box-simu-identity", + "priority": "P0", + "type": "runtime_blocker", + "scope": "m3-box-simu-identity", + "status": "open", + "source": "reports/dev-gate/dev-m3-hardware-loop.json", + "sourceIssue": "pikasTech/HWLAB#38", + "summary": "DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=res_boxsimu_1.", + "unblockOrder": 6, + "unblocks": [ + "pikasTech/HWLAB#38", + "pikasTech/HWLAB#39", + "pikasTech/HWLAB#64" + ], + "rationale": "M3 remains blocked until the real DEV hardware trusted loop proves DO1 -> patch-panel -> DI1 with operation, trace, audit, and evidence identifiers." + }, + { + "id": "dev-m3-hardware-loop:m3-gateway-simu-identity", + "priority": "P0", + "type": "runtime_blocker", + "scope": "m3-gateway-simu-identity", + "status": "open", + "source": "reports/dev-gate/dev-m3-hardware-loop.json", + "sourceIssue": "pikasTech/HWLAB#38", + "summary": "DEV exposes two gateway-simu endpoints, but live gateway identities are not distinct; observed identities=gateway-a:gws_gateway-a.", + "unblockOrder": 6, + "unblocks": [ + "pikasTech/HWLAB#38", + "pikasTech/HWLAB#39", + "pikasTech/HWLAB#64" + ], + "rationale": "M3 remains blocked until the real DEV hardware trusted loop proves DO1 -> patch-panel -> DI1 with operation, trace, audit, and evidence identifiers." + }, + { + "id": "dev-m3-hardware-loop:m3-patch-panel-wiring", + "priority": "P0", + "type": "runtime_blocker", + "scope": "m3-patch-panel-wiring", + "status": "open", + "source": "reports/dev-gate/dev-m3-hardware-loop.json", + "sourceIssue": "pikasTech/HWLAB#38", + "summary": "DEV patch-panel is callable, but live wiring does not contain res_boxsimu_1:DO1 -> res_boxsimu_2:DI1; active=res_boxsim_alpha:uart0->res_boxsim_beta:uart0, res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0; configured=res_boxsim_alpha:uart0->res_boxsim_beta:uart0, res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0.", + "unblockOrder": 6, + "unblocks": [ + "pikasTech/HWLAB#38", + "pikasTech/HWLAB#39", + "pikasTech/HWLAB#64" + ], + "rationale": "M3 remains blocked until the real DEV hardware trusted loop proves DO1 -> patch-panel -> DI1 with operation, trace, audit, and evidence identifiers." + }, { "id": "dev-mvp-gate-report:m3-hardware-loop-runtime", "priority": "P0", @@ -1308,6 +1402,36 @@ "pikasTech/HWLAB#64" ], "rationale": "M3 remains blocked until the real DEV hardware trusted loop proves DO1 -> patch-panel -> DI1 with operation, trace, audit, and evidence identifiers." + }, + { + "id": "dev-m4-agent-loop:agent-mgr-health", + "priority": "P3", + "type": "agent_blocker", + "scope": "agent-mgr-health", + "status": "open", + "source": "reports/dev-gate/dev-m4-agent-loop.json", + "sourceIssue": "pikasTech/HWLAB#37", + "summary": "hwlab-agent-mgr /health/live is degraded.", + "unblockOrder": 99, + "unblocks": [ + "pikasTech/HWLAB#39" + ], + "rationale": "Residual blocker that must be classified before claiming a green DEV gate." + }, + { + "id": "dev-m4-agent-loop:skills-commit-version-injection", + "priority": "P3", + "type": "agent_blocker", + "scope": "skills-commit-version-injection", + "status": "open", + "source": "reports/dev-gate/dev-m4-agent-loop.json", + "sourceIssue": "pikasTech/HWLAB#37", + "summary": "DEV skills injection is incomplete: missing hwlab-agent-skills.HWLAB_SKILLS_VERSION, worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV.", + "unblockOrder": 99, + "unblocks": [ + "pikasTech/HWLAB#39" + ], + "rationale": "Residual blocker that must be classified before claiming a green DEV gate." } ], "nextSteps": [ @@ -1334,8 +1458,7 @@ "blockerOrder": 4, "priority": "P1", "scopes": [ - "runner-kubeconfig-readonly-gap", - "m3-service-discovery" + "runner-kubeconfig-readonly-gap" ], "sourceIssues": [ "pikasTech/HWLAB#33", @@ -1374,6 +1497,9 @@ "blockerOrder": 6, "priority": "P0", "scopes": [ + "m3-box-simu-identity", + "m3-gateway-simu-identity", + "m3-patch-panel-wiring", "m3-hardware-loop-runtime" ], "sourceIssues": [ @@ -1382,8 +1508,24 @@ "pikasTech/HWLAB#64" ], "rationale": "M3 remains blocked until the real DEV hardware trusted loop proves DO1 -> patch-panel -> DI1 with operation, trace, audit, and evidence identifiers.", - "action": "Prove the real DEV M3 trusted loop res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, trace, audit, and evidence identifiers.", - "evidenceRequired": "Operation, trace, audit, and evidence IDs from the real DEV DO1 -> patch-panel -> DI1 trusted loop." + "action": "Fix DEV box-simu instance identity so direct endpoints expose distinct res_boxsimu_1 and res_boxsimu_2 resources.", + "evidenceRequired": "Read-only direct box-simu /health/live and /status output showing distinct res_boxsimu_1 and res_boxsimu_2 resources." + }, + { + "order": 5, + "blockerOrder": 99, + "priority": "P3", + "scopes": [ + "agent-mgr-health", + "skills-commit-version-injection" + ], + "sourceIssues": [ + "pikasTech/HWLAB#37", + "pikasTech/HWLAB#39" + ], + "rationale": "Residual blocker that must be classified before claiming a green DEV gate.", + "action": "Resolve the blocker and attach source/local/dry-run/DEV-live evidence at the correct level.", + "evidenceRequired": "A committed report with the exact evidence level and command used." } ], "validationCommands": [ diff --git a/reports/dev-gate/dev-m5-gate-aggregator-v2.md b/reports/dev-gate/dev-m5-gate-aggregator-v2.md index bcebd7c3..c88994cb 100644 --- a/reports/dev-gate/dev-m5-gate-aggregator-v2.md +++ b/reports/dev-gate/dev-m5-gate-aggregator-v2.md @@ -1,7 +1,7 @@ # HWLAB M5 DEV Gate Aggregator v2 Status: blocked -Generated from: `bf47a95a97a9` +Generated from: `c7de4745f491` Scope: DEV only, report-only Active frontend: `http://74.48.78.17:16666/` Active API/live: `http://74.48.78.17:16667/health/live` @@ -23,7 +23,7 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be | EDGE/ROUTE live | pass | DEV-LIVE | http://74.48.78.17:16666/, http://74.48.78.17:16667/health, and http://74.48.78.17:16667/health/live returned accepted HWLAB DEV responses in the active M2 read-only smoke. | Keep this separated from DB readiness, M3/M4 loop evidence, and M5 acceptance. | | DB live/degraded | blocked | BLOCKED | cloud-api DB status=degraded; configReady=true; ready=false; connected=false; liveDbEvidence=false. | Provide live DB connection evidence through redacted health output; route reachability alone is insufficient. | | D601 runner observability | blocked | DEV-LIVE | D601 public DEV endpoints are reachable, but the runner cannot read /etc/rancher/k3s/k3s.yaml; classify as #46 runner permission/mount or read-only observability gap, not D601 global offline. Alternate read-only cluster probes are readable. runnerKubeconfigReadable=false, runnerKubeconfigProbeExitCode=0, runnerKubeconfigProbeStderr=empty, d601PublicEndpointsReachable=true, d601K3sUnavailable=false. | Treat #46 runner kubeconfig/readonly gaps separately from D601 service health; rerun read-only observability after the mount or permission path is repaired. | -| M3 hardware trusted loop | blocked | BLOCKED | No live DEV operation has run yet. Blocker: DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.. | Only a real DEV res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 observation with operation/trace/audit/evidence can clear M3. | +| M3 hardware trusted loop | blocked | BLOCKED | No live DEV operation was attempted because read-only direct target checks did not prove the required patch-panel-owned M3 route. Blocker: DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=res_boxsimu_1. | Only a real DEV res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 observation with operation/trace/audit/evidence can clear M3. | | M4 agent loop | blocked | BLOCKED | Blocked at DB live readiness before scheduling a DEV agent task. | Do not schedule or claim the agent loop as live until DB live and required runtime/evidence preconditions pass. | | artifact/desired-state source | blocked | BLOCKED | artifact targetCovered=false; artifactSource=7e29522b65c8; target=8e89409dda5d; desiredApplyMode=dry-run; mutationAttempted=false. | Refresh artifact/source coverage for current origin/main and keep desired-state apply separate from read-only route proof. | @@ -67,7 +67,7 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be | Milestone | Status | Current level | Blocker class | Dependency | Required next proof | | --- | --- | --- | --- | --- | --- | -| M3 | blocked | BLOCKED | runner-readonly-observability-gap | Runner read-only service discovery must be repaired before the real DEV trusted loop can be observed; this does not mean D601/k3s is globally unavailable. | Repair the #46 runner readonly kubeconfig/service-discovery gap, then run the bounded DEV M3 live smoke only when direct simulator and patch-panel targets are discoverable. | +| M3 | blocked | BLOCKED | direct-target-identity-gap | Two distinct live box-simu resources and two distinct gateway-simu identities are required before M3 can run. | Fix DEV simulator instance identity so two box-simu pods report res_boxsimu_1/res_boxsimu_2 and two gateway-simu pods report distinct gateway identities. | | M4 | blocked | BLOCKED | db-live-readiness | Cloud API /health/live must report DB ready=true, connected=true, and liveDbEvidence=true before live agent scheduling/evidence closure. | Repair DB live readiness and rerun the M4 live preflight without scheduling a DEV agent task before preconditions pass. | | M5 | blocked | BLOCKED | composite-db-m3-m4-live | M5 needs DB live readiness, M3 trusted-loop DEV evidence, M4 live preflight/evidence closure, and current source/artifact coverage. | After DB/M3/M4 blockers are cleared, run only the bounded DEV MVP live gate command with explicit DEV/non-PROD confirmations. | @@ -78,13 +78,17 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be | P1 | 3 | contract_blocker | artifact-source-commit | source commit origin/main 8e89409 is not covered by artifact source 7e29522; target changes since the artifact source touch artifact build inputs. | | P1 | 3 | runtime_blocker | dev-artifact-publish | reports/dev-gate/dev-artifacts.json does not prove all required HWLAB service artifacts for origin/main 8e89409; current status is published with 13/13 required services published and source states resolved: 13 source-present, 0 intentionally-disabled. | | P1 | 4 | observability_blocker | runner-kubeconfig-readonly-gap | The runner cannot read /etc/rancher/k3s/k3s.yaml; classify this as a runner permission/mount gap, not a D601 global outage. Alternate read-only cluster probes and public DEV endpoint probes are reported separately. | -| P1 | 4 | observability_blocker | m3-service-discovery | DEV ingress is reachable on frozen :16667, but this runner cannot use /etc/rancher/k3s/k3s.yaml as readable kubeconfig to discover direct M3 service URLs (HWLAB_DEV_BOX_SIMU_1_URL, HWLAB_DEV_BOX_SIMU_2_URL, HWLAB_DEV_GATEWAY_SIMU_1_URL, HWLAB_DEV_GATEWAY_SIMU_2_URL, HWLAB_DEV_PATCH_PANEL_URL); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline. | | P1 | 5 | runtime_blocker | cloud-api-db | cloud-api DB env is injected, but runtime health has not attempted a live DB connection | | P1 | 5 | network_blocker | cloud-api-db | cloud-api DB env is injected, but runtime health has not attempted a live DB connection | | P1 | 5 | runtime_blocker | cloud-api-db-health-gate | cloud-api DB runtime env is not ready; missing HWLAB_CLOUD_DB_URL, HWLAB_CLOUD_DB_SSL_MODE. | | P1 | 5 | runtime_blocker | db-live | cloud-api /health/live reports DB degraded; connected=false; ready=false. | | P1 | 5 | runtime_blocker | db-live | cloud-api /health/live reports DB degraded; db.ready=false, db.connected=false, and live DB evidence is not established. | +| P0 | 6 | runtime_blocker | m3-box-simu-identity | DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=res_boxsimu_1. | +| P0 | 6 | runtime_blocker | m3-gateway-simu-identity | DEV exposes two gateway-simu endpoints, but live gateway identities are not distinct; observed identities=gateway-a:gws_gateway-a. | +| P0 | 6 | runtime_blocker | m3-patch-panel-wiring | DEV patch-panel is callable, but live wiring does not contain res_boxsimu_1:DO1 -> res_boxsimu_2:DI1; active=res_boxsim_alpha:uart0->res_boxsim_beta:uart0, res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0; configured=res_boxsim_alpha:uart0->res_boxsim_beta:uart0, res_boxsim_alpha:gpio0->res_boxsim_beta:gpio0. | | P0 | 6 | runtime_blocker | m3-hardware-loop-runtime | Live M3 smoke reached DEV simulators, but patch-panel active DO1 -> DI1 connection is missing. | +| P3 | 99 | agent_blocker | agent-mgr-health | hwlab-agent-mgr /health/live is degraded. | +| P3 | 99 | agent_blocker | skills-commit-version-injection | DEV skills injection is incomplete: missing hwlab-agent-skills.HWLAB_SKILLS_VERSION, worker-dry-run.HWLAB_SKILL_VERSION_FROM_DEV. | ## Next Unblock Order @@ -94,8 +98,10 @@ The latest accepted #99/#108 frontend DEV fact is revision `1e8805664970839b72be Evidence required: Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write. 3. Deploy cloud-api DB runtime readiness probe and/or repair DEV DB connectivity, then rerun the read-only health smoke without reading or printing the DB secret value. Evidence required: Cloud API health/live output showing DB env ready and redacted secret references, without secret material. -4. Prove the real DEV M3 trusted loop res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, trace, audit, and evidence identifiers. - Evidence required: Operation, trace, audit, and evidence IDs from the real DEV DO1 -> patch-panel -> DI1 trusted loop. +4. Fix DEV box-simu instance identity so direct endpoints expose distinct res_boxsimu_1 and res_boxsimu_2 resources. + Evidence required: Read-only direct box-simu /health/live and /status output showing distinct res_boxsimu_1 and res_boxsimu_2 resources. +5. Resolve the blocker and attach source/local/dry-run/DEV-live evidence at the correct level. + Evidence required: A committed report with the exact evidence level and command used. ## Validation diff --git a/scripts/dev-m3-hardware-loop-smoke.mjs b/scripts/dev-m3-hardware-loop-smoke.mjs index cb38f7dc..db01176b 100644 --- a/scripts/dev-m3-hardware-loop-smoke.mjs +++ b/scripts/dev-m3-hardware-loop-smoke.mjs @@ -20,6 +20,12 @@ const deployWorkloadsPath = "deploy/k8s/base/workloads.yaml"; const requiredManifestCardinalityCommand = "node scripts/validate-dev-m3-cardinality.mjs"; const requiredM3BoxIds = Object.freeze(["boxsimu_1", "boxsimu_2"]); const requiredM3BoxResources = Object.freeze(["res_boxsimu_1", "res_boxsimu_2"]); +const requiredM3Connection = Object.freeze({ + fromResourceId: "res_boxsimu_1", + fromPort: "DO1", + toResourceId: "res_boxsimu_2", + toPort: "DI1" +}); const runnerK3sKubeconfigPath = "/etc/rancher/k3s/k3s.yaml"; const serviceTargets = Object.freeze([ @@ -189,6 +195,9 @@ function collectD601RunnerObservability(ingress) { .filter(Boolean); const d601PublicEndpointsReachable = ingress.probes.some((item) => item.ok && isHwlabDevIdentity(item)); const runnerKubeconfigReadable = fileReadable(runnerK3sKubeconfigPath); + const classification = probe.exitCode === 0 + ? "read_only_kubectl_available_kubeconfig_file_unreadable" + : "runner_permission_mount_gap"; return { runnerKubeconfigReadable, @@ -197,11 +206,13 @@ function collectD601RunnerObservability(ingress) { runnerKubeconfigProbeStderr: stderr, d601PublicEndpointsReachable, d601K3sUnavailable: false, - classification: "runner_permission_mount_gap", + classification, sourceIssue: "pikasTech/HWLAB#46", command: runnerKubeconfigCommandText(args), stdoutSummary: probe.exitCode === 0 ? `${stdoutLines.length} pod name(s) observed` : undefined, - summary: "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; a kubectl success may come from alternate in-cluster config, so file readability and command exit code stay separate.", + summary: probe.exitCode === 0 + ? "Runner /etc/rancher/k3s/k3s.yaml is not readable through fs access, but read-only kubectl observation succeeds; file readability, kubectl reachability, and service discovery are recorded separately." + : "Runner /etc/rancher/k3s/k3s.yaml is not treated as readable from this container; kubectl read-only observation did not succeed, so this is a runner permission/mount gap, not proof that D601 is globally offline.", inferenceRule: "runnerKubeconfigReadable=false must not imply d601K3sUnavailable=true; public 16666/16667 endpoint reachability is tracked separately.", secretValuesRead: false, secretResourcesRead: false @@ -443,6 +454,23 @@ async function observeDevIngress() { for (const routePath of paths) { probes.push(await probeJson(joinUrl(DEV_ENDPOINT, routePath))); } + probes.push(await probeJson(joinUrl(DEV_FRONTEND_ENDPOINT, "/health/live"))); + probes.push(await probeJson(joinUrl(DEV_FRONTEND_ENDPOINT, "/health"))); + probes.push(await probeJson(joinUrl(DEV_ENDPOINT, "/json-rpc"), { + method: "POST", + body: { + jsonrpc: "2.0", + id: `req_dev_m3_${randomUUID()}`, + method: "system.health", + params: {}, + meta: { + traceId: `trc_dev_m3_${randomUUID()}`, + actorId: "system_code_queue_d601", + serviceId: "hwlab-cloud-api", + environment: ENVIRONMENT_DEV + } + } + })); const accepted = probes.find(isHwlabDevIdentity); return { @@ -471,6 +499,418 @@ function targetMapFromEnvironment() { return { missing, targets }; } +function serviceDefinitionByServiceId(serviceId) { + return serviceTargets.find((target) => target.serviceId === serviceId); +} + +function serviceEndpointUrl(address, port) { + return `http://${address}:${port}`; +} + +function endpointReady(endpoint) { + return endpoint.conditions?.ready !== false && endpoint.conditions?.serving !== false; +} + +function collectKubernetesDirectTargetDiscovery() { + if (!commandExists("kubectl")) { + return { + status: "unavailable", + source: "kubectl", + command: "command -v kubectl", + services: {}, + summary: "kubectl is not available in this runner, so Kubernetes endpoint-slice service discovery was not attempted." + }; + } + + const args = ["-n", namespace, "get", "endpointslices.discovery.k8s.io", "-o", "json"]; + const probe = commandResult("env", [`KUBECONFIG=${runnerK3sKubeconfigPath}`, "kubectl", ...args], { + timeoutMs: 8000, + maxChars: 100000 + }); + const command = runnerKubeconfigCommandText(args); + + if (probe.exitCode !== 0) { + return { + status: "blocked", + source: "kubernetes-endpointslices", + command, + exitCode: probe.exitCode, + services: {}, + summary: oneLine(probe.stderr || probe.stdout || "kubectl endpoint-slice discovery failed.") + }; + } + + let document; + try { + document = JSON.parse(probe.stdout); + } catch (error) { + return { + status: "blocked", + source: "kubernetes-endpointslices", + command, + exitCode: probe.exitCode, + services: {}, + summary: `kubectl endpoint-slice discovery returned non-JSON output: ${error.message}` + }; + } + + const requiredServices = new Set(["hwlab-box-simu", "hwlab-gateway-simu", "hwlab-patch-panel"]); + const services = {}; + + for (const item of document.items ?? []) { + const serviceId = item.metadata?.labels?.["kubernetes.io/service-name"]; + if (!requiredServices.has(serviceId)) { + continue; + } + const port = item.ports?.find((entry) => entry.name === "http")?.port ?? item.ports?.[0]?.port; + if (!Number.isInteger(port)) { + continue; + } + const endpoints = []; + for (const endpoint of item.endpoints ?? []) { + if (!endpointReady(endpoint)) { + continue; + } + for (const address of endpoint.addresses ?? []) { + endpoints.push({ + url: serviceEndpointUrl(address, port), + address, + port, + targetRef: endpoint.targetRef?.name ?? "unknown" + }); + } + } + services[serviceId] = { + endpointSlice: item.metadata?.name ?? "unknown", + port, + count: endpoints.length, + endpoints + }; + } + + return { + status: "observed", + source: "kubernetes-endpointslices", + command, + exitCode: probe.exitCode, + services, + summary: `Observed endpoint slices for ${Object.keys(services).join(", ") || "no M3 services"}.` + }; +} + +function candidateTargetsFromKubernetes(discovery) { + const candidates = []; + for (const [serviceId, service] of Object.entries(discovery.services ?? {})) { + const definition = serviceDefinitionByServiceId(serviceId); + if (!definition) { + continue; + } + let index = 1; + for (const endpoint of service.endpoints ?? []) { + candidates.push({ + ...definition, + id: `${definition.id}-candidate-${index}`, + baseUrl: endpoint.url, + discoverySource: discovery.source, + targetRef: endpoint.targetRef, + address: endpoint.address, + port: endpoint.port + }); + index += 1; + } + } + return candidates; +} + +function candidateTargetsFromEnvironment(targets) { + return [...targets.values()].map((target) => ({ + ...target, + discoverySource: "environment" + })); +} + +function resourceIdOfBoxStatus(status) { + return status?.resource?.resourceId ?? status?.resourceId ?? null; +} + +function gatewayIdentity(status) { + return `${status?.gatewayId ?? "unknown"}:${status?.session?.gatewaySessionId ?? status?.gatewaySessionId ?? "unknown"}`; +} + +function summarizeCandidate(candidate) { + const body = candidate.status?.json ?? {}; + return { + id: candidate.id, + serviceId: candidate.serviceId, + baseUrl: candidate.baseUrl, + targetRef: candidate.targetRef, + ok: candidate.ok, + statusCode: candidate.status?.status ?? null, + healthCode: candidate.health?.status ?? null, + boxId: body.boxId ?? null, + resourceId: resourceIdOfBoxStatus(body), + gatewayId: body.gatewayId ?? null, + gatewaySessionId: body.session?.gatewaySessionId ?? body.gatewaySessionId ?? null, + patchPanelState: body.state ?? null + }; +} + +function formatConnection(connection) { + if (!connection) return "none"; + if (connection.from && connection.to) { + return `${connection.from.resourceId}:${connection.from.port}->${connection.to.resourceId}:${connection.to.port}`; + } + return `${connection.fromResourceId}:${connection.fromPort}->${connection.toResourceId}:${connection.toPort}`; +} + +function hasRequiredM3Connection({ status, wiring }) { + const activeConnections = status?.activeConnections ?? []; + const wiringConnections = wiring?.connections ?? []; + const active = activeConnections.find( + (connection) => + connection.fromResourceId === requiredM3Connection.fromResourceId && + connection.fromPort === requiredM3Connection.fromPort && + connection.toResourceId === requiredM3Connection.toResourceId && + connection.toPort === requiredM3Connection.toPort + ); + const configured = wiringConnections.find( + (connection) => + connection.from?.resourceId === requiredM3Connection.fromResourceId && + connection.from?.port === requiredM3Connection.fromPort && + connection.to?.resourceId === requiredM3Connection.toResourceId && + connection.to?.port === requiredM3Connection.toPort && + connection.mode === "exclusive" + ); + return { active, configured }; +} + +async function probeDirectCandidate(candidate) { + const health = await probeJson(joinUrl(candidate.baseUrl, "/health/live")); + const status = await probeJson(joinUrl(candidate.baseUrl, candidate.statusPath)); + return { + ...candidate, + health, + status, + ok: health.ok && status.ok && health.json?.serviceId === candidate.serviceId + }; +} + +async function resolveDirectM3Targets({ environmentTargets, kubernetesDiscovery }) { + const allEnvironmentTargetsAvailable = environmentTargets.missing.length === 0; + const candidates = allEnvironmentTargetsAvailable + ? candidateTargetsFromEnvironment(environmentTargets.targets) + : candidateTargetsFromKubernetes(kubernetesDiscovery); + const probed = []; + + for (const candidate of candidates) { + probed.push(await probeDirectCandidate(candidate)); + } + + const boxCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-box-simu" && candidate.ok); + const gatewayCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-gateway-simu" && candidate.ok); + const patchCandidates = probed.filter((candidate) => candidate.serviceId === "hwlab-patch-panel" && candidate.ok); + const box1 = boxCandidates.find((candidate) => resourceIdOfBoxStatus(candidate.status.json) === "res_boxsimu_1"); + const box2 = boxCandidates.find( + (candidate) => + resourceIdOfBoxStatus(candidate.status.json) === "res_boxsimu_2" && + candidate.baseUrl !== box1?.baseUrl + ); + const distinctBoxResources = new Set(boxCandidates.map((candidate) => resourceIdOfBoxStatus(candidate.status.json)).filter(Boolean)); + const distinctGatewayIdentities = new Set(gatewayCandidates.map((candidate) => gatewayIdentity(candidate.status.json))); + const selectedGateways = []; + for (const candidate of gatewayCandidates) { + if (!selectedGateways.some((item) => gatewayIdentity(item.status.json) === gatewayIdentity(candidate.status.json))) { + selectedGateways.push(candidate); + } + } + const patch = patchCandidates[0] ?? null; + let patchWiring = null; + let m3Connection = { active: null, configured: null }; + + if (patch) { + patchWiring = await probeJson(joinUrl(patch.baseUrl, patch.wiringPath)); + m3Connection = hasRequiredM3Connection({ + status: patch.status.json, + wiring: patchWiring.json + }); + } + + const blockers = []; + const serviceDiscoveryBlocked = + !allEnvironmentTargetsAvailable && kubernetesDiscovery.status !== "observed"; + if (serviceDiscoveryBlocked) { + blockers.push({ + type: "observability_blocker", + scope: "m3-service-discovery", + status: "open", + classification: "runner_kubeconfig_readonly_gap", + sourceIssue: "pikasTech/HWLAB#46", + summary: `DEV ingress is reachable on frozen :16667, but direct M3 target discovery via ${kubernetesDiscovery.source} is ${kubernetesDiscovery.status}; environment URLs missing ${environmentTargets.missing.join(", ")}; discovery detail=${kubernetesDiscovery.summary}.` + }); + } else if (boxCandidates.length < 2 || gatewayCandidates.length < 2 || patchCandidates.length < 1) { + blockers.push({ + type: "observability_blocker", + scope: "m3-direct-target-missing", + status: "open", + classification: "direct_target_missing", + sourceIssue: "pikasTech/HWLAB#64", + summary: `Read-only service discovery did not expose enough callable DEV M3 direct targets: box-simu=${boxCandidates.length}/2, gateway-simu=${gatewayCandidates.length}/2, patch-panel=${patchCandidates.length}/1.` + }); + } + + if (boxCandidates.length >= 2 && (!box1 || !box2 || distinctBoxResources.size < 2)) { + blockers.push({ + type: "runtime_blocker", + scope: "m3-box-simu-identity", + status: "open", + classification: "direct_target_identity_gap", + sourceIssue: "pikasTech/HWLAB#64", + summary: `DEV exposes two box-simu endpoints, but live identities are not the required distinct resources res_boxsimu_1 and res_boxsimu_2; observed resources=${[...distinctBoxResources].join(", ") || "none"}.` + }); + } + + if (gatewayCandidates.length >= 2 && distinctGatewayIdentities.size < 2) { + blockers.push({ + type: "runtime_blocker", + scope: "m3-gateway-simu-identity", + status: "open", + classification: "direct_target_identity_gap", + sourceIssue: "pikasTech/HWLAB#64", + summary: `DEV exposes two gateway-simu endpoints, but live gateway identities are not distinct; observed identities=${[...distinctGatewayIdentities].join(", ") || "none"}.` + }); + } + + if (patch && (!m3Connection.active || !m3Connection.configured)) { + const observedActive = (patch.status.json?.activeConnections ?? []).map(formatConnection).join(", ") || "none"; + const observedConfigured = (patchWiring?.json?.connections ?? []).map(formatConnection).join(", ") || "none"; + blockers.push({ + type: "runtime_blocker", + scope: "m3-patch-panel-wiring", + status: "open", + classification: "direct_target_m3_wiring_missing", + sourceIssue: "pikasTech/HWLAB#64", + summary: `DEV patch-panel is callable, but live wiring does not contain ${requiredM3Connection.fromResourceId}:${requiredM3Connection.fromPort} -> ${requiredM3Connection.toResourceId}:${requiredM3Connection.toPort}; active=${observedActive}; configured=${observedConfigured}.` + }); + } + + const ok = + blockers.length === 0 && + box1 && + box2 && + selectedGateways.length >= 2 && + patch && + m3Connection.active && + m3Connection.configured; + + const targets = new Map(); + if (ok) { + targets.set("box-simu-1", { ...serviceTargets.find((item) => item.id === "box-simu-1"), baseUrl: box1.baseUrl }); + targets.set("box-simu-2", { ...serviceTargets.find((item) => item.id === "box-simu-2"), baseUrl: box2.baseUrl }); + targets.set("gateway-simu-1", { ...serviceTargets.find((item) => item.id === "gateway-simu-1"), baseUrl: selectedGateways[0].baseUrl }); + targets.set("gateway-simu-2", { ...serviceTargets.find((item) => item.id === "gateway-simu-2"), baseUrl: selectedGateways[1].baseUrl }); + targets.set("patch-panel", { ...serviceTargets.find((item) => item.id === "patch-panel"), baseUrl: patch.baseUrl }); + } + + return { + ok: Boolean(ok), + source: allEnvironmentTargetsAvailable ? "environment" : kubernetesDiscovery.source, + environmentMissing: environmentTargets.missing, + candidates: probed.map(summarizeCandidate), + counts: { + boxSimu: boxCandidates.length, + gatewaySimu: gatewayCandidates.length, + patchPanel: patchCandidates.length, + distinctBoxResources: distinctBoxResources.size, + distinctGatewayIdentities: distinctGatewayIdentities.size + }, + selected: { + boxSimu1: box1 ? summarizeCandidate(box1) : null, + boxSimu2: box2 ? summarizeCandidate(box2) : null, + gateways: selectedGateways.slice(0, 2).map(summarizeCandidate), + patchPanel: patch ? summarizeCandidate(patch) : null + }, + patchPanelWiring: patch + ? { + url: patchWiring?.url ?? joinUrl(patch.baseUrl, patch.wiringPath), + ok: patchWiring?.ok === true, + requiredConnection: requiredM3Connection, + activeObserved: (patch.status.json?.activeConnections ?? []).map(formatConnection), + configuredObserved: (patchWiring?.json?.connections ?? []).map(formatConnection), + hasRequiredActiveConnection: Boolean(m3Connection.active), + hasRequiredConfiguredConnection: Boolean(m3Connection.configured), + configSource: patch.status.json?.metadata?.configSource ?? "unknown", + syncableConnectionCount: patch.status.json?.metadata?.syncableConnectionCount ?? null + } + : null, + blockers, + targets + }; +} + +function addBlockedM3ChecksFromResolution(report, resolution) { + const identityBlocked = resolution.blockers.some((blocker) => + blocker.scope === "m3-box-simu-identity" || blocker.scope === "m3-gateway-simu-identity" + ); + const missingBlocked = resolution.blockers.some((blocker) => + blocker.scope === "m3-direct-target-missing" || blocker.scope === "m3-service-discovery" + ); + const wiringBlocked = resolution.blockers.some((blocker) => blocker.scope === "m3-patch-panel-wiring"); + const candidatesEvidence = [JSON.stringify({ source: resolution.source, counts: resolution.counts, selected: resolution.selected })]; + + report.liveChecks.push( + { + id: "two-box-simu-online", + status: missingBlocked || identityBlocked ? "blocked" : "pass", + blockerClass: missingBlocked ? "observability_blocker" : identityBlocked ? "runtime_blocker" : undefined, + summary: missingBlocked || identityBlocked + ? "Read-only direct target probes did not prove two distinct live DEV box-simu resources res_boxsimu_1 and res_boxsimu_2." + : "Read-only direct target probes proved both required DEV box-simu resources.", + evidence: candidatesEvidence + }, + { + id: "two-gateway-simu-online", + status: missingBlocked || identityBlocked ? "blocked" : "pass", + blockerClass: missingBlocked ? "observability_blocker" : identityBlocked ? "runtime_blocker" : undefined, + summary: missingBlocked || identityBlocked + ? "Read-only direct target probes did not prove two distinct live DEV gateway-simu identities." + : "Read-only direct target probes proved two distinct DEV gateway-simu identities.", + evidence: candidatesEvidence + }, + { + id: "patch-panel-healthy", + status: resolution.selected.patchPanel ? "pass" : "blocked", + blockerClass: resolution.selected.patchPanel ? undefined : "observability_blocker", + summary: resolution.selected.patchPanel + ? "DEV patch-panel direct target returned live status." + : "DEV patch-panel direct target was not callable.", + evidence: [JSON.stringify(resolution.selected.patchPanel ?? { status: "not_observed" })] + }, + { + id: "wiring-do1-di1-applied", + status: wiringBlocked ? "blocked" : resolution.ok ? "pass" : "not_run", + blockerClass: wiringBlocked ? "runtime_blocker" : undefined, + summary: wiringBlocked + ? "DEV patch-panel wiring is active but does not contain the required res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 route." + : resolution.ok + ? "DEV patch-panel active wiring matched res_boxsimu_1:DO1 -> res_boxsimu_2:DI1." + : "Patch-panel wiring check was not conclusive because earlier direct target checks were blocked.", + evidence: [JSON.stringify(resolution.patchPanelWiring ?? { status: "not_observed" })] + }, + { + id: "direct-call-do-write-di-read", + status: "not_run", + summary: "Not attempted because read-only DEV M3 preconditions were blocked; no box-simu loopback or SOURCE/DRY-RUN substitute was used.", + evidence: ["No live DEV write was sent."] + }, + { + id: "audit-evidence-traceable", + status: "not_run", + summary: "Not attempted because the patch-panel-bound live operation was not safe to trigger.", + evidence: ["operationId=not_observed", "traceId=not_observed", "auditId=not_observed", "evidenceId=not_observed"] + } + ); +} + function assertBoxState(state, label) { assert.equal(state?.serviceId, "hwlab-box-simu", `${label} serviceId`); assert.equal(state.live, true, `${label} live`); @@ -790,31 +1230,68 @@ async function main() { return; } - const { missing, targets } = targetMapFromEnvironment(); - if (missing.length > 0) { - addNotRunM3Checks(report, "Stopped before M3 direct checks because service target URLs were not provided."); - report.blockers.push({ - type: "observability_blocker", - scope: "m3-service-discovery", - status: "open", - classification: "runner permission/mount gap", - sourceIssue: "pikasTech/HWLAB#46", - summary: `DEV ingress is reachable on frozen :16667, but this runner cannot use ${runnerK3sKubeconfigPath} as readable kubeconfig to discover direct M3 service URLs (${missing.join(", ")}); this is a #46 runner permission/mount or read-only observability gap, not D601 global offline.` - }); + const environmentTargets = targetMapFromEnvironment(); + const kubernetesDiscovery = environmentTargets.missing.length === 0 + ? { + status: "not_run", + source: "environment", + command: "environment direct target variables", + services: {}, + summary: "All direct target URLs were supplied by environment variables." + } + : collectKubernetesDirectTargetDiscovery(); + report.directTargetDiscovery = { + environmentMissing: environmentTargets.missing, + kubernetes: { + status: kubernetesDiscovery.status, + source: kubernetesDiscovery.source, + command: kubernetesDiscovery.command, + exitCode: kubernetesDiscovery.exitCode ?? null, + services: kubernetesDiscovery.services, + summary: kubernetesDiscovery.summary + } + }; + const directTargets = await resolveDirectM3Targets({ + environmentTargets, + kubernetesDiscovery + }); + report.directTargetDiscovery = { + ...report.directTargetDiscovery, + source: directTargets.source, + counts: directTargets.counts, + selected: directTargets.selected, + candidates: directTargets.candidates, + patchPanelWiring: directTargets.patchPanelWiring, + summary: directTargets.ok + ? "Direct DEV M3 targets and patch-panel M3 wiring were discovered; live operation can be attempted." + : "Direct DEV M3 targets were probed read-only, but required identities or patch-panel M3 wiring were not satisfied." + }; + + if (!directTargets.ok) { + addBlockedM3ChecksFromResolution(report, directTargets); + report.liveOperation = { + status: "not_run", + operationId: "not_observed", + traceId: "not_observed", + auditId: "not_observed", + evidenceId: "not_observed", + summary: "No live DEV operation was attempted because read-only direct target checks did not prove the required patch-panel-owned M3 route." + }; + report.blockers.push(...directTargets.blockers); report.summary = { status: "blocked", - classification: "runner permission/mount gap", + classification: directTargets.blockers[0]?.classification ?? "direct_target_blocked", observedAt, - result: "DEV ingress was reachable, but live M3 simulator and patch-panel targets were not discoverable." + result: "DEV ingress and direct targets were reachable, but M3 DEV-LIVE was not triggered because required identities or patch-panel wiring were missing." }; await writeReport(report, reportPath); console.log(`[dev-m3-smoke] status=blocked report=${relativePath(reportPath)}`); - console.log("[dev-m3-smoke] blocker=observability_blocker scope=m3-service-discovery"); + console.log(`[dev-m3-smoke] blocker=${directTargets.blockers[0]?.type ?? "runtime_blocker"} scope=${directTargets.blockers[0]?.scope ?? "m3-direct-targets"}`); return; } try { - const live = await runLiveM3Targets(targets); + const live = await runLiveM3Targets(directTargets.targets); report.liveChecks.push( { id: "two-box-simu-online", diff --git a/scripts/src/dev-evidence-blocker-aggregator.mjs b/scripts/src/dev-evidence-blocker-aggregator.mjs index 9c4e8c78..ebfae73e 100644 --- a/scripts/src/dev-evidence-blocker-aggregator.mjs +++ b/scripts/src/dev-evidence-blocker-aggregator.mjs @@ -268,6 +268,7 @@ function applyPriority(blocker) { blocker.scope === "d601-k3s" || blocker.scope === "runner-kubeconfig-readonly-gap" || blocker.scope === "m3-service-discovery" || + blocker.scope === "m3-direct-target-missing" || blocker.scope.startsWith("d601-") ) { return { @@ -319,7 +320,12 @@ function applyPriority(blocker) { }; } - if (blocker.scope === "m3-hardware-loop-runtime") { + if ( + blocker.scope === "m3-hardware-loop-runtime" || + blocker.scope === "m3-patch-panel-wiring" || + blocker.scope === "m3-box-simu-identity" || + blocker.scope === "m3-gateway-simu-identity" + ) { return { ...blocker, priority: "P0", @@ -526,12 +532,16 @@ function m3TrustedLoopSummary(m3Report) { const operationSummary = m3Report.liveOperation?.summary ?? "No live M3 hardware operation was observed."; const blockerSummary = m3Report.blockers?.find((blocker) => blocker.scope === "m3-hardware-loop-runtime" || - blocker.scope === "m3-service-discovery" + blocker.scope === "m3-service-discovery" || + blocker.scope === "m3-direct-target-missing" || + blocker.scope === "m3-box-simu-identity" || + blocker.scope === "m3-gateway-simu-identity" || + blocker.scope === "m3-patch-panel-wiring" )?.summary; if (statusIsPass(m3Report.liveOperation?.status) || !blockerSummary) { return operationSummary; } - return `${operationSummary} Blocker: ${blockerSummary}.`; + return `${operationSummary} Blocker: ${blockerSummary.replace(/[.。]+$/u, "")}.`; } function collectM4Evidence(reports) { @@ -1009,13 +1019,23 @@ function buildMilestoneBlockerClassification(reports) { const m3Live = statusIsPass(reports.devM3Hardware.liveOperation?.status); const m4Live = statusIsPass(reports.devM4Agent.livePreflight?.status); const m3ServiceDiscoveryBlocked = reports.devM3Hardware.blockers?.some((blocker) => - blocker.scope === "m3-service-discovery" + blocker.scope === "m3-service-discovery" || blocker.scope === "m3-direct-target-missing" + ) === true; + const m3IdentityBlocked = reports.devM3Hardware.blockers?.some((blocker) => + blocker.scope === "m3-box-simu-identity" || blocker.scope === "m3-gateway-simu-identity" + ) === true; + const m3WiringBlocked = reports.devM3Hardware.blockers?.some((blocker) => + blocker.scope === "m3-patch-panel-wiring" ) === true; const m3BlockerClass = m3Live ? "cleared" : m3ServiceDiscoveryBlocked - ? "runner-readonly-observability-gap" - : "hardware-loop-runtime"; + ? "direct-target-missing" + : m3IdentityBlocked + ? "direct-target-identity-gap" + : m3WiringBlocked + ? "patch-panel-m3-wiring-missing" + : "hardware-loop-runtime"; return [ { @@ -1024,8 +1044,12 @@ function buildMilestoneBlockerClassification(reports) { currentLevel: m3Live ? "DEV-LIVE" : "BLOCKED", blockerClass: m3BlockerClass, dependency: m3ServiceDiscoveryBlocked - ? "Runner read-only service discovery must be repaired before the real DEV trusted loop can be observed; this does not mean D601/k3s is globally unavailable." - : "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.", + ? "Direct DEV simulator and patch-panel targets must be discoverable before the real trusted loop can be observed." + : m3IdentityBlocked + ? "Two distinct live box-simu resources and two distinct gateway-simu identities are required before M3 can run." + : m3WiringBlocked + ? "The callable DEV patch-panel must actively carry res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 before the write/read operation can run." + : "Real DEV trusted loop through res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1.", evidence: [ `operationId=${reports.devM3Hardware.liveOperation?.operationId ?? "not_observed"}`, `traceId=${reports.devM3Hardware.liveOperation?.traceId ?? "not_observed"}`, @@ -1033,11 +1057,18 @@ function buildMilestoneBlockerClassification(reports) { `evidenceId=${reports.devM3Hardware.liveOperation?.evidenceId ?? "not_observed"}`, `runnerKubeconfigReadable=${reports.devM3Hardware.d601Observability?.runnerKubeconfigReadable === true}`, `d601PublicEndpointsReachable=${reports.devM3Hardware.d601Observability?.d601PublicEndpointsReachable === true}`, - `d601K3sUnavailable=${reports.devM3Hardware.d601Observability?.d601K3sUnavailable === true}` + `d601K3sUnavailable=${reports.devM3Hardware.d601Observability?.d601K3sUnavailable === true}`, + `directTargetSource=${reports.devM3Hardware.directTargetDiscovery?.source ?? "not_observed"}`, + `directTargetCounts=${JSON.stringify(reports.devM3Hardware.directTargetDiscovery?.counts ?? {})}`, + `patchPanelM3Wiring=${reports.devM3Hardware.directTargetDiscovery?.patchPanelWiring?.hasRequiredConfiguredConnection === true}` ], nextRequired: m3ServiceDiscoveryBlocked - ? "Repair the #46 runner readonly kubeconfig/service-discovery gap, then run the bounded DEV M3 live smoke only when direct simulator and patch-panel targets are discoverable." - : "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.", + ? "Expose or document callable DEV direct targets for both simulators and the patch-panel, then rerun the read-only M3 preflight." + : m3IdentityBlocked + ? "Fix DEV simulator instance identity so two box-simu pods report res_boxsimu_1/res_boxsimu_2 and two gateway-simu pods report distinct gateway identities." + : m3WiringBlocked + ? "Load/apply DEV patch-panel wiring for res_boxsimu_1:DO1 -> res_boxsimu_2:DI1, then rerun the bounded DEV M3 smoke." + : "Run the bounded DEV M3 live smoke only after patch-panel topology can prove operation, trace, audit, and evidence IDs.", nonPromotionReason: "Public frontend, route, and artifact evidence do not prove the required hardware loop." }, { @@ -1125,8 +1156,11 @@ function fallbackAction(scope) { if (scope.includes("edge") || scope.includes("ingress") || scope.includes("frp")) return "Repair frp/master-edge/D601 router path and rerun read-only DEV edge health."; if (scope.includes("cloud-api-db")) return "Configure DEV cloud-api DB env readiness and rerun health/preflight without exposing secrets."; if (scope === "db-live") return "Repair DEV cloud-api DB live readiness, then rerun the read-only health and M4 preflight reports without exposing secret values."; + if (scope === "m3-patch-panel-wiring") return "Load/apply DEV patch-panel wiring for res_boxsimu_1:DO1 -> res_boxsimu_2:DI1, then rerun the bounded DEV M3 smoke."; + if (scope === "m3-box-simu-identity") return "Fix DEV box-simu instance identity so direct endpoints expose distinct res_boxsimu_1 and res_boxsimu_2 resources."; + if (scope === "m3-gateway-simu-identity") return "Fix DEV gateway-simu instance identity so direct endpoints expose two distinct gateway sessions."; if (scope === "m3-hardware-loop-runtime") return "Prove the real DEV M3 trusted loop res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1 with operation, trace, audit, and evidence identifiers."; - if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery") return "Repair the #46 runner kubeconfig mount/permission or document the approved alternate read-only KUBECONFIG/service-discovery path; do not classify this as D601 global offline."; + if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery" || scope === "m3-direct-target-missing") return "Repair the #46 runner kubeconfig mount/permission or document the approved alternate read-only KUBECONFIG/service-discovery path; do not classify this as D601 global offline."; if (scope.includes("kubectl") || scope.includes("k3s")) return "Provide read-only kubectl/kubeconfig observability for hwlab-dev."; return "Resolve the blocker and attach source/local/dry-run/DEV-live evidence at the correct level."; } @@ -1138,8 +1172,11 @@ function evidenceRequiredFor(scope) { if (scope.includes("kubectl") || scope.includes("k3s")) return "Read-only kubectl/k3s report proving pods/services/configmaps are observable in hwlab-dev without reading Secrets."; if (scope.includes("cloud-api-db")) return "Cloud API health/live output showing DB env ready and redacted secret references, without secret material."; if (scope === "db-live") return "Cloud API /health/live output with ready=true, connected=true, liveDbEvidence=true, and redacted secret references."; + if (scope === "m3-patch-panel-wiring") return "Read-only patch-panel /status and /wiring showing active res_boxsimu_1:DO1 -> res_boxsimu_2:DI1 before a bounded write/read smoke records operation, trace, audit, and evidence IDs."; + if (scope === "m3-box-simu-identity") return "Read-only direct box-simu /health/live and /status output showing distinct res_boxsimu_1 and res_boxsimu_2 resources."; + if (scope === "m3-gateway-simu-identity") return "Read-only direct gateway-simu /health/live and /status output showing two distinct gateway identities/sessions."; if (scope === "m3-hardware-loop-runtime") return "Operation, trace, audit, and evidence IDs from the real DEV DO1 -> patch-panel -> DI1 trusted loop."; - if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery") return "Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write."; + if (scope === "runner-kubeconfig-readonly-gap" || scope === "m3-service-discovery" || scope === "m3-direct-target-missing") return "Read-only report with runnerKubeconfigReadable, runnerKubeconfigProbeExitCode/stderr, d601PublicEndpointsReachable, and d601K3sUnavailable recorded separately, plus direct M3 service target discovery before any DO write."; if (scope.includes("edge") || scope.includes("ingress") || scope.includes("frp")) return "Read-only DEV route observation for :16667/frp/edge/router with HWLAB service identity and artifact identity."; return "A committed report with the exact evidence level and command used."; }