diff --git a/docs/dev-edge-health.md b/docs/dev-edge-health.md index e642af6b..3aa6c465 100644 --- a/docs/dev-edge-health.md +++ b/docs/dev-edge-health.md @@ -42,7 +42,15 @@ names so gate reports can identify `cloud-api-db-health-gate`. When the env is present but no live DB probe is attempted, the gate remains blocked with `connectionResult: "not_attempted"`. When a probe is attempted but `liveConnected` is false, the gate remains blocked/degraded with the redacted -connection classifier. Env presence alone is not live DB evidence. +connection classifier. `connectionResult: "dns_error"` with +`classification: "dns_resolution_failed"` means DB endpoint DNS failed from the +cloud-api runtime. Env presence alone is not live DB evidence. + +The Code Agent provider Secret for #143 is a separate blocker. The edge-health +report may record only key-presence evidence for +`hwlab-code-agent-provider/openai-api-key` and runtime error classification such +as `provider_secret_missing`; it must not read or print provider Secret values. +Do not collapse this #143 provider Secret gap into the DB live blocker. ## Route Classification @@ -75,9 +83,18 @@ Blocked `frp_blocker` and `edge_proxy_blocker` reports now include a The committed report at `reports/dev-gate/dev-edge-health.json` records a read-only live run where public `http://74.48.78.17:16667/health/live` returned -HWLAB DEV health from `hwlab-cloud-api`, DB config was present, but runtime -health still reported no live DB connection attempt/result from the deployed -image. Classification: `app_health_blocker` with likely layer `cloud-api-db`. +HWLAB DEV health from `hwlab-cloud-api`, DB config was present, and runtime +health attempted a redacted TCP DB probe. The probe was not connected: +`connected=false`, `liveConnected=false`, `liveDbEvidence=false`, +`connectionResult="dns_error"`, and +`classification="dns_resolution_failed"`. This is a DB live readiness blocker +for #49, not proof that DB is connected. + +The same report records the #143 Code Agent provider Secret gap separately as +`provider_secret_missing` when key-presence evidence for +`hwlab-code-agent-provider/openai-api-key` is absent. DB live readiness, +provider-backed Code Agent chat, M3 hardware trusted-loop proof, M4 live agent +loop, and M5 DEV-LIVE acceptance remain separate evidence layers. ## DEV FRP Contract diff --git a/docs/dev-evidence-blocker-aggregator.md b/docs/dev-evidence-blocker-aggregator.md index 14e8abd7..9d795214 100644 --- a/docs/dev-evidence-blocker-aggregator.md +++ b/docs/dev-evidence-blocker-aggregator.md @@ -50,7 +50,8 @@ without promoting the full gate: | --- | --- | | `Frontend DEV revision` | Latest accepted Cloud Workbench frontend revision on the active `http://74.48.78.17:16666/` browser endpoint. Frontend revision evidence is browser-only and cannot green DB, M3, M4, or M5. | | `EDGE/ROUTE live` | `16666` browser and `16667` health route evidence from read-only probes. | -| `DB live/degraded` | Cloud API DB readiness from redacted health evidence; route success does not imply DB live. | +| `DB live/degraded` | Cloud API DB readiness from redacted health evidence; route success and env presence do not imply DB live. A `dns_error`/`dns_resolution_failed` DB probe is a DB live blocker. | +| `Code Agent provider Secret` | #143 provider-backed chat readiness from redacted runtime error and Secret key-presence evidence only; missing provider Secret is independent from DB live. | | `M3 hardware trusted loop` | Real DEV `res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1` plus operation, trace, audit, and evidence identifiers. | | `M4 agent loop` | Agent loop live preflight and scheduling/evidence closure, gated by DB and runtime readiness. | | `artifact/desired-state source` | Artifact source coverage for the current target ref plus desired-state dry-run/apply status. | @@ -100,9 +101,9 @@ The v2 report reads committed report fixtures only: The report remains blocked until real DEV-LIVE evidence is attached for the required downstream HWLAB runtime services. Public route evidence on -`16666/16667` is necessary but not sufficient: DB live readiness, M3 trusted -loop evidence, M4 agent-loop evidence, and artifact/desired-state source -coverage stay separate. +`16666/16667` is necessary but not sufficient: DB live readiness, #143 Code +Agent provider Secret readiness, M3 trusted loop evidence, M4 agent-loop +evidence, and artifact/desired-state source coverage stay separate. The legacy base-image blocker is not current when the committed reports show an approved Node 20 builder base. Artifact/current remains stricter: a target ref diff --git a/reports/dev-gate/dev-edge-health.json b/reports/dev-gate/dev-edge-health.json index 9396ac60..3001da5f 100644 --- a/reports/dev-gate/dev-edge-health.json +++ b/reports/dev-gate/dev-edge-health.json @@ -4,12 +4,20 @@ "reportVersion": "v1", "issue": "pikasTech/HWLAB#36", "taskId": "dev-edge-health", - "commitId": "86d769525caf", + "commitId": "c7de4745f491", "acceptanceLevel": "dev_edge_health", "devOnly": true, "prodDisabled": true, + "reportLifecycle": { + "version": "v1", + "state": "active", + "activeEndpoint": "http://74.48.78.17:16667", + "activeBrowserEndpoint": "http://74.48.78.17:16666", + "deprecatedEndpoint": null, + "summary": "Current DEV edge-health report; deprecated public 6666/6667 endpoints are not valid active evidence." + }, "status": "blocked", - "generatedAt": "2026-05-22T04:30:36.613Z", + "generatedAt": "2026-05-22T13:45:42.982Z", "namespace": "hwlab-dev", "endpoint": "http://74.48.78.17:16667", "sourceContract": { @@ -28,6 +36,7 @@ "node scripts/validate-dev-gate-report.mjs", "node --check scripts/dev-edge-health-smoke.mjs", "node --check scripts/src/dev-edge-health-smoke-lib.mjs", + "node --test scripts/src/dev-edge-health-smoke-lib.test.mjs", "node scripts/dev-edge-health-smoke.mjs --live --write-report" ], "localSmoke": { @@ -58,18 +67,50 @@ "hwlab-dev hwlab-edge-proxy, hwlab-tunnel-client, hwlab-router, and hwlab-cloud-api are observable", "No PROD endpoint, secret read, restart, or UniDesk runtime substitute is used" ], - "summary": "cloud-api DB env is injected, but runtime health has not attempted a live DB connection" + "summary": "DB endpoint DNS resolution failed from the cloud-api runtime" }, "blockers": [ { "type": "runtime_blocker", "scope": "cloud-api-db", "status": "open", - "summary": "cloud-api DB env is injected, but runtime health has not attempted a live DB connection" + "summary": "DB endpoint DNS resolution failed from the cloud-api runtime", + "classification": "dns_resolution_failed", + "sourceIssue": "pikasTech/HWLAB#49", + "impact": { + "blocks": [ + "DB live readiness", + "M4 live agent scheduling precondition", + "M5 DEV-LIVE acceptance" + ], + "separateFrom": [ + "pikasTech/HWLAB#143 provider Secret", + "M3 hardware trusted-loop proof" + ] + } + }, + { + "type": "agent_blocker", + "scope": "code-agent-provider-secret", + "status": "open", + "summary": "Code Agent provider Secret hwlab-code-agent-provider/openai-api-key is not present as key-presence evidence; #143 real provider-backed chat remains blocked", + "classification": "provider_secret_missing", + "sourceIssue": "pikasTech/HWLAB#143", + "impact": { + "blocks": [ + "#143 provider-backed Code Agent chat", + "M4 provider-backed agent execution", + "M5 DEV-LIVE acceptance" + ], + "separateFrom": [ + "pikasTech/HWLAB#49 DB live readiness", + "M3 hardware trusted-loop proof" + ] + } } ], "edgeHealth": { - "generatedAt": "2026-05-22T04:30:36.613Z", + "generatedAt": "2026-05-22T13:45:42.982Z", "mode": "live-read-only", "endpoint": "http://74.48.78.17:16667", "safety": { @@ -101,7 +142,7 @@ "serviceCount": 13, "edgeProxy": { "serviceId": "hwlab-edge-proxy", - "image": "127.0.0.1:5000/hwlab/hwlab-edge-proxy:73b379f", + "image": "127.0.0.1:5000/hwlab/hwlab-edge-proxy:7e29522", "namespace": "hwlab-dev", "healthPath": "/health/live", "profile": "dev", @@ -113,7 +154,7 @@ }, "tunnelClient": { "serviceId": "hwlab-tunnel-client", - "image": "127.0.0.1:5000/hwlab/hwlab-tunnel-client:73b379f", + "image": "127.0.0.1:5000/hwlab/hwlab-tunnel-client:7e29522", "namespace": "hwlab-dev", "healthPath": "/health/live", "profile": "dev", @@ -134,6 +175,7 @@ "connectionChecked": false, "connectionAttempted": false, "connectionResult": "manifest_only_not_attempted", + "liveDbEvidence": false, "requiredEnv": [ { "name": "HWLAB_CLOUD_DB_URL", @@ -173,14 +215,46 @@ "durationMs": 0, "errorCode": null }, + "redaction": { + "valuesRedacted": true, + "endpointRedacted": true, + "valueRedacted": true, + "secretMaterialRead": false, + "secretRefsOnly": true + }, "blocker": "Manifest declares DB env/Secret references but does not prove runtime env injection or live DB connection", - "liveDbEvidence": false, "fixtureEvidence": false, "manifestConfigReady": true, "envInjected": false, "secretMaterialRead": false, "valuesRedacted": true, "note": "Manifest-only DB env placeholder; this is not runtime env injection or live DB evidence." + }, + "codeAgentProvider": { + "status": "degraded", + "sourceIssue": "pikasTech/HWLAB#143", + "requiredEnv": [ + { + "name": "OPENAI_API_KEY", + "present": true, + "redacted": true, + "source": "k8s-secret-ref" + } + ], + "secretRef": { + "env": "OPENAI_API_KEY", + "secretName": "hwlab-code-agent-provider", + "secretKey": "openai-api-key", + "present": true, + "envInjected": false, + "secretPresent": "not_observed_by_manifest", + "secretKeyPresent": "declared", + "secretValueRead": false, + "redacted": true + }, + "secretMaterialRead": false, + "valuesRedacted": true, + "note": "Manifest-only Code Agent provider Secret reference; this is not runtime Secret injection or provider connectivity evidence." } } }, @@ -189,19 +263,19 @@ "host": "74.48.78.17", "port": 16667, "status": "connected", - "durationMs": 228 + "durationMs": 206 }, { "host": "74.48.78.17", "port": 7000, "status": "connected", - "durationMs": 212 + "durationMs": 218 }, { "host": "74.48.78.17", "port": 7402, "status": "connected", - "durationMs": 216 + "durationMs": 207 } ], "publicHttp": [ @@ -214,7 +288,7 @@ "connection": "keep-alive", "content-length": "661", "content-type": "application/json; charset=utf-8", - "date": "Fri, 22 May 2026 04:30:35 GMT", + "date": "Fri, 22 May 2026 13:45:42 GMT", "keep-alive": "timeout=5" }, "json": { @@ -237,14 +311,14 @@ "digest": "unknown" }, "endpoint": "http://74.48.78.17:16667", - "observedAt": "2026-05-22T04:30:35.063Z", + "observedAt": "2026-05-22T13:45:42.602Z", "details": { "upstream": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667", "mode": "dev-edge-proxy" } }, "bodyPreview": "{\n \"serviceId\": \"hwlab-edge-proxy\",\n \"environment\": \"dev\",\n \"status\": \"ok\",\n \"service\": {\n \"id\": \"hwlab-edge-proxy\",\n \"role\": \"public-dev-ingress\",\n \"healthPath\": \"/health\",\n \"livePath\": \"/health/live\"\n },\n \"commit\": {\n \"id\": \"cb35ada6860653b27269b0a57991184118cbf4b1\",\n \"source\": \"runtime-env\"\n },\n \"image\": {\n \"reference\": \"ghcr.io/pikastech/hwlab-edge-proxy:cb35ada\",\n \"tag\": \"cb35ada\",\n \"digest\": \"unknown\"\n },\n \"endpoint\": \"http://74.48.78.17:16667\",\n \"obse", - "durationMs": 2000 + "durationMs": 837 }, { "url": "http://74.48.78.17:16667/health/live", @@ -255,7 +329,7 @@ "cache-control": "no-store", "connection": "keep-alive", "content-type": "application/json; charset=utf-8", - "date": "Fri, 22 May 2026 04:30:35 GMT", + "date": "Fri, 22 May 2026 13:45:42 GMT", "keep-alive": "timeout=5", "transfer-encoding": "chunked" }, @@ -270,25 +344,29 @@ "livePath": "/health/live" }, "commit": { - "id": "cb35ada", + "id": "c7de474", "source": "runtime-env" }, "image": { - "reference": "127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada", - "tag": "cb35ada", + "reference": "127.0.0.1:5000/hwlab/hwlab-cloud-api:c7de474", + "tag": "c7de474", "digest": "unknown" }, "endpoint": "http://74.48.78.17:16667", - "observedAt": "2026-05-22T04:30:35.064Z", + "observedAt": "2026-05-22T13:45:42.604Z", "db": { "contractVersion": "v1", "environment": "dev", "connected": false, - "connectionChecked": false, + "liveConnected": false, + "liveDbEvidence": false, + "connectionChecked": true, + "connectionAttempted": true, + "connectionResult": "dns_error", "configReady": true, "ready": false, "status": "degraded", - "mode": "configured_without_live_connection", + "mode": "live_connection_blocked", "fields": [ { "name": "HWLAB_CLOUD_DB_URL", @@ -312,17 +390,44 @@ "secretName": "hwlab-cloud-api-dev-db", "secretKey": "database-url", "present": true, + "envInjected": true, + "secretPresent": "not_observed_by_runtime", + "secretKeyPresent": "not_observed_by_runtime", "redacted": true } ], + "connection": { + "attempted": true, + "networkAttempted": true, + "probeType": "tcp-connect", + "endpointRedacted": true, + "valueRedacted": true, + "timeoutMs": 1200, + "durationMs": 1, + "missingEnv": [], + "result": "dns_error", + "classification": "dns_resolution_failed", + "errorCode": "ENOTFOUND" + }, + "redaction": { + "valuesRedacted": true, + "endpointRedacted": true, + "valueRedacted": true, + "secretMaterialRead": false, + "secretRefsOnly": true + }, "safety": { "devOnly": true, "prodAllowed": false, "secretsRead": false, + "secretMaterialRead": false, "valuesRedacted": true, - "liveDbEvidence": false + "endpointRedacted": true, + "liveDbEvidence": false, + "liveDbConnectedEvidence": false }, - "evidence": "env_presence_only_no_live_db" + "blocker": "DB endpoint DNS resolution failed from the cloud-api runtime", + "evidence": "live_db_tcp_connection_blocked" }, "runtime": { "adapter": "memory", @@ -339,8 +444,8 @@ } } }, - "bodyPreview": "{\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"status\":\"degraded\",\"service\":{\"id\":\"hwlab-cloud-api\",\"role\":\"cloud-api\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T04:30:35.064Z\",\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"c", - "durationMs": 820 + "bodyPreview": "{\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"status\":\"degraded\",\"service\":{\"id\":\"hwlab-cloud-api\",\"role\":\"cloud-api\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"c7de474\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"127.0.0.1:5000/hwlab/hwlab-cloud-api:c7de474\",\"tag\":\"c7de474\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T13:45:42.604Z\",\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"liveConnected\":false,\"liveD", + "durationMs": 810 } ], "kubernetes": { @@ -429,7 +534,7 @@ { "name": "hwlab-cloud-api", "readyAddresses": [ - "10.42.0.201" + "10.42.0.226" ], "notReadyAddresses": [], "ports": [ @@ -537,7 +642,7 @@ ] }, { - "name": "hwlab-cloud-api-f48dcdfb-9n6zh", + "name": "hwlab-cloud-api-74c969765d-kzvdd", "phase": "Running", "serviceId": "hwlab-cloud-api", "containers": [ @@ -545,13 +650,13 @@ "name": "hwlab-cloud-api", "ready": true, "restartCount": 0, - "image": "127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada", - "imageID": "127.0.0.1:5000/hwlab/hwlab-cloud-api@sha256:3946efdd7360409cea6a897cb03a6f3172f14819d3132df3aab821621909513f" + "image": "127.0.0.1:5000/hwlab/hwlab-cloud-api:c7de474", + "imageID": "127.0.0.1:5000/hwlab/hwlab-cloud-api@sha256:86fe92c0dd5d5883851b38805e6f5acc49d2876552599c18f85480cf0d3bf981" } ] }, { - "name": "hwlab-cloud-web-cd9655b98-gxtmc", + "name": "hwlab-cloud-web-7d48b84f4b-n6hw6", "phase": "Running", "serviceId": "hwlab-cloud-web", "containers": [ @@ -559,8 +664,8 @@ "name": "hwlab-cloud-web", "ready": true, "restartCount": 0, - "image": "127.0.0.1:5000/hwlab/hwlab-cloud-web:cb35ada", - "imageID": "127.0.0.1:5000/hwlab/hwlab-cloud-web@sha256:868e2a11c56cc1a27c77de3fa71a7b092f21ae98a4052104e9071bd93960abfa" + "image": "127.0.0.1:5000/hwlab/hwlab-cloud-web:c7de474", + "imageID": "127.0.0.1:5000/hwlab/hwlab-cloud-web@sha256:91cfacc1fc011a96bb47ff00d2ab09ee3c93003b7ec482308ec5ac35004e5627" } ] }, @@ -660,7 +765,22 @@ "command": "kubectl describe secret (key-presence-only)", "error": null }, - "notes": [] + "codeAgentProviderSecret": { + "observable": true, + "sourceIssue": "pikasTech/HWLAB#143", + "env": "OPENAI_API_KEY", + "secretName": "hwlab-code-agent-provider", + "secretKey": "openai-api-key", + "secretPresent": false, + "secretKeyPresent": false, + "secretValueRead": false, + "redacted": true, + "command": "kubectl get secret -o name", + "error": "Error from server (NotFound): secrets \"hwlab-code-agent-provider\" not found" + }, + "notes": [ + "kubectl Code Agent provider Secret presence probe failed: Error from server (NotFound): secrets \"hwlab-code-agent-provider\" not found" + ] }, "clusterDns": [ { @@ -709,19 +829,21 @@ "secretRefPresent": true, "envInjected": true }, - "connectionAttempted": false, - "connectionResult": "not_attempted", + "connectionAttempted": true, + "connectionResult": "dns_error", + "connectionClassification": "dns_resolution_failed", "liveConnected": false, "valuesRedacted": true, "secretMaterialRead": false, "liveDbEvidence": false, - "blocker": "cloud-api DB env is injected, but runtime health has not attempted a live DB connection", + "blocker": "DB endpoint DNS resolution failed from the cloud-api runtime", "evidence": [ { "configReady": true, "missingEnv": [], - "connectionAttempted": false, - "connectionResult": "not_attempted", + "connectionAttempted": true, + "connectionResult": "dns_error", + "classification": "dns_resolution_failed", "liveConnected": false, "endpointRedacted": true, "valueRedacted": true @@ -730,35 +852,127 @@ }, "status": "blocker", "classification": "app_health_blocker", - "blocker": "cloud-api DB env is injected, but runtime health has not attempted a live DB connection", + "blocker": "DB endpoint DNS resolution failed from the cloud-api runtime", "diagnosis": { "likelyLayer": "cloud-api-db", "confidence": "high", - "likelyCause": "cloud-api DB env is injected, but runtime health has not attempted a live DB connection", + "likelyCause": "DB endpoint DNS resolution failed from the cloud-api runtime", "evidence": [ { "configReady": true, "missingEnv": [], - "connectionAttempted": false, - "connectionResult": "not_attempted", + "connectionAttempted": true, + "connectionResult": "dns_error", + "classification": "dns_resolution_failed", "liveConnected": false, "endpointRedacted": true, "valueRedacted": true } ], "notProven": [ - "live DB endpoint reachability", "authenticated SQL query readiness" ], "nextTask": "Deploy cloud-api DB runtime readiness probe and/or repair DEV DB connectivity, then rerun the read-only health smoke without reading or printing the DB secret value." + }, + "providerSecretReadiness": { + "status": "blocked", + "sourceIssue": "pikasTech/HWLAB#143", + "provider": "openai-responses", + "requiredEnv": "OPENAI_API_KEY", + "manifestRefPresent": true, + "secret": { + "observable": true, + "sourceIssue": "pikasTech/HWLAB#143", + "env": "OPENAI_API_KEY", + "secretName": "hwlab-code-agent-provider", + "secretKey": "openai-api-key", + "secretPresent": false, + "secretKeyPresent": false, + "secretValueRead": false, + "redacted": true, + "command": "kubectl get secret -o name", + "error": "Error from server (NotFound): secrets \"hwlab-code-agent-provider\" not found" + }, + "providerCallAttempted": false, + "providerConnected": false, + "secretMaterialRead": false, + "valuesRedacted": true, + "classification": "provider_secret_missing", + "blocker": "Code Agent provider Secret hwlab-code-agent-provider/openai-api-key is not present as key-presence evidence; #143 real provider-backed chat remains blocked", + "evidence": [ + { + "source": "kubernetes-secret-presence", + "manifestRefPresent": true, + "secretPresent": false, + "secretKeyPresent": false, + "secretValueRead": false, + "redacted": true + } + ] + }, + "parallelBlockers": [ + { + "id": "cloud-api-db-live", + "type": "runtime_blocker", + "scope": "cloud-api-db", + "status": "open", + "sourceIssue": "pikasTech/HWLAB#49", + "classification": "dns_resolution_failed", + "summary": "DB endpoint DNS resolution failed from the cloud-api runtime", + "impact": { + "blocks": [ + "DB live readiness", + "M4 live agent scheduling precondition", + "M5 DEV-LIVE acceptance" + ], + "separateFrom": [ + "pikasTech/HWLAB#143 provider Secret", + "M3 hardware trusted-loop proof" + ] + } + }, + { + "id": "code-agent-provider-secret", + "type": "agent_blocker", + "scope": "code-agent-provider-secret", + "status": "open", + "sourceIssue": "pikasTech/HWLAB#143", + "classification": "provider_secret_missing", + "summary": "Code Agent provider Secret hwlab-code-agent-provider/openai-api-key is not present as key-presence evidence; #143 real provider-backed chat remains blocked", + "impact": { + "blocks": [ + "#143 provider-backed Code Agent chat", + "M4 provider-backed agent execution", + "M5 DEV-LIVE acceptance" + ], + "separateFrom": [ + "pikasTech/HWLAB#49 DB live readiness", + "M3 hardware trusted-loop proof" + ] + } + } + ], + "milestoneImpact": { + "M3": { + "status": "separate_blocker", + "summary": "M3 still requires real DEV hardware trusted-loop operation/trace/audit/evidence; DB DNS failure and #143 provider Secret absence do not prove or clear M3." + }, + "M4": { + "status": "blocked", + "blockedBy": [ + "cloud-api-db-live", + "code-agent-provider-secret" + ], + "summary": "M4 live agent-loop remains blocked until DB live readiness and provider-backed agent prerequisites are independently green." + }, + "M5": { + "status": "blocked", + "blockedBy": [ + "cloud-api-db-live", + "code-agent-provider-secret" + ], + "summary": "M5 DEV-LIVE acceptance cannot be promoted from route health, env presence, or provider manifest refs while DB live and #143 provider Secret blockers remain open." + } } - }, - "reportLifecycle": { - "version": "v1", - "state": "active", - "activeEndpoint": "http://74.48.78.17:16667", - "activeBrowserEndpoint": "http://74.48.78.17:16666", - "deprecatedEndpoint": null, - "summary": "Current DEV edge-health report; deprecated public 6666/6667 endpoints are not valid active evidence." } } diff --git a/scripts/src/dev-edge-health-smoke-lib.mjs b/scripts/src/dev-edge-health-smoke-lib.mjs index cf644b63..61432675 100644 --- a/scripts/src/dev-edge-health-smoke-lib.mjs +++ b/scripts/src/dev-edge-health-smoke-lib.mjs @@ -15,6 +15,12 @@ const publicHost = "74.48.78.17"; const publicPort = 16667; const namespace = "hwlab-dev"; const tcpPorts = [16667, 7000, 7402]; +const CODE_AGENT_PROVIDER_SECRET_CONTRACT = Object.freeze({ + sourceIssue: "pikasTech/HWLAB#143", + env: "OPENAI_API_KEY", + secretName: "hwlab-code-agent-provider", + secretKey: "openai-api-key" +}); const runtimeServices = [ { serviceId: "hwlab-cloud-api", port: 6667 }, { serviceId: "hwlab-router", port: 7401 }, @@ -69,11 +75,24 @@ export async function runDevEdgeHealthSmoke(argv) { }; edgeHealth.runtimeDbReadiness = inspectRuntimeDbReadiness(edgeHealth); Object.assign(edgeHealth, classify(edgeHealth)); + edgeHealth.providerSecretReadiness = inspectProviderSecretReadiness(edgeHealth); + edgeHealth.parallelBlockers = classifyParallelBlockers(edgeHealth); + if (edgeHealth.status === "pass" && edgeHealth.parallelBlockers.length > 0) { + Object.assign(edgeHealth, classifyParallelBlockerStatus(edgeHealth.parallelBlockers)); + } + edgeHealth.milestoneImpact = classifyMilestoneImpact(edgeHealth); const report = await createDevGateReport(edgeHealth); await maybeWriteReport(report, args); return { report, exitCode: edgeHealth.status === "pass" ? 0 : 2 }; } +export const __testHooks = Object.freeze({ + classifyMilestoneImpact, + classifyParallelBlockers, + inspectProviderSecretReadiness, + summarizeRuntimeDbReadiness +}); + function parseArgs(argv) { const args = { live: false, @@ -106,8 +125,7 @@ function nowIso() { async function createDevGateReport(edgeHealth) { const commitId = await gitCommitId(); const status = edgeHealth.status === "pass" ? "pass" : edgeHealth.status === "not_run" ? "not_run" : "blocked"; - const blockerType = status === "blocked" ? blockerTypeFor(edgeHealth.classification) : null; - const blockerScope = edgeHealth.diagnosis?.likelyLayer ?? edgeHealth.classification; + const blockers = reportBlockers(edgeHealth, status); return { $schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json", @@ -140,6 +158,7 @@ async function createDevGateReport(edgeHealth) { "node scripts/validate-dev-gate-report.mjs", "node --check scripts/dev-edge-health-smoke.mjs", "node --check scripts/src/dev-edge-health-smoke-lib.mjs", + "node --test scripts/src/dev-edge-health-smoke-lib.test.mjs", "node scripts/dev-edge-health-smoke.mjs --live --write-report" ], localSmoke: { @@ -172,16 +191,7 @@ async function createDevGateReport(edgeHealth) { ], summary: edgeHealth.blocker ?? "DEV edge health preconditions are satisfied." }, - blockers: blockerType - ? [ - { - type: blockerType, - scope: blockerScope, - status: "open", - summary: edgeHealth.blocker - } - ] - : [], + blockers, edgeHealth }; } @@ -204,6 +214,34 @@ function blockerTypeFor(classification) { return "network_blocker"; } +function reportBlockers(edgeHealth, status) { + if (status !== "blocked") { + return []; + } + + const parallelBlockers = Array.isArray(edgeHealth.parallelBlockers) ? edgeHealth.parallelBlockers : []; + if (parallelBlockers.length > 0) { + return parallelBlockers.map(({ type, scope, status: blockerStatus, summary, classification, sourceIssue, impact }) => ({ + type, + scope, + status: blockerStatus, + summary, + classification, + sourceIssue, + impact + })); + } + + return [ + { + type: blockerTypeFor(edgeHealth.classification), + scope: edgeHealth.diagnosis?.likelyLayer ?? edgeHealth.classification, + status: "open", + summary: edgeHealth.blocker + } + ]; +} + function requireDevEndpoint() { const endpoint = new URL(DEV_ENDPOINT); assert.equal(endpoint.protocol, "http:", "DEV endpoint must use http"); @@ -246,7 +284,8 @@ async function inspectContracts() { serviceCount: deploy.services?.length ?? 0, edgeProxy: deploy.services?.find((service) => service.serviceId === "hwlab-edge-proxy") ?? null, tunnelClient: deploy.services?.find((service) => service.serviceId === "hwlab-tunnel-client") ?? null, - cloudApiDb: inspectCloudApiDbContract(deploy) + cloudApiDb: inspectCloudApiDbContract(deploy), + codeAgentProvider: inspectCodeAgentProviderContract(deploy) } }; } @@ -322,6 +361,38 @@ function inspectCloudApiDbContract(deploy) { }; } +function inspectCodeAgentProviderContract(deploy) { + const env = deploy.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {}; + const expectedRef = `secretRef:${CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretName}/${CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretKey}`; + const secretRefPresent = env[CODE_AGENT_PROVIDER_SECRET_CONTRACT.env] === expectedRef; + return { + status: secretRefPresent ? "degraded" : "blocked", + sourceIssue: CODE_AGENT_PROVIDER_SECRET_CONTRACT.sourceIssue, + requiredEnv: [ + { + name: CODE_AGENT_PROVIDER_SECRET_CONTRACT.env, + present: Boolean(env[CODE_AGENT_PROVIDER_SECRET_CONTRACT.env]), + redacted: true, + source: "k8s-secret-ref" + } + ], + secretRef: { + env: CODE_AGENT_PROVIDER_SECRET_CONTRACT.env, + secretName: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretName, + secretKey: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretKey, + present: secretRefPresent, + envInjected: false, + secretPresent: "not_observed_by_manifest", + secretKeyPresent: secretRefPresent ? "declared" : "not_observed_by_manifest", + secretValueRead: false, + redacted: true + }, + secretMaterialRead: false, + valuesRedacted: true, + note: "Manifest-only Code Agent provider Secret reference; this is not runtime Secret injection or provider connectivity evidence." + }; +} + function notRunDbReadiness(blockerMessage) { return { status: "not_run", @@ -330,6 +401,7 @@ function notRunDbReadiness(blockerMessage) { secret: unknownDbSecretPresence("not_observed"), connectionAttempted: false, connectionResult: "not_run", + connectionClassification: "not_run", liveConnected: false, valuesRedacted: true, secretMaterialRead: false, @@ -359,6 +431,7 @@ function summarizeRuntimeDbReadiness(db, secretPresence = unknownDbSecretPresenc secret: secretPresence, connectionAttempted: false, connectionResult: "health_db_missing", + connectionClassification: "health_db_missing", liveConnected: false, valuesRedacted: true, secretMaterialRead: false, @@ -374,6 +447,8 @@ function summarizeRuntimeDbReadiness(db, secretPresence = unknownDbSecretPresenc ); const connectionAttempted = Boolean(db.connectionAttempted ?? db.connectionChecked); const connectionResult = db.connectionResult ?? db.connection?.result ?? (connectionAttempted ? "unknown" : "not_attempted"); + const connectionClassification = + db.connection?.classification ?? classificationForRuntimeDb(db, connectionAttempted, connectionResult); const liveConnected = Boolean(db.liveConnected ?? db.connected ?? db.ready); const blockerMessage = liveConnected ? null : db.blocker ?? blockerForRuntimeDb(db, connectionAttempted, connectionResult); const status = liveConnected ? "pass" : "blocked"; @@ -393,6 +468,7 @@ function summarizeRuntimeDbReadiness(db, secretPresence = unknownDbSecretPresenc }, connectionAttempted, connectionResult, + connectionClassification, liveConnected, valuesRedacted: db.redaction?.valuesRedacted !== false && db.safety?.valuesRedacted === true, secretMaterialRead: false, @@ -404,6 +480,7 @@ function summarizeRuntimeDbReadiness(db, secretPresence = unknownDbSecretPresenc missingEnv: Array.isArray(db.missingEnv) ? db.missingEnv : [], connectionAttempted, connectionResult, + classification: connectionClassification, liveConnected, endpointRedacted: db.connection?.endpointRedacted !== false, valueRedacted: true @@ -412,6 +489,34 @@ function summarizeRuntimeDbReadiness(db, secretPresence = unknownDbSecretPresenc }; } +function classificationForRuntimeDb(db, connectionAttempted, connectionResult) { + if (typeof db?.connectionClassification === "string") { + return db.connectionClassification; + } + if (typeof db?.classification === "string") { + return db.classification; + } + if (db?.configReady !== true) { + return "missing_runtime_env"; + } + if (!connectionAttempted) { + return "configured_no_live_attempt"; + } + if (connectionResult === "dns_error") { + return "dns_resolution_failed"; + } + if (connectionResult === "refused") { + return "tcp_refused"; + } + if (connectionResult === "timeout") { + return "tcp_timeout"; + } + if (connectionResult === "network_unreachable") { + return "network_unreachable"; + } + return connectionResult === "connected" ? "tcp_connected" : "tcp_error"; +} + function blockerForRuntimeDb(db, connectionAttempted, connectionResult) { if (db.configReady !== true) { const missing = Array.isArray(db.missingEnv) && db.missingEnv.length > 0 ? db.missingEnv.join(", ") : "required DB env"; @@ -436,6 +541,18 @@ function unknownDbSecretPresence(state) { }; } +function unknownProviderSecretPresence(state) { + return { + observable: false, + secretName: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretName, + secretKey: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretKey, + secretPresent: state, + secretKeyPresent: state, + secretValueRead: false, + redacted: true + }; +} + async function inspectKubernetes() { const hasKubectl = await commandExists("kubectl"); const result = { @@ -446,6 +563,7 @@ async function inspectKubernetes() { endpoints: [], pods: [], dbSecret: unknownDbSecretPresence("not_observed"), + codeAgentProviderSecret: unknownProviderSecretPresence("not_observed"), notes: [] }; if (!hasKubectl) { @@ -464,6 +582,7 @@ async function inspectKubernetes() { "-n", namespace, "get", "pods", "-l", "hwlab.pikastech.local/profile=dev", "-o", "json" ]); await collectDbSecretPresence(result); + await collectCodeAgentProviderSecretPresence(result); return result; } @@ -551,6 +670,48 @@ async function collectDbSecretPresence(result) { } } +async function collectCodeAgentProviderSecretPresence(result) { + const ref = CODE_AGENT_PROVIDER_SECRET_CONTRACT; + const base = { + observable: true, + sourceIssue: ref.sourceIssue, + env: ref.env, + secretName: ref.secretName, + secretKey: ref.secretKey, + secretPresent: false, + secretKeyPresent: false, + secretValueRead: false, + redacted: true + }; + const exists = await runCommand("kubectl", ["-n", namespace, "get", "secret", ref.secretName, "-o", "name"], { + timeoutMs: 5000 + }); + if (exists.exitCode !== 0) { + result.codeAgentProviderSecret = { + ...base, + command: "kubectl get secret -o name", + error: exists.stderr.trim() || "secret not observed" + }; + result.notes.push(`kubectl Code Agent provider Secret presence probe failed: ${exists.stderr.trim()}`); + return; + } + + const keyPresence = await runCommand("kubectl", ["-n", namespace, "describe", "secret", ref.secretName], { + timeoutMs: 5000 + }); + const keyLinePattern = new RegExp(`^\\s*${escapeRegExp(ref.secretKey)}:\\s+\\d+\\s+bytes\\s*$`, "mu"); + result.codeAgentProviderSecret = { + ...base, + secretPresent: true, + secretKeyPresent: keyPresence.exitCode === 0 && keyLinePattern.test(keyPresence.stdout), + command: "kubectl describe secret (key-presence-only)", + error: keyPresence.exitCode === 0 ? null : keyPresence.stderr.trim() + }; + if (keyPresence.exitCode !== 0) { + result.notes.push(`kubectl Code Agent provider Secret key presence probe failed: ${keyPresence.stderr.trim()}`); + } +} + async function inspectClusterDns() { const results = []; for (const service of runtimeServices) { @@ -615,6 +776,131 @@ function classifyReachableHealth(json, runtimeDb = null) { return { status: "pass", classification: "none", blocker: null }; } +function inspectProviderSecretReadiness(edgeHealth) { + const secret = edgeHealth.kubernetes?.codeAgentProviderSecret ?? unknownProviderSecretPresence("not_observed"); + const manifest = edgeHealth.contracts?.deploy?.codeAgentProvider ?? null; + const manifestRefPresent = manifest?.secretRef?.present === true; + const secretPresent = secret.secretPresent === true; + const secretKeyPresent = secret.secretKeyPresent === true; + const status = secret.observable && (!secretPresent || !secretKeyPresent) ? "blocked" : "not_proven"; + const classification = status === "blocked" ? "provider_secret_missing" : "provider_secret_presence_not_proven"; + return { + status, + sourceIssue: CODE_AGENT_PROVIDER_SECRET_CONTRACT.sourceIssue, + provider: "openai-responses", + requiredEnv: CODE_AGENT_PROVIDER_SECRET_CONTRACT.env, + manifestRefPresent, + secret: { + ...secret, + secretValueRead: false, + redacted: true + }, + providerCallAttempted: false, + providerConnected: false, + secretMaterialRead: false, + valuesRedacted: true, + classification, + blocker: status === "blocked" + ? `Code Agent provider Secret ${CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretName}/${CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretKey} is not present as key-presence evidence; #143 real provider-backed chat remains blocked` + : "Code Agent provider Secret presence was not proven by this edge-health run", + evidence: [ + { + source: secret.observable ? "kubernetes-secret-presence" : "runtime-not-observable", + manifestRefPresent, + secretPresent, + secretKeyPresent, + secretValueRead: false, + redacted: true + } + ] + }; +} + +function classifyParallelBlockers(edgeHealth) { + const blockers = []; + const dbReadiness = edgeHealth.runtimeDbReadiness; + if (dbReadiness?.status === "blocked") { + blockers.push({ + id: "cloud-api-db-live", + type: "runtime_blocker", + scope: "cloud-api-db", + status: "open", + sourceIssue: "pikasTech/HWLAB#49", + classification: dbReadiness.connectionClassification ?? dbReadiness.connectionResult, + summary: dbReadiness.blocker, + impact: { + blocks: ["DB live readiness", "M4 live agent scheduling precondition", "M5 DEV-LIVE acceptance"], + separateFrom: ["pikasTech/HWLAB#143 provider Secret", "M3 hardware trusted-loop proof"] + } + }); + } + + const provider = edgeHealth.providerSecretReadiness; + if (provider?.status === "blocked") { + blockers.push({ + id: "code-agent-provider-secret", + type: "agent_blocker", + scope: "code-agent-provider-secret", + status: "open", + sourceIssue: CODE_AGENT_PROVIDER_SECRET_CONTRACT.sourceIssue, + classification: provider.classification, + summary: provider.blocker, + impact: { + blocks: ["#143 provider-backed Code Agent chat", "M4 provider-backed agent execution", "M5 DEV-LIVE acceptance"], + separateFrom: ["pikasTech/HWLAB#49 DB live readiness", "M3 hardware trusted-loop proof"] + } + }); + } + return blockers; +} + +function classifyParallelBlockerStatus(parallelBlockers) { + return blocker( + "parallel_runtime_blocker", + parallelBlockers.map((item) => item.summary).join("; "), + { + likelyLayer: "parallel-dev-blockers", + confidence: "high", + likelyCause: "one or more independent DEV blockers remain open after route health passed", + evidence: parallelBlockers.map(({ id, scope, classification, sourceIssue }) => ({ + id, + scope, + classification, + sourceIssue + })), + notProven: ["DB-backed runtime readiness", "provider-backed Code Agent readiness", "M3/M4/M5 DEV-LIVE acceptance"], + nextTask: "Close each independent blocker with redacted evidence, then rerun the read-only edge smoke." + } + ); +} + +function classifyMilestoneImpact(edgeHealth) { + const dbBlocked = edgeHealth.runtimeDbReadiness?.status === "blocked"; + const providerBlocked = edgeHealth.providerSecretReadiness?.status === "blocked"; + return { + M3: { + status: "separate_blocker", + summary: "M3 still requires real DEV hardware trusted-loop operation/trace/audit/evidence; DB DNS failure and #143 provider Secret absence do not prove or clear M3." + }, + M4: { + status: dbBlocked || providerBlocked ? "blocked" : "not_proven", + blockedBy: [ + ...(dbBlocked ? ["cloud-api-db-live"] : []), + ...(providerBlocked ? ["code-agent-provider-secret"] : []) + ], + summary: "M4 live agent-loop remains blocked until DB live readiness and provider-backed agent prerequisites are independently green." + }, + M5: { + status: dbBlocked || providerBlocked ? "blocked" : "not_proven", + blockedBy: [ + ...(dbBlocked ? ["cloud-api-db-live"] : []), + ...(providerBlocked ? ["code-agent-provider-secret"] : []) + ], + summary: "M5 DEV-LIVE acceptance cannot be promoted from route health, env presence, or provider manifest refs while DB live and #143 provider Secret blockers remain open." + } + }; +} + function classifyRefusedPort(publicEdgePort, frpsControl, tunnelHealth) { if (frpsControl?.status === "error" && frpsControl.code === "ECONNREFUSED") { const allPortsRefused = tunnelHealth?.status === "error" && tunnelHealth.code === "ECONNREFUSED"; diff --git a/scripts/src/dev-edge-health-smoke-lib.test.mjs b/scripts/src/dev-edge-health-smoke-lib.test.mjs new file mode 100644 index 00000000..07bb2bde --- /dev/null +++ b/scripts/src/dev-edge-health-smoke-lib.test.mjs @@ -0,0 +1,111 @@ +import assert from "node:assert/strict"; +import test from "node:test"; + +import { __testHooks } from "./dev-edge-health-smoke-lib.mjs"; + +test("runtime DB readiness preserves dns resolution classification", () => { + const readiness = __testHooks.summarizeRuntimeDbReadiness( + { + configReady: true, + liveConnected: false, + liveDbEvidence: false, + connectionAttempted: true, + connectionResult: "dns_error", + fields: [ + { name: "HWLAB_CLOUD_DB_URL", present: true }, + { name: "HWLAB_CLOUD_DB_SSL_MODE", present: true } + ], + secretRefs: [ + { + env: "HWLAB_CLOUD_DB_URL", + secretName: "hwlab-cloud-api-dev-db", + secretKey: "database-url", + present: true, + envInjected: true + } + ], + connection: { + attempted: true, + result: "dns_error", + classification: "dns_resolution_failed", + endpointRedacted: true, + valueRedacted: true + }, + redaction: { valuesRedacted: true }, + safety: { valuesRedacted: true, liveDbEvidence: false }, + blocker: "DB endpoint DNS resolution failed from the cloud-api runtime" + }, + { + observable: true, + secretName: "hwlab-cloud-api-dev-db", + secretKey: "database-url", + secretPresent: true, + secretKeyPresent: true, + secretValueRead: false, + redacted: true + } + ); + + assert.equal(readiness.status, "blocked"); + assert.equal(readiness.connectionResult, "dns_error"); + assert.equal(readiness.connectionClassification, "dns_resolution_failed"); + assert.equal(readiness.liveConnected, false); + assert.equal(readiness.liveDbEvidence, false); + assert.equal(readiness.secretMaterialRead, false); + assert.equal(readiness.evidence[0].classification, "dns_resolution_failed"); +}); + +test("parallel blockers keep DB DNS and provider Secret gaps independent", () => { + const edgeHealth = { + runtimeDbReadiness: { + status: "blocked", + connectionResult: "dns_error", + connectionClassification: "dns_resolution_failed", + blocker: "DB endpoint DNS resolution failed from the cloud-api runtime" + }, + providerSecretReadiness: { + status: "blocked", + classification: "provider_secret_missing", + blocker: "Code Agent provider Secret hwlab-code-agent-provider/openai-api-key is not present as key-presence evidence" + } + }; + + const blockers = __testHooks.classifyParallelBlockers(edgeHealth); + assert.deepEqual( + blockers.map((blocker) => blocker.scope), + ["cloud-api-db", "code-agent-provider-secret"] + ); + assert.equal(blockers[0].classification, "dns_resolution_failed"); + assert.equal(blockers[0].sourceIssue, "pikasTech/HWLAB#49"); + assert.equal(blockers[1].classification, "provider_secret_missing"); + assert.equal(blockers[1].sourceIssue, "pikasTech/HWLAB#143"); + assert.ok(blockers[0].impact.separateFrom.includes("pikasTech/HWLAB#143 provider Secret")); + assert.ok(blockers[1].impact.separateFrom.includes("pikasTech/HWLAB#49 DB live readiness")); +}); + +test("provider Secret blocker remains open even if DB is not blocked", () => { + const blockers = __testHooks.classifyParallelBlockers({ + runtimeDbReadiness: { status: "pass" }, + providerSecretReadiness: { + status: "blocked", + classification: "provider_secret_missing", + blocker: "Code Agent provider Secret hwlab-code-agent-provider/openai-api-key is not present as key-presence evidence" + } + }); + + assert.equal(blockers.length, 1); + assert.equal(blockers[0].scope, "code-agent-provider-secret"); + assert.equal(blockers[0].type, "agent_blocker"); +}); + +test("milestone impact does not promote DB or provider evidence into M3/M4/M5", () => { + const impact = __testHooks.classifyMilestoneImpact({ + runtimeDbReadiness: { status: "blocked" }, + providerSecretReadiness: { status: "blocked" } + }); + + assert.equal(impact.M3.status, "separate_blocker"); + assert.deepEqual(impact.M4.blockedBy, ["cloud-api-db-live", "code-agent-provider-secret"]); + assert.deepEqual(impact.M5.blockedBy, ["cloud-api-db-live", "code-agent-provider-secret"]); + assert.match(impact.M5.summary, /cannot be promoted/u); +});