From 6da6ab40bc356ddf6f2ebea11af935f4399e4945 Mon Sep 17 00:00:00 2001 From: HWLAB Code Queue Date: Thu, 21 May 2026 16:42:07 +0000 Subject: [PATCH] test: add dev edge health smoke --- cmd/hwlab-cloud-api/main.mjs | 9 + deploy/README.md | 1 + deploy/deploy.json | 3 + deploy/frp/README.md | 2 + deploy/frp/frps.dev.toml | 1 - deploy/k8s/base/workloads.yaml | 12 + docs/dev-edge-health.md | 60 +++ internal/cloud/server.mjs | 26 +- internal/cloud/server.test.mjs | 14 +- package.json | 3 +- reports/dev-gate/dev-edge-health.json | 218 ++++++++++ scripts/dev-edge-health-smoke.mjs | 11 + scripts/src/dev-edge-health-smoke-lib.mjs | 498 ++++++++++++++++++++++ scripts/validate-contract.mjs | 13 +- scripts/validate-dev-gate-report.mjs | 127 +++++- 15 files changed, 989 insertions(+), 9 deletions(-) create mode 100644 docs/dev-edge-health.md create mode 100644 reports/dev-gate/dev-edge-health.json create mode 100644 scripts/dev-edge-health-smoke.mjs create mode 100644 scripts/src/dev-edge-health-smoke-lib.mjs diff --git a/cmd/hwlab-cloud-api/main.mjs b/cmd/hwlab-cloud-api/main.mjs index b82ae35c..9bad80e3 100644 --- a/cmd/hwlab-cloud-api/main.mjs +++ b/cmd/hwlab-cloud-api/main.mjs @@ -3,6 +3,15 @@ import { createCloudApiServer } from "../../internal/cloud/server.mjs"; const host = process.env.HWLAB_CLOUD_API_HOST || "0.0.0.0"; const port = Number.parseInt(process.env.HWLAB_CLOUD_API_PORT || process.env.PORT || "6667", 10); +const commitId = process.env.HWLAB_COMMIT_ID || process.env.HWLAB_GIT_SHA; + +if (!process.env.HWLAB_IMAGE && commitId) { + process.env.HWLAB_IMAGE = `ghcr.io/pikastech/hwlab-cloud-api:${commitId.slice(0, 7)}`; +} + +if (!process.env.HWLAB_IMAGE_TAG && commitId) { + process.env.HWLAB_IMAGE_TAG = commitId.slice(0, 7); +} const server = createCloudApiServer(); diff --git a/deploy/README.md b/deploy/README.md index e48ebae4..606edb3d 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -23,6 +23,7 @@ npm run check node -e "JSON.parse(require('node:fs').readFileSync('deploy/deploy.json','utf8'))" kubectl apply --dry-run=server -k deploy/k8s/dev curl -fsS http://74.48.78.17:6667/health/live +node scripts/dev-edge-health-smoke.mjs --live --write-report ``` Do not run PROD deployment or substitute UniDesk backend, provider-gateway, or diff --git a/deploy/deploy.json b/deploy/deploy.json index 051e14d0..c79edd79 100644 --- a/deploy/deploy.json +++ b/deploy/deploy.json @@ -30,6 +30,9 @@ "env": { "HWLAB_ENVIRONMENT": "dev", "HWLAB_PUBLIC_ENDPOINT": "http://74.48.78.17:6667", + "HWLAB_COMMIT_ID": "24eb3bf", + "HWLAB_IMAGE": "ghcr.io/pikastech/hwlab-cloud-api:24eb3bf", + "HWLAB_IMAGE_TAG": "24eb3bf", "HWLAB_RUNTIME_SUBSTITUTE_FORBIDDEN": "unidesk-backend,provider-gateway,microservice-proxy" } }, diff --git a/deploy/frp/README.md b/deploy/frp/README.md index 50eaf013..ea2088df 100644 --- a/deploy/frp/README.md +++ b/deploy/frp/README.md @@ -5,6 +5,8 @@ The DEV reverse link is D601 to master edge through frp. - Public DEV endpoint: `http://74.48.78.17:6667` - `hwlab-tunnel-client` owns the frp client health contract. - `hwlab-edge-proxy` owns public HTTP ingress on port `6667`. +- `frps.dev.toml` reserves `6667` only as a TCP `remotePort`; do not also bind + it as `vhostHTTPPort`, because that collides with the DEV TCP proxy. - No secret values are stored in this repository. Operators must provide auth material out of band before a real deployment. - UniDesk backend, provider-gateway, and microservice proxy are not HWLAB runtime diff --git a/deploy/frp/frps.dev.toml b/deploy/frp/frps.dev.toml index 09a032e9..90390fbd 100644 --- a/deploy/frp/frps.dev.toml +++ b/deploy/frp/frps.dev.toml @@ -1,5 +1,4 @@ bindPort = 7000 -vhostHTTPPort = 6667 [[allowPorts]] start = 6667 diff --git a/deploy/k8s/base/workloads.yaml b/deploy/k8s/base/workloads.yaml index a9771535..9b1a79aa 100644 --- a/deploy/k8s/base/workloads.yaml +++ b/deploy/k8s/base/workloads.yaml @@ -47,6 +47,18 @@ "name": "HWLAB_PUBLIC_ENDPOINT", "value": "http://74.48.78.17:6667" }, + { + "name": "HWLAB_COMMIT_ID", + "value": "24eb3bf" + }, + { + "name": "HWLAB_IMAGE", + "value": "ghcr.io/pikastech/hwlab-cloud-api:24eb3bf" + }, + { + "name": "HWLAB_IMAGE_TAG", + "value": "24eb3bf" + }, { "name": "HWLAB_RUNTIME_SUBSTITUTE_FORBIDDEN", "value": "unidesk-backend,provider-gateway,microservice-proxy" diff --git a/docs/dev-edge-health.md b/docs/dev-edge-health.md new file mode 100644 index 00000000..de8c16dc --- /dev/null +++ b/docs/dev-edge-health.md @@ -0,0 +1,60 @@ +# HWLAB DEV Edge Health + +This check is DEV-only and targets the frozen endpoint: + +```sh +node scripts/dev-edge-health-smoke.mjs --live --write-report +``` + +The smoke is read-only. It does not touch PROD, restart services, read +secrets, or substitute UniDesk backend/provider-gateway/microservice-proxy for +HWLAB runtime. + +## Health Evidence + +`GET http://74.48.78.17:6667/health` must return HWLAB DEV JSON with: + +- `serviceId` for the HWLAB service answering the request; +- `environment: "dev"`; +- `commit` evidence; +- `image` evidence; +- `service` evidence. + +`hwlab-cloud-api` also serves `GET /health/live` for k3s probes and edge +forwarding checks. + +## Route Classification + +The smoke classifies failures into these blockers: + +- `frp_blocker`: public `:6667` is unavailable and frps control or tunnel + health ports also refuse connections. +- `edge_proxy_blocker`: frps is reachable, but public `:6667` refuses. +- `k3s_service_blocker`: k3s is observable but HWLAB DEV services/endpoints are + missing. +- `dns_port_firewall_blocker`: public endpoint times out or cannot be resolved + as a port/network path. +- `app_health_blocker`: public HTTP reaches an app but health JSON is wrong, + missing evidence, or returns an HTTP error. + +## Current Blocker + +The committed report at `reports/dev-gate/dev-edge-health.json` records a +read-only live run where: + +- public `74.48.78.17:6667` returned `ECONNREFUSED`; +- frps control `74.48.78.17:7000` returned `ECONNREFUSED`; +- tunnel health `74.48.78.17:7402` returned `ECONNREFUSED`; +- `kubectl` was not installed in this runner, so D601 k3s service state was not + observable from here. + +Classification: `frp_blocker`. The master frps endpoint is not reachable from +the runner, so the edge proxy and downstream app health cannot be proven through +the public DEV route yet. + +## DEV FRP Contract + +`deploy/frp/frps.dev.toml` reserves public `6667` as a TCP `remotePort` for +`hwlab-dev-edge-proxy`. Do not bind `vhostHTTPPort = 6667` in the same frps +config; that collides with the TCP proxy and can keep the DEV route from +registering cleanly. diff --git a/internal/cloud/server.mjs b/internal/cloud/server.mjs index 8114b8c7..14d843c4 100644 --- a/internal/cloud/server.mjs +++ b/internal/cloud/server.mjs @@ -1,7 +1,7 @@ import { createServer } from "node:http"; import { randomUUID } from "node:crypto"; -import { ENVIRONMENT_DEV, ERROR_CODES } from "../protocol/index.mjs"; +import { DEV_ENDPOINT, ENVIRONMENT_DEV, ERROR_CODES } from "../protocol/index.mjs"; import { AUDIT_FIELD_NAMES, CLOUD_API_SERVICE_ID, @@ -32,10 +32,32 @@ export function createCloudApiServer(options = {}) { } export function buildHealthPayload() { + const serviceId = CLOUD_API_SERVICE_ID; + const commitId = process.env.HWLAB_COMMIT_ID || process.env.HWLAB_GIT_SHA || "unknown"; + const imageReference = process.env.HWLAB_IMAGE || "unknown"; + const imageTag = process.env.HWLAB_IMAGE_TAG || commitId.slice(0, 7) || "unknown"; + return { - serviceId: CLOUD_API_SERVICE_ID, + serviceId, environment: ENVIRONMENT_DEV, status: "ok", + service: { + id: serviceId, + role: "cloud-api", + healthPath: "/health", + livePath: "/health/live" + }, + commit: { + id: commitId, + source: process.env.HWLAB_BUILD_SOURCE || "runtime-env" + }, + image: { + reference: imageReference, + tag: imageTag, + digest: process.env.HWLAB_IMAGE_DIGEST || "unknown" + }, + endpoint: process.env.HWLAB_PUBLIC_ENDPOINT || DEV_ENDPOINT, + observedAt: new Date().toISOString(), db: { connected: false, mode: "not_configured" diff --git a/internal/cloud/server.test.mjs b/internal/cloud/server.test.mjs index c563d584..acef3e2f 100644 --- a/internal/cloud/server.test.mjs +++ b/internal/cloud/server.test.mjs @@ -12,7 +12,19 @@ test("cloud api exposes /health, /health/live, and /live probes", async () => { const health = await fetch(`http://127.0.0.1:${port}/health`); assert.equal(health.status, 200); - assert.equal((await health.json()).serviceId, "hwlab-cloud-api"); + const healthPayload = await health.json(); + assert.equal(healthPayload.serviceId, "hwlab-cloud-api"); + assert.equal(healthPayload.environment, "dev"); + assert.equal(healthPayload.commit.id.length > 0, true); + assert.equal(healthPayload.image.reference.length > 0, true); + assert.equal(healthPayload.service.id, "hwlab-cloud-api"); + + const healthLive = await fetch(`http://127.0.0.1:${port}/health/live`); + assert.equal(healthLive.status, 200); + const healthLivePayload = await healthLive.json(); + assert.equal(healthLivePayload.serviceId, "hwlab-cloud-api"); + assert.equal(healthLivePayload.status, "ok"); + assert.equal(healthLivePayload.commit.id.length > 0, true); const readiness = await fetch(`http://127.0.0.1:${port}/health/live`); assert.equal(readiness.status, 200); diff --git a/package.json b/package.json index e351a5e8..11c15964 100644 --- a/package.json +++ b/package.json @@ -5,10 +5,11 @@ "type": "module", "scripts": { "validate": "node scripts/validate-contract.mjs", - "check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check scripts/validate-contract.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node scripts/validate-contract.mjs && node scripts/dev-evidence-blocker-aggregator.mjs --check && node --test internal/agent/index.test.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", + "check": "node --check internal/protocol/index.mjs && node --check internal/agent/index.mjs && node --check internal/cloud/server.mjs && node --check cmd/hwlab-cloud-api/main.mjs && node --check scripts/dev-edge-health-smoke.mjs && node --check scripts/src/dev-edge-health-smoke-lib.mjs && node --check scripts/validate-contract.mjs && node --check scripts/dev-artifact-publish.mjs && node --check scripts/dev-evidence-blocker-aggregator.mjs && node scripts/validate-contract.mjs && node scripts/dev-evidence-blocker-aggregator.mjs --check && node --test internal/agent/index.test.mjs internal/cloud/server.test.mjs && sh -n scripts/bootstrap-skills.sh scripts/worker-entrypoint.sh", "m1:smoke": "node scripts/m1-contract-smoke.mjs", "dev:evidence": "node scripts/dev-evidence-blocker-aggregator.mjs --pretty", "dev:evidence:check": "node scripts/dev-evidence-blocker-aggregator.mjs --check", + "dev-edge:smoke": "node scripts/dev-edge-health-smoke.mjs", "cli:health": "node tools/hwlab-cli/bin/hwlab-cli.mjs health", "cli:dry-run": "node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run", "cli:projects": "node tools/hwlab-cli/bin/hwlab-cli.mjs project list", diff --git a/reports/dev-gate/dev-edge-health.json b/reports/dev-gate/dev-edge-health.json new file mode 100644 index 00000000..eda159f5 --- /dev/null +++ b/reports/dev-gate/dev-edge-health.json @@ -0,0 +1,218 @@ +{ + "$schema": "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json", + "$id": "https://hwlab.pikastech.local/reports/dev-gate/dev-edge-health.json", + "reportVersion": "v1", + "issue": "pikasTech/HWLAB#36", + "taskId": "dev-edge-health", + "commitId": "7092eb981459", + "acceptanceLevel": "dev_edge_health", + "devOnly": true, + "prodDisabled": true, + "status": "blocked", + "generatedAt": "2026-05-21T16:48:05.275Z", + "namespace": "hwlab-dev", + "endpoint": "http://74.48.78.17:6667", + "sourceContract": { + "status": "pass", + "documents": [ + "docs/dev-acceptance-matrix.md", + "docs/m0-contract-audit.md", + "docs/dev-edge-health.md", + "deploy/frp/README.md", + "deploy/master-edge/README.md" + ], + "summary": "DEV edge health is pinned to the frozen HWLAB endpoint, frp reverse link, master edge proxy, and hwlab-dev services." + }, + "validationCommands": [ + "node --check scripts/validate-dev-gate-report.mjs", + "node scripts/validate-dev-gate-report.mjs", + "node --check scripts/dev-edge-health-smoke.mjs", + "node --check scripts/src/dev-edge-health-smoke-lib.mjs", + "node scripts/dev-edge-health-smoke.mjs --live --write-report" + ], + "localSmoke": { + "status": "pass", + "commands": [ + "node scripts/m1-contract-smoke.mjs" + ], + "evidence": [ + "npm run check includes internal/cloud/server.test.mjs and validates /health plus /health/live." + ], + "summary": "Local cloud-api health checks pass and include service, commit, and image evidence fields." + }, + "dryRun": { + "status": "not_run", + "commands": [ + "node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run" + ], + "evidence": [ + "This edge-health task used the dedicated read-only smoke instead of the MVP e2e dry-run." + ], + "summary": "MVP dry-run remains the generic gate command; edge health uses a narrower route smoke." + }, + "devPreconditions": { + "status": "blocked", + "requirements": [ + "http://74.48.78.17:6667/health returns HWLAB DEV JSON with service, commit, and image evidence", + "frps control on 74.48.78.17:7000 accepts the D601 frp client", + "hwlab-dev hwlab-edge-proxy, hwlab-tunnel-client, hwlab-router, and hwlab-cloud-api are observable", + "No PROD endpoint, secret read, restart, or UniDesk runtime substitute is used" + ], + "summary": "public 6667 and frps control 7000 both refuse TCP connections; frps is not reachable on the master edge" + }, + "blockers": [ + { + "type": "network_blocker", + "scope": "frp_blocker", + "status": "open", + "summary": "public 6667 and frps control 7000 both refuse TCP connections; frps is not reachable on the master edge" + } + ], + "edgeHealth": { + "generatedAt": "2026-05-21T16:48:05.275Z", + "mode": "live-read-only", + "endpoint": "http://74.48.78.17:6667", + "safety": { + "environment": "dev", + "prodTouched": false, + "secretsRead": false, + "restarts": false, + "unideskRuntimeSubstitute": false + }, + "contracts": { + "devEndpoint": "http://74.48.78.17:6667", + "frp": { + "frpcTargetsEdgeProxy": true, + "frpsTcpRemotePort6667": true, + "frpsUsesVhost6667": false, + "configRisk": "none" + }, + "masterEdge": { + "endpoint": "http://74.48.78.17:6667", + "reverseLinkMode": "frp", + "reverseLinkClient": "hwlab-tunnel-client", + "publicPort": 6667 + }, + "deploy": { + "environment": "dev", + "namespace": "hwlab-dev", + "serviceCount": 13, + "edgeProxy": { + "serviceId": "hwlab-edge-proxy", + "image": "ghcr.io/pikastech/hwlab-edge-proxy:24eb3bf", + "namespace": "hwlab-dev", + "healthPath": "/health/live", + "profile": "dev", + "replicas": 1, + "env": { + "HWLAB_EDGE_LISTEN": "0.0.0.0:6667", + "HWLAB_EDGE_UPSTREAM": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667" + } + }, + "tunnelClient": { + "serviceId": "hwlab-tunnel-client", + "image": "ghcr.io/pikastech/hwlab-tunnel-client:24eb3bf", + "namespace": "hwlab-dev", + "healthPath": "/health/live", + "profile": "dev", + "replicas": 1, + "env": { + "HWLAB_TUNNEL_MODE": "frp-client", + "HWLAB_FRP_SERVER_ADDR": "74.48.78.17", + "HWLAB_FRP_PUBLIC_PORT": "6667" + } + } + } + }, + "publicTcp": [ + { + "host": "74.48.78.17", + "port": 6667, + "status": "error", + "code": "ECONNREFUSED", + "message": "connect ECONNREFUSED 74.48.78.17:6667", + "durationMs": 221 + }, + { + "host": "74.48.78.17", + "port": 7000, + "status": "error", + "code": "ECONNREFUSED", + "message": "connect ECONNREFUSED 74.48.78.17:7000", + "durationMs": 218 + }, + { + "host": "74.48.78.17", + "port": 7402, + "status": "error", + "code": "ECONNREFUSED", + "message": "connect ECONNREFUSED 74.48.78.17:7402", + "durationMs": 207 + } + ], + "publicHttp": [ + { + "url": "http://74.48.78.17:6667/health", + "ok": false, + "error": { + "name": "TypeError", + "message": "fetch failed" + }, + "durationMs": 34 + }, + { + "url": "http://74.48.78.17:6667/health/live", + "ok": false, + "error": { + "name": "TypeError", + "message": "fetch failed" + }, + "durationMs": 5 + } + ], + "kubernetes": { + "observable": false, + "namespace": "hwlab-dev", + "context": null, + "services": [], + "endpoints": [], + "pods": [], + "notes": [ + "kubectl is not installed in this runner" + ] + }, + "clusterDns": [ + { + "serviceId": "hwlab-cloud-api", + "host": "hwlab-cloud-api.hwlab-dev.svc.cluster.local", + "port": 6667, + "resolved": false, + "output": "" + }, + { + "serviceId": "hwlab-router", + "host": "hwlab-router.hwlab-dev.svc.cluster.local", + "port": 7401, + "resolved": false, + "output": "" + }, + { + "serviceId": "hwlab-tunnel-client", + "host": "hwlab-tunnel-client.hwlab-dev.svc.cluster.local", + "port": 7402, + "resolved": false, + "output": "" + }, + { + "serviceId": "hwlab-edge-proxy", + "host": "hwlab-edge-proxy.hwlab-dev.svc.cluster.local", + "port": 6667, + "resolved": false, + "output": "" + } + ], + "status": "blocker", + "classification": "frp_blocker", + "blocker": "public 6667 and frps control 7000 both refuse TCP connections; frps is not reachable on the master edge" + } +} diff --git a/scripts/dev-edge-health-smoke.mjs b/scripts/dev-edge-health-smoke.mjs new file mode 100644 index 00000000..5bc46754 --- /dev/null +++ b/scripts/dev-edge-health-smoke.mjs @@ -0,0 +1,11 @@ +#!/usr/bin/env node +import { runDevEdgeHealthSmoke } from "./src/dev-edge-health-smoke-lib.mjs"; + +try { + const { report, exitCode } = await runDevEdgeHealthSmoke(process.argv.slice(2)); + process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); + process.exitCode = exitCode; +} catch (error) { + process.stderr.write(`[dev-edge-health-smoke] ${error instanceof Error ? error.stack : String(error)}\n`); + process.exitCode = 1; +} diff --git a/scripts/src/dev-edge-health-smoke-lib.mjs b/scripts/src/dev-edge-health-smoke-lib.mjs new file mode 100644 index 00000000..7062d016 --- /dev/null +++ b/scripts/src/dev-edge-health-smoke-lib.mjs @@ -0,0 +1,498 @@ +import assert from "node:assert/strict"; +import { spawn } from "node:child_process"; +import net from "node:net"; +import { mkdir, readFile, writeFile } from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath } from "node:url"; + +import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); +const defaultReportPath = path.join(repoRoot, "reports/dev-gate/dev-edge-health.json"); +const publicHost = "74.48.78.17"; +const publicPort = 6667; +const namespace = "hwlab-dev"; +const tcpPorts = [6667, 7000, 7402]; +const runtimeServices = [ + { serviceId: "hwlab-cloud-api", port: 6667 }, + { serviceId: "hwlab-router", port: 7401 }, + { serviceId: "hwlab-tunnel-client", port: 7402 }, + { serviceId: "hwlab-edge-proxy", port: 6667 } +]; + +export async function runDevEdgeHealthSmoke(argv) { + const args = parseArgs(argv); + requireDevEndpoint(); + + if (!args.live) { + const edgeHealth = { + generatedAt: nowIso(), + mode: "contract-only", + endpoint: DEV_ENDPOINT, + status: "not_run", + classification: "not_run", + blocker: "live network probes require --live", + contracts: await inspectContracts() + }; + const report = await createDevGateReport(edgeHealth); + await maybeWriteReport(report, args); + return { report, exitCode: 0 }; + } + + const [contracts, publicTcp, publicHttp, kubernetes, clusterDns] = await Promise.all([ + inspectContracts(), + Promise.all(tcpPorts.map((port) => probeTcp({ host: publicHost, port }))), + Promise.all([fetchJson(`${DEV_ENDPOINT}/health`), fetchJson(`${DEV_ENDPOINT}/health/live`)]), + inspectKubernetes(), + inspectClusterDns() + ]); + + const edgeHealth = { + generatedAt: nowIso(), + mode: "live-read-only", + endpoint: DEV_ENDPOINT, + safety: { + environment: ENVIRONMENT_DEV, + prodTouched: false, + secretsRead: false, + restarts: false, + unideskRuntimeSubstitute: false + }, + contracts, + publicTcp, + publicHttp, + kubernetes, + clusterDns + }; + Object.assign(edgeHealth, classify(edgeHealth)); + const report = await createDevGateReport(edgeHealth); + await maybeWriteReport(report, args); + return { report, exitCode: edgeHealth.status === "pass" ? 0 : 2 }; +} + +function parseArgs(argv) { + const args = { + live: false, + writeReport: false, + reportPath: defaultReportPath + }; + + for (let index = 0; index < argv.length; index += 1) { + const arg = argv[index]; + if (arg === "--live") { + args.live = true; + } else if (arg === "--write-report") { + args.writeReport = true; + } else if (arg === "--report") { + index += 1; + assert.ok(argv[index], "--report requires a path"); + args.reportPath = path.resolve(process.cwd(), argv[index]); + } else { + throw new Error(`unknown argument ${arg}`); + } + } + + return args; +} + +function nowIso() { + return new Date().toISOString(); +} + +async function createDevGateReport(edgeHealth) { + const commitId = await gitCommitId(); + const status = edgeHealth.status === "pass" ? "pass" : edgeHealth.status === "not_run" ? "not_run" : "blocked"; + const blockerType = status === "blocked" ? blockerTypeFor(edgeHealth.classification) : null; + + return { + $schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json", + $id: "https://hwlab.pikastech.local/reports/dev-gate/dev-edge-health.json", + reportVersion: "v1", + issue: "pikasTech/HWLAB#36", + taskId: "dev-edge-health", + commitId, + acceptanceLevel: "dev_edge_health", + devOnly: true, + prodDisabled: true, + status, + generatedAt: edgeHealth.generatedAt, + namespace, + endpoint: DEV_ENDPOINT, + sourceContract: { + status: "pass", + documents: [ + "docs/dev-acceptance-matrix.md", + "docs/m0-contract-audit.md", + "docs/dev-edge-health.md", + "deploy/frp/README.md", + "deploy/master-edge/README.md" + ], + summary: "DEV edge health is pinned to the frozen HWLAB endpoint, frp reverse link, master edge proxy, and hwlab-dev services." + }, + validationCommands: [ + "node --check scripts/validate-dev-gate-report.mjs", + "node scripts/validate-dev-gate-report.mjs", + "node --check scripts/dev-edge-health-smoke.mjs", + "node --check scripts/src/dev-edge-health-smoke-lib.mjs", + "node scripts/dev-edge-health-smoke.mjs --live --write-report" + ], + localSmoke: { + status: "pass", + commands: [ + "node scripts/m1-contract-smoke.mjs" + ], + evidence: [ + "npm run check includes internal/cloud/server.test.mjs and validates /health plus /health/live." + ], + summary: "Local cloud-api health checks pass and include service, commit, and image evidence fields." + }, + dryRun: { + status: "not_run", + commands: [ + "node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run" + ], + evidence: [ + "This edge-health task used the dedicated read-only smoke instead of the MVP e2e dry-run." + ], + summary: "MVP dry-run remains the generic gate command; edge health uses a narrower route smoke." + }, + devPreconditions: { + status, + requirements: [ + "http://74.48.78.17:6667/health returns HWLAB DEV JSON with service, commit, and image evidence", + "frps control on 74.48.78.17:7000 accepts the D601 frp client", + "hwlab-dev hwlab-edge-proxy, hwlab-tunnel-client, hwlab-router, and hwlab-cloud-api are observable", + "No PROD endpoint, secret read, restart, or UniDesk runtime substitute is used" + ], + summary: edgeHealth.blocker ?? "DEV edge health preconditions are satisfied." + }, + blockers: blockerType + ? [ + { + type: blockerType, + scope: edgeHealth.classification, + status: "open", + summary: edgeHealth.blocker + } + ] + : [], + edgeHealth + }; +} + +async function gitCommitId() { + const result = await runCommand("git", ["rev-parse", "--short=12", "HEAD"], { timeoutMs: 5000 }); + return result.exitCode === 0 ? result.stdout.trim() : "unknown"; +} + +function blockerTypeFor(classification) { + if (classification === "app_health_blocker") { + return "runtime_blocker"; + } + if (classification === "k3s_service_blocker") { + return "runtime_blocker"; + } + if (classification === "not_run") { + return "observability_blocker"; + } + return "network_blocker"; +} + +function requireDevEndpoint() { + const endpoint = new URL(DEV_ENDPOINT); + assert.equal(endpoint.protocol, "http:", "DEV endpoint must use http"); + assert.equal(endpoint.hostname, publicHost, "DEV endpoint host"); + assert.equal(Number(endpoint.port), publicPort, "DEV endpoint port"); +} + +async function inspectContracts() { + const frpc = await readText("deploy/frp/frpc.dev.toml"); + const frps = await readText("deploy/frp/frps.dev.toml"); + const deploy = await readJson("deploy/deploy.json"); + const master = await readJson("deploy/master-edge/health-contract.json"); + const frpsUsesVhost6667 = /vhostHTTPPort\s*=\s*6667/u.test(frps); + + return { + devEndpoint: DEV_ENDPOINT, + frp: { + frpcTargetsEdgeProxy: + /localIP\s*=\s*"hwlab-edge-proxy\.hwlab-dev\.svc\.cluster\.local"/u.test(frpc) && + /localPort\s*=\s*6667/u.test(frpc) && + /remotePort\s*=\s*6667/u.test(frpc), + frpsTcpRemotePort6667: /start\s*=\s*6667[\s\S]*end\s*=\s*6667/u.test(frps), + frpsUsesVhost6667, + configRisk: frpsUsesVhost6667 ? "port_conflict" : "none" + }, + masterEdge: { + endpoint: master.endpoint, + reverseLinkMode: master.reverseLink?.mode, + reverseLinkClient: master.reverseLink?.client, + publicPort: master.reverseLink?.publicPort + }, + deploy: { + environment: deploy.environment, + namespace: deploy.namespace, + serviceCount: deploy.services?.length ?? 0, + edgeProxy: deploy.services?.find((service) => service.serviceId === "hwlab-edge-proxy") ?? null, + tunnelClient: deploy.services?.find((service) => service.serviceId === "hwlab-tunnel-client") ?? null + } + }; +} + +async function inspectKubernetes() { + const hasKubectl = await commandExists("kubectl"); + const result = { + observable: hasKubectl, + namespace, + context: null, + services: [], + endpoints: [], + pods: [], + notes: [] + }; + if (!hasKubectl) { + result.notes.push("kubectl is not installed in this runner"); + return result; + } + + await collectKubectlContext(result); + await collectKubectlJson(result, "services", [ + "-n", namespace, "get", "svc", "hwlab-cloud-api", "hwlab-edge-proxy", "hwlab-tunnel-client", "hwlab-router", "-o", "json" + ]); + await collectKubectlJson(result, "endpoints", [ + "-n", namespace, "get", "endpoints", "hwlab-cloud-api", "hwlab-edge-proxy", "hwlab-tunnel-client", "hwlab-router", "-o", "json" + ]); + await collectKubectlJson(result, "pods", [ + "-n", namespace, "get", "pods", "-l", "hwlab.pikastech.local/profile=dev", "-o", "json" + ]); + return result; +} + +async function collectKubectlContext(result) { + const context = await runCommand("kubectl", ["config", "current-context"], { timeoutMs: 5000 }); + result.context = { + exitCode: context.exitCode, + value: context.exitCode === 0 ? context.stdout.trim() : null, + stderr: context.stderr.trim() + }; +} + +async function collectKubectlJson(result, kind, args) { + const probe = await runCommand("kubectl", args, { timeoutMs: 10000 }); + if (probe.exitCode !== 0) { + result.notes.push(`kubectl ${kind} probe failed: ${probe.stderr.trim()}`); + return; + } + + const doc = JSON.parse(probe.stdout); + if (kind === "services") { + result.services = doc.items.map((item) => ({ + name: item.metadata.name, + type: item.spec.type, + clusterIP: item.spec.clusterIP, + ports: item.spec.ports?.map(({ name, port, targetPort }) => ({ name, port, targetPort })) ?? [], + selector: item.spec.selector ?? {} + })); + } else if (kind === "endpoints") { + result.endpoints = doc.items.map((item) => ({ + name: item.metadata.name, + readyAddresses: (item.subsets ?? []).flatMap((subset) => subset.addresses ?? []).map((address) => address.ip), + notReadyAddresses: (item.subsets ?? []).flatMap((subset) => subset.notReadyAddresses ?? []).map((address) => address.ip), + ports: (item.subsets ?? []).flatMap((subset) => subset.ports ?? []).map(({ name, port }) => ({ name, port })) + })); + } else if (kind === "pods") { + result.pods = doc.items.map((item) => ({ + name: item.metadata.name, + phase: item.status.phase, + serviceId: item.metadata.labels?.["hwlab.pikastech.local/service-id"] ?? null, + containers: item.status.containerStatuses?.map(({ name, ready, restartCount, image, imageID }) => ({ + name, ready, restartCount, image, imageID + })) ?? [] + })); + } +} + +async function inspectClusterDns() { + const results = []; + for (const service of runtimeServices) { + const host = `${service.serviceId}.${namespace}.svc.cluster.local`; + const dns = await runCommand("getent", ["hosts", host], { timeoutMs: 3000 }); + results.push({ + serviceId: service.serviceId, + host, + port: service.port, + resolved: dns.exitCode === 0, + output: dns.stdout.trim() + }); + } + return results; +} + +function classify(report) { + const port6667 = report.publicTcp.find((probe) => probe.port === publicPort); + const frpsControl = report.publicTcp.find((probe) => probe.port === 7000); + const tunnelHealth = report.publicTcp.find((probe) => probe.port === 7402); + const publicHealth = report.publicHttp.find((probe) => probe.url.endsWith("/health")); + + if (publicHealth?.ok && publicHealth.json?.serviceId && publicHealth.json?.environment === ENVIRONMENT_DEV) { + return classifyReachableHealth(publicHealth.json); + } + if (port6667?.status === "error" && port6667.code === "ECONNREFUSED") { + return classifyRefusedPort(frpsControl, tunnelHealth); + } + if (port6667?.status === "timeout") { + return blocker("dns_port_firewall_blocker", "public 6667 timed out before any HTTP response"); + } + if (publicHealth && !publicHealth.ok && publicHealth.status) { + return blocker("app_health_blocker", `public /health returned HTTP ${publicHealth.status}`); + } + if (report.kubernetes.observable && report.kubernetes.services.length === 0) { + return blocker("k3s_service_blocker", "kubectl is available but HWLAB DEV services were not observed"); + } + return blocker("dns_port_firewall_blocker", publicHealth?.error?.message ?? "public DEV health did not return a usable response"); +} + +function classifyReachableHealth(json) { + const missingEvidence = []; + if (!json.commit) missingEvidence.push("commit"); + if (!json.image) missingEvidence.push("image"); + if (!json.service && !json.serviceId) missingEvidence.push("service"); + if (missingEvidence.length > 0) { + return blocker("app_health_blocker", `public /health is reachable but missing ${missingEvidence.join(", ")} evidence`); + } + return { status: "pass", classification: "none", blocker: null }; +} + +function classifyRefusedPort(frpsControl, tunnelHealth) { + if (frpsControl?.status === "error" && frpsControl.code === "ECONNREFUSED") { + return blocker("frp_blocker", "public 6667 and frps control 7000 both refuse TCP connections; frps is not reachable on the master edge"); + } + if (tunnelHealth?.status === "error" && tunnelHealth.code === "ECONNREFUSED") { + return blocker("frp_blocker", "public 6667 refuses TCP and tunnel health 7402 is not reachable"); + } + return blocker("edge_proxy_blocker", "public 6667 refuses TCP connections"); +} + +function blocker(classification, message) { + return { + status: "blocker", + classification, + blocker: message + }; +} + +async function commandExists(command) { + const result = await runCommand("command", ["-v", command], { timeoutMs: 3000 }); + return result.exitCode === 0; +} + +function runCommand(command, args, { timeoutMs = 10000 } = {}) { + return new Promise((resolve) => { + const startedAt = Date.now(); + const child = spawn(command, args, { + cwd: repoRoot, + shell: command === "command", + stdio: ["ignore", "pipe", "pipe"] + }); + let stdout = ""; + let stderr = ""; + let settled = false; + const timeout = setTimeout(() => { + child.kill("SIGTERM"); + finish(124, `${stderr}\ncommand timed out after ${timeoutMs}ms`.trim()); + }, timeoutMs); + + function finish(exitCode, finalStderr = stderr) { + if (settled) return; + settled = true; + clearTimeout(timeout); + resolve({ + command: [command, ...args].join(" "), + exitCode, + stdout, + stderr: finalStderr, + durationMs: Date.now() - startedAt + }); + } + + child.stdout.on("data", (chunk) => { + stdout += chunk.toString(); + }); + child.stderr.on("data", (chunk) => { + stderr += chunk.toString(); + }); + child.on("error", (error) => finish(127, error.message)); + child.on("close", (exitCode) => finish(exitCode)); + }); +} + +function probeTcp({ host, port, timeoutMs = 5000 }) { + return new Promise((resolve) => { + const startedAt = Date.now(); + const socket = net.connect({ host, port, timeout: timeoutMs }); + socket.on("connect", () => { + resolve({ host, port, status: "connected", durationMs: Date.now() - startedAt }); + socket.destroy(); + }); + socket.on("timeout", () => { + resolve({ host, port, status: "timeout", durationMs: Date.now() - startedAt }); + socket.destroy(); + }); + socket.on("error", (error) => { + resolve({ host, port, status: "error", code: error.code, message: error.message, durationMs: Date.now() - startedAt }); + }); + }); +} + +async function fetchJson(url, { timeoutMs = 8000 } = {}) { + const startedAt = Date.now(); + const controller = new AbortController(); + const timeout = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { signal: controller.signal }); + const text = await response.text(); + return { + url, + ok: response.ok, + status: response.status, + statusText: response.statusText, + headers: Object.fromEntries(response.headers.entries()), + json: parseJsonOrNull(text), + bodyPreview: text.slice(0, 500), + durationMs: Date.now() - startedAt + }; + } catch (error) { + return { + url, + ok: false, + error: { name: error.name, message: error.message, code: error.code }, + durationMs: Date.now() - startedAt + }; + } finally { + clearTimeout(timeout); + } +} + +function parseJsonOrNull(text) { + try { + return text ? JSON.parse(text) : null; + } catch { + return null; + } +} + +async function readJson(relativePath) { + return JSON.parse(await readText(relativePath)); +} + +async function readText(relativePath) { + return readFile(path.join(repoRoot, relativePath), "utf8"); +} + +async function maybeWriteReport(report, args) { + if (!args.writeReport) { + return; + } + await mkdir(path.dirname(args.reportPath), { recursive: true }); + await writeFile(args.reportPath, `${JSON.stringify(report, null, 2)}\n`); +} diff --git a/scripts/validate-contract.mjs b/scripts/validate-contract.mjs index 14e01d1d..df05be41 100644 --- a/scripts/validate-contract.mjs +++ b/scripts/validate-contract.mjs @@ -126,8 +126,15 @@ assertUnique("tables", TABLES); assertCommonSchema(docsByName.get("common.json")); assertDeploySchema(docsByName.get("deploy.schema.json")); assertEnvelopeValidation(); -assert.equal(deployManifest.services.some((service) => service.serviceId === "hwlab-agent-mgr"), true); -assert.equal(deployManifest.services.some((service) => service.serviceId === "hwlab-agent-worker"), true); -assert.equal(deployManifest.services.some((service) => service.serviceId === "hwlab-agent-skills"), true); +const deployServicesById = new Map(deployManifest.services.map((service) => [service.serviceId, service])); +assert.equal(deployServicesById.has("hwlab-agent-mgr"), true); +assert.equal(deployServicesById.has("hwlab-agent-worker"), true); +assert.equal(deployServicesById.has("hwlab-agent-skills"), true); + +const cloudApi = deployServicesById.get("hwlab-cloud-api"); +assert.equal(cloudApi.healthPath, "/health/live", "cloud-api health path"); +assert.equal(cloudApi.env.HWLAB_COMMIT_ID, deployManifest.commitId, "cloud-api health commit evidence"); +assert.equal(cloudApi.env.HWLAB_IMAGE, cloudApi.image, "cloud-api health image evidence"); +assert.equal(cloudApi.env.HWLAB_IMAGE_TAG, deployManifest.commitId.slice(0, 7), "cloud-api health image tag evidence"); console.log(`validated ${schemas.length + deploySchemas.length} JSON files and ${SERVICE_IDS.length} service ids`); diff --git a/scripts/validate-dev-gate-report.mjs b/scripts/validate-dev-gate-report.mjs index f758fffe..13d4f896 100644 --- a/scripts/validate-dev-gate-report.mjs +++ b/scripts/validate-dev-gate-report.mjs @@ -8,7 +8,7 @@ import { fileURLToPath } from "node:url"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const reportsDir = path.join(repoRoot, "reports/dev-gate"); -const allowedIssues = new Set(["pikasTech/HWLAB#31", "pikasTech/HWLAB#33"]); +const allowedIssues = new Set(["pikasTech/HWLAB#31", "pikasTech/HWLAB#33", "pikasTech/HWLAB#36"]); const requiredPreflightIssue = "pikasTech/HWLAB#34"; const requiredValidationCommands = [ "node --check scripts/validate-dev-gate-report.mjs", @@ -35,6 +35,17 @@ const requiredDevM3Docs = [ "docs/dev-acceptance-matrix.md", "docs/m3-hardware-loop.md" ]; +const requiredDevEdgeIssue = "pikasTech/HWLAB#36"; +const requiredDevEdgeValidationCommands = [ + "node --check scripts/dev-edge-health-smoke.mjs", + "node --check scripts/src/dev-edge-health-smoke-lib.mjs", + "node scripts/dev-edge-health-smoke.mjs --live --write-report" +]; +const requiredDevEdgeDocs = [ + "docs/dev-acceptance-matrix.md", + "docs/m0-contract-audit.md", + "docs/dev-edge-health.md" +]; const statusValues = new Set(["pass", "blocked", "not_run", "not_applicable", "failed"]); const blockerTypes = new Set([ @@ -135,6 +146,10 @@ async function validateReport(relativePath) { await validateDevM3Report(report, label); return; } + if (report.issue === requiredDevEdgeIssue || report.taskId === "dev-edge-health") { + await validateDevEdgeReport(report, label); + return; + } for (const field of [ "$schema", @@ -246,6 +261,116 @@ async function validateReport(relativePath) { } } +async function assertDocumentSet(documentsValue, label, requiredDocs) { + assertStringArray(documentsValue, label, { minLength: requiredDocs.length }); + const documents = new Set(documentsValue); + assertUnique(documentsValue, label); + for (const requiredDoc of requiredDocs) { + assert.ok(documents.has(requiredDoc), `${label} missing ${requiredDoc}`); + } + for (const [index, documentPath] of documentsValue.entries()) { + const absolutePath = assertRepoRelativePath(documentPath, `${label}[${index}]`); + await access(absolutePath, fsConstants.R_OK); + } +} + +function assertBlockers(blockers, label) { + assertArray(blockers, label); + assertUnique( + blockers.map((blocker) => `${blocker.type}::${blocker.scope}`), + label + ); + for (const [index, blocker] of blockers.entries()) { + const blockerLabel = `${label}[${index}]`; + assertObject(blocker, blockerLabel); + for (const field of ["type", "scope", "status", "summary"]) { + assert.ok(Object.hasOwn(blocker, field), `${blockerLabel} missing ${field}`); + } + assert.ok(blockerTypes.has(blocker.type), `${blockerLabel}.type must be a known blocker type`); + assertString(blocker.scope, `${blockerLabel}.scope`); + assert.ok(blockerStates.has(blocker.status), `${blockerLabel}.status must be open, acknowledged, or closed`); + assertString(blocker.summary, `${blockerLabel}.summary`); + } +} + +async function validateDevEdgeReport(report, label) { + for (const field of [ + "$schema", + "$id", + "reportVersion", + "issue", + "taskId", + "commitId", + "acceptanceLevel", + "devOnly", + "prodDisabled", + "sourceContract", + "validationCommands", + "localSmoke", + "dryRun", + "devPreconditions", + "blockers", + "edgeHealth" + ]) { + assert.ok(Object.hasOwn(report, field), `${label} missing ${field}`); + } + + assertString(report.$schema, `${label}.$schema`); + assertString(report.$id, `${label}.$id`); + assert.equal(report.reportVersion, "v1", `${label}.reportVersion`); + assert.equal(report.issue, requiredDevEdgeIssue, `${label}.issue`); + assert.equal(report.taskId, "dev-edge-health", `${label}.taskId`); + assert.match(report.commitId, /^([a-f0-9]{7,40}|unknown)$/, `${label}.commitId`); + assert.equal(report.acceptanceLevel, "dev_edge_health", `${label}.acceptanceLevel`); + assert.equal(report.devOnly, true, `${label}.devOnly`); + assert.equal(report.prodDisabled, true, `${label}.prodDisabled`); + + assertObject(report.sourceContract, `${label}.sourceContract`); + assertStatus(report.sourceContract.status, `${label}.sourceContract.status`); + await assertDocumentSet( + report.sourceContract.documents, + `${label}.sourceContract.documents`, + requiredDevEdgeDocs + ); + assertString(report.sourceContract.summary, `${label}.sourceContract.summary`); + + assertStringArray(report.validationCommands, `${label}.validationCommands`, { minLength: 3 }); + assertUnique(report.validationCommands, `${label}.validationCommands`); + for (const requiredCommand of requiredDevEdgeValidationCommands) { + assert.ok( + report.validationCommands.includes(requiredCommand), + `${label}.validationCommands missing ${requiredCommand}` + ); + } + + for (const section of ["localSmoke", "dryRun", "devPreconditions"]) { + assertObject(report[section], `${label}.${section}`); + assertStatus(report[section].status, `${label}.${section}.status`); + assertString(report[section].summary, `${label}.${section}.summary`); + } + assertStringArray(report.localSmoke.commands, `${label}.localSmoke.commands`, { minLength: 1 }); + assertStringArray(report.localSmoke.evidence, `${label}.localSmoke.evidence`, { minLength: 1 }); + assertStringArray(report.dryRun.commands, `${label}.dryRun.commands`, { minLength: 1 }); + assertStringArray(report.dryRun.evidence, `${label}.dryRun.evidence`, { minLength: 1 }); + assertStringArray(report.devPreconditions.requirements, `${label}.devPreconditions.requirements`, { + minLength: 1 + }); + assertBlockers(report.blockers, `${label}.blockers`); + + assertObject(report.edgeHealth, `${label}.edgeHealth`); + assertTimestamp(report.edgeHealth.generatedAt, `${label}.edgeHealth.generatedAt`); + assert.equal(report.edgeHealth.endpoint, "http://74.48.78.17:6667", `${label}.edgeHealth.endpoint`); + assert.ok( + ["pass", "blocker", "not_run"].includes(report.edgeHealth.status), + `${label}.edgeHealth.status must be pass, blocker, or not_run` + ); + assertString(report.edgeHealth.classification, `${label}.edgeHealth.classification`); + assertObject(report.edgeHealth.contracts, `${label}.edgeHealth.contracts`); + assertArray(report.edgeHealth.publicTcp ?? [], `${label}.edgeHealth.publicTcp`); + assertArray(report.edgeHealth.publicHttp ?? [], `${label}.edgeHealth.publicHttp`); + assertObject(report.edgeHealth.kubernetes ?? {}, `${label}.edgeHealth.kubernetes`); +} + async function validateDevM3Report(report, label) { for (const field of [ "$schema",