From e9e71a7045fd5be3e7bec7ca13e0fc7961965be6 Mon Sep 17 00:00:00 2001 From: HWLAB Code Queue Date: Thu, 21 May 2026 17:38:14 +0000 Subject: [PATCH] fix: add structured edge frp diagnosis --- docs/dev-edge-health.md | 12 ++++ reports/dev-gate/dev-edge-health.json | 56 ++++++++++++---- reports/dev-gate/dev-preflight-report.json | 14 ++-- scripts/src/dev-edge-health-smoke-lib.mjs | 76 +++++++++++++++++++--- scripts/src/dev-gate-preflight.mjs | 4 +- 5 files changed, 134 insertions(+), 28 deletions(-) diff --git a/docs/dev-edge-health.md b/docs/dev-edge-health.md index de8c16dc..3392e7ca 100644 --- a/docs/dev-edge-health.md +++ b/docs/dev-edge-health.md @@ -37,6 +37,18 @@ The smoke classifies failures into these blockers: - `app_health_blocker`: public HTTP reaches an app but health JSON is wrong, missing evidence, or returns an HTTP error. +## Machine-Readable Diagnosis + +Blocked `frp_blocker` and `edge_proxy_blocker` reports now include a +`diagnosis` object in `reports/dev-gate/dev-edge-health.json` with: + +- `likelyLayer`: the most likely broken layer, such as `master-edge/frps`; +- `confidence`: coarse confidence for the diagnosis; +- `likelyCause`: a single-sentence blocker summary; +- `evidence`: the TCP probes that support the diagnosis; +- `notProven`: layers this probe did not rule out; +- `nextTask`: the smallest read-only follow-up or ops action. + ## Current Blocker The committed report at `reports/dev-gate/dev-edge-health.json` records a diff --git a/reports/dev-gate/dev-edge-health.json b/reports/dev-gate/dev-edge-health.json index eda159f5..233bd30b 100644 --- a/reports/dev-gate/dev-edge-health.json +++ b/reports/dev-gate/dev-edge-health.json @@ -4,12 +4,12 @@ "reportVersion": "v1", "issue": "pikasTech/HWLAB#36", "taskId": "dev-edge-health", - "commitId": "7092eb981459", + "commitId": "cef2befdd156", "acceptanceLevel": "dev_edge_health", "devOnly": true, "prodDisabled": true, "status": "blocked", - "generatedAt": "2026-05-21T16:48:05.275Z", + "generatedAt": "2026-05-21T17:35:52.483Z", "namespace": "hwlab-dev", "endpoint": "http://74.48.78.17:6667", "sourceContract": { @@ -58,18 +58,18 @@ "hwlab-dev hwlab-edge-proxy, hwlab-tunnel-client, hwlab-router, and hwlab-cloud-api are observable", "No PROD endpoint, secret read, restart, or UniDesk runtime substitute is used" ], - "summary": "public 6667 and frps control 7000 both refuse TCP connections; frps is not reachable on the master edge" + "summary": "public 6667, frps control 7000, and tunnel health 7402 all refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed" }, "blockers": [ { "type": "network_blocker", - "scope": "frp_blocker", + "scope": "master-edge/frps", "status": "open", - "summary": "public 6667 and frps control 7000 both refuse TCP connections; frps is not reachable on the master edge" + "summary": "public 6667, frps control 7000, and tunnel health 7402 all refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed" } ], "edgeHealth": { - "generatedAt": "2026-05-21T16:48:05.275Z", + "generatedAt": "2026-05-21T17:35:52.483Z", "mode": "live-read-only", "endpoint": "http://74.48.78.17:6667", "safety": { @@ -131,7 +131,7 @@ "status": "error", "code": "ECONNREFUSED", "message": "connect ECONNREFUSED 74.48.78.17:6667", - "durationMs": 221 + "durationMs": 223 }, { "host": "74.48.78.17", @@ -139,7 +139,7 @@ "status": "error", "code": "ECONNREFUSED", "message": "connect ECONNREFUSED 74.48.78.17:7000", - "durationMs": 218 + "durationMs": 209 }, { "host": "74.48.78.17", @@ -147,7 +147,7 @@ "status": "error", "code": "ECONNREFUSED", "message": "connect ECONNREFUSED 74.48.78.17:7402", - "durationMs": 207 + "durationMs": 1222 } ], "publicHttp": [ @@ -158,7 +158,7 @@ "name": "TypeError", "message": "fetch failed" }, - "durationMs": 34 + "durationMs": 40 }, { "url": "http://74.48.78.17:6667/health/live", @@ -213,6 +213,40 @@ ], "status": "blocker", "classification": "frp_blocker", - "blocker": "public 6667 and frps control 7000 both refuse TCP connections; frps is not reachable on the master edge" + "blocker": "public 6667, frps control 7000, and tunnel health 7402 all refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed", + "diagnosis": { + "likelyLayer": "master-edge/frps", + "confidence": "high", + "likelyCause": "public 6667, frps control 7000, and tunnel health 7402 all refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed", + "evidence": [ + { + "host": "74.48.78.17", + "port": 6667, + "status": "error", + "code": "ECONNREFUSED", + "message": "connect ECONNREFUSED 74.48.78.17:6667" + }, + { + "host": "74.48.78.17", + "port": 7000, + "status": "error", + "code": "ECONNREFUSED", + "message": "connect ECONNREFUSED 74.48.78.17:7000" + }, + { + "host": "74.48.78.17", + "port": 7402, + "status": "error", + "code": "ECONNREFUSED", + "message": "connect ECONNREFUSED 74.48.78.17:7402" + } + ], + "notProven": [ + "k3s service readiness", + "frpc connectivity", + "edge proxy deployment" + ], + "nextTask": "Restore or reopen frps on 74.48.78.17, then rerun the read-only edge smoke." + } } } diff --git a/reports/dev-gate/dev-preflight-report.json b/reports/dev-gate/dev-preflight-report.json index c1069fb4..5739745a 100644 --- a/reports/dev-gate/dev-preflight-report.json +++ b/reports/dev-gate/dev-preflight-report.json @@ -15,10 +15,10 @@ ], "target": { "ref": "origin/main", - "commitId": "6da6ab40bc356ddf6f2ebea11af935f4399e4945", - "shortCommitId": "6da6ab4" + "commitId": "cef2befdd156d3d4028f087a96e9c118b9174bf1", + "shortCommitId": "cef2bef" }, - "generatedAt": "2026-05-21T16:53:29.568Z", + "generatedAt": "2026-05-21T17:35:52.507Z", "mode": "read-only", "devOnly": true, "prodDisabled": true, @@ -51,7 +51,7 @@ "id": "target-commit-pinning", "category": "contract", "status": "blocked", - "summary": "deploy/catalog commitId 24eb3bf does not match origin/main 6da6ab4.", + "summary": "deploy/catalog commitId 24eb3bf does not match origin/main cef2bef.", "evidence": [] }, { @@ -72,7 +72,7 @@ "id": "dev-artifact-publish-report", "category": "registry", "status": "blocked", - "summary": "DEV artifact publish report status is blocked with 9/13 published for source 1e8d009e9531d71cef0762998663b4e3731032c6.", + "summary": "DEV artifact publish report status is blocked with 0/13 published for source 70bb9168ead272ec1f77b752df8e178e70858df4.", "evidence": [] }, { @@ -294,7 +294,7 @@ "status": "open", "type": "contract_blocker", "scope": "deploy-target", - "summary": "deploy/deploy.json and deploy/artifact-catalog.dev.json still target 24eb3bf, not origin/main 6da6ab4.", + "summary": "deploy/deploy.json and deploy/artifact-catalog.dev.json still target 24eb3bf, not origin/main cef2bef.", "nextTask": "Publish or select a DEV artifact set for the current origin/main commit and update deploy/deploy.json plus deploy/artifact-catalog.dev.json to that immutable commit/tag." }, { @@ -308,7 +308,7 @@ "status": "open", "type": "runtime_blocker", "scope": "dev-artifact-publish", - "summary": "reports/dev-gate/dev-artifacts.json does not prove all HWLAB service artifacts for origin/main 6da6ab4; current status is blocked with 9/13 published.", + "summary": "reports/dev-gate/dev-artifacts.json does not prove all HWLAB service artifacts for origin/main cef2bef; current status is blocked with 0/13 published.", "nextTask": "Complete DEV artifact publishing for every frozen HWLAB service at the current origin/main commit and record immutable registry digests." }, { diff --git a/scripts/src/dev-edge-health-smoke-lib.mjs b/scripts/src/dev-edge-health-smoke-lib.mjs index 7062d016..9e5ff071 100644 --- a/scripts/src/dev-edge-health-smoke-lib.mjs +++ b/scripts/src/dev-edge-health-smoke-lib.mjs @@ -103,6 +103,7 @@ async function createDevGateReport(edgeHealth) { const commitId = await gitCommitId(); const status = edgeHealth.status === "pass" ? "pass" : edgeHealth.status === "not_run" ? "not_run" : "blocked"; const blockerType = status === "blocked" ? blockerTypeFor(edgeHealth.classification) : null; + const blockerScope = edgeHealth.diagnosis?.likelyLayer ?? edgeHealth.classification; return { $schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json", @@ -170,7 +171,7 @@ async function createDevGateReport(edgeHealth) { ? [ { type: blockerType, - scope: edgeHealth.classification, + scope: blockerScope, status: "open", summary: edgeHealth.blocker } @@ -338,7 +339,7 @@ function classify(report) { return classifyReachableHealth(publicHealth.json); } if (port6667?.status === "error" && port6667.code === "ECONNREFUSED") { - return classifyRefusedPort(frpsControl, tunnelHealth); + return classifyRefusedPort(port6667, frpsControl, tunnelHealth); } if (port6667?.status === "timeout") { return blocker("dns_port_firewall_blocker", "public 6667 timed out before any HTTP response"); @@ -363,21 +364,80 @@ function classifyReachableHealth(json) { return { status: "pass", classification: "none", blocker: null }; } -function classifyRefusedPort(frpsControl, tunnelHealth) { +function classifyRefusedPort(port6667, frpsControl, tunnelHealth) { if (frpsControl?.status === "error" && frpsControl.code === "ECONNREFUSED") { - return blocker("frp_blocker", "public 6667 and frps control 7000 both refuse TCP connections; frps is not reachable on the master edge"); + const allPortsRefused = tunnelHealth?.status === "error" && tunnelHealth.code === "ECONNREFUSED"; + const diagnosis = { + likelyLayer: "master-edge/frps", + confidence: "high", + likelyCause: allPortsRefused + ? "public 6667, frps control 7000, and tunnel health 7402 all refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed" + : "public 6667 and frps control 7000 both refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed", + evidence: [ + summarizeTcpProbe(port6667), + summarizeTcpProbe(frpsControl), + summarizeTcpProbe(tunnelHealth) + ], + notProven: [ + "k3s service readiness", + "frpc connectivity", + "edge proxy deployment" + ], + nextTask: "Restore or reopen frps on 74.48.78.17, then rerun the read-only edge smoke." + }; + return blocker("frp_blocker", diagnosis.likelyCause, diagnosis); } if (tunnelHealth?.status === "error" && tunnelHealth.code === "ECONNREFUSED") { - return blocker("frp_blocker", "public 6667 refuses TCP and tunnel health 7402 is not reachable"); + return blocker("frp_blocker", "public 6667 refuses TCP and tunnel health 7402 is not reachable", { + likelyLayer: "d601/frpc", + confidence: "medium", + likelyCause: "public 6667 and tunnel health 7402 both refuse TCP connections; the D601 frp client or its remote port is the likely break", + evidence: [ + summarizeTcpProbe(port6667), + summarizeTcpProbe(tunnelHealth) + ], + notProven: [ + "master-edge frps reachability", + "k3s service readiness" + ], + nextTask: "Restore the D601 tunnel client or its remote port, then rerun the read-only edge smoke." + }); } - return blocker("edge_proxy_blocker", "public 6667 refuses TCP connections"); + return blocker("edge_proxy_blocker", "public 6667 refuses TCP connections", { + likelyLayer: "master-edge/edge-proxy", + confidence: "medium", + likelyCause: "public 6667 refuses TCP while frps control 7000 stays reachable, so the public edge proxy or its listener is the likely break", + evidence: [ + summarizeTcpProbe(port6667), + summarizeTcpProbe(frpsControl) + ], + notProven: [ + "k3s service readiness" + ], + nextTask: "Repair the edge proxy or its public listener, then rerun the read-only edge smoke." + }); } -function blocker(classification, message) { +function blocker(classification, message, diagnosis = null) { return { status: "blocker", classification, - blocker: message + blocker: message, + ...(diagnosis ? { diagnosis } : {}) + }; +} + +function summarizeTcpProbe(probe) { + if (!probe) { + return null; + } + + return { + host: probe.host, + port: probe.port, + status: probe.status, + code: probe.code ?? null, + message: probe.message ?? null }; } diff --git a/scripts/src/dev-gate-preflight.mjs b/scripts/src/dev-gate-preflight.mjs index a3e7dfaf..235d46bb 100644 --- a/scripts/src/dev-gate-preflight.mjs +++ b/scripts/src/dev-gate-preflight.mjs @@ -431,9 +431,9 @@ function validateEdgeHealthReport(reporter, edgeReport) { reporter.check("dev-edge-health-report", "edge", "blocked", `Committed DEV edge health report is ${edgeReport.status} with classification ${classification}.`, evidence); reporter.block({ type: "network_blocker", - scope: "dev-edge-health", + scope: edgeHealth?.diagnosis?.likelyLayer ?? "dev-edge-health", summary, - nextTask: "Repair the master frps/public DEV edge route and rerun the read-only DEV edge health smoke before real deployment." + nextTask: edgeHealth?.diagnosis?.nextTask ?? "Repair the master frps/public DEV edge route and rerun the read-only DEV edge health smoke before real deployment." }); }