fix: freeze dev ports and edge health
This commit is contained in:
lyon
@@ -3,7 +3,7 @@ import { constants as fsConstants } from "node:fs";
|
||||
import { access, readFile, writeFile } from "node:fs/promises";
|
||||
import { request as httpRequest } from "node:http";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
import {
|
||||
@@ -470,30 +470,40 @@ async function checkCloudApiDb(deploy, workloads, blockers) {
|
||||
);
|
||||
}
|
||||
|
||||
let health = null;
|
||||
try {
|
||||
const modulePath = pathToFileURL(path.join(repoRoot, "internal/cloud/server.mjs")).href;
|
||||
const { buildHealthPayload } = await import(modulePath);
|
||||
health = buildHealthPayload();
|
||||
if (health.db?.ready !== true) {
|
||||
addBlocker(
|
||||
blockers,
|
||||
"runtime_blocker",
|
||||
"cloud-api-db",
|
||||
`cloud-api health reports DB config blocked; missing ${health.db?.missingEnv?.join(", ") ?? "unknown DB env"}`
|
||||
);
|
||||
const configReady = missingManifest.length === 0 && missingK8s.length === 0 && missingSecretRefs.length === 0;
|
||||
const health = {
|
||||
status: configReady ? "degraded" : "blocked",
|
||||
ready: false,
|
||||
configReady,
|
||||
connected: false,
|
||||
connectionChecked: false,
|
||||
fields: requiredEnv.map((env) => ({
|
||||
name: env.name,
|
||||
present: env.manifestPresent && env.k8sPresent && (!env.secretRef || env.secretRef.present),
|
||||
redacted: env.redacted
|
||||
})),
|
||||
missingEnv: uniqueStrings([...missingManifest, ...missingK8s]),
|
||||
secretRefs: requiredEnv
|
||||
.filter((env) => env.secretRef)
|
||||
.map((env) => ({
|
||||
env: env.name,
|
||||
secretName: env.secretRef.secretName,
|
||||
secretKey: env.secretRef.secretKey,
|
||||
present: env.secretRef.present,
|
||||
redacted: true
|
||||
})),
|
||||
safety: {
|
||||
liveDbEvidence: false
|
||||
}
|
||||
} catch (error) {
|
||||
addBlocker(blockers, "observability_blocker", "cloud-api-health", `Cannot inspect cloud-api health payload: ${error.message}`);
|
||||
}
|
||||
};
|
||||
|
||||
return {
|
||||
status: missingManifest.length === 0 && missingK8s.length === 0 && missingSecretRefs.length === 0 ? "contract-ready" : "contract-blocked",
|
||||
status: configReady ? "contract-ready" : "contract-blocked",
|
||||
requiredEnv,
|
||||
missingManifest,
|
||||
missingK8s,
|
||||
missingSecretRefs,
|
||||
runtimeHealth: health?.db ? summarizeDbContract(health.db) : null,
|
||||
runtimeHealth: summarizeDbContract(health),
|
||||
secretMaterialRead: false,
|
||||
valuesRedacted: true,
|
||||
liveDbEvidence: false,
|
||||
|
||||
@@ -11,9 +11,9 @@ import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const defaultReportPath = path.join(repoRoot, "reports/dev-gate/dev-edge-health.json");
|
||||
const publicHost = "74.48.78.17";
|
||||
const publicPort = 6667;
|
||||
const publicPort = 16667;
|
||||
const namespace = "hwlab-dev";
|
||||
const tcpPorts = [6667, 7000, 7402];
|
||||
const tcpPorts = [16667, 7000, 7402];
|
||||
const runtimeServices = [
|
||||
{ serviceId: "hwlab-cloud-api", port: 6667 },
|
||||
{ serviceId: "hwlab-router", port: 7401 },
|
||||
@@ -161,7 +161,7 @@ async function createDevGateReport(edgeHealth) {
|
||||
devPreconditions: {
|
||||
status,
|
||||
requirements: [
|
||||
"http://74.48.78.17:6667/health returns HWLAB DEV JSON with service, commit, and image evidence",
|
||||
"http://74.48.78.17:16667/health returns HWLAB DEV JSON with service, commit, and image evidence",
|
||||
"frps control on 74.48.78.17:7000 accepts the D601 frp client",
|
||||
"hwlab-dev hwlab-edge-proxy, hwlab-tunnel-client, hwlab-router, and hwlab-cloud-api are observable",
|
||||
"No PROD endpoint, secret read, restart, or UniDesk runtime substitute is used"
|
||||
@@ -212,7 +212,7 @@ async function inspectContracts() {
|
||||
const frps = await readText("deploy/frp/frps.dev.toml");
|
||||
const deploy = await readJson("deploy/deploy.json");
|
||||
const master = await readJson("deploy/master-edge/health-contract.json");
|
||||
const frpsUsesVhost6667 = /vhostHTTPPort\s*=\s*6667/u.test(frps);
|
||||
const frpsUsesVhostEdgePort = /vhostHTTPPort\s*=\s*(16667|6667)/u.test(frps);
|
||||
|
||||
return {
|
||||
devEndpoint: DEV_ENDPOINT,
|
||||
@@ -220,10 +220,15 @@ async function inspectContracts() {
|
||||
frpcTargetsEdgeProxy:
|
||||
/localIP\s*=\s*"hwlab-edge-proxy\.hwlab-dev\.svc\.cluster\.local"/u.test(frpc) &&
|
||||
/localPort\s*=\s*6667/u.test(frpc) &&
|
||||
/remotePort\s*=\s*6667/u.test(frpc),
|
||||
frpsTcpRemotePort6667: /start\s*=\s*6667[\s\S]*end\s*=\s*6667/u.test(frps),
|
||||
frpsUsesVhost6667,
|
||||
configRisk: frpsUsesVhost6667 ? "port_conflict" : "none"
|
||||
/remotePort\s*=\s*16667/u.test(frpc),
|
||||
frpcTargetsCloudWeb:
|
||||
/localIP\s*=\s*"hwlab-cloud-web\.hwlab-dev\.svc\.cluster\.local"/u.test(frpc) &&
|
||||
/localPort\s*=\s*8080/u.test(frpc) &&
|
||||
/remotePort\s*=\s*16666/u.test(frpc),
|
||||
frpsTcpRemotePort16667: /start\s*=\s*16667[\s\S]*end\s*=\s*16667/u.test(frps),
|
||||
frpsTcpRemotePort16666: /start\s*=\s*16666[\s\S]*end\s*=\s*16666/u.test(frps),
|
||||
frpsUsesVhostEdgePort,
|
||||
configRisk: frpsUsesVhostEdgePort ? "port_conflict" : "none"
|
||||
},
|
||||
masterEdge: {
|
||||
endpoint: master.endpoint,
|
||||
@@ -375,7 +380,7 @@ async function inspectClusterDns() {
|
||||
}
|
||||
|
||||
function classify(report) {
|
||||
const port6667 = report.publicTcp.find((probe) => probe.port === publicPort);
|
||||
const publicEdgePort = report.publicTcp.find((probe) => probe.port === publicPort);
|
||||
const frpsControl = report.publicTcp.find((probe) => probe.port === 7000);
|
||||
const tunnelHealth = report.publicTcp.find((probe) => probe.port === 7402);
|
||||
const publicHealth = report.publicHttp.find((probe) => probe.url.endsWith("/health"));
|
||||
@@ -383,11 +388,11 @@ function classify(report) {
|
||||
if (publicHealth?.ok && publicHealth.json?.serviceId && publicHealth.json?.environment === ENVIRONMENT_DEV) {
|
||||
return classifyReachableHealth(publicHealth.json);
|
||||
}
|
||||
if (port6667?.status === "error" && port6667.code === "ECONNREFUSED") {
|
||||
return classifyRefusedPort(port6667, frpsControl, tunnelHealth);
|
||||
if (publicEdgePort?.status === "error" && publicEdgePort.code === "ECONNREFUSED") {
|
||||
return classifyRefusedPort(publicEdgePort, frpsControl, tunnelHealth);
|
||||
}
|
||||
if (port6667?.status === "timeout") {
|
||||
return blocker("dns_port_firewall_blocker", "public 6667 timed out before any HTTP response");
|
||||
if (publicEdgePort?.status === "timeout") {
|
||||
return blocker("dns_port_firewall_blocker", "public 16667 timed out before any HTTP response");
|
||||
}
|
||||
if (publicHealth && !publicHealth.ok && publicHealth.status) {
|
||||
return blocker("app_health_blocker", `public /health returned HTTP ${publicHealth.status}`);
|
||||
@@ -409,17 +414,17 @@ function classifyReachableHealth(json) {
|
||||
return { status: "pass", classification: "none", blocker: null };
|
||||
}
|
||||
|
||||
function classifyRefusedPort(port6667, frpsControl, tunnelHealth) {
|
||||
function classifyRefusedPort(publicEdgePort, frpsControl, tunnelHealth) {
|
||||
if (frpsControl?.status === "error" && frpsControl.code === "ECONNREFUSED") {
|
||||
const allPortsRefused = tunnelHealth?.status === "error" && tunnelHealth.code === "ECONNREFUSED";
|
||||
const diagnosis = {
|
||||
likelyLayer: "master-edge/frps",
|
||||
confidence: "high",
|
||||
likelyCause: allPortsRefused
|
||||
? "public 6667, frps control 7000, and tunnel health 7402 all refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed"
|
||||
: "public 6667 and frps control 7000 both refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed",
|
||||
? "public 16667, frps control 7000, and tunnel health 7402 all refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed"
|
||||
: "public 16667 and frps control 7000 both refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed",
|
||||
evidence: [
|
||||
summarizeTcpProbe(port6667),
|
||||
summarizeTcpProbe(publicEdgePort),
|
||||
summarizeTcpProbe(frpsControl),
|
||||
summarizeTcpProbe(tunnelHealth)
|
||||
],
|
||||
@@ -433,12 +438,12 @@ function classifyRefusedPort(port6667, frpsControl, tunnelHealth) {
|
||||
return blocker("frp_blocker", diagnosis.likelyCause, diagnosis);
|
||||
}
|
||||
if (tunnelHealth?.status === "error" && tunnelHealth.code === "ECONNREFUSED") {
|
||||
return blocker("frp_blocker", "public 6667 refuses TCP and tunnel health 7402 is not reachable", {
|
||||
return blocker("frp_blocker", "public 16667 refuses TCP and tunnel health 7402 is not reachable", {
|
||||
likelyLayer: "d601/frpc",
|
||||
confidence: "medium",
|
||||
likelyCause: "public 6667 and tunnel health 7402 both refuse TCP connections; the D601 frp client or its remote port is the likely break",
|
||||
likelyCause: "public 16667 and tunnel health 7402 both refuse TCP connections; the D601 frp client or its remote port is the likely break",
|
||||
evidence: [
|
||||
summarizeTcpProbe(port6667),
|
||||
summarizeTcpProbe(publicEdgePort),
|
||||
summarizeTcpProbe(tunnelHealth)
|
||||
],
|
||||
notProven: [
|
||||
@@ -448,12 +453,12 @@ function classifyRefusedPort(port6667, frpsControl, tunnelHealth) {
|
||||
nextTask: "Restore the D601 tunnel client or its remote port, then rerun the read-only edge smoke."
|
||||
});
|
||||
}
|
||||
return blocker("edge_proxy_blocker", "public 6667 refuses TCP connections", {
|
||||
return blocker("edge_proxy_blocker", "public 16667 refuses TCP connections", {
|
||||
likelyLayer: "master-edge/edge-proxy",
|
||||
confidence: "medium",
|
||||
likelyCause: "public 6667 refuses TCP while frps control 7000 stays reachable, so the public edge proxy or its listener is the likely break",
|
||||
likelyCause: "public 16667 refuses TCP while frps control 7000 stays reachable, so the public edge proxy or its listener is the likely break",
|
||||
evidence: [
|
||||
summarizeTcpProbe(port6667),
|
||||
summarizeTcpProbe(publicEdgePort),
|
||||
summarizeTcpProbe(frpsControl)
|
||||
],
|
||||
notProven: [
|
||||
|
||||
@@ -245,7 +245,7 @@ function applyPriority(blocker) {
|
||||
priority: "P2",
|
||||
unblockOrder: 6,
|
||||
unblocks: [issue(36), issue(38), issue(37), issue(39)],
|
||||
rationale: "The :6667/frp path must be reachable before any DEV-LIVE M3, M4, or M5 evidence can be collected."
|
||||
rationale: "The :16667/frp path must be reachable before any DEV-LIVE M3, M4, or M5 evidence can be collected."
|
||||
};
|
||||
}
|
||||
|
||||
@@ -606,7 +606,7 @@ function buildDoD(reports, milestones, blockers) {
|
||||
id: "dev-edge-frp-6667",
|
||||
status: hasBlocker(blockers, (blocker) => blocker.scope.includes("edge") || blocker.scope.includes("ingress") || blocker.summary.includes("6667")) ? "blocked" : "pass",
|
||||
evidenceLevel: "BLOCKED",
|
||||
summary: "No committed report proves live HTTP 200/JSON on http://74.48.78.17:6667."
|
||||
summary: "No committed report proves live HTTP 200/JSON on http://74.48.78.17:16667."
|
||||
},
|
||||
{
|
||||
id: "cloud-api-db-ready",
|
||||
@@ -673,7 +673,7 @@ function evidenceRequiredFor(scope) {
|
||||
if (scope.includes("artifact") || scope === "ghcr") return "Artifact publish report with ciPublished=true, registryVerified=true, and sha256 digest for each frozen service ID.";
|
||||
if (scope.includes("kubectl") || scope.includes("k3s")) return "Read-only kubectl/k3s report proving pods/services/configmaps are observable in hwlab-dev without reading Secrets.";
|
||||
if (scope.includes("cloud-api-db")) return "Cloud API health/live output showing DB env ready and redacted secret references, without secret material.";
|
||||
if (scope.includes("edge") || scope.includes("ingress")) return "Read-only DEV route observation for :6667/frp/edge/router with HWLAB service identity and artifact identity.";
|
||||
if (scope.includes("edge") || scope.includes("ingress")) return "Read-only DEV route observation for :16667/frp/edge/router with HWLAB service identity and artifact identity.";
|
||||
return "A committed report with the exact evidence level and command used.";
|
||||
}
|
||||
|
||||
|
||||
@@ -351,9 +351,11 @@ function assertFrpStatic(frpc, frps, masterEdge) {
|
||||
assert.equal(frpc.serverAddr, "74.48.78.17");
|
||||
assert.equal(frpc.serverPort, 7000);
|
||||
assert.equal(frps.bindPort, 7000);
|
||||
assert.equal(Object.hasOwn(frps, "vhostHTTPPort"), false, "frps must not bind vhostHTTPPort because 6667 is reserved for the TCP proxy");
|
||||
assert.ok(frpc.proxies.some((proxy) => proxy.name === "hwlab-dev-edge-proxy" && proxy.remotePort === 6667));
|
||||
assert.ok(frps.allowPorts.some((port) => port.start === 6667 && port.end === 6667));
|
||||
assert.equal(Object.hasOwn(frps, "vhostHTTPPort"), false, "frps must not bind vhostHTTPPort because 16667 is reserved for the TCP proxy");
|
||||
assert.ok(frpc.proxies.some((proxy) => proxy.name === "hwlab-dev-edge-proxy" && proxy.remotePort === 16667));
|
||||
assert.ok(frpc.proxies.some((proxy) => proxy.name === "hwlab-dev-cloud-web" && proxy.remotePort === 16666));
|
||||
assert.ok(frps.allowPorts.some((port) => port.start === 16667 && port.end === 16667));
|
||||
assert.ok(frps.allowPorts.some((port) => port.start === 16666 && port.end === 16666));
|
||||
assert.equal(masterEdge.environment, ENVIRONMENT_DEV);
|
||||
assert.equal(masterEdge.endpoint, DEV_ENDPOINT);
|
||||
assert.equal(masterEdge.reverseLink.mode, "frp");
|
||||
@@ -731,7 +733,7 @@ async function validateEdgeContracts(reporter, masterEdge) {
|
||||
parseToml("deploy/frp/frps.dev.toml")
|
||||
]);
|
||||
assertFrpStatic(frpc, frps, masterEdge);
|
||||
reporter.check("frp-master-edge-static", "edge", "pass", "FRP and master-edge contracts describe D601-to-master DEV on port 6667.");
|
||||
reporter.check("frp-master-edge-static", "edge", "pass", "FRP and master-edge contracts describe D601-to-master DEV on public ports 16666/16667.");
|
||||
} catch (error) {
|
||||
reporter.check("frp-master-edge-static", "edge", "blocked", error.message);
|
||||
reporter.block({
|
||||
|
||||
Reference in New Issue
Block a user