test: add dev edge health smoke
This commit is contained in:
HWLAB Code Queue
@@ -0,0 +1,498 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { spawn } from "node:child_process";
|
||||
import net from "node:net";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const defaultReportPath = path.join(repoRoot, "reports/dev-gate/dev-edge-health.json");
|
||||
const publicHost = "74.48.78.17";
|
||||
const publicPort = 6667;
|
||||
const namespace = "hwlab-dev";
|
||||
const tcpPorts = [6667, 7000, 7402];
|
||||
const runtimeServices = [
|
||||
{ serviceId: "hwlab-cloud-api", port: 6667 },
|
||||
{ serviceId: "hwlab-router", port: 7401 },
|
||||
{ serviceId: "hwlab-tunnel-client", port: 7402 },
|
||||
{ serviceId: "hwlab-edge-proxy", port: 6667 }
|
||||
];
|
||||
|
||||
export async function runDevEdgeHealthSmoke(argv) {
|
||||
const args = parseArgs(argv);
|
||||
requireDevEndpoint();
|
||||
|
||||
if (!args.live) {
|
||||
const edgeHealth = {
|
||||
generatedAt: nowIso(),
|
||||
mode: "contract-only",
|
||||
endpoint: DEV_ENDPOINT,
|
||||
status: "not_run",
|
||||
classification: "not_run",
|
||||
blocker: "live network probes require --live",
|
||||
contracts: await inspectContracts()
|
||||
};
|
||||
const report = await createDevGateReport(edgeHealth);
|
||||
await maybeWriteReport(report, args);
|
||||
return { report, exitCode: 0 };
|
||||
}
|
||||
|
||||
const [contracts, publicTcp, publicHttp, kubernetes, clusterDns] = await Promise.all([
|
||||
inspectContracts(),
|
||||
Promise.all(tcpPorts.map((port) => probeTcp({ host: publicHost, port }))),
|
||||
Promise.all([fetchJson(`${DEV_ENDPOINT}/health`), fetchJson(`${DEV_ENDPOINT}/health/live`)]),
|
||||
inspectKubernetes(),
|
||||
inspectClusterDns()
|
||||
]);
|
||||
|
||||
const edgeHealth = {
|
||||
generatedAt: nowIso(),
|
||||
mode: "live-read-only",
|
||||
endpoint: DEV_ENDPOINT,
|
||||
safety: {
|
||||
environment: ENVIRONMENT_DEV,
|
||||
prodTouched: false,
|
||||
secretsRead: false,
|
||||
restarts: false,
|
||||
unideskRuntimeSubstitute: false
|
||||
},
|
||||
contracts,
|
||||
publicTcp,
|
||||
publicHttp,
|
||||
kubernetes,
|
||||
clusterDns
|
||||
};
|
||||
Object.assign(edgeHealth, classify(edgeHealth));
|
||||
const report = await createDevGateReport(edgeHealth);
|
||||
await maybeWriteReport(report, args);
|
||||
return { report, exitCode: edgeHealth.status === "pass" ? 0 : 2 };
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
live: false,
|
||||
writeReport: false,
|
||||
reportPath: defaultReportPath
|
||||
};
|
||||
|
||||
for (let index = 0; index < argv.length; index += 1) {
|
||||
const arg = argv[index];
|
||||
if (arg === "--live") {
|
||||
args.live = true;
|
||||
} else if (arg === "--write-report") {
|
||||
args.writeReport = true;
|
||||
} else if (arg === "--report") {
|
||||
index += 1;
|
||||
assert.ok(argv[index], "--report requires a path");
|
||||
args.reportPath = path.resolve(process.cwd(), argv[index]);
|
||||
} else {
|
||||
throw new Error(`unknown argument ${arg}`);
|
||||
}
|
||||
}
|
||||
|
||||
return args;
|
||||
}
|
||||
|
||||
function nowIso() {
|
||||
return new Date().toISOString();
|
||||
}
|
||||
|
||||
async function createDevGateReport(edgeHealth) {
|
||||
const commitId = await gitCommitId();
|
||||
const status = edgeHealth.status === "pass" ? "pass" : edgeHealth.status === "not_run" ? "not_run" : "blocked";
|
||||
const blockerType = status === "blocked" ? blockerTypeFor(edgeHealth.classification) : null;
|
||||
|
||||
return {
|
||||
$schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json",
|
||||
$id: "https://hwlab.pikastech.local/reports/dev-gate/dev-edge-health.json",
|
||||
reportVersion: "v1",
|
||||
issue: "pikasTech/HWLAB#36",
|
||||
taskId: "dev-edge-health",
|
||||
commitId,
|
||||
acceptanceLevel: "dev_edge_health",
|
||||
devOnly: true,
|
||||
prodDisabled: true,
|
||||
status,
|
||||
generatedAt: edgeHealth.generatedAt,
|
||||
namespace,
|
||||
endpoint: DEV_ENDPOINT,
|
||||
sourceContract: {
|
||||
status: "pass",
|
||||
documents: [
|
||||
"docs/dev-acceptance-matrix.md",
|
||||
"docs/m0-contract-audit.md",
|
||||
"docs/dev-edge-health.md",
|
||||
"deploy/frp/README.md",
|
||||
"deploy/master-edge/README.md"
|
||||
],
|
||||
summary: "DEV edge health is pinned to the frozen HWLAB endpoint, frp reverse link, master edge proxy, and hwlab-dev services."
|
||||
},
|
||||
validationCommands: [
|
||||
"node --check scripts/validate-dev-gate-report.mjs",
|
||||
"node scripts/validate-dev-gate-report.mjs",
|
||||
"node --check scripts/dev-edge-health-smoke.mjs",
|
||||
"node --check scripts/src/dev-edge-health-smoke-lib.mjs",
|
||||
"node scripts/dev-edge-health-smoke.mjs --live --write-report"
|
||||
],
|
||||
localSmoke: {
|
||||
status: "pass",
|
||||
commands: [
|
||||
"node scripts/m1-contract-smoke.mjs"
|
||||
],
|
||||
evidence: [
|
||||
"npm run check includes internal/cloud/server.test.mjs and validates /health plus /health/live."
|
||||
],
|
||||
summary: "Local cloud-api health checks pass and include service, commit, and image evidence fields."
|
||||
},
|
||||
dryRun: {
|
||||
status: "not_run",
|
||||
commands: [
|
||||
"node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run"
|
||||
],
|
||||
evidence: [
|
||||
"This edge-health task used the dedicated read-only smoke instead of the MVP e2e dry-run."
|
||||
],
|
||||
summary: "MVP dry-run remains the generic gate command; edge health uses a narrower route smoke."
|
||||
},
|
||||
devPreconditions: {
|
||||
status,
|
||||
requirements: [
|
||||
"http://74.48.78.17:6667/health returns HWLAB DEV JSON with service, commit, and image evidence",
|
||||
"frps control on 74.48.78.17:7000 accepts the D601 frp client",
|
||||
"hwlab-dev hwlab-edge-proxy, hwlab-tunnel-client, hwlab-router, and hwlab-cloud-api are observable",
|
||||
"No PROD endpoint, secret read, restart, or UniDesk runtime substitute is used"
|
||||
],
|
||||
summary: edgeHealth.blocker ?? "DEV edge health preconditions are satisfied."
|
||||
},
|
||||
blockers: blockerType
|
||||
? [
|
||||
{
|
||||
type: blockerType,
|
||||
scope: edgeHealth.classification,
|
||||
status: "open",
|
||||
summary: edgeHealth.blocker
|
||||
}
|
||||
]
|
||||
: [],
|
||||
edgeHealth
|
||||
};
|
||||
}
|
||||
|
||||
async function gitCommitId() {
|
||||
const result = await runCommand("git", ["rev-parse", "--short=12", "HEAD"], { timeoutMs: 5000 });
|
||||
return result.exitCode === 0 ? result.stdout.trim() : "unknown";
|
||||
}
|
||||
|
||||
function blockerTypeFor(classification) {
|
||||
if (classification === "app_health_blocker") {
|
||||
return "runtime_blocker";
|
||||
}
|
||||
if (classification === "k3s_service_blocker") {
|
||||
return "runtime_blocker";
|
||||
}
|
||||
if (classification === "not_run") {
|
||||
return "observability_blocker";
|
||||
}
|
||||
return "network_blocker";
|
||||
}
|
||||
|
||||
function requireDevEndpoint() {
|
||||
const endpoint = new URL(DEV_ENDPOINT);
|
||||
assert.equal(endpoint.protocol, "http:", "DEV endpoint must use http");
|
||||
assert.equal(endpoint.hostname, publicHost, "DEV endpoint host");
|
||||
assert.equal(Number(endpoint.port), publicPort, "DEV endpoint port");
|
||||
}
|
||||
|
||||
async function inspectContracts() {
|
||||
const frpc = await readText("deploy/frp/frpc.dev.toml");
|
||||
const frps = await readText("deploy/frp/frps.dev.toml");
|
||||
const deploy = await readJson("deploy/deploy.json");
|
||||
const master = await readJson("deploy/master-edge/health-contract.json");
|
||||
const frpsUsesVhost6667 = /vhostHTTPPort\s*=\s*6667/u.test(frps);
|
||||
|
||||
return {
|
||||
devEndpoint: DEV_ENDPOINT,
|
||||
frp: {
|
||||
frpcTargetsEdgeProxy:
|
||||
/localIP\s*=\s*"hwlab-edge-proxy\.hwlab-dev\.svc\.cluster\.local"/u.test(frpc) &&
|
||||
/localPort\s*=\s*6667/u.test(frpc) &&
|
||||
/remotePort\s*=\s*6667/u.test(frpc),
|
||||
frpsTcpRemotePort6667: /start\s*=\s*6667[\s\S]*end\s*=\s*6667/u.test(frps),
|
||||
frpsUsesVhost6667,
|
||||
configRisk: frpsUsesVhost6667 ? "port_conflict" : "none"
|
||||
},
|
||||
masterEdge: {
|
||||
endpoint: master.endpoint,
|
||||
reverseLinkMode: master.reverseLink?.mode,
|
||||
reverseLinkClient: master.reverseLink?.client,
|
||||
publicPort: master.reverseLink?.publicPort
|
||||
},
|
||||
deploy: {
|
||||
environment: deploy.environment,
|
||||
namespace: deploy.namespace,
|
||||
serviceCount: deploy.services?.length ?? 0,
|
||||
edgeProxy: deploy.services?.find((service) => service.serviceId === "hwlab-edge-proxy") ?? null,
|
||||
tunnelClient: deploy.services?.find((service) => service.serviceId === "hwlab-tunnel-client") ?? null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
async function inspectKubernetes() {
|
||||
const hasKubectl = await commandExists("kubectl");
|
||||
const result = {
|
||||
observable: hasKubectl,
|
||||
namespace,
|
||||
context: null,
|
||||
services: [],
|
||||
endpoints: [],
|
||||
pods: [],
|
||||
notes: []
|
||||
};
|
||||
if (!hasKubectl) {
|
||||
result.notes.push("kubectl is not installed in this runner");
|
||||
return result;
|
||||
}
|
||||
|
||||
await collectKubectlContext(result);
|
||||
await collectKubectlJson(result, "services", [
|
||||
"-n", namespace, "get", "svc", "hwlab-cloud-api", "hwlab-edge-proxy", "hwlab-tunnel-client", "hwlab-router", "-o", "json"
|
||||
]);
|
||||
await collectKubectlJson(result, "endpoints", [
|
||||
"-n", namespace, "get", "endpoints", "hwlab-cloud-api", "hwlab-edge-proxy", "hwlab-tunnel-client", "hwlab-router", "-o", "json"
|
||||
]);
|
||||
await collectKubectlJson(result, "pods", [
|
||||
"-n", namespace, "get", "pods", "-l", "hwlab.pikastech.local/profile=dev", "-o", "json"
|
||||
]);
|
||||
return result;
|
||||
}
|
||||
|
||||
async function collectKubectlContext(result) {
|
||||
const context = await runCommand("kubectl", ["config", "current-context"], { timeoutMs: 5000 });
|
||||
result.context = {
|
||||
exitCode: context.exitCode,
|
||||
value: context.exitCode === 0 ? context.stdout.trim() : null,
|
||||
stderr: context.stderr.trim()
|
||||
};
|
||||
}
|
||||
|
||||
async function collectKubectlJson(result, kind, args) {
|
||||
const probe = await runCommand("kubectl", args, { timeoutMs: 10000 });
|
||||
if (probe.exitCode !== 0) {
|
||||
result.notes.push(`kubectl ${kind} probe failed: ${probe.stderr.trim()}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const doc = JSON.parse(probe.stdout);
|
||||
if (kind === "services") {
|
||||
result.services = doc.items.map((item) => ({
|
||||
name: item.metadata.name,
|
||||
type: item.spec.type,
|
||||
clusterIP: item.spec.clusterIP,
|
||||
ports: item.spec.ports?.map(({ name, port, targetPort }) => ({ name, port, targetPort })) ?? [],
|
||||
selector: item.spec.selector ?? {}
|
||||
}));
|
||||
} else if (kind === "endpoints") {
|
||||
result.endpoints = doc.items.map((item) => ({
|
||||
name: item.metadata.name,
|
||||
readyAddresses: (item.subsets ?? []).flatMap((subset) => subset.addresses ?? []).map((address) => address.ip),
|
||||
notReadyAddresses: (item.subsets ?? []).flatMap((subset) => subset.notReadyAddresses ?? []).map((address) => address.ip),
|
||||
ports: (item.subsets ?? []).flatMap((subset) => subset.ports ?? []).map(({ name, port }) => ({ name, port }))
|
||||
}));
|
||||
} else if (kind === "pods") {
|
||||
result.pods = doc.items.map((item) => ({
|
||||
name: item.metadata.name,
|
||||
phase: item.status.phase,
|
||||
serviceId: item.metadata.labels?.["hwlab.pikastech.local/service-id"] ?? null,
|
||||
containers: item.status.containerStatuses?.map(({ name, ready, restartCount, image, imageID }) => ({
|
||||
name, ready, restartCount, image, imageID
|
||||
})) ?? []
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
async function inspectClusterDns() {
|
||||
const results = [];
|
||||
for (const service of runtimeServices) {
|
||||
const host = `${service.serviceId}.${namespace}.svc.cluster.local`;
|
||||
const dns = await runCommand("getent", ["hosts", host], { timeoutMs: 3000 });
|
||||
results.push({
|
||||
serviceId: service.serviceId,
|
||||
host,
|
||||
port: service.port,
|
||||
resolved: dns.exitCode === 0,
|
||||
output: dns.stdout.trim()
|
||||
});
|
||||
}
|
||||
return results;
|
||||
}
|
||||
|
||||
function classify(report) {
|
||||
const port6667 = report.publicTcp.find((probe) => probe.port === publicPort);
|
||||
const frpsControl = report.publicTcp.find((probe) => probe.port === 7000);
|
||||
const tunnelHealth = report.publicTcp.find((probe) => probe.port === 7402);
|
||||
const publicHealth = report.publicHttp.find((probe) => probe.url.endsWith("/health"));
|
||||
|
||||
if (publicHealth?.ok && publicHealth.json?.serviceId && publicHealth.json?.environment === ENVIRONMENT_DEV) {
|
||||
return classifyReachableHealth(publicHealth.json);
|
||||
}
|
||||
if (port6667?.status === "error" && port6667.code === "ECONNREFUSED") {
|
||||
return classifyRefusedPort(frpsControl, tunnelHealth);
|
||||
}
|
||||
if (port6667?.status === "timeout") {
|
||||
return blocker("dns_port_firewall_blocker", "public 6667 timed out before any HTTP response");
|
||||
}
|
||||
if (publicHealth && !publicHealth.ok && publicHealth.status) {
|
||||
return blocker("app_health_blocker", `public /health returned HTTP ${publicHealth.status}`);
|
||||
}
|
||||
if (report.kubernetes.observable && report.kubernetes.services.length === 0) {
|
||||
return blocker("k3s_service_blocker", "kubectl is available but HWLAB DEV services were not observed");
|
||||
}
|
||||
return blocker("dns_port_firewall_blocker", publicHealth?.error?.message ?? "public DEV health did not return a usable response");
|
||||
}
|
||||
|
||||
function classifyReachableHealth(json) {
|
||||
const missingEvidence = [];
|
||||
if (!json.commit) missingEvidence.push("commit");
|
||||
if (!json.image) missingEvidence.push("image");
|
||||
if (!json.service && !json.serviceId) missingEvidence.push("service");
|
||||
if (missingEvidence.length > 0) {
|
||||
return blocker("app_health_blocker", `public /health is reachable but missing ${missingEvidence.join(", ")} evidence`);
|
||||
}
|
||||
return { status: "pass", classification: "none", blocker: null };
|
||||
}
|
||||
|
||||
function classifyRefusedPort(frpsControl, tunnelHealth) {
|
||||
if (frpsControl?.status === "error" && frpsControl.code === "ECONNREFUSED") {
|
||||
return blocker("frp_blocker", "public 6667 and frps control 7000 both refuse TCP connections; frps is not reachable on the master edge");
|
||||
}
|
||||
if (tunnelHealth?.status === "error" && tunnelHealth.code === "ECONNREFUSED") {
|
||||
return blocker("frp_blocker", "public 6667 refuses TCP and tunnel health 7402 is not reachable");
|
||||
}
|
||||
return blocker("edge_proxy_blocker", "public 6667 refuses TCP connections");
|
||||
}
|
||||
|
||||
function blocker(classification, message) {
|
||||
return {
|
||||
status: "blocker",
|
||||
classification,
|
||||
blocker: message
|
||||
};
|
||||
}
|
||||
|
||||
async function commandExists(command) {
|
||||
const result = await runCommand("command", ["-v", command], { timeoutMs: 3000 });
|
||||
return result.exitCode === 0;
|
||||
}
|
||||
|
||||
function runCommand(command, args, { timeoutMs = 10000 } = {}) {
|
||||
return new Promise((resolve) => {
|
||||
const startedAt = Date.now();
|
||||
const child = spawn(command, args, {
|
||||
cwd: repoRoot,
|
||||
shell: command === "command",
|
||||
stdio: ["ignore", "pipe", "pipe"]
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
let settled = false;
|
||||
const timeout = setTimeout(() => {
|
||||
child.kill("SIGTERM");
|
||||
finish(124, `${stderr}\ncommand timed out after ${timeoutMs}ms`.trim());
|
||||
}, timeoutMs);
|
||||
|
||||
function finish(exitCode, finalStderr = stderr) {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
clearTimeout(timeout);
|
||||
resolve({
|
||||
command: [command, ...args].join(" "),
|
||||
exitCode,
|
||||
stdout,
|
||||
stderr: finalStderr,
|
||||
durationMs: Date.now() - startedAt
|
||||
});
|
||||
}
|
||||
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk.toString();
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk.toString();
|
||||
});
|
||||
child.on("error", (error) => finish(127, error.message));
|
||||
child.on("close", (exitCode) => finish(exitCode));
|
||||
});
|
||||
}
|
||||
|
||||
function probeTcp({ host, port, timeoutMs = 5000 }) {
|
||||
return new Promise((resolve) => {
|
||||
const startedAt = Date.now();
|
||||
const socket = net.connect({ host, port, timeout: timeoutMs });
|
||||
socket.on("connect", () => {
|
||||
resolve({ host, port, status: "connected", durationMs: Date.now() - startedAt });
|
||||
socket.destroy();
|
||||
});
|
||||
socket.on("timeout", () => {
|
||||
resolve({ host, port, status: "timeout", durationMs: Date.now() - startedAt });
|
||||
socket.destroy();
|
||||
});
|
||||
socket.on("error", (error) => {
|
||||
resolve({ host, port, status: "error", code: error.code, message: error.message, durationMs: Date.now() - startedAt });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function fetchJson(url, { timeoutMs = 8000 } = {}) {
|
||||
const startedAt = Date.now();
|
||||
const controller = new AbortController();
|
||||
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
||||
try {
|
||||
const response = await fetch(url, { signal: controller.signal });
|
||||
const text = await response.text();
|
||||
return {
|
||||
url,
|
||||
ok: response.ok,
|
||||
status: response.status,
|
||||
statusText: response.statusText,
|
||||
headers: Object.fromEntries(response.headers.entries()),
|
||||
json: parseJsonOrNull(text),
|
||||
bodyPreview: text.slice(0, 500),
|
||||
durationMs: Date.now() - startedAt
|
||||
};
|
||||
} catch (error) {
|
||||
return {
|
||||
url,
|
||||
ok: false,
|
||||
error: { name: error.name, message: error.message, code: error.code },
|
||||
durationMs: Date.now() - startedAt
|
||||
};
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
}
|
||||
|
||||
function parseJsonOrNull(text) {
|
||||
try {
|
||||
return text ? JSON.parse(text) : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function readJson(relativePath) {
|
||||
return JSON.parse(await readText(relativePath));
|
||||
}
|
||||
|
||||
async function readText(relativePath) {
|
||||
return readFile(path.join(repoRoot, relativePath), "utf8");
|
||||
}
|
||||
|
||||
async function maybeWriteReport(report, args) {
|
||||
if (!args.writeReport) {
|
||||
return;
|
||||
}
|
||||
await mkdir(path.dirname(args.reportPath), { recursive: true });
|
||||
await writeFile(args.reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
Reference in New Issue
Block a user