Codex Agent
1208 lines
44 KiB
JavaScript
1208 lines
44 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { spawn } from "node:child_process";
|
|
import net from "node:net";
|
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import {
|
|
DEV_DB_ENV_CONTRACT,
|
|
summarizeDbContract
|
|
} from "../../internal/cloud/db-contract.ts";
|
|
import {
|
|
DEV_CODE_AGENT_PROVIDER_CONTRACT,
|
|
codeAgentSecretRefPlaceholder
|
|
} from "../../internal/cloud/code-agent-contract.ts";
|
|
import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs";
|
|
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
|
import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs";
|
|
import { readStructuredFile } from "./structured-config.mjs";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
const defaultReportPath = tempReportPath("dev-edge-health.json");
|
|
const publicHost = "74.48.78.17";
|
|
const publicPort = 16667;
|
|
const namespace = "hwlab-dev";
|
|
const d601KubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
|
|
const tcpPorts = [16667, 7000, 7402];
|
|
const CODE_AGENT_PROVIDER_SECRET_CONTRACT = Object.freeze({
|
|
sourceIssue: "pikasTech/HWLAB#143",
|
|
env: DEV_CODE_AGENT_PROVIDER_CONTRACT.secretRefs[0].env,
|
|
secretName: DEV_CODE_AGENT_PROVIDER_CONTRACT.secretRefs[0].secretName,
|
|
secretKey: DEV_CODE_AGENT_PROVIDER_CONTRACT.secretRefs[0].secretKey
|
|
});
|
|
const runtimeServices = [
|
|
{ serviceId: "hwlab-cloud-api", port: 6667 },
|
|
{ serviceId: "hwlab-router", port: 7401 },
|
|
{ serviceId: "hwlab-tunnel-client", port: 7402 },
|
|
{ serviceId: "hwlab-edge-proxy", port: 6667 }
|
|
];
|
|
|
|
function nativeKubectlArgs(args) {
|
|
return [`KUBECONFIG=${d601KubeconfigPath}`, "kubectl", ...args];
|
|
}
|
|
|
|
function nativeKubectlCommand(args) {
|
|
return nativeKubectlArgs(args).join(" ");
|
|
}
|
|
|
|
function runKubectl(args, options) {
|
|
return runCommand("env", nativeKubectlArgs(args), options);
|
|
}
|
|
|
|
export async function runDevEdgeHealthSmoke(argv) {
|
|
const args = parseArgs(argv);
|
|
requireDevEndpoint();
|
|
|
|
if (!args.live) {
|
|
const edgeHealth = {
|
|
generatedAt: nowIso(),
|
|
mode: "contract-only",
|
|
endpoint: DEV_ENDPOINT,
|
|
status: "not_run",
|
|
classification: "not_run",
|
|
blocker: "live network probes require --live",
|
|
contracts: await inspectContracts(),
|
|
runtimeDbReadiness: notRunDbReadiness("live network probes require --live")
|
|
};
|
|
const report = await createDevGateReport(edgeHealth);
|
|
await maybeWriteReport(report, args);
|
|
return { report, exitCode: 0 };
|
|
}
|
|
|
|
const [contracts, publicTcp, publicHttp, kubernetes, clusterDns] = await Promise.all([
|
|
inspectContracts(),
|
|
Promise.all(tcpPorts.map((port) => probeTcp({ host: publicHost, port }))),
|
|
Promise.all([fetchJson(`${DEV_ENDPOINT}/health`), fetchJson(`${DEV_ENDPOINT}/health/live`)]),
|
|
inspectKubernetes(),
|
|
inspectClusterDns()
|
|
]);
|
|
|
|
const edgeHealth = {
|
|
generatedAt: nowIso(),
|
|
mode: "live-read-only",
|
|
endpoint: DEV_ENDPOINT,
|
|
safety: {
|
|
environment: ENVIRONMENT_DEV,
|
|
prodTouched: false,
|
|
secretsRead: false,
|
|
restarts: false,
|
|
unideskRuntimeSubstitute: false
|
|
},
|
|
contracts,
|
|
publicTcp,
|
|
publicHttp,
|
|
kubernetes,
|
|
clusterDns
|
|
};
|
|
edgeHealth.runtimeDbReadiness = inspectRuntimeDbReadiness(edgeHealth);
|
|
Object.assign(edgeHealth, classify(edgeHealth));
|
|
edgeHealth.providerSecretReadiness = inspectProviderSecretReadiness(edgeHealth);
|
|
edgeHealth.parallelBlockers = classifyParallelBlockers(edgeHealth);
|
|
if (edgeHealth.status === "pass" && edgeHealth.parallelBlockers.length > 0) {
|
|
Object.assign(edgeHealth, classifyParallelBlockerStatus(edgeHealth.parallelBlockers));
|
|
}
|
|
edgeHealth.milestoneImpact = classifyMilestoneImpact(edgeHealth);
|
|
const report = await createDevGateReport(edgeHealth);
|
|
await maybeWriteReport(report, args);
|
|
return { report, exitCode: edgeHealth.status === "pass" ? 0 : 2 };
|
|
}
|
|
|
|
export const __testHooks = Object.freeze({
|
|
classifyMilestoneImpact,
|
|
classifyParallelBlockers,
|
|
inspectProviderSecretReadiness,
|
|
summarizeRuntimeDbReadiness
|
|
});
|
|
|
|
function parseArgs(argv) {
|
|
const args = {
|
|
live: false,
|
|
writeReport: false,
|
|
reportPath: defaultReportPath
|
|
};
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const arg = argv[index];
|
|
if (arg === "--live") {
|
|
args.live = true;
|
|
} else if (arg === "--report") {
|
|
index += 1;
|
|
assert.ok(argv[index], "--report requires a path");
|
|
args.reportPath = argv[index];
|
|
args.writeReport = true;
|
|
} else {
|
|
throw new Error(`unknown argument ${arg}`);
|
|
}
|
|
}
|
|
|
|
return args;
|
|
}
|
|
|
|
function nowIso() {
|
|
return new Date().toISOString();
|
|
}
|
|
|
|
async function createDevGateReport(edgeHealth) {
|
|
const commitId = await gitCommitId();
|
|
const status = edgeHealth.status === "pass" ? "pass" : edgeHealth.status === "not_run" ? "not_run" : "blocked";
|
|
const blockers = reportBlockers(edgeHealth, status);
|
|
|
|
return {
|
|
$schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json",
|
|
$id: "https://hwlab.pikastech.local/dev-gate/dev-edge-health.json",
|
|
reportVersion: "v1",
|
|
issue: "pikasTech/HWLAB#36",
|
|
taskId: "dev-edge-health",
|
|
commitId,
|
|
acceptanceLevel: "dev_edge_health",
|
|
devOnly: true,
|
|
prodDisabled: true,
|
|
reportLifecycle: activeReportLifecycle("Current DEV edge-health report; deprecated public 6666/6667 endpoints are not valid active evidence."),
|
|
status,
|
|
generatedAt: edgeHealth.generatedAt,
|
|
namespace,
|
|
endpoint: DEV_ENDPOINT,
|
|
sourceContract: {
|
|
status: "pass",
|
|
documents: [
|
|
"docs/reference/dev-runtime-boundary.md",
|
|
"docs/reference/spec-v02-documentation-governance.md",
|
|
"docs/reference/node-gitops-cicd.md",
|
|
"deploy/frp/README.md",
|
|
"deploy/master-edge/README.md"
|
|
],
|
|
summary: "DEV edge health is pinned to the frozen HWLAB endpoint, frp reverse link, master edge proxy, and hwlab-dev services."
|
|
},
|
|
validationCommands: [
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs",
|
|
"node --check scripts/dev-edge-health-smoke.mjs",
|
|
"node --check scripts/src/dev-edge-health-smoke-lib.mjs",
|
|
"node --test scripts/src/dev-edge-health-smoke-lib.test.mjs",
|
|
`node scripts/dev-edge-health-smoke.mjs --live --report ${defaultReportPath}`
|
|
],
|
|
localSmoke: {
|
|
status: "pass",
|
|
commands: [],
|
|
evidence: [
|
|
"npm run check includes internal/cloud/server-health.test.ts and validates /health plus /health/live."
|
|
],
|
|
summary: "Local cloud-api health checks pass and include service, commit, and image evidence fields."
|
|
},
|
|
dryRun: {
|
|
status: "not_run",
|
|
commands: [
|
|
"node scripts/run-bun.mjs tools/hwlab-cli/bin/hwlab-cli.ts client workbench summary --base-url http://74.48.78.17:19666"
|
|
],
|
|
evidence: [
|
|
"This edge-health task used the dedicated read-only smoke instead of the MVP e2e dry-run."
|
|
],
|
|
summary: "MVP dry-run remains the generic gate command; edge health uses a narrower route smoke."
|
|
},
|
|
devPreconditions: {
|
|
status,
|
|
requirements: [
|
|
"http://74.48.78.17:16667/health returns HWLAB DEV JSON with service, commit, and image evidence",
|
|
"frps control on 74.48.78.17:7000 accepts the D601 frp client",
|
|
"hwlab-dev hwlab-edge-proxy, hwlab-tunnel-client, hwlab-router, and hwlab-cloud-api are observable",
|
|
"No PROD endpoint, secret read, restart, or UniDesk runtime substitute is used"
|
|
],
|
|
summary: edgeHealth.blocker ?? "DEV edge health preconditions are satisfied."
|
|
},
|
|
blockers,
|
|
edgeHealth
|
|
};
|
|
}
|
|
|
|
async function gitCommitId() {
|
|
const result = await runCommand("git", ["rev-parse", "--short=12", "HEAD"], { timeoutMs: 5000 });
|
|
return result.exitCode === 0 ? result.stdout.trim() : "unknown";
|
|
}
|
|
|
|
function blockerTypeFor(classification) {
|
|
if (classification === "app_health_blocker") {
|
|
return "runtime_blocker";
|
|
}
|
|
if (classification === "k3s_service_blocker") {
|
|
return "runtime_blocker";
|
|
}
|
|
if (classification === "not_run") {
|
|
return "observability_blocker";
|
|
}
|
|
return "network_blocker";
|
|
}
|
|
|
|
function reportBlockers(edgeHealth, status) {
|
|
if (status !== "blocked") {
|
|
return [];
|
|
}
|
|
|
|
const parallelBlockers = Array.isArray(edgeHealth.parallelBlockers) ? edgeHealth.parallelBlockers : [];
|
|
if (parallelBlockers.length > 0) {
|
|
return parallelBlockers.map(({ type, scope, status: blockerStatus, summary, classification, sourceIssue, impact }) => ({
|
|
type,
|
|
scope,
|
|
status: blockerStatus,
|
|
summary,
|
|
classification,
|
|
sourceIssue,
|
|
impact
|
|
}));
|
|
}
|
|
|
|
return [
|
|
{
|
|
type: blockerTypeFor(edgeHealth.classification),
|
|
scope: edgeHealth.diagnosis?.likelyLayer ?? edgeHealth.classification,
|
|
status: "open",
|
|
summary: edgeHealth.blocker
|
|
}
|
|
];
|
|
}
|
|
|
|
function requireDevEndpoint() {
|
|
const endpoint = new URL(DEV_ENDPOINT);
|
|
assert.equal(endpoint.protocol, "http:", "DEV endpoint must use http");
|
|
assert.equal(endpoint.hostname, publicHost, "DEV endpoint host");
|
|
assert.equal(Number(endpoint.port), publicPort, "DEV endpoint port");
|
|
}
|
|
|
|
async function inspectContracts() {
|
|
const frpc = await readText("deploy/frp/frpc.dev.toml");
|
|
const frps = await readText("deploy/frp/frps.dev.toml");
|
|
const deploy = await readJson("deploy/deploy.yaml");
|
|
const master = await readJson("deploy/master-edge/health-contract.json");
|
|
const frpsUsesVhostEdgePort = /vhostHTTPPort\s*=\s*(16667|6667)/u.test(frps);
|
|
|
|
return {
|
|
devEndpoint: DEV_ENDPOINT,
|
|
frp: {
|
|
frpcTargetsEdgeProxy:
|
|
/localIP\s*=\s*"hwlab-edge-proxy\.hwlab-dev\.svc\.cluster\.local"/u.test(frpc) &&
|
|
/localPort\s*=\s*6667/u.test(frpc) &&
|
|
/remotePort\s*=\s*16667/u.test(frpc),
|
|
frpcTargetsCloudWeb:
|
|
/localIP\s*=\s*"hwlab-cloud-web\.hwlab-dev\.svc\.cluster\.local"/u.test(frpc) &&
|
|
/localPort\s*=\s*8080/u.test(frpc) &&
|
|
/remotePort\s*=\s*16666/u.test(frpc),
|
|
frpsTcpRemotePort16667: /start\s*=\s*16667[\s\S]*end\s*=\s*16667/u.test(frps),
|
|
frpsTcpRemotePort16666: /start\s*=\s*16666[\s\S]*end\s*=\s*16666/u.test(frps),
|
|
frpsUsesVhostEdgePort,
|
|
configRisk: frpsUsesVhostEdgePort ? "port_conflict" : "none"
|
|
},
|
|
masterEdge: {
|
|
endpoint: master.endpoint,
|
|
reverseLinkMode: master.reverseLink?.mode,
|
|
reverseLinkClient: master.reverseLink?.client,
|
|
publicPort: master.reverseLink?.publicPort
|
|
},
|
|
deploy: {
|
|
environment: deploy.environment,
|
|
namespace: deploy.namespace,
|
|
serviceCount: deploy.services?.length ?? 0,
|
|
edgeProxy: deploy.services?.find((service) => service.serviceId === "hwlab-edge-proxy") ?? null,
|
|
tunnelClient: deploy.services?.find((service) => service.serviceId === "hwlab-tunnel-client") ?? null,
|
|
cloudApiDb: inspectCloudApiDbContract(deploy),
|
|
codeAgentProvider: inspectCodeAgentProviderContract(deploy)
|
|
}
|
|
};
|
|
}
|
|
|
|
function inspectCloudApiDbContract(deploy) {
|
|
const env = deploy.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {};
|
|
const configReady = Boolean(env.HWLAB_CLOUD_DB_URL && env.HWLAB_CLOUD_DB_SSL_MODE);
|
|
const connection = {
|
|
attempted: false,
|
|
networkAttempted: false,
|
|
result: "manifest_only_not_attempted",
|
|
classification: "manifest_only",
|
|
endpointSource: DEV_DB_ENV_CONTRACT.endpointAuthority.source,
|
|
probeType: "tcp-connect",
|
|
endpointRedacted: true,
|
|
valueRedacted: true,
|
|
timeoutMs: null,
|
|
durationMs: 0,
|
|
errorCode: null
|
|
};
|
|
const db = {
|
|
status: configReady ? "degraded" : "blocked",
|
|
ready: false,
|
|
configReady,
|
|
connected: false,
|
|
liveConnected: false,
|
|
connectionChecked: false,
|
|
connectionAttempted: false,
|
|
connectionResult: connection.result,
|
|
endpointSource: DEV_DB_ENV_CONTRACT.endpointAuthority.source,
|
|
endpoint: {
|
|
authoritative: {
|
|
source: DEV_DB_ENV_CONTRACT.endpointAuthority.source,
|
|
env: DEV_DB_ENV_CONTRACT.endpointAuthority.env,
|
|
requiredForReadiness: true,
|
|
usedForProbe: true,
|
|
endpointRedacted: true,
|
|
valueRedacted: true
|
|
}
|
|
},
|
|
optionalPublicDnsAlias: {
|
|
source: DEV_DB_ENV_CONTRACT.dns.source,
|
|
serviceName: DEV_DB_ENV_CONTRACT.dns.serviceName,
|
|
namespace: DEV_DB_ENV_CONTRACT.dns.namespace,
|
|
port: DEV_DB_ENV_CONTRACT.dns.port,
|
|
portName: DEV_DB_ENV_CONTRACT.dns.portName,
|
|
requiredForReadiness: false,
|
|
usedForProbe: false,
|
|
envPresent: false,
|
|
readyGate: "not_readiness_authority"
|
|
},
|
|
fields: [
|
|
{
|
|
name: "HWLAB_CLOUD_DB_URL",
|
|
present: Boolean(env.HWLAB_CLOUD_DB_URL),
|
|
redacted: true,
|
|
source: "k8s-secret-ref"
|
|
},
|
|
{
|
|
name: "HWLAB_CLOUD_DB_SSL_MODE",
|
|
present: Boolean(env.HWLAB_CLOUD_DB_SSL_MODE),
|
|
redacted: false,
|
|
source: "runtime-env"
|
|
}
|
|
],
|
|
missingEnv: ["HWLAB_CLOUD_DB_URL", "HWLAB_CLOUD_DB_SSL_MODE"].filter((name) => !env[name]),
|
|
secretRefs: [
|
|
{
|
|
env: "HWLAB_CLOUD_DB_URL",
|
|
secretName: "hwlab-cloud-api-dev-db",
|
|
secretKey: "database-url",
|
|
present: env.HWLAB_CLOUD_DB_URL === "secretRef:hwlab-cloud-api-dev-db/database-url",
|
|
envInjected: false,
|
|
secretPresent: "not_observed_by_manifest",
|
|
secretKeyPresent: "declared",
|
|
redacted: true
|
|
}
|
|
],
|
|
connection,
|
|
safety: {
|
|
secretsRead: false,
|
|
valuesRedacted: true,
|
|
liveDbEvidence: false
|
|
},
|
|
readinessLayers: Object.fromEntries(DEV_DB_ENV_CONTRACT.readinessLayers.map((layer) => [layer, { status: "not_proven" }])),
|
|
blocker: configReady
|
|
? "Manifest declares DB env/Secret references but does not prove runtime env injection or live DB connection"
|
|
: "Manifest is missing DB env/Secret references"
|
|
};
|
|
return {
|
|
...summarizeDbContract(db),
|
|
manifestConfigReady: configReady,
|
|
envInjected: false,
|
|
secretMaterialRead: false,
|
|
valuesRedacted: true,
|
|
note: "Manifest-only DB env placeholder; this is not runtime env injection or live DB evidence."
|
|
};
|
|
}
|
|
|
|
function inspectCodeAgentProviderContract(deploy) {
|
|
const env = deploy.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {};
|
|
const expectedRef = codeAgentSecretRefPlaceholder();
|
|
const secretRefPresent = env[CODE_AGENT_PROVIDER_SECRET_CONTRACT.env] === expectedRef;
|
|
const egressBaseUrl = env.HWLAB_CODE_AGENT_OPENAI_BASE_URL;
|
|
const egressReady = egressBaseUrl === DEV_CODE_AGENT_PROVIDER_CONTRACT.egress.defaultBaseUrl;
|
|
const directPublicOpenAi = egressBaseUrl === DEV_CODE_AGENT_PROVIDER_CONTRACT.egress.forbiddenDirectBaseUrl;
|
|
return {
|
|
status: secretRefPresent && egressReady ? "degraded" : "blocked",
|
|
sourceIssue: CODE_AGENT_PROVIDER_SECRET_CONTRACT.sourceIssue,
|
|
provider: DEV_CODE_AGENT_PROVIDER_CONTRACT.runtimeProvider,
|
|
requiredEnv: [
|
|
{
|
|
name: CODE_AGENT_PROVIDER_SECRET_CONTRACT.env,
|
|
present: Boolean(env[CODE_AGENT_PROVIDER_SECRET_CONTRACT.env]),
|
|
redacted: true,
|
|
source: "k8s-secret-ref"
|
|
},
|
|
{
|
|
name: DEV_CODE_AGENT_PROVIDER_CONTRACT.egress.env,
|
|
present: Boolean(egressBaseUrl),
|
|
redacted: false,
|
|
source: "runtime-env"
|
|
}
|
|
],
|
|
secretRef: {
|
|
env: CODE_AGENT_PROVIDER_SECRET_CONTRACT.env,
|
|
secretName: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretName,
|
|
secretKey: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretKey,
|
|
present: secretRefPresent,
|
|
envInjected: false,
|
|
secretPresent: "not_observed_by_manifest",
|
|
secretKeyPresent: secretRefPresent ? "declared" : "not_observed_by_manifest",
|
|
secretValueRead: false,
|
|
redacted: true
|
|
},
|
|
egress: {
|
|
env: DEV_CODE_AGENT_PROVIDER_CONTRACT.egress.env,
|
|
target: DEV_CODE_AGENT_PROVIDER_CONTRACT.egress.target,
|
|
present: Boolean(egressBaseUrl),
|
|
ready: egressReady,
|
|
directPublicOpenAi,
|
|
valueRedacted: true
|
|
},
|
|
secretMaterialRead: false,
|
|
valuesRedacted: true,
|
|
note: "Manifest-only Code Agent provider Secret reference and DEV egress/base-url contract; this is not runtime Secret injection or provider connectivity evidence."
|
|
};
|
|
}
|
|
|
|
function notRunDbReadiness(blockerMessage) {
|
|
return {
|
|
status: "not_run",
|
|
configReady: false,
|
|
envInjected: false,
|
|
secret: unknownDbSecretPresence("not_observed"),
|
|
connectionAttempted: false,
|
|
connectionResult: "not_run",
|
|
connectionClassification: "not_run",
|
|
liveConnected: false,
|
|
valuesRedacted: true,
|
|
secretMaterialRead: false,
|
|
liveDbEvidence: false,
|
|
blocker: blockerMessage,
|
|
evidence: []
|
|
};
|
|
}
|
|
|
|
function inspectRuntimeDbReadiness(edgeHealth) {
|
|
const db = selectRuntimeDb(edgeHealth.publicHttp);
|
|
return summarizeRuntimeDbReadiness(db, edgeHealth.kubernetes?.dbSecret ?? unknownDbSecretPresence("not_observed"));
|
|
}
|
|
|
|
function selectRuntimeDb(publicHttp) {
|
|
const live = publicHttp.find((probe) => probe.url.endsWith("/health/live") && probe.ok && probe.json?.db);
|
|
const health = publicHttp.find((probe) => probe.url.endsWith("/health") && probe.ok && probe.json?.db);
|
|
return live?.json?.db ?? health?.json?.db ?? null;
|
|
}
|
|
|
|
function summarizeRuntimeDbReadiness(db, secretPresence = unknownDbSecretPresence("not_observed")) {
|
|
if (!db) {
|
|
return {
|
|
status: "blocked",
|
|
configReady: false,
|
|
envInjected: false,
|
|
secret: secretPresence,
|
|
connectionAttempted: false,
|
|
connectionResult: "health_db_missing",
|
|
connectionClassification: "health_db_missing",
|
|
liveConnected: false,
|
|
valuesRedacted: true,
|
|
secretMaterialRead: false,
|
|
liveDbEvidence: false,
|
|
blocker: "cloud-api health is reachable but does not expose redacted DB readiness fields",
|
|
evidence: []
|
|
};
|
|
}
|
|
|
|
const requiredEnv = Array.isArray(db.fields) ? db.fields : [];
|
|
const envInjected = DEV_DB_ENV_CONTRACT.requiredEnv.every((name) =>
|
|
requiredEnv.some((field) => field.name === name && field.present === true)
|
|
);
|
|
const connectionAttempted = Boolean(db.connectionAttempted ?? db.connectionChecked);
|
|
const connectionResult = db.connectionResult ?? db.connection?.result ?? (connectionAttempted ? "unknown" : "not_attempted");
|
|
const connectionClassification =
|
|
db.connection?.classification ?? classificationForRuntimeDb(db, connectionAttempted, connectionResult);
|
|
const endpointSource =
|
|
db.endpointSource ?? db.connection?.endpointSource ?? db.endpoint?.authoritative?.source ?? DEV_DB_ENV_CONTRACT.endpointAuthority.source;
|
|
const optionalPublicDnsAlias = db.optionalPublicDnsAlias ?? db.endpoint?.optionalPublicDnsAlias ?? {
|
|
source: DEV_DB_ENV_CONTRACT.dns.source,
|
|
serviceName: DEV_DB_ENV_CONTRACT.dns.serviceName,
|
|
namespace: DEV_DB_ENV_CONTRACT.dns.namespace,
|
|
port: DEV_DB_ENV_CONTRACT.dns.port,
|
|
portName: DEV_DB_ENV_CONTRACT.dns.portName,
|
|
requiredForReadiness: false,
|
|
usedForProbe: false,
|
|
envPresent: false,
|
|
readyGate: "not_readiness_authority"
|
|
};
|
|
const liveConnected = Boolean(db.liveConnected ?? db.connected ?? db.ready);
|
|
const authorityReady = endpointSource === DEV_DB_ENV_CONTRACT.endpointAuthority.source;
|
|
const blockerMessage = liveConnected && authorityReady
|
|
? null
|
|
: db.blocker ?? (authorityReady ? blockerForRuntimeDb(db, connectionAttempted, connectionResult) : "DB readiness did not use the Secret URL host authority");
|
|
const status = liveConnected && authorityReady ? "pass" : "blocked";
|
|
const secretRef = Array.isArray(db.secretRefs) ? db.secretRefs[0] : null;
|
|
|
|
return {
|
|
status,
|
|
configReady: Boolean(db.configReady),
|
|
envInjected,
|
|
secret: {
|
|
...secretPresence,
|
|
secretName: secretRef?.secretName ?? DEV_DB_ENV_CONTRACT.secretRefs[0].secretName,
|
|
secretKey: secretRef?.secretKey ?? DEV_DB_ENV_CONTRACT.secretRefs[0].secretKey,
|
|
secretRefPresent: Boolean(secretRef?.present),
|
|
envInjected: Boolean(secretRef?.envInjected ?? secretRef?.present),
|
|
redacted: true
|
|
},
|
|
connectionAttempted,
|
|
connectionResult,
|
|
connectionClassification,
|
|
endpointSource,
|
|
optionalPublicDnsAlias: {
|
|
source: optionalPublicDnsAlias.source ?? DEV_DB_ENV_CONTRACT.dns.source,
|
|
serviceName: optionalPublicDnsAlias.serviceName ?? DEV_DB_ENV_CONTRACT.dns.serviceName,
|
|
namespace: optionalPublicDnsAlias.namespace ?? DEV_DB_ENV_CONTRACT.dns.namespace,
|
|
port: optionalPublicDnsAlias.port ?? DEV_DB_ENV_CONTRACT.dns.port,
|
|
portName: optionalPublicDnsAlias.portName ?? DEV_DB_ENV_CONTRACT.dns.portName,
|
|
requiredForReadiness: optionalPublicDnsAlias.requiredForReadiness === true,
|
|
usedForProbe: optionalPublicDnsAlias.usedForProbe === true,
|
|
envPresent: optionalPublicDnsAlias.envPresent === true,
|
|
readyGate: optionalPublicDnsAlias.readyGate ?? "not_readiness_authority"
|
|
},
|
|
liveConnected,
|
|
valuesRedacted: db.redaction?.valuesRedacted !== false && db.safety?.valuesRedacted === true,
|
|
secretMaterialRead: false,
|
|
liveDbEvidence: Boolean(db.liveDbEvidence ?? db.safety?.liveDbEvidence),
|
|
blocker: blockerMessage,
|
|
evidence: [
|
|
{
|
|
configReady: Boolean(db.configReady),
|
|
missingEnv: Array.isArray(db.missingEnv) ? db.missingEnv : [],
|
|
connectionAttempted,
|
|
connectionResult,
|
|
classification: connectionClassification,
|
|
endpointSource,
|
|
liveConnected,
|
|
endpointRedacted: db.connection?.endpointRedacted !== false,
|
|
valueRedacted: true
|
|
}
|
|
]
|
|
};
|
|
}
|
|
|
|
function classificationForRuntimeDb(db, connectionAttempted, connectionResult) {
|
|
if (typeof db?.connectionClassification === "string") {
|
|
return db.connectionClassification;
|
|
}
|
|
if (typeof db?.classification === "string") {
|
|
return db.classification;
|
|
}
|
|
if (db?.configReady !== true) {
|
|
return "missing_runtime_env";
|
|
}
|
|
if (!connectionAttempted) {
|
|
return "configured_no_live_attempt";
|
|
}
|
|
if (connectionResult === "dns_error") {
|
|
return "dns_resolution_failed";
|
|
}
|
|
if (connectionResult === "refused") {
|
|
return "tcp_refused";
|
|
}
|
|
if (connectionResult === "timeout") {
|
|
return "tcp_timeout";
|
|
}
|
|
if (connectionResult === "network_unreachable") {
|
|
return "network_unreachable";
|
|
}
|
|
return connectionResult === "connected" ? "tcp_connected" : "tcp_error";
|
|
}
|
|
|
|
function blockerForRuntimeDb(db, connectionAttempted, connectionResult) {
|
|
if (db.configReady !== true) {
|
|
const missing = Array.isArray(db.missingEnv) && db.missingEnv.length > 0 ? db.missingEnv.join(", ") : "required DB env";
|
|
return `cloud-api DB runtime env is not ready; missing ${missing}`;
|
|
}
|
|
if (!connectionAttempted) {
|
|
return "cloud-api DB env is injected, but runtime health has not attempted a live DB connection";
|
|
}
|
|
return `cloud-api DB live connection is blocked; connectionResult=${connectionResult}`;
|
|
}
|
|
|
|
function unknownDbSecretPresence(state) {
|
|
const ref = DEV_DB_ENV_CONTRACT.secretRefs[0];
|
|
return {
|
|
observable: false,
|
|
secretName: ref.secretName,
|
|
secretKey: ref.secretKey,
|
|
secretPresent: state,
|
|
secretKeyPresent: state,
|
|
secretValueRead: false,
|
|
redacted: true
|
|
};
|
|
}
|
|
|
|
function unknownProviderSecretPresence(state) {
|
|
return {
|
|
observable: false,
|
|
secretName: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretName,
|
|
secretKey: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretKey,
|
|
secretPresent: state,
|
|
secretKeyPresent: state,
|
|
secretValueRead: false,
|
|
redacted: true
|
|
};
|
|
}
|
|
|
|
async function inspectKubernetes() {
|
|
const hasKubectl = await commandExists("kubectl");
|
|
const result = {
|
|
observable: hasKubectl,
|
|
namespace,
|
|
kubeconfig: d601KubeconfigPath,
|
|
context: null,
|
|
nodeNames: [],
|
|
services: [],
|
|
endpoints: [],
|
|
pods: [],
|
|
dbSecret: unknownDbSecretPresence("not_observed"),
|
|
codeAgentProviderSecret: unknownProviderSecretPresence("not_observed"),
|
|
notes: []
|
|
};
|
|
if (!hasKubectl) {
|
|
result.notes.push("kubectl is not installed in this runner");
|
|
return result;
|
|
}
|
|
|
|
await collectKubectlContext(result);
|
|
await collectKubectlNodeIdentity(result);
|
|
await collectKubectlJson(result, "services", [
|
|
"-n", namespace, "get", "svc", "hwlab-cloud-api", "hwlab-edge-proxy", "hwlab-tunnel-client", "hwlab-router", "-o", "json"
|
|
]);
|
|
await collectKubectlJson(result, "endpoints", [
|
|
"-n", namespace, "get", "endpoints", "hwlab-cloud-api", "hwlab-edge-proxy", "hwlab-tunnel-client", "hwlab-router", "-o", "json"
|
|
]);
|
|
await collectKubectlJson(result, "pods", [
|
|
"-n", namespace, "get", "pods", "-l", "hwlab.pikastech.local/profile=dev", "-o", "json"
|
|
]);
|
|
await collectDbSecretPresence(result);
|
|
await collectCodeAgentProviderSecretPresence(result);
|
|
return result;
|
|
}
|
|
|
|
async function collectKubectlContext(result) {
|
|
const context = await runKubectl(["config", "current-context"], { timeoutMs: 5000 });
|
|
result.context = {
|
|
exitCode: context.exitCode,
|
|
value: context.exitCode === 0 ? context.stdout.trim() : null,
|
|
command: nativeKubectlCommand(["config", "current-context"]),
|
|
stderr: context.stderr.trim()
|
|
};
|
|
}
|
|
|
|
async function collectKubectlNodeIdentity(result) {
|
|
const args = ["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"];
|
|
const nodes = await runKubectl(args, { timeoutMs: 5000 });
|
|
if (nodes.exitCode !== 0) {
|
|
result.notes.push(`kubectl node identity probe failed: ${nodes.stderr.trim()}`);
|
|
return;
|
|
}
|
|
result.nodeNames = nodes.stdout.trim().split(/\s+/u).filter(Boolean);
|
|
if (!result.nodeNames.includes("d601")) {
|
|
result.notes.push(`kubectl node identity did not include d601: ${result.nodeNames.join(",") || "none"}`);
|
|
}
|
|
}
|
|
|
|
async function collectKubectlJson(result, kind, args) {
|
|
const probe = await runKubectl(args, { timeoutMs: 10000 });
|
|
if (probe.exitCode !== 0) {
|
|
result.notes.push(`kubectl ${kind} probe failed: ${probe.stderr.trim()}`);
|
|
return;
|
|
}
|
|
|
|
const doc = JSON.parse(probe.stdout);
|
|
if (kind === "services") {
|
|
result.services = doc.items.map((item) => ({
|
|
name: item.metadata.name,
|
|
type: item.spec.type,
|
|
clusterIP: item.spec.clusterIP,
|
|
ports: item.spec.ports?.map(({ name, port, targetPort }) => ({ name, port, targetPort })) ?? [],
|
|
selector: item.spec.selector ?? {}
|
|
}));
|
|
} else if (kind === "endpoints") {
|
|
result.endpoints = doc.items.map((item) => ({
|
|
name: item.metadata.name,
|
|
readyAddresses: (item.subsets ?? []).flatMap((subset) => subset.addresses ?? []).map((address) => address.ip),
|
|
notReadyAddresses: (item.subsets ?? []).flatMap((subset) => subset.notReadyAddresses ?? []).map((address) => address.ip),
|
|
ports: (item.subsets ?? []).flatMap((subset) => subset.ports ?? []).map(({ name, port }) => ({ name, port }))
|
|
}));
|
|
} else if (kind === "pods") {
|
|
result.pods = doc.items.map((item) => ({
|
|
name: item.metadata.name,
|
|
phase: item.status.phase,
|
|
serviceId: item.metadata.labels?.["hwlab.pikastech.local/service-id"] ?? null,
|
|
containers: item.status.containerStatuses?.map(({ name, ready, restartCount, image, imageID }) => ({
|
|
name, ready, restartCount, image, imageID
|
|
})) ?? []
|
|
}));
|
|
}
|
|
}
|
|
|
|
async function collectDbSecretPresence(result) {
|
|
const ref = DEV_DB_ENV_CONTRACT.secretRefs[0];
|
|
const base = {
|
|
observable: true,
|
|
secretName: ref.secretName,
|
|
secretKey: ref.secretKey,
|
|
secretPresent: false,
|
|
secretKeyPresent: false,
|
|
secretValueRead: false,
|
|
redacted: true
|
|
};
|
|
const exists = await runKubectl(["-n", namespace, "get", "secret", ref.secretName, "-o", "name"], {
|
|
timeoutMs: 5000
|
|
});
|
|
if (exists.exitCode !== 0) {
|
|
result.dbSecret = {
|
|
...base,
|
|
command: nativeKubectlCommand(["-n", namespace, "get", "secret", ref.secretName, "-o", "name"]),
|
|
error: exists.stderr.trim() || "secret not observed"
|
|
};
|
|
result.notes.push(`kubectl DB Secret presence probe failed: ${exists.stderr.trim()}`);
|
|
return;
|
|
}
|
|
|
|
const keyPresence = await runKubectl(["-n", namespace, "describe", "secret", ref.secretName], {
|
|
timeoutMs: 5000
|
|
});
|
|
const keyLinePattern = new RegExp(`^\\s*${escapeRegExp(ref.secretKey)}:\\s+\\d+\\s+bytes\\s*$`, "mu");
|
|
result.dbSecret = {
|
|
...base,
|
|
secretPresent: true,
|
|
secretKeyPresent: keyPresence.exitCode === 0 && keyLinePattern.test(keyPresence.stdout),
|
|
command: `${nativeKubectlCommand(["-n", namespace, "describe", "secret", ref.secretName])} (key-presence-only)`,
|
|
error: keyPresence.exitCode === 0 ? null : keyPresence.stderr.trim()
|
|
};
|
|
if (keyPresence.exitCode !== 0) {
|
|
result.notes.push(`kubectl DB Secret key presence probe failed: ${keyPresence.stderr.trim()}`);
|
|
}
|
|
}
|
|
|
|
async function collectCodeAgentProviderSecretPresence(result) {
|
|
const ref = CODE_AGENT_PROVIDER_SECRET_CONTRACT;
|
|
const base = {
|
|
observable: true,
|
|
sourceIssue: ref.sourceIssue,
|
|
env: ref.env,
|
|
secretName: ref.secretName,
|
|
secretKey: ref.secretKey,
|
|
secretPresent: false,
|
|
secretKeyPresent: false,
|
|
secretValueRead: false,
|
|
redacted: true
|
|
};
|
|
const exists = await runKubectl(["-n", namespace, "get", "secret", ref.secretName, "-o", "name"], {
|
|
timeoutMs: 5000
|
|
});
|
|
if (exists.exitCode !== 0) {
|
|
result.codeAgentProviderSecret = {
|
|
...base,
|
|
command: nativeKubectlCommand(["-n", namespace, "get", "secret", ref.secretName, "-o", "name"]),
|
|
error: exists.stderr.trim() || "secret not observed"
|
|
};
|
|
result.notes.push(`kubectl Code Agent provider Secret presence probe failed: ${exists.stderr.trim()}`);
|
|
return;
|
|
}
|
|
|
|
const keyPresence = await runKubectl(["-n", namespace, "describe", "secret", ref.secretName], {
|
|
timeoutMs: 5000
|
|
});
|
|
const keyLinePattern = new RegExp(`^\\s*${escapeRegExp(ref.secretKey)}:\\s+\\d+\\s+bytes\\s*$`, "mu");
|
|
result.codeAgentProviderSecret = {
|
|
...base,
|
|
secretPresent: true,
|
|
secretKeyPresent: keyPresence.exitCode === 0 && keyLinePattern.test(keyPresence.stdout),
|
|
command: `${nativeKubectlCommand(["-n", namespace, "describe", "secret", ref.secretName])} (key-presence-only)`,
|
|
error: keyPresence.exitCode === 0 ? null : keyPresence.stderr.trim()
|
|
};
|
|
if (keyPresence.exitCode !== 0) {
|
|
result.notes.push(`kubectl Code Agent provider Secret key presence probe failed: ${keyPresence.stderr.trim()}`);
|
|
}
|
|
}
|
|
|
|
async function inspectClusterDns() {
|
|
const results = [];
|
|
for (const service of runtimeServices) {
|
|
const host = `${service.serviceId}.${namespace}.svc.cluster.local`;
|
|
const dns = await runCommand("getent", ["hosts", host], { timeoutMs: 3000 });
|
|
results.push({
|
|
serviceId: service.serviceId,
|
|
host,
|
|
port: service.port,
|
|
resolved: dns.exitCode === 0,
|
|
output: dns.stdout.trim()
|
|
});
|
|
}
|
|
return results;
|
|
}
|
|
|
|
function classify(report) {
|
|
const publicEdgePort = report.publicTcp.find((probe) => probe.port === publicPort);
|
|
const frpsControl = report.publicTcp.find((probe) => probe.port === 7000);
|
|
const tunnelHealth = report.publicTcp.find((probe) => probe.port === 7402);
|
|
const publicHealth = report.publicHttp.find((probe) => probe.url.endsWith("/health"));
|
|
|
|
if (publicHealth?.ok && publicHealth.json?.serviceId && publicHealth.json?.environment === ENVIRONMENT_DEV) {
|
|
return classifyReachableHealth(publicHealth.json, selectRuntimeDb(report.publicHttp));
|
|
}
|
|
if (publicEdgePort?.status === "error" && publicEdgePort.code === "ECONNREFUSED") {
|
|
return classifyRefusedPort(publicEdgePort, frpsControl, tunnelHealth);
|
|
}
|
|
if (publicEdgePort?.status === "timeout") {
|
|
return blocker("dns_port_firewall_blocker", "public 16667 timed out before any HTTP response");
|
|
}
|
|
if (publicHealth && !publicHealth.ok && publicHealth.status) {
|
|
return blocker("app_health_blocker", `public /health returned HTTP ${publicHealth.status}`);
|
|
}
|
|
if (report.kubernetes.observable && report.kubernetes.services.length === 0) {
|
|
return blocker("k3s_service_blocker", "kubectl is available but HWLAB DEV services were not observed");
|
|
}
|
|
return blocker("dns_port_firewall_blocker", publicHealth?.error?.message ?? "public DEV health did not return a usable response");
|
|
}
|
|
|
|
function classifyReachableHealth(json, runtimeDb = null) {
|
|
const missingEvidence = [];
|
|
if (!json.commit) missingEvidence.push("commit");
|
|
if (!json.image) missingEvidence.push("image");
|
|
if (!json.service && !json.serviceId) missingEvidence.push("service");
|
|
if (missingEvidence.length > 0) {
|
|
return blocker("app_health_blocker", `public /health is reachable but missing ${missingEvidence.join(", ")} evidence`);
|
|
}
|
|
const dbReadiness = summarizeRuntimeDbReadiness(runtimeDb ?? json.db);
|
|
if (dbReadiness.status !== "pass") {
|
|
return blocker("app_health_blocker", dbReadiness.blocker, {
|
|
likelyLayer: "cloud-api-db",
|
|
confidence: dbReadiness.configReady ? "high" : "medium",
|
|
likelyCause: dbReadiness.blocker,
|
|
evidence: dbReadiness.evidence,
|
|
notProven: dbReadiness.connectionAttempted
|
|
? ["authenticated SQL query readiness"]
|
|
: ["live DB endpoint reachability", "authenticated SQL query readiness"],
|
|
nextTask: "Deploy cloud-api DB runtime readiness probe and/or repair DEV DB connectivity, then rerun the read-only health smoke without reading or printing the DB secret value."
|
|
});
|
|
}
|
|
return { status: "pass", classification: "none", blocker: null };
|
|
}
|
|
|
|
function inspectProviderSecretReadiness(edgeHealth) {
|
|
const secret = edgeHealth.kubernetes?.codeAgentProviderSecret ?? unknownProviderSecretPresence("not_observed");
|
|
const manifest = edgeHealth.contracts?.deploy?.codeAgentProvider ?? null;
|
|
const manifestRefPresent = manifest?.secretRef?.present === true;
|
|
const secretPresent = secret.secretPresent === true;
|
|
const secretKeyPresent = secret.secretKeyPresent === true;
|
|
const status = secret.observable && (!secretPresent || !secretKeyPresent) ? "blocked" : "not_proven";
|
|
const classification = status === "blocked" ? "provider_secret_missing" : "provider_secret_presence_not_proven";
|
|
return {
|
|
status,
|
|
sourceIssue: CODE_AGENT_PROVIDER_SECRET_CONTRACT.sourceIssue,
|
|
provider: "openai-responses",
|
|
requiredEnv: CODE_AGENT_PROVIDER_SECRET_CONTRACT.env,
|
|
manifestRefPresent,
|
|
secret: {
|
|
...secret,
|
|
secretValueRead: false,
|
|
redacted: true
|
|
},
|
|
providerCallAttempted: false,
|
|
providerConnected: false,
|
|
secretMaterialRead: false,
|
|
valuesRedacted: true,
|
|
classification,
|
|
blocker: status === "blocked"
|
|
? `Code Agent provider Secret ${CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretName}/${CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretKey} is not present as key-presence evidence; #143 real provider-backed chat remains blocked`
|
|
: "Code Agent provider Secret presence was not proven by this edge-health run",
|
|
evidence: [
|
|
{
|
|
source: secret.observable ? "kubernetes-secret-presence" : "runtime-not-observable",
|
|
manifestRefPresent,
|
|
secretPresent,
|
|
secretKeyPresent,
|
|
secretValueRead: false,
|
|
redacted: true
|
|
}
|
|
]
|
|
};
|
|
}
|
|
|
|
function classifyParallelBlockers(edgeHealth) {
|
|
const blockers = [];
|
|
const dbReadiness = edgeHealth.runtimeDbReadiness;
|
|
if (dbReadiness?.status === "blocked") {
|
|
blockers.push({
|
|
id: "cloud-api-db-live",
|
|
type: "runtime_blocker",
|
|
scope: "cloud-api-db",
|
|
status: "open",
|
|
sourceIssue: "pikasTech/HWLAB#49",
|
|
classification: dbReadiness.connectionClassification ?? dbReadiness.connectionResult,
|
|
summary: dbReadiness.blocker,
|
|
impact: {
|
|
blocks: ["DB live readiness", "M4 live agent scheduling precondition", "M5 DEV-LIVE acceptance"],
|
|
separateFrom: ["pikasTech/HWLAB#143 provider Secret", "M3 hardware trusted-loop proof"]
|
|
}
|
|
});
|
|
}
|
|
|
|
const provider = edgeHealth.providerSecretReadiness;
|
|
if (provider?.status === "blocked") {
|
|
blockers.push({
|
|
id: "code-agent-provider-secret",
|
|
type: "agent_blocker",
|
|
scope: "code-agent-provider-secret",
|
|
status: "open",
|
|
sourceIssue: CODE_AGENT_PROVIDER_SECRET_CONTRACT.sourceIssue,
|
|
classification: provider.classification,
|
|
summary: provider.blocker,
|
|
impact: {
|
|
blocks: ["#143 provider-backed Code Agent chat", "M4 provider-backed agent execution", "M5 DEV-LIVE acceptance"],
|
|
separateFrom: ["pikasTech/HWLAB#49 DB live readiness", "M3 hardware trusted-loop proof"]
|
|
}
|
|
});
|
|
}
|
|
return blockers;
|
|
}
|
|
|
|
function classifyParallelBlockerStatus(parallelBlockers) {
|
|
return blocker(
|
|
"parallel_runtime_blocker",
|
|
parallelBlockers.map((item) => item.summary).join("; "),
|
|
{
|
|
likelyLayer: "parallel-dev-blockers",
|
|
confidence: "high",
|
|
likelyCause: "one or more independent DEV blockers remain open after route health passed",
|
|
evidence: parallelBlockers.map(({ id, scope, classification, sourceIssue }) => ({
|
|
id,
|
|
scope,
|
|
classification,
|
|
sourceIssue
|
|
})),
|
|
notProven: ["DB-backed runtime readiness", "provider-backed Code Agent readiness", "M3/M4/M5 DEV-LIVE acceptance"],
|
|
nextTask: "Close each independent blocker with redacted evidence, then rerun the read-only edge smoke."
|
|
}
|
|
);
|
|
}
|
|
|
|
function classifyMilestoneImpact(edgeHealth) {
|
|
const dbBlocked = edgeHealth.runtimeDbReadiness?.status === "blocked";
|
|
const providerBlocked = edgeHealth.providerSecretReadiness?.status === "blocked";
|
|
return {
|
|
M3: {
|
|
status: "separate_blocker",
|
|
summary: "M3 still requires real DEV hardware trusted-loop operation/trace/audit/evidence; DB Secret URL host readiness failure and #143 provider Secret absence do not prove or clear M3."
|
|
},
|
|
M4: {
|
|
status: dbBlocked || providerBlocked ? "blocked" : "not_proven",
|
|
blockedBy: [
|
|
...(dbBlocked ? ["cloud-api-db-live"] : []),
|
|
...(providerBlocked ? ["code-agent-provider-secret"] : [])
|
|
],
|
|
summary: "M4 live agent-loop remains blocked until DB live readiness and provider-backed agent prerequisites are independently green."
|
|
},
|
|
M5: {
|
|
status: dbBlocked || providerBlocked ? "blocked" : "not_proven",
|
|
blockedBy: [
|
|
...(dbBlocked ? ["cloud-api-db-live"] : []),
|
|
...(providerBlocked ? ["code-agent-provider-secret"] : [])
|
|
],
|
|
summary: "M5 DEV-LIVE acceptance cannot be promoted from route health, env presence, or provider manifest refs while DB live and #143 provider Secret blockers remain open."
|
|
}
|
|
};
|
|
}
|
|
|
|
function classifyRefusedPort(publicEdgePort, frpsControl, tunnelHealth) {
|
|
if (frpsControl?.status === "error" && frpsControl.code === "ECONNREFUSED") {
|
|
const allPortsRefused = tunnelHealth?.status === "error" && tunnelHealth.code === "ECONNREFUSED";
|
|
const diagnosis = {
|
|
likelyLayer: "master-edge/frps",
|
|
confidence: "high",
|
|
likelyCause: allPortsRefused
|
|
? "public 16667, frps control 7000, and tunnel health 7402 all refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed"
|
|
: "public 16667 and frps control 7000 both refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed",
|
|
evidence: [
|
|
summarizeTcpProbe(publicEdgePort),
|
|
summarizeTcpProbe(frpsControl),
|
|
summarizeTcpProbe(tunnelHealth)
|
|
],
|
|
notProven: [
|
|
"k3s service readiness",
|
|
"frpc connectivity",
|
|
"edge proxy deployment"
|
|
],
|
|
nextTask: "Restore or reopen frps on 74.48.78.17, then rerun the read-only edge smoke."
|
|
};
|
|
return blocker("frp_blocker", diagnosis.likelyCause, diagnosis);
|
|
}
|
|
if (tunnelHealth?.status === "error" && tunnelHealth.code === "ECONNREFUSED") {
|
|
return blocker("frp_blocker", "public 16667 refuses TCP and tunnel health 7402 is not reachable", {
|
|
likelyLayer: "d601/frpc",
|
|
confidence: "medium",
|
|
likelyCause: "public 16667 and tunnel health 7402 both refuse TCP connections; the D601 frp client or its remote port is the likely break",
|
|
evidence: [
|
|
summarizeTcpProbe(publicEdgePort),
|
|
summarizeTcpProbe(tunnelHealth)
|
|
],
|
|
notProven: [
|
|
"master-edge frps reachability",
|
|
"k3s service readiness"
|
|
],
|
|
nextTask: "Restore the D601 tunnel client or its remote port, then rerun the read-only edge smoke."
|
|
});
|
|
}
|
|
return blocker("edge_proxy_blocker", "public 16667 refuses TCP connections", {
|
|
likelyLayer: "master-edge/edge-proxy",
|
|
confidence: "medium",
|
|
likelyCause: "public 16667 refuses TCP while frps control 7000 stays reachable, so the public edge proxy or its listener is the likely break",
|
|
evidence: [
|
|
summarizeTcpProbe(publicEdgePort),
|
|
summarizeTcpProbe(frpsControl)
|
|
],
|
|
notProven: [
|
|
"k3s service readiness"
|
|
],
|
|
nextTask: "Repair the edge proxy or its public listener, then rerun the read-only edge smoke."
|
|
});
|
|
}
|
|
|
|
function blocker(classification, message, diagnosis = null) {
|
|
return {
|
|
status: "blocker",
|
|
classification,
|
|
blocker: message,
|
|
...(diagnosis ? { diagnosis } : {})
|
|
};
|
|
}
|
|
|
|
function summarizeTcpProbe(probe) {
|
|
if (!probe) {
|
|
return null;
|
|
}
|
|
|
|
return {
|
|
host: probe.host,
|
|
port: probe.port,
|
|
status: probe.status,
|
|
code: probe.code ?? null,
|
|
message: probe.message ?? null
|
|
};
|
|
}
|
|
|
|
function escapeRegExp(value) {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
|
}
|
|
|
|
async function commandExists(command) {
|
|
const result = await runCommand("command", ["-v", command], { timeoutMs: 3000 });
|
|
return result.exitCode === 0;
|
|
}
|
|
|
|
function runCommand(command, args, { timeoutMs = 10000 } = {}) {
|
|
return new Promise((resolve) => {
|
|
const startedAt = Date.now();
|
|
const child = spawn(command, args, {
|
|
cwd: repoRoot,
|
|
shell: command === "command",
|
|
stdio: ["ignore", "pipe", "pipe"]
|
|
});
|
|
let stdout = "";
|
|
let stderr = "";
|
|
let settled = false;
|
|
const timeout = setTimeout(() => {
|
|
child.kill("SIGTERM");
|
|
finish(124, `${stderr}\ncommand timed out after ${timeoutMs}ms`.trim());
|
|
}, timeoutMs);
|
|
|
|
function finish(exitCode, finalStderr = stderr) {
|
|
if (settled) return;
|
|
settled = true;
|
|
clearTimeout(timeout);
|
|
resolve({
|
|
command: [command, ...args].join(" "),
|
|
exitCode,
|
|
stdout,
|
|
stderr: finalStderr,
|
|
durationMs: Date.now() - startedAt
|
|
});
|
|
}
|
|
|
|
child.stdout.on("data", (chunk) => {
|
|
stdout += chunk.toString();
|
|
});
|
|
child.stderr.on("data", (chunk) => {
|
|
stderr += chunk.toString();
|
|
});
|
|
child.on("error", (error) => finish(127, error.message));
|
|
child.on("close", (exitCode) => finish(exitCode));
|
|
});
|
|
}
|
|
|
|
function probeTcp({ host, port, timeoutMs = 5000 }) {
|
|
return new Promise((resolve) => {
|
|
const startedAt = Date.now();
|
|
const socket = net.connect({ host, port, timeout: timeoutMs });
|
|
socket.on("connect", () => {
|
|
resolve({ host, port, status: "connected", durationMs: Date.now() - startedAt });
|
|
socket.destroy();
|
|
});
|
|
socket.on("timeout", () => {
|
|
resolve({ host, port, status: "timeout", durationMs: Date.now() - startedAt });
|
|
socket.destroy();
|
|
});
|
|
socket.on("error", (error) => {
|
|
resolve({ host, port, status: "error", code: error.code, message: error.message, durationMs: Date.now() - startedAt });
|
|
});
|
|
});
|
|
}
|
|
|
|
async function fetchJson(url, { timeoutMs = 8000 } = {}) {
|
|
const startedAt = Date.now();
|
|
const controller = new AbortController();
|
|
const timeout = setTimeout(() => controller.abort(), timeoutMs);
|
|
try {
|
|
const response = await fetch(url, { signal: controller.signal });
|
|
const text = await response.text();
|
|
return {
|
|
url,
|
|
ok: response.ok,
|
|
status: response.status,
|
|
statusText: response.statusText,
|
|
headers: Object.fromEntries(response.headers.entries()),
|
|
json: parseJsonOrNull(text),
|
|
bodyPreview: text.slice(0, 500),
|
|
durationMs: Date.now() - startedAt
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
url,
|
|
ok: false,
|
|
error: { name: error.name, message: error.message, code: error.code },
|
|
durationMs: Date.now() - startedAt
|
|
};
|
|
} finally {
|
|
clearTimeout(timeout);
|
|
}
|
|
}
|
|
|
|
function parseJsonOrNull(text) {
|
|
try {
|
|
return text ? JSON.parse(text) : null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function readJson(relativePath) {
|
|
return readStructuredFile(repoRoot, relativePath);
|
|
}
|
|
|
|
async function readText(relativePath) {
|
|
return readFile(path.join(repoRoot, relativePath), "utf8");
|
|
}
|
|
|
|
async function maybeWriteReport(report, args) {
|
|
if (!args.writeReport) {
|
|
return;
|
|
}
|
|
const absoluteReportPath = ensureNotRepoReportsPath(repoRoot, args.reportPath, "--report");
|
|
await mkdir(path.dirname(absoluteReportPath), { recursive: true });
|
|
await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
}
|