import assert from "node:assert/strict"; import { spawn } from "node:child_process"; import net from "node:net"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { DEV_DB_ENV_CONTRACT, summarizeDbContract } from "../../internal/cloud/db-contract.ts"; import { DEV_CODE_AGENT_PROVIDER_CONTRACT, codeAgentSecretRefPlaceholder } from "../../internal/cloud/code-agent-contract.ts"; import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs"; import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs"; import { readStructuredFile } from "./structured-config.mjs"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const defaultReportPath = tempReportPath("dev-edge-health.json"); const publicHost = "74.48.78.17"; const publicPort = 16667; const namespace = "hwlab-dev"; const d601KubeconfigPath = "/etc/rancher/k3s/k3s.yaml"; const tcpPorts = [16667, 7000, 7402]; const CODE_AGENT_PROVIDER_SECRET_CONTRACT = Object.freeze({ sourceIssue: "pikasTech/HWLAB#143", env: DEV_CODE_AGENT_PROVIDER_CONTRACT.secretRefs[0].env, secretName: DEV_CODE_AGENT_PROVIDER_CONTRACT.secretRefs[0].secretName, secretKey: DEV_CODE_AGENT_PROVIDER_CONTRACT.secretRefs[0].secretKey }); const runtimeServices = [ { serviceId: "hwlab-cloud-api", port: 6667 }, { serviceId: "hwlab-router", port: 7401 }, { serviceId: "hwlab-tunnel-client", port: 7402 }, { serviceId: "hwlab-edge-proxy", port: 6667 } ]; function nativeKubectlArgs(args) { return [`KUBECONFIG=${d601KubeconfigPath}`, "kubectl", ...args]; } function nativeKubectlCommand(args) { return nativeKubectlArgs(args).join(" "); } function runKubectl(args, options) { return runCommand("env", nativeKubectlArgs(args), options); } export async function runDevEdgeHealthSmoke(argv) { const args = parseArgs(argv); requireDevEndpoint(); if (!args.live) { const edgeHealth = { generatedAt: nowIso(), mode: "contract-only", endpoint: DEV_ENDPOINT, status: "not_run", classification: "not_run", blocker: "live network probes require --live", contracts: await inspectContracts(), runtimeDbReadiness: notRunDbReadiness("live network probes require --live") }; const report = await createDevGateReport(edgeHealth); await maybeWriteReport(report, args); return { report, exitCode: 0 }; } const [contracts, publicTcp, publicHttp, kubernetes, clusterDns] = await Promise.all([ inspectContracts(), Promise.all(tcpPorts.map((port) => probeTcp({ host: publicHost, port }))), Promise.all([fetchJson(`${DEV_ENDPOINT}/health`), fetchJson(`${DEV_ENDPOINT}/health/live`)]), inspectKubernetes(), inspectClusterDns() ]); const edgeHealth = { generatedAt: nowIso(), mode: "live-read-only", endpoint: DEV_ENDPOINT, safety: { environment: ENVIRONMENT_DEV, prodTouched: false, secretsRead: false, restarts: false, unideskRuntimeSubstitute: false }, contracts, publicTcp, publicHttp, kubernetes, clusterDns }; edgeHealth.runtimeDbReadiness = inspectRuntimeDbReadiness(edgeHealth); Object.assign(edgeHealth, classify(edgeHealth)); edgeHealth.providerSecretReadiness = inspectProviderSecretReadiness(edgeHealth); edgeHealth.parallelBlockers = classifyParallelBlockers(edgeHealth); if (edgeHealth.status === "pass" && edgeHealth.parallelBlockers.length > 0) { Object.assign(edgeHealth, classifyParallelBlockerStatus(edgeHealth.parallelBlockers)); } edgeHealth.milestoneImpact = classifyMilestoneImpact(edgeHealth); const report = await createDevGateReport(edgeHealth); await maybeWriteReport(report, args); return { report, exitCode: edgeHealth.status === "pass" ? 0 : 2 }; } export const __testHooks = Object.freeze({ classifyMilestoneImpact, classifyParallelBlockers, inspectProviderSecretReadiness, summarizeRuntimeDbReadiness }); function parseArgs(argv) { const args = { live: false, writeReport: false, reportPath: defaultReportPath }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === "--live") { args.live = true; } else if (arg === "--report") { index += 1; assert.ok(argv[index], "--report requires a path"); args.reportPath = argv[index]; args.writeReport = true; } else { throw new Error(`unknown argument ${arg}`); } } return args; } function nowIso() { return new Date().toISOString(); } async function createDevGateReport(edgeHealth) { const commitId = await gitCommitId(); const status = edgeHealth.status === "pass" ? "pass" : edgeHealth.status === "not_run" ? "not_run" : "blocked"; const blockers = reportBlockers(edgeHealth, status); return { $schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json", $id: "https://hwlab.pikastech.local/dev-gate/dev-edge-health.json", reportVersion: "v1", issue: "pikasTech/HWLAB#36", taskId: "dev-edge-health", commitId, acceptanceLevel: "dev_edge_health", devOnly: true, prodDisabled: true, reportLifecycle: activeReportLifecycle("Current DEV edge-health report; deprecated public 6666/6667 endpoints are not valid active evidence."), status, generatedAt: edgeHealth.generatedAt, namespace, endpoint: DEV_ENDPOINT, sourceContract: { status: "pass", documents: [ "docs/reference/dev-runtime-boundary.md", "docs/reference/spec-v02-documentation-governance.md", "docs/reference/node-gitops-cicd.md", "deploy/frp/README.md", "deploy/master-edge/README.md" ], summary: "DEV edge health is pinned to the frozen HWLAB endpoint, frp reverse link, master edge proxy, and hwlab-dev services." }, validationCommands: [ "node --check scripts/validate-dev-gate-report.mjs", "node scripts/validate-dev-gate-report.mjs", "node --check scripts/dev-edge-health-smoke.mjs", "node --check scripts/src/dev-edge-health-smoke-lib.mjs", "node --test scripts/src/dev-edge-health-smoke-lib.test.mjs", `node scripts/dev-edge-health-smoke.mjs --live --report ${defaultReportPath}` ], localSmoke: { status: "pass", commands: [], evidence: [ "npm run check includes internal/cloud/server-health.test.ts and validates /health plus /health/live." ], summary: "Local cloud-api health checks pass and include service, commit, and image evidence fields." }, dryRun: { status: "not_run", commands: [ "node scripts/run-bun.mjs tools/hwlab-cli/bin/hwlab-cli.ts client workbench summary --base-url http://74.48.78.17:19666" ], evidence: [ "This edge-health task used the dedicated read-only smoke instead of the MVP e2e dry-run." ], summary: "MVP dry-run remains the generic gate command; edge health uses a narrower route smoke." }, devPreconditions: { status, requirements: [ "http://74.48.78.17:16667/health returns HWLAB DEV JSON with service, commit, and image evidence", "frps control on 74.48.78.17:7000 accepts the D601 frp client", "hwlab-dev hwlab-edge-proxy, hwlab-tunnel-client, hwlab-router, and hwlab-cloud-api are observable", "No PROD endpoint, secret read, restart, or UniDesk runtime substitute is used" ], summary: edgeHealth.blocker ?? "DEV edge health preconditions are satisfied." }, blockers, edgeHealth }; } async function gitCommitId() { const result = await runCommand("git", ["rev-parse", "--short=12", "HEAD"], { timeoutMs: 5000 }); return result.exitCode === 0 ? result.stdout.trim() : "unknown"; } function blockerTypeFor(classification) { if (classification === "app_health_blocker") { return "runtime_blocker"; } if (classification === "k3s_service_blocker") { return "runtime_blocker"; } if (classification === "not_run") { return "observability_blocker"; } return "network_blocker"; } function reportBlockers(edgeHealth, status) { if (status !== "blocked") { return []; } const parallelBlockers = Array.isArray(edgeHealth.parallelBlockers) ? edgeHealth.parallelBlockers : []; if (parallelBlockers.length > 0) { return parallelBlockers.map(({ type, scope, status: blockerStatus, summary, classification, sourceIssue, impact }) => ({ type, scope, status: blockerStatus, summary, classification, sourceIssue, impact })); } return [ { type: blockerTypeFor(edgeHealth.classification), scope: edgeHealth.diagnosis?.likelyLayer ?? edgeHealth.classification, status: "open", summary: edgeHealth.blocker } ]; } function requireDevEndpoint() { const endpoint = new URL(DEV_ENDPOINT); assert.equal(endpoint.protocol, "http:", "DEV endpoint must use http"); assert.equal(endpoint.hostname, publicHost, "DEV endpoint host"); assert.equal(Number(endpoint.port), publicPort, "DEV endpoint port"); } async function inspectContracts() { const frpc = await readText("deploy/frp/frpc.dev.toml"); const frps = await readText("deploy/frp/frps.dev.toml"); const deploy = await readJson("deploy/deploy.yaml"); const master = await readJson("deploy/master-edge/health-contract.json"); const frpsUsesVhostEdgePort = /vhostHTTPPort\s*=\s*(16667|6667)/u.test(frps); return { devEndpoint: DEV_ENDPOINT, frp: { frpcTargetsEdgeProxy: /localIP\s*=\s*"hwlab-edge-proxy\.hwlab-dev\.svc\.cluster\.local"/u.test(frpc) && /localPort\s*=\s*6667/u.test(frpc) && /remotePort\s*=\s*16667/u.test(frpc), frpcTargetsCloudWeb: /localIP\s*=\s*"hwlab-cloud-web\.hwlab-dev\.svc\.cluster\.local"/u.test(frpc) && /localPort\s*=\s*8080/u.test(frpc) && /remotePort\s*=\s*16666/u.test(frpc), frpsTcpRemotePort16667: /start\s*=\s*16667[\s\S]*end\s*=\s*16667/u.test(frps), frpsTcpRemotePort16666: /start\s*=\s*16666[\s\S]*end\s*=\s*16666/u.test(frps), frpsUsesVhostEdgePort, configRisk: frpsUsesVhostEdgePort ? "port_conflict" : "none" }, masterEdge: { endpoint: master.endpoint, reverseLinkMode: master.reverseLink?.mode, reverseLinkClient: master.reverseLink?.client, publicPort: master.reverseLink?.publicPort }, deploy: { environment: deploy.environment, namespace: deploy.namespace, serviceCount: deploy.services?.length ?? 0, edgeProxy: deploy.services?.find((service) => service.serviceId === "hwlab-edge-proxy") ?? null, tunnelClient: deploy.services?.find((service) => service.serviceId === "hwlab-tunnel-client") ?? null, cloudApiDb: inspectCloudApiDbContract(deploy), codeAgentProvider: inspectCodeAgentProviderContract(deploy) } }; } function inspectCloudApiDbContract(deploy) { const env = deploy.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {}; const configReady = Boolean(env.HWLAB_CLOUD_DB_URL && env.HWLAB_CLOUD_DB_SSL_MODE); const connection = { attempted: false, networkAttempted: false, result: "manifest_only_not_attempted", classification: "manifest_only", endpointSource: DEV_DB_ENV_CONTRACT.endpointAuthority.source, probeType: "tcp-connect", endpointRedacted: true, valueRedacted: true, timeoutMs: null, durationMs: 0, errorCode: null }; const db = { status: configReady ? "degraded" : "blocked", ready: false, configReady, connected: false, liveConnected: false, connectionChecked: false, connectionAttempted: false, connectionResult: connection.result, endpointSource: DEV_DB_ENV_CONTRACT.endpointAuthority.source, endpoint: { authoritative: { source: DEV_DB_ENV_CONTRACT.endpointAuthority.source, env: DEV_DB_ENV_CONTRACT.endpointAuthority.env, requiredForReadiness: true, usedForProbe: true, endpointRedacted: true, valueRedacted: true } }, optionalPublicDnsAlias: { source: DEV_DB_ENV_CONTRACT.dns.source, serviceName: DEV_DB_ENV_CONTRACT.dns.serviceName, namespace: DEV_DB_ENV_CONTRACT.dns.namespace, port: DEV_DB_ENV_CONTRACT.dns.port, portName: DEV_DB_ENV_CONTRACT.dns.portName, requiredForReadiness: false, usedForProbe: false, envPresent: false, readyGate: "not_readiness_authority" }, fields: [ { name: "HWLAB_CLOUD_DB_URL", present: Boolean(env.HWLAB_CLOUD_DB_URL), redacted: true, source: "k8s-secret-ref" }, { name: "HWLAB_CLOUD_DB_SSL_MODE", present: Boolean(env.HWLAB_CLOUD_DB_SSL_MODE), redacted: false, source: "runtime-env" } ], missingEnv: ["HWLAB_CLOUD_DB_URL", "HWLAB_CLOUD_DB_SSL_MODE"].filter((name) => !env[name]), secretRefs: [ { env: "HWLAB_CLOUD_DB_URL", secretName: "hwlab-cloud-api-dev-db", secretKey: "database-url", present: env.HWLAB_CLOUD_DB_URL === "secretRef:hwlab-cloud-api-dev-db/database-url", envInjected: false, secretPresent: "not_observed_by_manifest", secretKeyPresent: "declared", redacted: true } ], connection, safety: { secretsRead: false, valuesRedacted: true, liveDbEvidence: false }, readinessLayers: Object.fromEntries(DEV_DB_ENV_CONTRACT.readinessLayers.map((layer) => [layer, { status: "not_proven" }])), blocker: configReady ? "Manifest declares DB env/Secret references but does not prove runtime env injection or live DB connection" : "Manifest is missing DB env/Secret references" }; return { ...summarizeDbContract(db), manifestConfigReady: configReady, envInjected: false, secretMaterialRead: false, valuesRedacted: true, note: "Manifest-only DB env placeholder; this is not runtime env injection or live DB evidence." }; } function inspectCodeAgentProviderContract(deploy) { const env = deploy.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {}; const expectedRef = codeAgentSecretRefPlaceholder(); const secretRefPresent = env[CODE_AGENT_PROVIDER_SECRET_CONTRACT.env] === expectedRef; const egressBaseUrl = env.HWLAB_CODE_AGENT_OPENAI_BASE_URL; const egressReady = egressBaseUrl === DEV_CODE_AGENT_PROVIDER_CONTRACT.egress.defaultBaseUrl; const directPublicOpenAi = egressBaseUrl === DEV_CODE_AGENT_PROVIDER_CONTRACT.egress.forbiddenDirectBaseUrl; return { status: secretRefPresent && egressReady ? "degraded" : "blocked", sourceIssue: CODE_AGENT_PROVIDER_SECRET_CONTRACT.sourceIssue, provider: DEV_CODE_AGENT_PROVIDER_CONTRACT.runtimeProvider, requiredEnv: [ { name: CODE_AGENT_PROVIDER_SECRET_CONTRACT.env, present: Boolean(env[CODE_AGENT_PROVIDER_SECRET_CONTRACT.env]), redacted: true, source: "k8s-secret-ref" }, { name: DEV_CODE_AGENT_PROVIDER_CONTRACT.egress.env, present: Boolean(egressBaseUrl), redacted: false, source: "runtime-env" } ], secretRef: { env: CODE_AGENT_PROVIDER_SECRET_CONTRACT.env, secretName: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretName, secretKey: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretKey, present: secretRefPresent, envInjected: false, secretPresent: "not_observed_by_manifest", secretKeyPresent: secretRefPresent ? "declared" : "not_observed_by_manifest", secretValueRead: false, redacted: true }, egress: { env: DEV_CODE_AGENT_PROVIDER_CONTRACT.egress.env, target: DEV_CODE_AGENT_PROVIDER_CONTRACT.egress.target, present: Boolean(egressBaseUrl), ready: egressReady, directPublicOpenAi, valueRedacted: true }, secretMaterialRead: false, valuesRedacted: true, note: "Manifest-only Code Agent provider Secret reference and DEV egress/base-url contract; this is not runtime Secret injection or provider connectivity evidence." }; } function notRunDbReadiness(blockerMessage) { return { status: "not_run", configReady: false, envInjected: false, secret: unknownDbSecretPresence("not_observed"), connectionAttempted: false, connectionResult: "not_run", connectionClassification: "not_run", liveConnected: false, valuesRedacted: true, secretMaterialRead: false, liveDbEvidence: false, blocker: blockerMessage, evidence: [] }; } function inspectRuntimeDbReadiness(edgeHealth) { const db = selectRuntimeDb(edgeHealth.publicHttp); return summarizeRuntimeDbReadiness(db, edgeHealth.kubernetes?.dbSecret ?? unknownDbSecretPresence("not_observed")); } function selectRuntimeDb(publicHttp) { const live = publicHttp.find((probe) => probe.url.endsWith("/health/live") && probe.ok && probe.json?.db); const health = publicHttp.find((probe) => probe.url.endsWith("/health") && probe.ok && probe.json?.db); return live?.json?.db ?? health?.json?.db ?? null; } function summarizeRuntimeDbReadiness(db, secretPresence = unknownDbSecretPresence("not_observed")) { if (!db) { return { status: "blocked", configReady: false, envInjected: false, secret: secretPresence, connectionAttempted: false, connectionResult: "health_db_missing", connectionClassification: "health_db_missing", liveConnected: false, valuesRedacted: true, secretMaterialRead: false, liveDbEvidence: false, blocker: "cloud-api health is reachable but does not expose redacted DB readiness fields", evidence: [] }; } const requiredEnv = Array.isArray(db.fields) ? db.fields : []; const envInjected = DEV_DB_ENV_CONTRACT.requiredEnv.every((name) => requiredEnv.some((field) => field.name === name && field.present === true) ); const connectionAttempted = Boolean(db.connectionAttempted ?? db.connectionChecked); const connectionResult = db.connectionResult ?? db.connection?.result ?? (connectionAttempted ? "unknown" : "not_attempted"); const connectionClassification = db.connection?.classification ?? classificationForRuntimeDb(db, connectionAttempted, connectionResult); const endpointSource = db.endpointSource ?? db.connection?.endpointSource ?? db.endpoint?.authoritative?.source ?? DEV_DB_ENV_CONTRACT.endpointAuthority.source; const optionalPublicDnsAlias = db.optionalPublicDnsAlias ?? db.endpoint?.optionalPublicDnsAlias ?? { source: DEV_DB_ENV_CONTRACT.dns.source, serviceName: DEV_DB_ENV_CONTRACT.dns.serviceName, namespace: DEV_DB_ENV_CONTRACT.dns.namespace, port: DEV_DB_ENV_CONTRACT.dns.port, portName: DEV_DB_ENV_CONTRACT.dns.portName, requiredForReadiness: false, usedForProbe: false, envPresent: false, readyGate: "not_readiness_authority" }; const liveConnected = Boolean(db.liveConnected ?? db.connected ?? db.ready); const authorityReady = endpointSource === DEV_DB_ENV_CONTRACT.endpointAuthority.source; const blockerMessage = liveConnected && authorityReady ? null : db.blocker ?? (authorityReady ? blockerForRuntimeDb(db, connectionAttempted, connectionResult) : "DB readiness did not use the Secret URL host authority"); const status = liveConnected && authorityReady ? "pass" : "blocked"; const secretRef = Array.isArray(db.secretRefs) ? db.secretRefs[0] : null; return { status, configReady: Boolean(db.configReady), envInjected, secret: { ...secretPresence, secretName: secretRef?.secretName ?? DEV_DB_ENV_CONTRACT.secretRefs[0].secretName, secretKey: secretRef?.secretKey ?? DEV_DB_ENV_CONTRACT.secretRefs[0].secretKey, secretRefPresent: Boolean(secretRef?.present), envInjected: Boolean(secretRef?.envInjected ?? secretRef?.present), redacted: true }, connectionAttempted, connectionResult, connectionClassification, endpointSource, optionalPublicDnsAlias: { source: optionalPublicDnsAlias.source ?? DEV_DB_ENV_CONTRACT.dns.source, serviceName: optionalPublicDnsAlias.serviceName ?? DEV_DB_ENV_CONTRACT.dns.serviceName, namespace: optionalPublicDnsAlias.namespace ?? DEV_DB_ENV_CONTRACT.dns.namespace, port: optionalPublicDnsAlias.port ?? DEV_DB_ENV_CONTRACT.dns.port, portName: optionalPublicDnsAlias.portName ?? DEV_DB_ENV_CONTRACT.dns.portName, requiredForReadiness: optionalPublicDnsAlias.requiredForReadiness === true, usedForProbe: optionalPublicDnsAlias.usedForProbe === true, envPresent: optionalPublicDnsAlias.envPresent === true, readyGate: optionalPublicDnsAlias.readyGate ?? "not_readiness_authority" }, liveConnected, valuesRedacted: db.redaction?.valuesRedacted !== false && db.safety?.valuesRedacted === true, secretMaterialRead: false, liveDbEvidence: Boolean(db.liveDbEvidence ?? db.safety?.liveDbEvidence), blocker: blockerMessage, evidence: [ { configReady: Boolean(db.configReady), missingEnv: Array.isArray(db.missingEnv) ? db.missingEnv : [], connectionAttempted, connectionResult, classification: connectionClassification, endpointSource, liveConnected, endpointRedacted: db.connection?.endpointRedacted !== false, valueRedacted: true } ] }; } function classificationForRuntimeDb(db, connectionAttempted, connectionResult) { if (typeof db?.connectionClassification === "string") { return db.connectionClassification; } if (typeof db?.classification === "string") { return db.classification; } if (db?.configReady !== true) { return "missing_runtime_env"; } if (!connectionAttempted) { return "configured_no_live_attempt"; } if (connectionResult === "dns_error") { return "dns_resolution_failed"; } if (connectionResult === "refused") { return "tcp_refused"; } if (connectionResult === "timeout") { return "tcp_timeout"; } if (connectionResult === "network_unreachable") { return "network_unreachable"; } return connectionResult === "connected" ? "tcp_connected" : "tcp_error"; } function blockerForRuntimeDb(db, connectionAttempted, connectionResult) { if (db.configReady !== true) { const missing = Array.isArray(db.missingEnv) && db.missingEnv.length > 0 ? db.missingEnv.join(", ") : "required DB env"; return `cloud-api DB runtime env is not ready; missing ${missing}`; } if (!connectionAttempted) { return "cloud-api DB env is injected, but runtime health has not attempted a live DB connection"; } return `cloud-api DB live connection is blocked; connectionResult=${connectionResult}`; } function unknownDbSecretPresence(state) { const ref = DEV_DB_ENV_CONTRACT.secretRefs[0]; return { observable: false, secretName: ref.secretName, secretKey: ref.secretKey, secretPresent: state, secretKeyPresent: state, secretValueRead: false, redacted: true }; } function unknownProviderSecretPresence(state) { return { observable: false, secretName: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretName, secretKey: CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretKey, secretPresent: state, secretKeyPresent: state, secretValueRead: false, redacted: true }; } async function inspectKubernetes() { const hasKubectl = await commandExists("kubectl"); const result = { observable: hasKubectl, namespace, kubeconfig: d601KubeconfigPath, context: null, nodeNames: [], services: [], endpoints: [], pods: [], dbSecret: unknownDbSecretPresence("not_observed"), codeAgentProviderSecret: unknownProviderSecretPresence("not_observed"), notes: [] }; if (!hasKubectl) { result.notes.push("kubectl is not installed in this runner"); return result; } await collectKubectlContext(result); await collectKubectlNodeIdentity(result); await collectKubectlJson(result, "services", [ "-n", namespace, "get", "svc", "hwlab-cloud-api", "hwlab-edge-proxy", "hwlab-tunnel-client", "hwlab-router", "-o", "json" ]); await collectKubectlJson(result, "endpoints", [ "-n", namespace, "get", "endpoints", "hwlab-cloud-api", "hwlab-edge-proxy", "hwlab-tunnel-client", "hwlab-router", "-o", "json" ]); await collectKubectlJson(result, "pods", [ "-n", namespace, "get", "pods", "-l", "hwlab.pikastech.local/profile=dev", "-o", "json" ]); await collectDbSecretPresence(result); await collectCodeAgentProviderSecretPresence(result); return result; } async function collectKubectlContext(result) { const context = await runKubectl(["config", "current-context"], { timeoutMs: 5000 }); result.context = { exitCode: context.exitCode, value: context.exitCode === 0 ? context.stdout.trim() : null, command: nativeKubectlCommand(["config", "current-context"]), stderr: context.stderr.trim() }; } async function collectKubectlNodeIdentity(result) { const args = ["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"]; const nodes = await runKubectl(args, { timeoutMs: 5000 }); if (nodes.exitCode !== 0) { result.notes.push(`kubectl node identity probe failed: ${nodes.stderr.trim()}`); return; } result.nodeNames = nodes.stdout.trim().split(/\s+/u).filter(Boolean); if (!result.nodeNames.includes("d601")) { result.notes.push(`kubectl node identity did not include d601: ${result.nodeNames.join(",") || "none"}`); } } async function collectKubectlJson(result, kind, args) { const probe = await runKubectl(args, { timeoutMs: 10000 }); if (probe.exitCode !== 0) { result.notes.push(`kubectl ${kind} probe failed: ${probe.stderr.trim()}`); return; } const doc = JSON.parse(probe.stdout); if (kind === "services") { result.services = doc.items.map((item) => ({ name: item.metadata.name, type: item.spec.type, clusterIP: item.spec.clusterIP, ports: item.spec.ports?.map(({ name, port, targetPort }) => ({ name, port, targetPort })) ?? [], selector: item.spec.selector ?? {} })); } else if (kind === "endpoints") { result.endpoints = doc.items.map((item) => ({ name: item.metadata.name, readyAddresses: (item.subsets ?? []).flatMap((subset) => subset.addresses ?? []).map((address) => address.ip), notReadyAddresses: (item.subsets ?? []).flatMap((subset) => subset.notReadyAddresses ?? []).map((address) => address.ip), ports: (item.subsets ?? []).flatMap((subset) => subset.ports ?? []).map(({ name, port }) => ({ name, port })) })); } else if (kind === "pods") { result.pods = doc.items.map((item) => ({ name: item.metadata.name, phase: item.status.phase, serviceId: item.metadata.labels?.["hwlab.pikastech.local/service-id"] ?? null, containers: item.status.containerStatuses?.map(({ name, ready, restartCount, image, imageID }) => ({ name, ready, restartCount, image, imageID })) ?? [] })); } } async function collectDbSecretPresence(result) { const ref = DEV_DB_ENV_CONTRACT.secretRefs[0]; const base = { observable: true, secretName: ref.secretName, secretKey: ref.secretKey, secretPresent: false, secretKeyPresent: false, secretValueRead: false, redacted: true }; const exists = await runKubectl(["-n", namespace, "get", "secret", ref.secretName, "-o", "name"], { timeoutMs: 5000 }); if (exists.exitCode !== 0) { result.dbSecret = { ...base, command: nativeKubectlCommand(["-n", namespace, "get", "secret", ref.secretName, "-o", "name"]), error: exists.stderr.trim() || "secret not observed" }; result.notes.push(`kubectl DB Secret presence probe failed: ${exists.stderr.trim()}`); return; } const keyPresence = await runKubectl(["-n", namespace, "describe", "secret", ref.secretName], { timeoutMs: 5000 }); const keyLinePattern = new RegExp(`^\\s*${escapeRegExp(ref.secretKey)}:\\s+\\d+\\s+bytes\\s*$`, "mu"); result.dbSecret = { ...base, secretPresent: true, secretKeyPresent: keyPresence.exitCode === 0 && keyLinePattern.test(keyPresence.stdout), command: `${nativeKubectlCommand(["-n", namespace, "describe", "secret", ref.secretName])} (key-presence-only)`, error: keyPresence.exitCode === 0 ? null : keyPresence.stderr.trim() }; if (keyPresence.exitCode !== 0) { result.notes.push(`kubectl DB Secret key presence probe failed: ${keyPresence.stderr.trim()}`); } } async function collectCodeAgentProviderSecretPresence(result) { const ref = CODE_AGENT_PROVIDER_SECRET_CONTRACT; const base = { observable: true, sourceIssue: ref.sourceIssue, env: ref.env, secretName: ref.secretName, secretKey: ref.secretKey, secretPresent: false, secretKeyPresent: false, secretValueRead: false, redacted: true }; const exists = await runKubectl(["-n", namespace, "get", "secret", ref.secretName, "-o", "name"], { timeoutMs: 5000 }); if (exists.exitCode !== 0) { result.codeAgentProviderSecret = { ...base, command: nativeKubectlCommand(["-n", namespace, "get", "secret", ref.secretName, "-o", "name"]), error: exists.stderr.trim() || "secret not observed" }; result.notes.push(`kubectl Code Agent provider Secret presence probe failed: ${exists.stderr.trim()}`); return; } const keyPresence = await runKubectl(["-n", namespace, "describe", "secret", ref.secretName], { timeoutMs: 5000 }); const keyLinePattern = new RegExp(`^\\s*${escapeRegExp(ref.secretKey)}:\\s+\\d+\\s+bytes\\s*$`, "mu"); result.codeAgentProviderSecret = { ...base, secretPresent: true, secretKeyPresent: keyPresence.exitCode === 0 && keyLinePattern.test(keyPresence.stdout), command: `${nativeKubectlCommand(["-n", namespace, "describe", "secret", ref.secretName])} (key-presence-only)`, error: keyPresence.exitCode === 0 ? null : keyPresence.stderr.trim() }; if (keyPresence.exitCode !== 0) { result.notes.push(`kubectl Code Agent provider Secret key presence probe failed: ${keyPresence.stderr.trim()}`); } } async function inspectClusterDns() { const results = []; for (const service of runtimeServices) { const host = `${service.serviceId}.${namespace}.svc.cluster.local`; const dns = await runCommand("getent", ["hosts", host], { timeoutMs: 3000 }); results.push({ serviceId: service.serviceId, host, port: service.port, resolved: dns.exitCode === 0, output: dns.stdout.trim() }); } return results; } function classify(report) { const publicEdgePort = report.publicTcp.find((probe) => probe.port === publicPort); const frpsControl = report.publicTcp.find((probe) => probe.port === 7000); const tunnelHealth = report.publicTcp.find((probe) => probe.port === 7402); const publicHealth = report.publicHttp.find((probe) => probe.url.endsWith("/health")); if (publicHealth?.ok && publicHealth.json?.serviceId && publicHealth.json?.environment === ENVIRONMENT_DEV) { return classifyReachableHealth(publicHealth.json, selectRuntimeDb(report.publicHttp)); } if (publicEdgePort?.status === "error" && publicEdgePort.code === "ECONNREFUSED") { return classifyRefusedPort(publicEdgePort, frpsControl, tunnelHealth); } if (publicEdgePort?.status === "timeout") { return blocker("dns_port_firewall_blocker", "public 16667 timed out before any HTTP response"); } if (publicHealth && !publicHealth.ok && publicHealth.status) { return blocker("app_health_blocker", `public /health returned HTTP ${publicHealth.status}`); } if (report.kubernetes.observable && report.kubernetes.services.length === 0) { return blocker("k3s_service_blocker", "kubectl is available but HWLAB DEV services were not observed"); } return blocker("dns_port_firewall_blocker", publicHealth?.error?.message ?? "public DEV health did not return a usable response"); } function classifyReachableHealth(json, runtimeDb = null) { const missingEvidence = []; if (!json.commit) missingEvidence.push("commit"); if (!json.image) missingEvidence.push("image"); if (!json.service && !json.serviceId) missingEvidence.push("service"); if (missingEvidence.length > 0) { return blocker("app_health_blocker", `public /health is reachable but missing ${missingEvidence.join(", ")} evidence`); } const dbReadiness = summarizeRuntimeDbReadiness(runtimeDb ?? json.db); if (dbReadiness.status !== "pass") { return blocker("app_health_blocker", dbReadiness.blocker, { likelyLayer: "cloud-api-db", confidence: dbReadiness.configReady ? "high" : "medium", likelyCause: dbReadiness.blocker, evidence: dbReadiness.evidence, notProven: dbReadiness.connectionAttempted ? ["authenticated SQL query readiness"] : ["live DB endpoint reachability", "authenticated SQL query readiness"], nextTask: "Deploy cloud-api DB runtime readiness probe and/or repair DEV DB connectivity, then rerun the read-only health smoke without reading or printing the DB secret value." }); } return { status: "pass", classification: "none", blocker: null }; } function inspectProviderSecretReadiness(edgeHealth) { const secret = edgeHealth.kubernetes?.codeAgentProviderSecret ?? unknownProviderSecretPresence("not_observed"); const manifest = edgeHealth.contracts?.deploy?.codeAgentProvider ?? null; const manifestRefPresent = manifest?.secretRef?.present === true; const secretPresent = secret.secretPresent === true; const secretKeyPresent = secret.secretKeyPresent === true; const status = secret.observable && (!secretPresent || !secretKeyPresent) ? "blocked" : "not_proven"; const classification = status === "blocked" ? "provider_secret_missing" : "provider_secret_presence_not_proven"; return { status, sourceIssue: CODE_AGENT_PROVIDER_SECRET_CONTRACT.sourceIssue, provider: "openai-responses", requiredEnv: CODE_AGENT_PROVIDER_SECRET_CONTRACT.env, manifestRefPresent, secret: { ...secret, secretValueRead: false, redacted: true }, providerCallAttempted: false, providerConnected: false, secretMaterialRead: false, valuesRedacted: true, classification, blocker: status === "blocked" ? `Code Agent provider Secret ${CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretName}/${CODE_AGENT_PROVIDER_SECRET_CONTRACT.secretKey} is not present as key-presence evidence; #143 real provider-backed chat remains blocked` : "Code Agent provider Secret presence was not proven by this edge-health run", evidence: [ { source: secret.observable ? "kubernetes-secret-presence" : "runtime-not-observable", manifestRefPresent, secretPresent, secretKeyPresent, secretValueRead: false, redacted: true } ] }; } function classifyParallelBlockers(edgeHealth) { const blockers = []; const dbReadiness = edgeHealth.runtimeDbReadiness; if (dbReadiness?.status === "blocked") { blockers.push({ id: "cloud-api-db-live", type: "runtime_blocker", scope: "cloud-api-db", status: "open", sourceIssue: "pikasTech/HWLAB#49", classification: dbReadiness.connectionClassification ?? dbReadiness.connectionResult, summary: dbReadiness.blocker, impact: { blocks: ["DB live readiness", "M4 live agent scheduling precondition", "M5 DEV-LIVE acceptance"], separateFrom: ["pikasTech/HWLAB#143 provider Secret", "M3 hardware trusted-loop proof"] } }); } const provider = edgeHealth.providerSecretReadiness; if (provider?.status === "blocked") { blockers.push({ id: "code-agent-provider-secret", type: "agent_blocker", scope: "code-agent-provider-secret", status: "open", sourceIssue: CODE_AGENT_PROVIDER_SECRET_CONTRACT.sourceIssue, classification: provider.classification, summary: provider.blocker, impact: { blocks: ["#143 provider-backed Code Agent chat", "M4 provider-backed agent execution", "M5 DEV-LIVE acceptance"], separateFrom: ["pikasTech/HWLAB#49 DB live readiness", "M3 hardware trusted-loop proof"] } }); } return blockers; } function classifyParallelBlockerStatus(parallelBlockers) { return blocker( "parallel_runtime_blocker", parallelBlockers.map((item) => item.summary).join("; "), { likelyLayer: "parallel-dev-blockers", confidence: "high", likelyCause: "one or more independent DEV blockers remain open after route health passed", evidence: parallelBlockers.map(({ id, scope, classification, sourceIssue }) => ({ id, scope, classification, sourceIssue })), notProven: ["DB-backed runtime readiness", "provider-backed Code Agent readiness", "M3/M4/M5 DEV-LIVE acceptance"], nextTask: "Close each independent blocker with redacted evidence, then rerun the read-only edge smoke." } ); } function classifyMilestoneImpact(edgeHealth) { const dbBlocked = edgeHealth.runtimeDbReadiness?.status === "blocked"; const providerBlocked = edgeHealth.providerSecretReadiness?.status === "blocked"; return { M3: { status: "separate_blocker", summary: "M3 still requires real DEV hardware trusted-loop operation/trace/audit/evidence; DB Secret URL host readiness failure and #143 provider Secret absence do not prove or clear M3." }, M4: { status: dbBlocked || providerBlocked ? "blocked" : "not_proven", blockedBy: [ ...(dbBlocked ? ["cloud-api-db-live"] : []), ...(providerBlocked ? ["code-agent-provider-secret"] : []) ], summary: "M4 live agent-loop remains blocked until DB live readiness and provider-backed agent prerequisites are independently green." }, M5: { status: dbBlocked || providerBlocked ? "blocked" : "not_proven", blockedBy: [ ...(dbBlocked ? ["cloud-api-db-live"] : []), ...(providerBlocked ? ["code-agent-provider-secret"] : []) ], summary: "M5 DEV-LIVE acceptance cannot be promoted from route health, env presence, or provider manifest refs while DB live and #143 provider Secret blockers remain open." } }; } function classifyRefusedPort(publicEdgePort, frpsControl, tunnelHealth) { if (frpsControl?.status === "error" && frpsControl.code === "ECONNREFUSED") { const allPortsRefused = tunnelHealth?.status === "error" && tunnelHealth.code === "ECONNREFUSED"; const diagnosis = { likelyLayer: "master-edge/frps", confidence: "high", likelyCause: allPortsRefused ? "public 16667, frps control 7000, and tunnel health 7402 all refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed" : "public 16667 and frps control 7000 both refuse TCP connections; the master-edge frps listener or its ports are most likely down or closed", evidence: [ summarizeTcpProbe(publicEdgePort), summarizeTcpProbe(frpsControl), summarizeTcpProbe(tunnelHealth) ], notProven: [ "k3s service readiness", "frpc connectivity", "edge proxy deployment" ], nextTask: "Restore or reopen frps on 74.48.78.17, then rerun the read-only edge smoke." }; return blocker("frp_blocker", diagnosis.likelyCause, diagnosis); } if (tunnelHealth?.status === "error" && tunnelHealth.code === "ECONNREFUSED") { return blocker("frp_blocker", "public 16667 refuses TCP and tunnel health 7402 is not reachable", { likelyLayer: "d601/frpc", confidence: "medium", likelyCause: "public 16667 and tunnel health 7402 both refuse TCP connections; the D601 frp client or its remote port is the likely break", evidence: [ summarizeTcpProbe(publicEdgePort), summarizeTcpProbe(tunnelHealth) ], notProven: [ "master-edge frps reachability", "k3s service readiness" ], nextTask: "Restore the D601 tunnel client or its remote port, then rerun the read-only edge smoke." }); } return blocker("edge_proxy_blocker", "public 16667 refuses TCP connections", { likelyLayer: "master-edge/edge-proxy", confidence: "medium", likelyCause: "public 16667 refuses TCP while frps control 7000 stays reachable, so the public edge proxy or its listener is the likely break", evidence: [ summarizeTcpProbe(publicEdgePort), summarizeTcpProbe(frpsControl) ], notProven: [ "k3s service readiness" ], nextTask: "Repair the edge proxy or its public listener, then rerun the read-only edge smoke." }); } function blocker(classification, message, diagnosis = null) { return { status: "blocker", classification, blocker: message, ...(diagnosis ? { diagnosis } : {}) }; } function summarizeTcpProbe(probe) { if (!probe) { return null; } return { host: probe.host, port: probe.port, status: probe.status, code: probe.code ?? null, message: probe.message ?? null }; } function escapeRegExp(value) { return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&"); } async function commandExists(command) { const result = await runCommand("command", ["-v", command], { timeoutMs: 3000 }); return result.exitCode === 0; } function runCommand(command, args, { timeoutMs = 10000 } = {}) { return new Promise((resolve) => { const startedAt = Date.now(); const child = spawn(command, args, { cwd: repoRoot, shell: command === "command", stdio: ["ignore", "pipe", "pipe"] }); let stdout = ""; let stderr = ""; let settled = false; const timeout = setTimeout(() => { child.kill("SIGTERM"); finish(124, `${stderr}\ncommand timed out after ${timeoutMs}ms`.trim()); }, timeoutMs); function finish(exitCode, finalStderr = stderr) { if (settled) return; settled = true; clearTimeout(timeout); resolve({ command: [command, ...args].join(" "), exitCode, stdout, stderr: finalStderr, durationMs: Date.now() - startedAt }); } child.stdout.on("data", (chunk) => { stdout += chunk.toString(); }); child.stderr.on("data", (chunk) => { stderr += chunk.toString(); }); child.on("error", (error) => finish(127, error.message)); child.on("close", (exitCode) => finish(exitCode)); }); } function probeTcp({ host, port, timeoutMs = 5000 }) { return new Promise((resolve) => { const startedAt = Date.now(); const socket = net.connect({ host, port, timeout: timeoutMs }); socket.on("connect", () => { resolve({ host, port, status: "connected", durationMs: Date.now() - startedAt }); socket.destroy(); }); socket.on("timeout", () => { resolve({ host, port, status: "timeout", durationMs: Date.now() - startedAt }); socket.destroy(); }); socket.on("error", (error) => { resolve({ host, port, status: "error", code: error.code, message: error.message, durationMs: Date.now() - startedAt }); }); }); } async function fetchJson(url, { timeoutMs = 8000 } = {}) { const startedAt = Date.now(); const controller = new AbortController(); const timeout = setTimeout(() => controller.abort(), timeoutMs); try { const response = await fetch(url, { signal: controller.signal }); const text = await response.text(); return { url, ok: response.ok, status: response.status, statusText: response.statusText, headers: Object.fromEntries(response.headers.entries()), json: parseJsonOrNull(text), bodyPreview: text.slice(0, 500), durationMs: Date.now() - startedAt }; } catch (error) { return { url, ok: false, error: { name: error.name, message: error.message, code: error.code }, durationMs: Date.now() - startedAt }; } finally { clearTimeout(timeout); } } function parseJsonOrNull(text) { try { return text ? JSON.parse(text) : null; } catch { return null; } } async function readJson(relativePath) { return readStructuredFile(repoRoot, relativePath); } async function readText(relativePath) { return readFile(path.join(repoRoot, relativePath), "utf8"); } async function maybeWriteReport(report, args) { if (!args.writeReport) { return; } const absoluteReportPath = ensureNotRepoReportsPath(repoRoot, args.reportPath, "--report"); await mkdir(path.dirname(absoluteReportPath), { recursive: true }); await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`); }