fix: layer dev DB readiness evidence
Refs #49 Merged by commander after reviewing Code Queue task codex_1779422762669_1. Adds redacted DB readiness layering and refreshes DEV endpoint/report evidence. Follow-up: deploy/verify the new readiness probe in DEV via standard image CI/CD.
This commit is contained in:
Lyon
+2
-1
@@ -16,7 +16,8 @@ future compatibility, but PROD deployment is not part of MVP acceptance.
|
||||
`HWLAB_CLOUD_DB_URL` from Secret reference
|
||||
`hwlab-cloud-api-dev-db/database-url` and non-secret
|
||||
`HWLAB_CLOUD_DB_SSL_MODE=require`. The repository records names only, never
|
||||
secret values or a live DB connection string.
|
||||
secret values or a live DB connection string; runtime health reports redacted
|
||||
env injection and DB connection result classifiers.
|
||||
- `deploy/k8s/prod` is a disabled placeholder gate only.
|
||||
- `deploy/frp` describes the D601-to-master reverse link without secrets.
|
||||
- `deploy/master-edge` describes public edge ownership and health boundaries.
|
||||
|
||||
@@ -12,7 +12,7 @@
|
||||
"data": {
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"cloud-api": "GET /health/live via hwlab-edge-proxy then hwlab-cloud-api:6667",
|
||||
"cloud-api-db": "DEV DB config gate requires HWLAB_CLOUD_DB_URL from Secret hwlab-cloud-api-dev-db/database-url plus HWLAB_CLOUD_DB_SSL_MODE=require; health reports only env presence and redacted Secret reference names, not live DB evidence",
|
||||
"cloud-api-db": "DEV DB config gate requires HWLAB_CLOUD_DB_URL from Secret hwlab-cloud-api-dev-db/database-url plus HWLAB_CLOUD_DB_SSL_MODE=require; health reports env presence, env injection, redacted connection attempt/result, and liveConnected without exposing secret values",
|
||||
"cloud-web": "GET /health/live on hwlab-cloud-web:8080; consumes cloud-api only",
|
||||
"agent": "hwlab-agent-mgr readiness gates worker template creation; hwlab-agent-worker is suspended until a session is scheduled",
|
||||
"sim": "hwlab-gateway-simu:7101 and hwlab-box-simu:7201 expose /health/live for DEV smoke",
|
||||
|
||||
@@ -16,8 +16,9 @@ and verifies:
|
||||
|
||||
- `GET /health` reports `status: "degraded"` when live DB persistence is not
|
||||
connected.
|
||||
- The DB contract reports required env names and redacted Secret refs without
|
||||
exposing secret values.
|
||||
- The DB contract reports required env names, redacted Secret refs, env
|
||||
injection, connection attempt status, connection result, and `liveConnected`
|
||||
without exposing DB URL or secret values.
|
||||
- `gateway.session.register`, `box.resource.register`,
|
||||
`box.capability.report`, `hardware.invoke.shell`, `audit.event.query`, and
|
||||
`evidence.record.query` round-trip through the same runtime store.
|
||||
@@ -35,8 +36,19 @@ The current runtime store is process-local memory:
|
||||
|
||||
This is intentional for L1. If `HWLAB_CLOUD_DB_URL` and
|
||||
`HWLAB_CLOUD_DB_SSL_MODE` are absent, health reports `db.status: "blocked"`.
|
||||
If both env vars are present, health reports `db.status: "degraded"` and still
|
||||
keeps `db.connected: false`, `db.ready: false`, and `db.connectionChecked:
|
||||
false`; env presence is not live DB evidence.
|
||||
If both env vars are present, cloud-api attempts a short read-only TCP
|
||||
connection to the DB endpoint from the injected runtime env. The health payload
|
||||
keeps all endpoint details redacted and reports only:
|
||||
|
||||
- `db.configReady`: required env names are present.
|
||||
- `db.connectionAttempted`: the runtime tried the live connection path.
|
||||
- `db.connectionResult`: redacted classifier such as `connected`, `refused`,
|
||||
`timeout`, `dns_error`, `invalid_url`, or `unsupported_protocol`.
|
||||
- `db.liveConnected`: the TCP connection completed.
|
||||
- `db.ready`: currently aliases `db.liveConnected`.
|
||||
|
||||
Env presence alone is not treated as green. If the runtime has env injection
|
||||
but `db.connectionAttempted` is false or `db.liveConnected` is false, the gate
|
||||
must stay blocked/degraded with the reported blocker.
|
||||
|
||||
Fixture: `fixtures/cloud-api-runtime/minimal-runtime.json`.
|
||||
|
||||
+14
-16
@@ -25,8 +25,10 @@ forwarding checks.
|
||||
|
||||
`db` in the cloud-api health payload is a redacted configuration gate. It
|
||||
reports required env names, presence, missing names, and Secret reference names
|
||||
only. It must not expose a connection string, token, password, or claim fixture
|
||||
output as live DB evidence.
|
||||
only. Runtime health additionally reports redacted DB readiness layers:
|
||||
`configReady`, `envInjected`, `connectionAttempted`, `connectionResult`, and
|
||||
`liveConnected`. It must not expose a connection string, token, password, DB
|
||||
host, username, or claim fixture output as live DB evidence.
|
||||
|
||||
Required DEV DB config names:
|
||||
|
||||
@@ -37,8 +39,10 @@ Required DEV DB config names:
|
||||
When the Secret/env is absent, the health JSON should keep
|
||||
`db.connected: false`, set `db.status: "blocked"`, and list the missing env
|
||||
names so gate reports can identify `cloud-api-db-health-gate`. When the env is
|
||||
present but no live DB probe is connected, `db.status` remains `"degraded"` and
|
||||
`db.ready` remains `false`; env presence alone is not live DB evidence.
|
||||
present but no live DB probe is attempted, the gate remains blocked with
|
||||
`connectionResult: "not_attempted"`. When a probe is attempted but
|
||||
`liveConnected` is false, the gate remains blocked/degraded with the redacted
|
||||
connection classifier. Env presence alone is not live DB evidence.
|
||||
|
||||
## Route Classification
|
||||
|
||||
@@ -52,7 +56,8 @@ The smoke classifies failures into these blockers:
|
||||
- `dns_port_firewall_blocker`: public endpoint times out or cannot be resolved
|
||||
as a port/network path.
|
||||
- `app_health_blocker`: public HTTP reaches an app but health JSON is wrong,
|
||||
missing evidence, or returns an HTTP error.
|
||||
missing evidence, returns an HTTP error, or reports DB env injected without a
|
||||
live DB connection.
|
||||
|
||||
## Machine-Readable Diagnosis
|
||||
|
||||
@@ -69,17 +74,10 @@ Blocked `frp_blocker` and `edge_proxy_blocker` reports now include a
|
||||
## Current Blocker
|
||||
|
||||
The committed report at `reports/dev-gate/dev-edge-health.json` records a
|
||||
read-only live run where:
|
||||
|
||||
- public `74.48.78.17:16667` returned `ECONNREFUSED`;
|
||||
- frps control `74.48.78.17:7000` returned `ECONNREFUSED`;
|
||||
- tunnel health `74.48.78.17:7402` returned `ECONNREFUSED`;
|
||||
- `kubectl` was not installed in this runner, so D601 k3s service state was not
|
||||
observable from here.
|
||||
|
||||
Classification: `frp_blocker`. The master frps endpoint is not reachable from
|
||||
the runner, so the edge proxy and downstream app health cannot be proven through
|
||||
the public DEV route yet.
|
||||
read-only live run where public `http://74.48.78.17:16667/health/live` returned
|
||||
HWLAB DEV health from `hwlab-cloud-api`, DB config was present, but runtime
|
||||
health still reported no live DB connection attempt/result from the deployed
|
||||
image. Classification: `app_health_blocker` with likely layer `cloud-api-db`.
|
||||
|
||||
## DEV FRP Contract
|
||||
|
||||
|
||||
@@ -53,8 +53,11 @@ Every report must include:
|
||||
objects.
|
||||
- DB gate extensions may add redacted `db`, `cloudApiDb`, or check evidence
|
||||
objects. These must include only env names, presence/missing status, and
|
||||
Secret reference names. They must not include connection strings, passwords,
|
||||
tokens, or fixture output presented as live DB evidence.
|
||||
Secret reference names plus redacted readiness layers such as `configReady`,
|
||||
`envInjected`, `connectionAttempted`, `connectionResult`, and
|
||||
`liveConnected`. They must not include connection strings, passwords, tokens,
|
||||
DB hostnames from the secret URL, or fixture output presented as live DB
|
||||
evidence.
|
||||
|
||||
## Validation
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"smokeId": "m2-dev-deploy-smoke",
|
||||
"issue": "pikasTech/HWLAB#23",
|
||||
"environment": "dev",
|
||||
"endpoint": "http://74.48.78.17:6667",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"defaultMode": "dry-run",
|
||||
"safetyGates": {
|
||||
"dryRunOnly": true,
|
||||
|
||||
@@ -3,6 +3,6 @@
|
||||
This evidence fixture is a mock artifact used by the CLI and cloud web skeleton.
|
||||
|
||||
- Environment: dev
|
||||
- Endpoint: http://74.48.78.17:6667
|
||||
- Endpoint: http://74.48.78.17:16667
|
||||
- Scope: browser/CLI/gateway through cloud-web/cloud-api
|
||||
- Status: dry-run only
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
"issue": "pikasTech/HWLAB#26",
|
||||
"mode": "dry-run",
|
||||
"environment": "dev",
|
||||
"endpoint": "http://74.48.78.17:6667",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"safety": {
|
||||
"allowNetwork": false,
|
||||
"allowDeploy": false,
|
||||
@@ -248,7 +248,7 @@
|
||||
"component": "DEV ingress",
|
||||
"status": "healthy",
|
||||
"deployEnv": "dev",
|
||||
"endpoint": "http://74.48.78.17:6667/health",
|
||||
"endpoint": "http://74.48.78.17:16667/health",
|
||||
"healthTimestamp": "2026-05-21T00:00:00.000Z",
|
||||
"observedBy": "fixture",
|
||||
"dryRun": true,
|
||||
@@ -263,7 +263,7 @@
|
||||
"component": "master edge proxy",
|
||||
"status": "healthy",
|
||||
"deployEnv": "dev",
|
||||
"endpoint": "http://74.48.78.17:6667",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"healthTimestamp": "2026-05-21T00:00:00.000Z",
|
||||
"observedBy": "fixture",
|
||||
"dryRun": true,
|
||||
@@ -324,7 +324,7 @@
|
||||
"component": "Cloud Web",
|
||||
"status": "healthy",
|
||||
"deployEnv": "dev",
|
||||
"endpoint": "http://74.48.78.17:6667",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"healthTimestamp": "2026-05-21T00:00:00.000Z",
|
||||
"observedBy": "fixture",
|
||||
"dryRun": true,
|
||||
@@ -441,7 +441,7 @@
|
||||
"component": "CLI",
|
||||
"status": "healthy",
|
||||
"deployEnv": "dev",
|
||||
"endpoint": "http://74.48.78.17:6667",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"healthTimestamp": "2026-05-21T00:00:00.000Z",
|
||||
"observedBy": "fixture",
|
||||
"dryRun": true,
|
||||
@@ -470,7 +470,7 @@
|
||||
"projectId": "proj_m5-mvp-e2e",
|
||||
"serviceId": "hwlab-gateway",
|
||||
"gatewayId": "gtw_m5-dev-boundary",
|
||||
"endpoint": "http://74.48.78.17:6667/gateway",
|
||||
"endpoint": "http://74.48.78.17:16667/gateway",
|
||||
"status": "connected",
|
||||
"environment": "dev",
|
||||
"startedAt": "2026-05-21T00:00:00.000Z",
|
||||
|
||||
@@ -48,7 +48,7 @@
|
||||
"worker cleanup"
|
||||
],
|
||||
"endpoints": {
|
||||
"dev": "http://74.48.78.17:6667"
|
||||
"dev": "http://74.48.78.17:16667"
|
||||
},
|
||||
"loops": [
|
||||
{
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import net from "node:net";
|
||||
|
||||
import { ENVIRONMENT_DEV } from "../protocol/index.mjs";
|
||||
|
||||
export const DEV_DB_ENV_CONTRACT = Object.freeze({
|
||||
@@ -25,6 +27,8 @@ export const DEV_DB_ENV_CONTRACT = Object.freeze({
|
||||
});
|
||||
|
||||
const secretEnvNames = new Set(DEV_DB_ENV_CONTRACT.secretRefs.map((item) => item.env));
|
||||
const supportedDbProtocols = new Set(["postgres:", "postgresql:"]);
|
||||
const defaultProbeTimeoutMs = 1200;
|
||||
|
||||
export function buildDbHealthContract(env = process.env) {
|
||||
const fields = DEV_DB_ENV_CONTRACT.requiredEnv.map((name) => {
|
||||
@@ -39,6 +43,7 @@ export function buildDbHealthContract(env = process.env) {
|
||||
});
|
||||
const missingEnv = fields.filter((field) => !field.present).map((field) => field.name);
|
||||
const configReady = missingEnv.length === 0;
|
||||
const connection = buildNotAttemptedConnection(configReady, missingEnv);
|
||||
const connected = false;
|
||||
const connectionChecked = false;
|
||||
const status = configReady ? "degraded" : "blocked";
|
||||
@@ -47,11 +52,14 @@ export function buildDbHealthContract(env = process.env) {
|
||||
contractVersion: DEV_DB_ENV_CONTRACT.contractVersion,
|
||||
environment: DEV_DB_ENV_CONTRACT.environment,
|
||||
connected,
|
||||
liveConnected: connected,
|
||||
connectionChecked,
|
||||
connectionAttempted: connection.attempted,
|
||||
connectionResult: connection.result,
|
||||
configReady,
|
||||
ready: connected,
|
||||
status,
|
||||
mode: configReady ? "configured_without_live_connection" : "not_configured",
|
||||
mode: configReady ? "configured_without_live_connection_attempt" : "not_configured",
|
||||
fields,
|
||||
missingEnv,
|
||||
secretRefs: DEV_DB_ENV_CONTRACT.secretRefs.map(({ env: envName, secretName, secretKey }) => ({
|
||||
@@ -59,8 +67,12 @@ export function buildDbHealthContract(env = process.env) {
|
||||
secretName,
|
||||
secretKey,
|
||||
present: hasEnvValue(env, envName),
|
||||
envInjected: hasEnvValue(env, envName),
|
||||
secretPresent: "not_observed_by_runtime",
|
||||
secretKeyPresent: "not_observed_by_runtime",
|
||||
redacted: true
|
||||
})),
|
||||
connection,
|
||||
safety: {
|
||||
devOnly: true,
|
||||
prodAllowed: DEV_DB_ENV_CONTRACT.forbidden.prodAllowed,
|
||||
@@ -68,10 +80,29 @@ export function buildDbHealthContract(env = process.env) {
|
||||
valuesRedacted: true,
|
||||
liveDbEvidence: false
|
||||
},
|
||||
blocker: configReady ? "live DB connection has not been attempted by this health contract" : missingEnvBlocker(missingEnv),
|
||||
evidence: configReady ? "env_presence_only_no_live_db" : "env_contract_blocked"
|
||||
};
|
||||
}
|
||||
|
||||
export async function buildDbRuntimeReadiness(env = process.env, options = {}) {
|
||||
const base = buildDbHealthContract(env);
|
||||
if (!base.configReady) {
|
||||
return base;
|
||||
}
|
||||
|
||||
if (probeDisabled(env, options)) {
|
||||
return {
|
||||
...base,
|
||||
blocker: "live DB probe is disabled for this runtime",
|
||||
evidence: "env_presence_only_probe_disabled"
|
||||
};
|
||||
}
|
||||
|
||||
const connection = await probeDbConnection(env, options);
|
||||
return applyConnectionResult(base, connection);
|
||||
}
|
||||
|
||||
export function buildDbEnvManifestPlaceholder() {
|
||||
return {
|
||||
requiredEnv: [...DEV_DB_ENV_CONTRACT.requiredEnv],
|
||||
@@ -90,20 +121,40 @@ export function summarizeDbContract(db = buildDbHealthContract()) {
|
||||
ready: db.ready,
|
||||
configReady: db.configReady,
|
||||
connected: db.connected,
|
||||
liveConnected: db.liveConnected ?? db.connected,
|
||||
connectionChecked: db.connectionChecked,
|
||||
connectionAttempted: db.connectionAttempted ?? db.connectionChecked,
|
||||
connectionResult: db.connectionResult ?? db.connection?.result ?? "unknown",
|
||||
requiredEnv: db.fields.map((field) => ({
|
||||
name: field.name,
|
||||
present: field.present,
|
||||
redacted: field.redacted
|
||||
redacted: field.redacted,
|
||||
source: field.source
|
||||
})),
|
||||
missingEnv: [...db.missingEnv],
|
||||
secretRefs: db.secretRefs.map(({ env: envName, secretName, secretKey, present, redacted }) => ({
|
||||
env: envName,
|
||||
secretName,
|
||||
secretKey,
|
||||
present,
|
||||
redacted
|
||||
})),
|
||||
secretRefs: db.secretRefs.map(
|
||||
({
|
||||
env: envName,
|
||||
secretName,
|
||||
secretKey,
|
||||
present,
|
||||
envInjected,
|
||||
secretPresent,
|
||||
secretKeyPresent,
|
||||
redacted
|
||||
}) => ({
|
||||
env: envName,
|
||||
secretName,
|
||||
secretKey,
|
||||
present,
|
||||
envInjected: envInjected ?? present,
|
||||
secretPresent: secretPresent ?? "not_observed",
|
||||
secretKeyPresent: secretKeyPresent ?? "not_observed",
|
||||
redacted
|
||||
})
|
||||
),
|
||||
connection: summarizeConnection(db.connection),
|
||||
blocker: db.blocker ?? null,
|
||||
liveDbEvidence: db.safety.liveDbEvidence,
|
||||
fixtureEvidence: false
|
||||
};
|
||||
@@ -112,3 +163,252 @@ export function summarizeDbContract(db = buildDbHealthContract()) {
|
||||
function hasEnvValue(env, name) {
|
||||
return typeof env?.[name] === "string" && env[name].trim().length > 0;
|
||||
}
|
||||
|
||||
function probeDisabled(env, options) {
|
||||
return options.probe === false || env?.HWLAB_CLOUD_DB_PROBE_DISABLED === "1";
|
||||
}
|
||||
|
||||
function buildNotAttemptedConnection(configReady, missingEnv) {
|
||||
return {
|
||||
attempted: false,
|
||||
networkAttempted: false,
|
||||
result: configReady ? "not_attempted" : "not_attempted_missing_env",
|
||||
classification: configReady ? "configured_no_live_attempt" : "missing_runtime_env",
|
||||
probeType: "tcp-connect",
|
||||
endpointRedacted: true,
|
||||
valueRedacted: true,
|
||||
timeoutMs: null,
|
||||
durationMs: 0,
|
||||
errorCode: null,
|
||||
missingEnv: [...missingEnv]
|
||||
};
|
||||
}
|
||||
|
||||
async function probeDbConnection(env, options) {
|
||||
const timeoutMs = normalizeTimeoutMs(options.timeoutMs ?? env?.HWLAB_CLOUD_DB_PROBE_TIMEOUT_MS);
|
||||
const parsed = parseDbTarget(env?.HWLAB_CLOUD_DB_URL);
|
||||
if (!parsed.ok) {
|
||||
return {
|
||||
attempted: true,
|
||||
networkAttempted: false,
|
||||
result: parsed.result,
|
||||
classification: parsed.classification,
|
||||
probeType: "tcp-connect",
|
||||
endpointRedacted: true,
|
||||
valueRedacted: true,
|
||||
timeoutMs,
|
||||
durationMs: 0,
|
||||
errorCode: parsed.errorCode,
|
||||
missingEnv: []
|
||||
};
|
||||
}
|
||||
|
||||
return tcpConnect({
|
||||
host: parsed.host,
|
||||
port: parsed.port,
|
||||
timeoutMs
|
||||
});
|
||||
}
|
||||
|
||||
function parseDbTarget(rawUrl) {
|
||||
try {
|
||||
const url = new URL(rawUrl);
|
||||
if (!supportedDbProtocols.has(url.protocol)) {
|
||||
return {
|
||||
ok: false,
|
||||
result: "unsupported_protocol",
|
||||
classification: "db_url_unsupported_protocol",
|
||||
errorCode: "UNSUPPORTED_PROTOCOL"
|
||||
};
|
||||
}
|
||||
if (!url.hostname) {
|
||||
return {
|
||||
ok: false,
|
||||
result: "invalid_url",
|
||||
classification: "db_url_missing_host",
|
||||
errorCode: "MISSING_HOST"
|
||||
};
|
||||
}
|
||||
const port = url.port ? Number.parseInt(url.port, 10) : 5432;
|
||||
if (!Number.isInteger(port) || port <= 0 || port > 65535) {
|
||||
return {
|
||||
ok: false,
|
||||
result: "invalid_url",
|
||||
classification: "db_url_invalid_port",
|
||||
errorCode: "INVALID_PORT"
|
||||
};
|
||||
}
|
||||
return {
|
||||
ok: true,
|
||||
host: url.hostname,
|
||||
port
|
||||
};
|
||||
} catch {
|
||||
return {
|
||||
ok: false,
|
||||
result: "invalid_url",
|
||||
classification: "db_url_parse_error",
|
||||
errorCode: "INVALID_URL"
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function tcpConnect({ host, port, timeoutMs }) {
|
||||
return new Promise((resolve) => {
|
||||
const startedAt = Date.now();
|
||||
const socket = net.connect({ host, port, timeout: timeoutMs });
|
||||
let settled = false;
|
||||
|
||||
function finish(result) {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
socket.destroy();
|
||||
resolve({
|
||||
attempted: true,
|
||||
networkAttempted: true,
|
||||
probeType: "tcp-connect",
|
||||
endpointRedacted: true,
|
||||
valueRedacted: true,
|
||||
timeoutMs,
|
||||
durationMs: Date.now() - startedAt,
|
||||
missingEnv: [],
|
||||
...result
|
||||
});
|
||||
}
|
||||
|
||||
socket.on("connect", () => {
|
||||
finish({
|
||||
result: "connected",
|
||||
classification: "tcp_connected",
|
||||
errorCode: null
|
||||
});
|
||||
});
|
||||
socket.on("timeout", () => {
|
||||
finish({
|
||||
result: "timeout",
|
||||
classification: "tcp_timeout",
|
||||
errorCode: "ETIMEDOUT"
|
||||
});
|
||||
});
|
||||
socket.on("error", (error) => {
|
||||
finish(classifySocketError(error));
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function classifySocketError(error) {
|
||||
const code = typeof error?.code === "string" ? error.code : "UNKNOWN";
|
||||
if (code === "ECONNREFUSED") {
|
||||
return {
|
||||
result: "refused",
|
||||
classification: "tcp_refused",
|
||||
errorCode: code
|
||||
};
|
||||
}
|
||||
if (code === "ENOTFOUND" || code === "EAI_AGAIN") {
|
||||
return {
|
||||
result: "dns_error",
|
||||
classification: "dns_resolution_failed",
|
||||
errorCode: code
|
||||
};
|
||||
}
|
||||
if (code === "ETIMEDOUT") {
|
||||
return {
|
||||
result: "timeout",
|
||||
classification: "tcp_timeout",
|
||||
errorCode: code
|
||||
};
|
||||
}
|
||||
if (code === "EHOSTUNREACH" || code === "ENETUNREACH") {
|
||||
return {
|
||||
result: "network_unreachable",
|
||||
classification: "network_unreachable",
|
||||
errorCode: code
|
||||
};
|
||||
}
|
||||
return {
|
||||
result: "error",
|
||||
classification: "tcp_error",
|
||||
errorCode: code
|
||||
};
|
||||
}
|
||||
|
||||
function applyConnectionResult(base, connection) {
|
||||
const liveConnected = connection.result === "connected";
|
||||
return {
|
||||
...base,
|
||||
connected: liveConnected,
|
||||
liveConnected,
|
||||
connectionChecked: connection.attempted,
|
||||
connectionAttempted: connection.attempted,
|
||||
connectionResult: connection.result,
|
||||
ready: liveConnected,
|
||||
status: liveConnected ? "ready" : "degraded",
|
||||
mode: liveConnected ? "live_connection_ready" : "live_connection_blocked",
|
||||
connection,
|
||||
safety: {
|
||||
...base.safety,
|
||||
liveDbEvidence: connection.attempted,
|
||||
liveDbConnectedEvidence: liveConnected
|
||||
},
|
||||
blocker: liveConnected ? null : blockerForConnection(connection),
|
||||
evidence: liveConnected ? "live_db_tcp_connection_ready" : "live_db_tcp_connection_blocked"
|
||||
};
|
||||
}
|
||||
|
||||
function blockerForConnection(connection) {
|
||||
if (connection.result === "invalid_url") {
|
||||
return "DB URL is injected but cannot be parsed as a supported postgres URL";
|
||||
}
|
||||
if (connection.result === "unsupported_protocol") {
|
||||
return "DB URL is injected but does not use a supported postgres protocol";
|
||||
}
|
||||
if (connection.result === "refused") {
|
||||
return "DB endpoint refused a TCP connection from the cloud-api runtime";
|
||||
}
|
||||
if (connection.result === "timeout") {
|
||||
return "DB endpoint TCP connection timed out from the cloud-api runtime";
|
||||
}
|
||||
if (connection.result === "dns_error") {
|
||||
return "DB endpoint DNS resolution failed from the cloud-api runtime";
|
||||
}
|
||||
if (connection.result === "network_unreachable") {
|
||||
return "DB endpoint network is unreachable from the cloud-api runtime";
|
||||
}
|
||||
return "DB endpoint TCP connection did not complete from the cloud-api runtime";
|
||||
}
|
||||
|
||||
function missingEnvBlocker(missingEnv) {
|
||||
return `DB runtime config is missing ${missingEnv.join(", ")}`;
|
||||
}
|
||||
|
||||
function normalizeTimeoutMs(value) {
|
||||
const parsed = Number.parseInt(String(value ?? ""), 10);
|
||||
if (!Number.isInteger(parsed) || parsed <= 0) {
|
||||
return defaultProbeTimeoutMs;
|
||||
}
|
||||
return Math.min(parsed, 5000);
|
||||
}
|
||||
|
||||
function summarizeConnection(connection) {
|
||||
if (!connection) {
|
||||
return {
|
||||
attempted: false,
|
||||
result: "unknown",
|
||||
endpointRedacted: true,
|
||||
valueRedacted: true
|
||||
};
|
||||
}
|
||||
return {
|
||||
attempted: Boolean(connection.attempted),
|
||||
networkAttempted: Boolean(connection.networkAttempted),
|
||||
result: connection.result,
|
||||
classification: connection.classification,
|
||||
probeType: connection.probeType,
|
||||
endpointRedacted: true,
|
||||
valueRedacted: true,
|
||||
timeoutMs: connection.timeoutMs,
|
||||
durationMs: connection.durationMs,
|
||||
errorCode: connection.errorCode ?? null
|
||||
};
|
||||
}
|
||||
|
||||
@@ -14,7 +14,7 @@ import {
|
||||
createAuditRecord,
|
||||
deriveActorFromMeta
|
||||
} from "../audit/index.mjs";
|
||||
import { buildDbHealthContract } from "./db-contract.mjs";
|
||||
import { buildDbRuntimeReadiness } from "./db-contract.mjs";
|
||||
import { createCloudRuntimeStore } from "../db/runtime-store.mjs";
|
||||
|
||||
export const SUPPORTED_RPC_METHODS = Object.freeze([
|
||||
@@ -217,24 +217,24 @@ export function createResponseMeta(requestMeta = {}) {
|
||||
return meta;
|
||||
}
|
||||
|
||||
function handleSystemHealth(params, envelope, context) {
|
||||
async function handleSystemHealth(params, envelope, context) {
|
||||
return {
|
||||
status: healthStatus(),
|
||||
serviceId: CLOUD_API_SERVICE_ID,
|
||||
environment: ENVIRONMENT_DEV,
|
||||
db: buildDbHealthContract(),
|
||||
db: await buildDbRuntimeReadiness(process.env, context.dbProbe),
|
||||
runtime: getRuntimeStore(context).summary()
|
||||
};
|
||||
}
|
||||
|
||||
function handleAdapterDescribe(params, envelope, context) {
|
||||
async function handleAdapterDescribe(params, envelope, context) {
|
||||
return {
|
||||
serviceId: CLOUD_API_SERVICE_ID,
|
||||
rpcEndpoint: "POST /rpc",
|
||||
restEndpoint: "POST /v1/rpc/{method}",
|
||||
methods: SUPPORTED_RPC_METHODS,
|
||||
auditFields: AUDIT_FIELD_NAMES,
|
||||
db: buildDbHealthContract(),
|
||||
db: await buildDbRuntimeReadiness(process.env, context.dbProbe),
|
||||
runtime: getRuntimeStore(context).summary()
|
||||
};
|
||||
}
|
||||
|
||||
@@ -12,7 +12,7 @@ import {
|
||||
createErrorEnvelope,
|
||||
handleJsonRpcRequest
|
||||
} from "./json-rpc.mjs";
|
||||
import { buildDbHealthContract } from "./db-contract.mjs";
|
||||
import { buildDbRuntimeReadiness } from "./db-contract.mjs";
|
||||
import { createCloudRuntimeStore } from "../db/runtime-store.mjs";
|
||||
|
||||
const DEFAULT_BODY_LIMIT_BYTES = 1024 * 1024;
|
||||
@@ -34,12 +34,12 @@ export function createCloudApiServer(options = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
export function buildHealthPayload(options = {}) {
|
||||
export async function buildHealthPayload(options = {}) {
|
||||
const serviceId = CLOUD_API_SERVICE_ID;
|
||||
const commitId = process.env.HWLAB_COMMIT_ID || process.env.HWLAB_GIT_SHA || "unknown";
|
||||
const imageReference = process.env.HWLAB_IMAGE || "unknown";
|
||||
const imageTag = process.env.HWLAB_IMAGE_TAG || commitId.slice(0, 7) || "unknown";
|
||||
const db = buildDbHealthContract();
|
||||
const db = await buildDbRuntimeReadiness(process.env, options.dbProbe);
|
||||
|
||||
return {
|
||||
serviceId,
|
||||
@@ -71,7 +71,7 @@ async function routeRequest(request, response, options) {
|
||||
const url = new URL(request.url || "/", "http://hwlab-cloud-api.local");
|
||||
|
||||
if (request.method === "GET" && (url.pathname === "/health" || url.pathname === "/health/live")) {
|
||||
sendJson(response, 200, buildHealthPayload(options));
|
||||
sendJson(response, 200, await buildHealthPayload(options));
|
||||
return;
|
||||
}
|
||||
|
||||
@@ -148,7 +148,7 @@ async function handleRestAdapter(request, response, url, options) {
|
||||
rpcBridge: "POST /v1/rpc/{method}",
|
||||
methods: SUPPORTED_RPC_METHODS,
|
||||
auditFields: AUDIT_FIELD_NAMES,
|
||||
db: buildDbHealthContract(),
|
||||
db: await buildDbRuntimeReadiness(process.env, options.dbProbe),
|
||||
runtime: options.runtimeStore.summary()
|
||||
});
|
||||
return;
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { createServer as createTcpServer } from "node:net";
|
||||
import test from "node:test";
|
||||
|
||||
import { createCloudApiServer } from "./server.mjs";
|
||||
@@ -26,6 +27,9 @@ test("cloud api exposes /health, /health/live, and /live probes", async () => {
|
||||
assert.equal(healthPayload.service.id, "hwlab-cloud-api");
|
||||
assert.equal(healthPayload.db.status, "blocked");
|
||||
assert.equal(healthPayload.db.connected, false);
|
||||
assert.equal(healthPayload.db.liveConnected, false);
|
||||
assert.equal(healthPayload.db.connectionAttempted, false);
|
||||
assert.equal(healthPayload.db.connectionResult, "not_attempted_missing_env");
|
||||
assert.equal(healthPayload.runtime.durable, false);
|
||||
assert.equal(healthPayload.runtime.status, "degraded");
|
||||
assert.deepEqual(healthPayload.db.missingEnv, ["HWLAB_CLOUD_DB_URL", "HWLAB_CLOUD_DB_SSL_MODE"]);
|
||||
@@ -64,10 +68,13 @@ test("cloud api exposes /health, /health/live, and /live probes", async () => {
|
||||
}
|
||||
});
|
||||
|
||||
test("cloud api health reports DB env presence without leaking values", async () => {
|
||||
test("cloud api health reports DB env presence and live connection classification without leaking values", async () => {
|
||||
const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
|
||||
const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
|
||||
process.env.HWLAB_CLOUD_DB_URL = "postgres://user:password@db.example.invalid/hwlab";
|
||||
const fakeDb = createTcpServer((socket) => socket.end());
|
||||
await new Promise((resolve) => fakeDb.listen(0, "127.0.0.1", resolve));
|
||||
const dbPort = fakeDb.address().port;
|
||||
process.env.HWLAB_CLOUD_DB_URL = `postgres://user:password@127.0.0.1:${dbPort}/hwlab`;
|
||||
process.env.HWLAB_CLOUD_DB_SSL_MODE = "require";
|
||||
|
||||
const server = createCloudApiServer();
|
||||
@@ -78,16 +85,21 @@ test("cloud api health reports DB env presence without leaking values", async ()
|
||||
const response = await fetch(`http://127.0.0.1:${port}/health`);
|
||||
const payload = await response.json();
|
||||
assert.equal(payload.status, "degraded");
|
||||
assert.equal(payload.db.status, "degraded");
|
||||
assert.equal(payload.db.connected, false);
|
||||
assert.equal(payload.db.connectionChecked, false);
|
||||
assert.equal(payload.db.status, "ready");
|
||||
assert.equal(payload.db.connected, true);
|
||||
assert.equal(payload.db.liveConnected, true);
|
||||
assert.equal(payload.db.connectionChecked, true);
|
||||
assert.equal(payload.db.connectionAttempted, true);
|
||||
assert.equal(payload.db.connectionResult, "connected");
|
||||
assert.equal(payload.db.configReady, true);
|
||||
assert.equal(payload.db.ready, false);
|
||||
assert.equal(payload.db.evidence, "env_presence_only_no_live_db");
|
||||
assert.equal(payload.db.ready, true);
|
||||
assert.equal(payload.db.evidence, "live_db_tcp_connection_ready");
|
||||
assert.deepEqual(payload.db.missingEnv, []);
|
||||
assert.equal(JSON.stringify(payload.db).includes("password"), false);
|
||||
assert.equal(JSON.stringify(payload.db).includes("db.example.invalid"), false);
|
||||
assert.equal(JSON.stringify(payload.db).includes("127.0.0.1"), false);
|
||||
assert.equal(JSON.stringify(payload.db).includes(String(dbPort)), false);
|
||||
assert.equal(payload.db.fields.find((field) => field.name === "HWLAB_CLOUD_DB_URL").redacted, true);
|
||||
assert.equal(payload.db.connection.endpointRedacted, true);
|
||||
} finally {
|
||||
if (originalUrl === undefined) {
|
||||
delete process.env.HWLAB_CLOUD_DB_URL;
|
||||
@@ -102,5 +114,8 @@ test("cloud api health reports DB env presence without leaking values", async ()
|
||||
await new Promise((resolve, reject) => {
|
||||
server.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
await new Promise((resolve, reject) => {
|
||||
fakeDb.close((error) => (error ? reject(error) : resolve()));
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
@@ -3,13 +3,13 @@
|
||||
"environment": "dev",
|
||||
"commitId": "6509a35804ee97dd8d711bdde334c46ca37c55b9",
|
||||
"namespace": "hwlab-dev",
|
||||
"endpoint": "http://74.48.78.17:6667",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"profiles": {
|
||||
"dev": {
|
||||
"name": "dev",
|
||||
"enabled": true,
|
||||
"namespace": "hwlab-dev",
|
||||
"endpoint": "http://74.48.78.17:6667",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"notes": "M0 contract example only; not a deploy manifest."
|
||||
}
|
||||
},
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
"guard": "hwlab-runtime-boundary",
|
||||
"schemaVersion": "v1",
|
||||
"environment": "dev",
|
||||
"endpoint": "http://74.48.78.17:6667",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"namespace": "hwlab-dev",
|
||||
"scope": "static-runtime-boundary",
|
||||
"policy": {
|
||||
|
||||
@@ -8,7 +8,7 @@ heavy e2e test suite.
|
||||
All MVP acceptance traffic targets:
|
||||
|
||||
```text
|
||||
http://74.48.78.17:6667
|
||||
http://74.48.78.17:16667
|
||||
```
|
||||
|
||||
## Minimal Flow
|
||||
|
||||
@@ -4,12 +4,12 @@
|
||||
"reportVersion": "v1",
|
||||
"issue": "pikasTech/HWLAB#36",
|
||||
"taskId": "dev-edge-health",
|
||||
"commitId": "73b379f4441d",
|
||||
"commitId": "cb35ada68606",
|
||||
"acceptanceLevel": "dev_edge_health",
|
||||
"devOnly": true,
|
||||
"prodDisabled": true,
|
||||
"status": "pass",
|
||||
"generatedAt": "2026-05-22T03:38:24.622Z",
|
||||
"status": "blocked",
|
||||
"generatedAt": "2026-05-22T04:22:14.774Z",
|
||||
"namespace": "hwlab-dev",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"sourceContract": {
|
||||
@@ -51,18 +51,25 @@
|
||||
"summary": "MVP dry-run remains the generic gate command; edge health uses a narrower route smoke."
|
||||
},
|
||||
"devPreconditions": {
|
||||
"status": "pass",
|
||||
"status": "blocked",
|
||||
"requirements": [
|
||||
"http://74.48.78.17:16667/health returns HWLAB DEV JSON with service, commit, and image evidence",
|
||||
"frps control on 74.48.78.17:7000 accepts the D601 frp client",
|
||||
"hwlab-dev hwlab-edge-proxy, hwlab-tunnel-client, hwlab-router, and hwlab-cloud-api are observable",
|
||||
"No PROD endpoint, secret read, restart, or UniDesk runtime substitute is used"
|
||||
],
|
||||
"summary": "DEV edge health preconditions are satisfied."
|
||||
"summary": "cloud-api DB env is injected, but runtime health has not attempted a live DB connection"
|
||||
},
|
||||
"blockers": [],
|
||||
"blockers": [
|
||||
{
|
||||
"type": "runtime_blocker",
|
||||
"scope": "cloud-api-db",
|
||||
"status": "open",
|
||||
"summary": "cloud-api DB env is injected, but runtime health has not attempted a live DB connection"
|
||||
}
|
||||
],
|
||||
"edgeHealth": {
|
||||
"generatedAt": "2026-05-22T03:38:24.622Z",
|
||||
"generatedAt": "2026-05-22T04:22:14.774Z",
|
||||
"mode": "live-read-only",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"safety": {
|
||||
@@ -123,17 +130,22 @@
|
||||
"ready": false,
|
||||
"configReady": true,
|
||||
"connected": false,
|
||||
"liveConnected": false,
|
||||
"connectionChecked": false,
|
||||
"connectionAttempted": false,
|
||||
"connectionResult": "manifest_only_not_attempted",
|
||||
"requiredEnv": [
|
||||
{
|
||||
"name": "HWLAB_CLOUD_DB_URL",
|
||||
"present": true,
|
||||
"redacted": true
|
||||
"redacted": true,
|
||||
"source": "k8s-secret-ref"
|
||||
},
|
||||
{
|
||||
"name": "HWLAB_CLOUD_DB_SSL_MODE",
|
||||
"present": true,
|
||||
"redacted": false
|
||||
"redacted": false,
|
||||
"source": "runtime-env"
|
||||
}
|
||||
],
|
||||
"missingEnv": [],
|
||||
@@ -143,14 +155,32 @@
|
||||
"secretName": "hwlab-cloud-api-dev-db",
|
||||
"secretKey": "database-url",
|
||||
"present": true,
|
||||
"envInjected": false,
|
||||
"secretPresent": "not_observed_by_manifest",
|
||||
"secretKeyPresent": "declared",
|
||||
"redacted": true
|
||||
}
|
||||
],
|
||||
"connection": {
|
||||
"attempted": false,
|
||||
"networkAttempted": false,
|
||||
"result": "manifest_only_not_attempted",
|
||||
"classification": "manifest_only",
|
||||
"probeType": "tcp-connect",
|
||||
"endpointRedacted": true,
|
||||
"valueRedacted": true,
|
||||
"timeoutMs": null,
|
||||
"durationMs": 0,
|
||||
"errorCode": null
|
||||
},
|
||||
"blocker": "Manifest declares DB env/Secret references but does not prove runtime env injection or live DB connection",
|
||||
"liveDbEvidence": false,
|
||||
"fixtureEvidence": false,
|
||||
"manifestConfigReady": true,
|
||||
"envInjected": false,
|
||||
"secretMaterialRead": false,
|
||||
"valuesRedacted": true,
|
||||
"note": "Manifest-only DB env placeholder; this is not live DB evidence."
|
||||
"note": "Manifest-only DB env placeholder; this is not runtime env injection or live DB evidence."
|
||||
}
|
||||
}
|
||||
},
|
||||
@@ -159,19 +189,19 @@
|
||||
"host": "74.48.78.17",
|
||||
"port": 16667,
|
||||
"status": "connected",
|
||||
"durationMs": 205
|
||||
"durationMs": 212
|
||||
},
|
||||
{
|
||||
"host": "74.48.78.17",
|
||||
"port": 7000,
|
||||
"status": "connected",
|
||||
"durationMs": 1231
|
||||
"durationMs": 1205
|
||||
},
|
||||
{
|
||||
"host": "74.48.78.17",
|
||||
"port": 7402,
|
||||
"status": "connected",
|
||||
"durationMs": 199
|
||||
"durationMs": 232
|
||||
}
|
||||
],
|
||||
"publicHttp": [
|
||||
@@ -184,7 +214,7 @@
|
||||
"connection": "keep-alive",
|
||||
"content-length": "661",
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
"date": "Fri, 22 May 2026 03:38:23 GMT",
|
||||
"date": "Fri, 22 May 2026 04:22:14 GMT",
|
||||
"keep-alive": "timeout=5"
|
||||
},
|
||||
"json": {
|
||||
@@ -198,23 +228,23 @@
|
||||
"livePath": "/health/live"
|
||||
},
|
||||
"commit": {
|
||||
"id": "73b379f4441d47a99dfd978a80fd43727410af51",
|
||||
"id": "cb35ada6860653b27269b0a57991184118cbf4b1",
|
||||
"source": "runtime-env"
|
||||
},
|
||||
"image": {
|
||||
"reference": "ghcr.io/pikastech/hwlab-edge-proxy:73b379f",
|
||||
"tag": "73b379f",
|
||||
"reference": "ghcr.io/pikastech/hwlab-edge-proxy:cb35ada",
|
||||
"tag": "cb35ada",
|
||||
"digest": "unknown"
|
||||
},
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"observedAt": "2026-05-22T03:38:23.861Z",
|
||||
"observedAt": "2026-05-22T04:22:14.011Z",
|
||||
"details": {
|
||||
"upstream": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667",
|
||||
"mode": "dev-edge-proxy"
|
||||
}
|
||||
},
|
||||
"bodyPreview": "{\n \"serviceId\": \"hwlab-edge-proxy\",\n \"environment\": \"dev\",\n \"status\": \"ok\",\n \"service\": {\n \"id\": \"hwlab-edge-proxy\",\n \"role\": \"public-dev-ingress\",\n \"healthPath\": \"/health\",\n \"livePath\": \"/health/live\"\n },\n \"commit\": {\n \"id\": \"73b379f4441d47a99dfd978a80fd43727410af51\",\n \"source\": \"runtime-env\"\n },\n \"image\": {\n \"reference\": \"ghcr.io/pikastech/hwlab-edge-proxy:73b379f\",\n \"tag\": \"73b379f\",\n \"digest\": \"unknown\"\n },\n \"endpoint\": \"http://74.48.78.17:16667\",\n \"obse",
|
||||
"durationMs": 656
|
||||
"bodyPreview": "{\n \"serviceId\": \"hwlab-edge-proxy\",\n \"environment\": \"dev\",\n \"status\": \"ok\",\n \"service\": {\n \"id\": \"hwlab-edge-proxy\",\n \"role\": \"public-dev-ingress\",\n \"healthPath\": \"/health\",\n \"livePath\": \"/health/live\"\n },\n \"commit\": {\n \"id\": \"cb35ada6860653b27269b0a57991184118cbf4b1\",\n \"source\": \"runtime-env\"\n },\n \"image\": {\n \"reference\": \"ghcr.io/pikastech/hwlab-edge-proxy:cb35ada\",\n \"tag\": \"cb35ada\",\n \"digest\": \"unknown\"\n },\n \"endpoint\": \"http://74.48.78.17:16667\",\n \"obse",
|
||||
"durationMs": 821
|
||||
},
|
||||
{
|
||||
"url": "http://74.48.78.17:16667/health/live",
|
||||
@@ -225,7 +255,7 @@
|
||||
"cache-control": "no-store",
|
||||
"connection": "keep-alive",
|
||||
"content-type": "application/json; charset=utf-8",
|
||||
"date": "Fri, 22 May 2026 03:38:23 GMT",
|
||||
"date": "Fri, 22 May 2026 04:22:14 GMT",
|
||||
"keep-alive": "timeout=5",
|
||||
"transfer-encoding": "chunked"
|
||||
},
|
||||
@@ -240,16 +270,16 @@
|
||||
"livePath": "/health/live"
|
||||
},
|
||||
"commit": {
|
||||
"id": "73b379f",
|
||||
"id": "cb35ada",
|
||||
"source": "runtime-env"
|
||||
},
|
||||
"image": {
|
||||
"reference": "127.0.0.1:5000/hwlab/hwlab-cloud-api:73b379f",
|
||||
"tag": "73b379f",
|
||||
"reference": "127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada",
|
||||
"tag": "cb35ada",
|
||||
"digest": "unknown"
|
||||
},
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"observedAt": "2026-05-22T03:38:23.864Z",
|
||||
"observedAt": "2026-05-22T04:22:14.013Z",
|
||||
"db": {
|
||||
"contractVersion": "v1",
|
||||
"environment": "dev",
|
||||
@@ -309,8 +339,8 @@
|
||||
}
|
||||
}
|
||||
},
|
||||
"bodyPreview": "{\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"status\":\"degraded\",\"service\":{\"id\":\"hwlab-cloud-api\",\"role\":\"cloud-api\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"73b379f\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"127.0.0.1:5000/hwlab/hwlab-cloud-api:73b379f\",\"tag\":\"73b379f\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T03:38:23.864Z\",\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"c",
|
||||
"durationMs": 651
|
||||
"bodyPreview": "{\"serviceId\":\"hwlab-cloud-api\",\"environment\":\"dev\",\"status\":\"degraded\",\"service\":{\"id\":\"hwlab-cloud-api\",\"role\":\"cloud-api\",\"healthPath\":\"/health\",\"livePath\":\"/health/live\"},\"commit\":{\"id\":\"cb35ada\",\"source\":\"runtime-env\"},\"image\":{\"reference\":\"127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada\",\"tag\":\"cb35ada\",\"digest\":\"unknown\"},\"endpoint\":\"http://74.48.78.17:16667\",\"observedAt\":\"2026-05-22T04:22:14.013Z\",\"db\":{\"contractVersion\":\"v1\",\"environment\":\"dev\",\"connected\":false,\"connectionChecked\":false,\"c",
|
||||
"durationMs": 793
|
||||
}
|
||||
],
|
||||
"kubernetes": {
|
||||
@@ -318,7 +348,7 @@
|
||||
"namespace": "hwlab-dev",
|
||||
"context": {
|
||||
"exitCode": 0,
|
||||
"value": "default",
|
||||
"value": "in-cluster",
|
||||
"stderr": ""
|
||||
},
|
||||
"services": [
|
||||
@@ -399,7 +429,7 @@
|
||||
{
|
||||
"name": "hwlab-cloud-api",
|
||||
"readyAddresses": [
|
||||
"10.42.0.188"
|
||||
"10.42.0.201"
|
||||
],
|
||||
"notReadyAddresses": [],
|
||||
"ports": [
|
||||
@@ -412,7 +442,7 @@
|
||||
{
|
||||
"name": "hwlab-edge-proxy",
|
||||
"readyAddresses": [
|
||||
"10.42.0.191"
|
||||
"10.42.0.203"
|
||||
],
|
||||
"notReadyAddresses": [],
|
||||
"ports": [
|
||||
@@ -425,7 +455,7 @@
|
||||
{
|
||||
"name": "hwlab-tunnel-client",
|
||||
"readyAddresses": [
|
||||
"10.42.0.194"
|
||||
"10.42.0.207"
|
||||
],
|
||||
"notReadyAddresses": [],
|
||||
"ports": [
|
||||
@@ -438,7 +468,7 @@
|
||||
{
|
||||
"name": "hwlab-router",
|
||||
"readyAddresses": [
|
||||
"10.42.0.193"
|
||||
"10.42.0.206"
|
||||
],
|
||||
"notReadyAddresses": [],
|
||||
"ports": [
|
||||
@@ -451,7 +481,7 @@
|
||||
],
|
||||
"pods": [
|
||||
{
|
||||
"name": "hwlab-agent-mgr-6b4976cb77-rjc2f",
|
||||
"name": "hwlab-agent-mgr-75544c7747-5dzt9",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-agent-mgr",
|
||||
"containers": [
|
||||
@@ -459,13 +489,13 @@
|
||||
"name": "hwlab-agent-mgr",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-agent-mgr:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-agent-mgr@sha256:ae6ca76cd09a7d21e46e0076e12dea1843417a0d59d57b3115915c831e88ebef"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-agent-mgr:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-agent-mgr@sha256:c16c733306a28ae9385087653b79f5bb2f28724d917a89bb296646f6a02c5318"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hwlab-agent-skills-76b5d4f4bb-8x55z",
|
||||
"name": "hwlab-agent-skills-849d9976f4-2nsmw",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-agent-skills",
|
||||
"containers": [
|
||||
@@ -473,13 +503,13 @@
|
||||
"name": "hwlab-agent-skills",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-agent-skills:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-agent-skills@sha256:5bdf9a4d17f8f4c7f3ecead99dcbd4de0ac346a2c28fef020b8853cf494df573"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-agent-skills:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-agent-skills@sha256:246fb7bb48597a7179aab62cb091e8213e47b71b6871ba94e6028032992e916c"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hwlab-box-simu-5d574d4675-jb6vb",
|
||||
"name": "hwlab-box-simu-55688cbcc7-pftxc",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-box-simu",
|
||||
"containers": [
|
||||
@@ -487,13 +517,13 @@
|
||||
"name": "hwlab-box-simu",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-box-simu:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-box-simu@sha256:2a12c1c462b2c01a9fa15efa75a6e1f17ecc8432dfa88bca53f0d8d4b6355daf"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-box-simu:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-box-simu@sha256:ae5fcc06a276918561c2d752f2221156bd2bd4db7fb93967ae76af97bfdc51bf"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hwlab-box-simu-5d574d4675-qwp84",
|
||||
"name": "hwlab-box-simu-55688cbcc7-sq5fx",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-box-simu",
|
||||
"containers": [
|
||||
@@ -501,13 +531,13 @@
|
||||
"name": "hwlab-box-simu",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-box-simu:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-box-simu@sha256:2a12c1c462b2c01a9fa15efa75a6e1f17ecc8432dfa88bca53f0d8d4b6355daf"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-box-simu:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-box-simu@sha256:ae5fcc06a276918561c2d752f2221156bd2bd4db7fb93967ae76af97bfdc51bf"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hwlab-cloud-api-769848bfd8-k2hdb",
|
||||
"name": "hwlab-cloud-api-f48dcdfb-9n6zh",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-cloud-api",
|
||||
"containers": [
|
||||
@@ -515,13 +545,13 @@
|
||||
"name": "hwlab-cloud-api",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-cloud-api:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-cloud-api@sha256:561e50494bf13bf61f83d28b371bc39b860b21333ef9991711853a095bd535bf"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-cloud-api:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-cloud-api@sha256:3946efdd7360409cea6a897cb03a6f3172f14819d3132df3aab821621909513f"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hwlab-cloud-web-bc56bc464-l7mdl",
|
||||
"name": "hwlab-cloud-web-cd9655b98-gxtmc",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-cloud-web",
|
||||
"containers": [
|
||||
@@ -529,13 +559,13 @@
|
||||
"name": "hwlab-cloud-web",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-cloud-web:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-cloud-web@sha256:c7939658af9f1109e5384619163dd0d11e1a1de14e7d15d4cbde7c4400b495c2"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-cloud-web:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-cloud-web@sha256:868e2a11c56cc1a27c77de3fa71a7b092f21ae98a4052104e9071bd93960abfa"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hwlab-edge-proxy-544886546-656nq",
|
||||
"name": "hwlab-edge-proxy-567f978cdd-8fwc8",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-edge-proxy",
|
||||
"containers": [
|
||||
@@ -543,13 +573,13 @@
|
||||
"name": "hwlab-edge-proxy",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-edge-proxy:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-edge-proxy@sha256:480dea831051d45b6d63a489db9c0fa314f88bae35dbf72be4186bf1de356f38"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-edge-proxy:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-edge-proxy@sha256:384832fc579063b9b0a940973690734bb402607720fd36f4662e4ee6fa5a57d2"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hwlab-gateway-simu-744ffffbbd-66q2c",
|
||||
"name": "hwlab-gateway-simu-699fd486b-j9pk7",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-gateway-simu",
|
||||
"containers": [
|
||||
@@ -557,13 +587,13 @@
|
||||
"name": "hwlab-gateway-simu",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-gateway-simu:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-gateway-simu@sha256:f5ccff29da331e424a482f92394d071e1be90c0244e7137e07ac33d9a7cc4082"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-gateway-simu:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-gateway-simu@sha256:00599721b910b5667b6be43c416cab4c410c6cc29b7f56ecdfb6eff71123d01c"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hwlab-gateway-simu-744ffffbbd-rmlzq",
|
||||
"name": "hwlab-gateway-simu-699fd486b-phcgp",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-gateway-simu",
|
||||
"containers": [
|
||||
@@ -571,13 +601,13 @@
|
||||
"name": "hwlab-gateway-simu",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-gateway-simu:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-gateway-simu@sha256:f5ccff29da331e424a482f92394d071e1be90c0244e7137e07ac33d9a7cc4082"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-gateway-simu:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-gateway-simu@sha256:00599721b910b5667b6be43c416cab4c410c6cc29b7f56ecdfb6eff71123d01c"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hwlab-patch-panel-7d9487bb6c-82qqk",
|
||||
"name": "hwlab-patch-panel-757f9f44d5-7np8r",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-patch-panel",
|
||||
"containers": [
|
||||
@@ -585,13 +615,13 @@
|
||||
"name": "hwlab-patch-panel",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-patch-panel:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-patch-panel@sha256:5bc9a1c7ce65026cd7bf6d20a311226b35c796a4d52a914266244bf3c1a17286"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-patch-panel:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-patch-panel@sha256:5b3aa6505492c2ed398d21cbd35a8754672114acd0fa97faa3351d4b6a6dfd5a"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hwlab-router-77465567b6-9qs6t",
|
||||
"name": "hwlab-router-5d9cbdf89b-p7r94",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-router",
|
||||
"containers": [
|
||||
@@ -599,13 +629,13 @@
|
||||
"name": "hwlab-router",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-router:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-router@sha256:356f37a7d78cc2f86230c2037f43215ad66979fbe4a949cb9241b5957692d9d8"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-router:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-router@sha256:217b1fdb0f704c4183e9d8503644177f0c8bae674da0433985c19ef909f0e342"
|
||||
}
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "hwlab-tunnel-client-6449fcb5fb-27sk2",
|
||||
"name": "hwlab-tunnel-client-65c7858c8-wjqsh",
|
||||
"phase": "Running",
|
||||
"serviceId": "hwlab-tunnel-client",
|
||||
"containers": [
|
||||
@@ -613,12 +643,23 @@
|
||||
"name": "hwlab-tunnel-client",
|
||||
"ready": true,
|
||||
"restartCount": 0,
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-tunnel-client:73b379f",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-tunnel-client@sha256:8ca553408a633d7f287ca9440f822fb1458dd9a028eb4db178b466b0b5505c49"
|
||||
"image": "127.0.0.1:5000/hwlab/hwlab-tunnel-client:cb35ada",
|
||||
"imageID": "127.0.0.1:5000/hwlab/hwlab-tunnel-client@sha256:9bcdd8a58132d7bc4dbfc71297f802fc0ab407a1b5c1bb1ea58e0791147a243b"
|
||||
}
|
||||
]
|
||||
}
|
||||
],
|
||||
"dbSecret": {
|
||||
"observable": true,
|
||||
"secretName": "hwlab-cloud-api-dev-db",
|
||||
"secretKey": "database-url",
|
||||
"secretPresent": true,
|
||||
"secretKeyPresent": true,
|
||||
"secretValueRead": false,
|
||||
"redacted": true,
|
||||
"command": "kubectl describe secret <name> (key-presence-only)",
|
||||
"error": null
|
||||
},
|
||||
"notes": []
|
||||
},
|
||||
"clusterDns": [
|
||||
@@ -626,33 +667,90 @@
|
||||
"serviceId": "hwlab-cloud-api",
|
||||
"host": "hwlab-cloud-api.hwlab-dev.svc.cluster.local",
|
||||
"port": 6667,
|
||||
"resolved": false,
|
||||
"output": ""
|
||||
"resolved": true,
|
||||
"output": "10.43.88.144 hwlab-cloud-api.hwlab-dev.svc.cluster.local"
|
||||
},
|
||||
{
|
||||
"serviceId": "hwlab-router",
|
||||
"host": "hwlab-router.hwlab-dev.svc.cluster.local",
|
||||
"port": 7401,
|
||||
"resolved": false,
|
||||
"output": ""
|
||||
"resolved": true,
|
||||
"output": "10.43.6.240 hwlab-router.hwlab-dev.svc.cluster.local"
|
||||
},
|
||||
{
|
||||
"serviceId": "hwlab-tunnel-client",
|
||||
"host": "hwlab-tunnel-client.hwlab-dev.svc.cluster.local",
|
||||
"port": 7402,
|
||||
"resolved": false,
|
||||
"output": ""
|
||||
"resolved": true,
|
||||
"output": "10.43.129.225 hwlab-tunnel-client.hwlab-dev.svc.cluster.local"
|
||||
},
|
||||
{
|
||||
"serviceId": "hwlab-edge-proxy",
|
||||
"host": "hwlab-edge-proxy.hwlab-dev.svc.cluster.local",
|
||||
"port": 6667,
|
||||
"resolved": false,
|
||||
"output": ""
|
||||
"resolved": true,
|
||||
"output": "10.43.104.107 hwlab-edge-proxy.hwlab-dev.svc.cluster.local"
|
||||
}
|
||||
],
|
||||
"status": "pass",
|
||||
"classification": "none",
|
||||
"blocker": null
|
||||
"runtimeDbReadiness": {
|
||||
"status": "blocked",
|
||||
"configReady": true,
|
||||
"envInjected": true,
|
||||
"secret": {
|
||||
"observable": true,
|
||||
"secretName": "hwlab-cloud-api-dev-db",
|
||||
"secretKey": "database-url",
|
||||
"secretPresent": true,
|
||||
"secretKeyPresent": true,
|
||||
"secretValueRead": false,
|
||||
"redacted": true,
|
||||
"command": "kubectl describe secret <name> (key-presence-only)",
|
||||
"error": null,
|
||||
"secretRefPresent": true,
|
||||
"envInjected": true
|
||||
},
|
||||
"connectionAttempted": false,
|
||||
"connectionResult": "not_attempted",
|
||||
"liveConnected": false,
|
||||
"valuesRedacted": true,
|
||||
"secretMaterialRead": false,
|
||||
"liveDbEvidence": false,
|
||||
"blocker": "cloud-api DB env is injected, but runtime health has not attempted a live DB connection",
|
||||
"evidence": [
|
||||
{
|
||||
"configReady": true,
|
||||
"missingEnv": [],
|
||||
"connectionAttempted": false,
|
||||
"connectionResult": "not_attempted",
|
||||
"liveConnected": false,
|
||||
"endpointRedacted": true,
|
||||
"valueRedacted": true
|
||||
}
|
||||
]
|
||||
},
|
||||
"status": "blocker",
|
||||
"classification": "app_health_blocker",
|
||||
"blocker": "cloud-api DB env is injected, but runtime health has not attempted a live DB connection",
|
||||
"diagnosis": {
|
||||
"likelyLayer": "cloud-api-db",
|
||||
"confidence": "high",
|
||||
"likelyCause": "cloud-api DB env is injected, but runtime health has not attempted a live DB connection",
|
||||
"evidence": [
|
||||
{
|
||||
"configReady": true,
|
||||
"missingEnv": [],
|
||||
"connectionAttempted": false,
|
||||
"connectionResult": "not_attempted",
|
||||
"liveConnected": false,
|
||||
"endpointRedacted": true,
|
||||
"valueRedacted": true
|
||||
}
|
||||
],
|
||||
"notProven": [
|
||||
"live DB endpoint reachability",
|
||||
"authenticated SQL query readiness"
|
||||
],
|
||||
"nextTask": "Deploy cloud-api DB runtime readiness probe and/or repair DEV DB connectivity, then rerun the read-only health smoke without reading or printing the DB secret value."
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -43,7 +43,7 @@
|
||||
"devPreconditions": {
|
||||
"status": "blocked",
|
||||
"requirements": [
|
||||
"DEV route remains frozen at http://74.48.78.17:6667",
|
||||
"DEV route remains frozen at http://74.48.78.17:16667",
|
||||
"Runtime evidence names frozen HWLAB service IDs",
|
||||
"No PROD path, secret read, or real deployment is attempted"
|
||||
],
|
||||
|
||||
@@ -26,7 +26,7 @@
|
||||
"node scripts/validate-dev-gate-report.mjs"
|
||||
],
|
||||
"runtimeTarget": {
|
||||
"endpoint": "http://74.48.78.17:6667",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"namespace": "hwlab-dev",
|
||||
"environment": "dev",
|
||||
"requiredBoxSimulators": 2,
|
||||
@@ -57,21 +57,21 @@
|
||||
{
|
||||
"id": "endpoint-freeze",
|
||||
"status": "pass",
|
||||
"summary": "DEV endpoint is frozen at http://74.48.78.17:6667.",
|
||||
"summary": "DEV endpoint is frozen at http://74.48.78.17:16667.",
|
||||
"evidence": [
|
||||
"http://74.48.78.17:6667"
|
||||
"http://74.48.78.17:16667"
|
||||
]
|
||||
},
|
||||
{
|
||||
"id": "dev-ingress-health",
|
||||
"status": "blocked",
|
||||
"blockerClass": "network_blocker",
|
||||
"summary": "DEV ingress at http://74.48.78.17:6667 did not return reachable HWLAB DEV health.",
|
||||
"summary": "DEV ingress at http://74.48.78.17:16667 did not return reachable HWLAB DEV health.",
|
||||
"evidence": [
|
||||
"{\"url\":\"http://74.48.78.17:6667/health/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-21T17:29:33.794Z\",\"observedAt\":\"2026-05-21T17:29:33.964Z\",\"reached\":false,\"ok\":false,\"error\":{\"name\":\"Error\",\"message\":\"connect ECONNREFUSED 74.48.78.17:6667\",\"code\":\"ECONNREFUSED\"}}",
|
||||
"{\"url\":\"http://74.48.78.17:6667/health\",\"method\":\"GET\",\"startedAt\":\"2026-05-21T17:29:33.964Z\",\"observedAt\":\"2026-05-21T17:29:34.132Z\",\"reached\":false,\"ok\":false,\"error\":{\"name\":\"Error\",\"message\":\"connect ECONNREFUSED 74.48.78.17:6667\",\"code\":\"ECONNREFUSED\"}}",
|
||||
"{\"url\":\"http://74.48.78.17:6667/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-21T17:29:34.132Z\",\"observedAt\":\"2026-05-21T17:29:34.313Z\",\"reached\":false,\"ok\":false,\"error\":{\"name\":\"Error\",\"message\":\"connect ECONNREFUSED 74.48.78.17:6667\",\"code\":\"ECONNREFUSED\"}}",
|
||||
"{\"url\":\"http://74.48.78.17:6667/v1\",\"method\":\"GET\",\"startedAt\":\"2026-05-21T17:29:34.314Z\",\"observedAt\":\"2026-05-21T17:29:34.479Z\",\"reached\":false,\"ok\":false,\"error\":{\"name\":\"Error\",\"message\":\"connect ECONNREFUSED 74.48.78.17:6667\",\"code\":\"ECONNREFUSED\"}}"
|
||||
"{\"url\":\"http://74.48.78.17:16667/health/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-21T17:29:33.794Z\",\"observedAt\":\"2026-05-21T17:29:33.964Z\",\"reached\":false,\"ok\":false,\"error\":{\"name\":\"Error\",\"message\":\"connect ECONNREFUSED 74.48.78.17:16667\",\"code\":\"ECONNREFUSED\"}}",
|
||||
"{\"url\":\"http://74.48.78.17:16667/health\",\"method\":\"GET\",\"startedAt\":\"2026-05-21T17:29:33.964Z\",\"observedAt\":\"2026-05-21T17:29:34.132Z\",\"reached\":false,\"ok\":false,\"error\":{\"name\":\"Error\",\"message\":\"connect ECONNREFUSED 74.48.78.17:16667\",\"code\":\"ECONNREFUSED\"}}",
|
||||
"{\"url\":\"http://74.48.78.17:16667/live\",\"method\":\"GET\",\"startedAt\":\"2026-05-21T17:29:34.132Z\",\"observedAt\":\"2026-05-21T17:29:34.313Z\",\"reached\":false,\"ok\":false,\"error\":{\"name\":\"Error\",\"message\":\"connect ECONNREFUSED 74.48.78.17:16667\",\"code\":\"ECONNREFUSED\"}}",
|
||||
"{\"url\":\"http://74.48.78.17:16667/v1\",\"method\":\"GET\",\"startedAt\":\"2026-05-21T17:29:34.314Z\",\"observedAt\":\"2026-05-21T17:29:34.479Z\",\"reached\":false,\"ok\":false,\"error\":{\"name\":\"Error\",\"message\":\"connect ECONNREFUSED 74.48.78.17:16667\",\"code\":\"ECONNREFUSED\"}}"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
||||
@@ -47,18 +47,18 @@
|
||||
"devPreconditions": {
|
||||
"status": "blocked",
|
||||
"requirements": [
|
||||
"DEV route remains frozen at http://74.48.78.17:6667",
|
||||
"DEV route remains frozen at http://74.48.78.17:16667",
|
||||
"No secret reads, real deployment, or long-running agent task is attempted",
|
||||
"Agent manager, worker, and skills are reachable only through the cloud API boundary"
|
||||
],
|
||||
"summary": "Blocked before agent scheduling because http://74.48.78.17:6667 is unreachable from this runner."
|
||||
"summary": "Blocked before agent scheduling because http://74.48.78.17:16667 is unreachable from this runner."
|
||||
},
|
||||
"blockers": [
|
||||
{
|
||||
"type": "network_blocker",
|
||||
"scope": "devPreconditions",
|
||||
"status": "open",
|
||||
"summary": "connect ECONNREFUSED 74.48.78.17:6667"
|
||||
"summary": "connect ECONNREFUSED 74.48.78.17:16667"
|
||||
}
|
||||
],
|
||||
"livePreflight": {
|
||||
@@ -69,7 +69,7 @@
|
||||
"evidence": [
|
||||
"No live DEV observation was recorded."
|
||||
],
|
||||
"summary": "Blocked before agent scheduling because http://74.48.78.17:6667 is unreachable from this runner."
|
||||
"summary": "Blocked before agent scheduling because http://74.48.78.17:16667 is unreachable from this runner."
|
||||
},
|
||||
"notes": "DEV M4 agent loop live preflight report. No secret material is read."
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@
|
||||
"generatedAt": "2026-05-21T18:27:07.237Z",
|
||||
"generatedFromCommit": "1e9591a6611e",
|
||||
"environment": "dev",
|
||||
"endpoint": "http://74.48.78.17:6667",
|
||||
"endpoint": "http://74.48.78.17:16667",
|
||||
"devOnly": true,
|
||||
"prodDisabled": true,
|
||||
"safety": {
|
||||
@@ -128,7 +128,7 @@
|
||||
"id": "dev-edge-frp-6667",
|
||||
"status": "blocked",
|
||||
"evidenceLevel": "BLOCKED",
|
||||
"summary": "No committed report proves live HTTP 200/JSON on http://74.48.78.17:6667."
|
||||
"summary": "No committed report proves live HTTP 200/JSON on http://74.48.78.17:16667."
|
||||
},
|
||||
{
|
||||
"id": "cloud-api-db-ready",
|
||||
@@ -459,7 +459,7 @@
|
||||
"evidence": [
|
||||
"No live DEV observation was recorded."
|
||||
],
|
||||
"summary": "Blocked before agent scheduling because http://74.48.78.17:6667 is unreachable from this runner."
|
||||
"summary": "Blocked before agent scheduling because http://74.48.78.17:16667 is unreachable from this runner."
|
||||
},
|
||||
{
|
||||
"milestone": "M5",
|
||||
@@ -476,7 +476,7 @@
|
||||
"node scripts/m5-mvp-e2e-dry-run.mjs"
|
||||
],
|
||||
"evidence": [
|
||||
"hwlab-cli health -> status: ok, environment: dev, endpoint: http://74.48.78.17:6667, projects: 2",
|
||||
"hwlab-cli health -> status: ok, environment: dev, endpoint: http://74.48.78.17:16667, projects: 2",
|
||||
"hwlab-cli dry-run -> 9 steps, 13 artifacts, 14 health contracts, 2 hardware operations, 2 evidence records, 0 network calls",
|
||||
"M5 MVP E2E dry-run passed: 9 steps, 13 artifacts, 14 health contracts, 2 hardware operations, 2 evidence records, 0 network calls"
|
||||
],
|
||||
@@ -492,8 +492,8 @@
|
||||
"status": "blocked",
|
||||
"category": "mvp-e2e-live",
|
||||
"commands": [
|
||||
"curl -fsS --max-time 5 http://74.48.78.17:6667/health",
|
||||
"curl -fsS --max-time 5 http://74.48.78.17:6667/live"
|
||||
"curl -fsS --max-time 5 http://74.48.78.17:16667/health",
|
||||
"curl -fsS --max-time 5 http://74.48.78.17:16667/live"
|
||||
],
|
||||
"evidence": [
|
||||
"curl exit 7 for /health: could not connect to 74.48.78.17 port 6667",
|
||||
@@ -631,7 +631,7 @@
|
||||
"status": "blocked",
|
||||
"category": "agent-loop-live-preflight",
|
||||
"reportPath": "reports/dev-gate/dev-m4-agent-loop.json",
|
||||
"summary": "Blocked before agent scheduling because http://74.48.78.17:6667 is unreachable from this runner."
|
||||
"summary": "Blocked before agent scheduling because http://74.48.78.17:16667 is unreachable from this runner."
|
||||
},
|
||||
{
|
||||
"milestone": "M5",
|
||||
@@ -779,7 +779,7 @@
|
||||
"scope": "dev-edge",
|
||||
"sourceIssue": "pikasTech/HWLAB#34",
|
||||
"source": "reports/dev-gate/dev-preflight-report.json",
|
||||
"summary": "http://74.48.78.17:6667/health/live is not reachable from this runner.",
|
||||
"summary": "http://74.48.78.17:16667/health/live is not reachable from this runner.",
|
||||
"nextTask": "Bring up or repair the D601-to-master frp route and hwlab-edge-proxy, then rerun the health probe."
|
||||
},
|
||||
{
|
||||
@@ -808,7 +808,7 @@
|
||||
"scope": "devPreconditions",
|
||||
"sourceIssue": "pikasTech/HWLAB#37",
|
||||
"source": "reports/dev-gate/dev-m4-agent-loop.json",
|
||||
"summary": "connect ECONNREFUSED 74.48.78.17:6667"
|
||||
"summary": "connect ECONNREFUSED 74.48.78.17:16667"
|
||||
},
|
||||
{
|
||||
"priority": "P2",
|
||||
@@ -817,7 +817,7 @@
|
||||
"scope": "devPreconditions",
|
||||
"sourceIssue": "pikasTech/HWLAB#39",
|
||||
"source": "reports/dev-gate/dev-mvp-gate-report.json",
|
||||
"summary": "http://74.48.78.17:6667/health and /live both fail with curl exit 7; live DEV ingress evidence is missing."
|
||||
"summary": "http://74.48.78.17:16667/health and /live both fail with curl exit 7; live DEV ingress evidence is missing."
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1068,7 +1068,7 @@
|
||||
"status": "open",
|
||||
"source": "reports/dev-gate/dev-preflight-report.json",
|
||||
"sourceIssue": "pikasTech/HWLAB#34",
|
||||
"summary": "http://74.48.78.17:6667/health/live is not reachable from this runner.",
|
||||
"summary": "http://74.48.78.17:16667/health/live is not reachable from this runner.",
|
||||
"nextTask": "Bring up or repair the D601-to-master frp route and hwlab-edge-proxy, then rerun the health probe.",
|
||||
"unblockOrder": 6,
|
||||
"unblocks": [
|
||||
@@ -1124,7 +1124,7 @@
|
||||
"status": "open",
|
||||
"source": "reports/dev-gate/dev-m4-agent-loop.json",
|
||||
"sourceIssue": "pikasTech/HWLAB#37",
|
||||
"summary": "connect ECONNREFUSED 74.48.78.17:6667",
|
||||
"summary": "connect ECONNREFUSED 74.48.78.17:16667",
|
||||
"unblockOrder": 6,
|
||||
"unblocks": [
|
||||
"pikasTech/HWLAB#36",
|
||||
@@ -1142,7 +1142,7 @@
|
||||
"status": "open",
|
||||
"source": "reports/dev-gate/dev-mvp-gate-report.json",
|
||||
"sourceIssue": "pikasTech/HWLAB#39",
|
||||
"summary": "http://74.48.78.17:6667/health and /live both fail with curl exit 7; live DEV ingress evidence is missing.",
|
||||
"summary": "http://74.48.78.17:16667/health and /live both fail with curl exit 7; live DEV ingress evidence is missing.",
|
||||
"unblockOrder": 6,
|
||||
"unblocks": [
|
||||
"pikasTech/HWLAB#36",
|
||||
|
||||
@@ -27,7 +27,7 @@ SOURCE, LOCAL, and DRY-RUN evidence exists, but DEV-LIVE acceptance is blocked b
|
||||
| m1-local-smoke | pass | LOCAL | M1 local smoke is not a live DEV substitute. |
|
||||
| artifact-publish-digests | blocked | BLOCKED | artifactState=contract-skeleton, ciPublished=false, registryVerified=false, sha256=0, not_published=13 |
|
||||
| d601-k3s-observability | blocked | BLOCKED | D601 runner lacks kubectl/k3s/kubeconfig observability for hwlab-dev. |
|
||||
| dev-edge-frp-6667 | blocked | BLOCKED | No committed report proves live HTTP 200/JSON on http://74.48.78.17:6667. |
|
||||
| dev-edge-frp-6667 | blocked | BLOCKED | No committed report proves live HTTP 200/JSON on http://74.48.78.17:16667. |
|
||||
| cloud-api-db-ready | blocked | BLOCKED | Manifest-level DB env exists, but live DB health readiness is still blocked/missing. |
|
||||
| m5-mvp-dev-live | blocked | BLOCKED | M5 dry-run passed; bounded DEV-LIVE MVP e2e has not passed. |
|
||||
|
||||
@@ -48,11 +48,11 @@ SOURCE, LOCAL, and DRY-RUN evidence exists, but DEV-LIVE acceptance is blocked b
|
||||
| P1 | 4 | environment_blocker | d601-k3s | D601 runner lacks kubectl and no default kubeconfig was used by this preflight. |
|
||||
| P1 | 5 | runtime_blocker | cloud-api-db | cloud-api health reports DB config blocked; missing HWLAB_CLOUD_DB_URL, HWLAB_CLOUD_DB_SSL_MODE |
|
||||
| P1 | 5 | runtime_blocker | cloud-api-db-health-gate | cloud-api DB runtime env is not ready; missing HWLAB_CLOUD_DB_URL, HWLAB_CLOUD_DB_SSL_MODE. |
|
||||
| P2 | 6 | network_blocker | dev-edge | http://74.48.78.17:6667/health/live is not reachable from this runner. |
|
||||
| P2 | 6 | network_blocker | dev-edge | http://74.48.78.17:16667/health/live is not reachable from this runner. |
|
||||
| P2 | 6 | network_blocker | dev-edge-health | live network probes require --live |
|
||||
| P2 | 6 | network_blocker | dev-ingress-health | Blocked at the #33 DEV runtime readiness condition: public DEV ingress does not accept HWLAB health requests, so #36/M3 hardware-loop checks were not reached. |
|
||||
| P2 | 6 | network_blocker | devPreconditions | connect ECONNREFUSED 74.48.78.17:6667 |
|
||||
| P2 | 6 | network_blocker | devPreconditions | http://74.48.78.17:6667/health and /live both fail with curl exit 7; live DEV ingress evidence is missing. |
|
||||
| P2 | 6 | network_blocker | devPreconditions | connect ECONNREFUSED 74.48.78.17:16667 |
|
||||
| P2 | 6 | network_blocker | devPreconditions | http://74.48.78.17:16667/health and /live both fail with curl exit 7; live DEV ingress evidence is missing. |
|
||||
|
||||
## Next Unblock Order
|
||||
|
||||
|
||||
@@ -58,7 +58,7 @@
|
||||
"validated 2 evidence chain fixtures and 2 evidence records",
|
||||
"[m1-smoke] passed",
|
||||
"[m2-smoke] validated 11 service contracts",
|
||||
"[m2-smoke] mode=dry-run endpoint=http://74.48.78.17:6667",
|
||||
"[m2-smoke] mode=dry-run endpoint=http://74.48.78.17:16667",
|
||||
"[m2-smoke] no real DEV/PROD request was made",
|
||||
"M3 hardware loop smoke passed",
|
||||
"[m4-smoke] ok m4-agent-automation-loop-local-contract-smoke"
|
||||
@@ -73,7 +73,7 @@
|
||||
"node scripts/m5-mvp-e2e-dry-run.mjs"
|
||||
],
|
||||
"evidence": [
|
||||
"hwlab-cli health -> status: ok, environment: dev, endpoint: http://74.48.78.17:6667, projects: 2",
|
||||
"hwlab-cli health -> status: ok, environment: dev, endpoint: http://74.48.78.17:16667, projects: 2",
|
||||
"hwlab-cli dry-run -> 9 steps, 13 artifacts, 14 health contracts, 2 hardware operations, 2 evidence records, 0 network calls",
|
||||
"M5 MVP E2E dry-run passed: 9 steps, 13 artifacts, 14 health contracts, 2 hardware operations, 2 evidence records, 0 network calls"
|
||||
],
|
||||
@@ -82,13 +82,13 @@
|
||||
"devPreconditions": {
|
||||
"status": "blocked",
|
||||
"requirements": [
|
||||
"Obtain a live HTTP 200/JSON health observation from http://74.48.78.17:6667/health and /live",
|
||||
"Obtain a live HTTP 200/JSON health observation from http://74.48.78.17:16667/health and /live",
|
||||
"Record live route observations for hwlab-edge-proxy, hwlab-tunnel-client, hwlab-router, hwlab-cloud-api/web, hwlab-gateway, hwlab-gateway-simu, hwlab-box-simu, hwlab-patch-panel, hwlab-agent-mgr, hwlab-agent-worker, and hwlab-agent-skills",
|
||||
"Keep the run DEV-only and avoid PROD, heavy e2e, secrets, restarts, or UniDesk substitution"
|
||||
],
|
||||
"commands": [
|
||||
"curl -fsS --max-time 5 http://74.48.78.17:6667/health",
|
||||
"curl -fsS --max-time 5 http://74.48.78.17:6667/live"
|
||||
"curl -fsS --max-time 5 http://74.48.78.17:16667/health",
|
||||
"curl -fsS --max-time 5 http://74.48.78.17:16667/live"
|
||||
],
|
||||
"evidence": [
|
||||
"curl exit 7 for /health: could not connect to 74.48.78.17 port 6667",
|
||||
@@ -101,7 +101,7 @@
|
||||
"type": "network_blocker",
|
||||
"scope": "devPreconditions",
|
||||
"status": "open",
|
||||
"summary": "http://74.48.78.17:6667/health and /live both fail with curl exit 7; live DEV ingress evidence is missing."
|
||||
"summary": "http://74.48.78.17:16667/health and /live both fail with curl exit 7; live DEV ingress evidence is missing."
|
||||
}
|
||||
],
|
||||
"milestones": [
|
||||
@@ -148,7 +148,7 @@
|
||||
],
|
||||
"evidence": [
|
||||
"[m2-smoke] validated 11 service contracts",
|
||||
"[m2-smoke] mode=dry-run endpoint=http://74.48.78.17:6667",
|
||||
"[m2-smoke] mode=dry-run endpoint=http://74.48.78.17:16667",
|
||||
"[m2-smoke] no real DEV/PROD request was made"
|
||||
],
|
||||
"summary": "The deploy smoke remains fixture-only and validates the frozen route phases and artifact metadata."
|
||||
@@ -186,7 +186,7 @@
|
||||
"node scripts/m5-mvp-e2e-dry-run.mjs"
|
||||
],
|
||||
"evidence": [
|
||||
"hwlab-cli health -> status: ok, environment: dev, endpoint: http://74.48.78.17:6667, projects: 2",
|
||||
"hwlab-cli health -> status: ok, environment: dev, endpoint: http://74.48.78.17:16667, projects: 2",
|
||||
"hwlab-cli dry-run -> 9 steps, 13 artifacts, 14 health contracts, 2 hardware operations, 2 evidence records, 0 network calls",
|
||||
"M5 MVP E2E dry-run passed: 9 steps, 13 artifacts, 14 health contracts, 2 hardware operations, 2 evidence records, 0 network calls"
|
||||
],
|
||||
|
||||
@@ -7,7 +7,7 @@ Scope: DEV only
|
||||
|
||||
## Summary
|
||||
|
||||
M0-M5 local contract, smoke, and dry-run checks pass on the frozen `origin/main` baseline. The live DEV gate is still blocked because `http://74.48.78.17:6667` is not reachable from this runner.
|
||||
M0-M5 local contract, smoke, and dry-run checks pass on the frozen `origin/main` baseline. The live DEV gate is still blocked because `http://74.48.78.17:16667` is not reachable from this runner.
|
||||
|
||||
## Evidence
|
||||
|
||||
@@ -24,8 +24,8 @@ M0-M5 local contract, smoke, and dry-run checks pass on the frozen `origin/main`
|
||||
- `node tools/hwlab-cli/bin/hwlab-cli.mjs health`
|
||||
- `node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run`
|
||||
- `node scripts/m5-mvp-e2e-dry-run.mjs`
|
||||
- `curl -fsS --max-time 5 http://74.48.78.17:6667/health`
|
||||
- `curl -fsS --max-time 5 http://74.48.78.17:6667/live`
|
||||
- `curl -fsS --max-time 5 http://74.48.78.17:16667/health`
|
||||
- `curl -fsS --max-time 5 http://74.48.78.17:16667/live`
|
||||
|
||||
## Result
|
||||
|
||||
@@ -39,7 +39,7 @@ M0-M5 local contract, smoke, and dry-run checks pass on the frozen `origin/main`
|
||||
|
||||
## Minimal Next Step
|
||||
|
||||
Obtain live `200`/JSON health observations from `http://74.48.78.17:6667/health` and `/live`, then capture the live route observations for the DEV ingress chain without switching to PROD or heavy e2e.
|
||||
Obtain live `200`/JSON health observations from `http://74.48.78.17:16667/health` and `/live`, then capture the live route observations for the DEV ingress chain without switching to PROD or heavy e2e.
|
||||
|
||||
## Blocker
|
||||
|
||||
|
||||
@@ -613,7 +613,7 @@
|
||||
"evidence": [
|
||||
{
|
||||
"ok": false,
|
||||
"url": "http://74.48.78.17:6667/health/live",
|
||||
"url": "http://74.48.78.17:16667/health/live",
|
||||
"method": "GET",
|
||||
"error": "fetch failed"
|
||||
}
|
||||
@@ -796,7 +796,7 @@
|
||||
"status": "open",
|
||||
"type": "network_blocker",
|
||||
"scope": "dev-edge",
|
||||
"summary": "http://74.48.78.17:6667/health/live is not reachable from this runner.",
|
||||
"summary": "http://74.48.78.17:16667/health/live is not reachable from this runner.",
|
||||
"nextTask": "Bring up or repair the D601-to-master frp route and hwlab-edge-proxy, then rerun the health probe."
|
||||
},
|
||||
{
|
||||
|
||||
@@ -86,6 +86,9 @@ async function run() {
|
||||
assert.equal(health.body.status, "degraded");
|
||||
assert.equal(health.body.db.status, "blocked");
|
||||
assert.equal(health.body.db.connected, false);
|
||||
assert.equal(health.body.db.liveConnected, false);
|
||||
assert.equal(health.body.db.connectionAttempted, false);
|
||||
assert.equal(health.body.db.connectionResult, "not_attempted_missing_env");
|
||||
assert.equal(health.body.db.ready, false);
|
||||
assert.deepEqual(health.body.db.missingEnv, ["HWLAB_CLOUD_DB_URL", "HWLAB_CLOUD_DB_SSL_MODE"]);
|
||||
assert.equal(health.body.runtime.durable, false);
|
||||
|
||||
@@ -153,6 +153,9 @@ async function smokeCloudApi() {
|
||||
assert.equal(health.body.environment, ENVIRONMENT_DEV);
|
||||
assert.equal(health.body.status, "degraded");
|
||||
assert.equal(health.body.db.connected, false);
|
||||
assert.equal(health.body.db.liveConnected, false);
|
||||
assert.equal(health.body.db.connectionAttempted, false);
|
||||
assert.equal(health.body.db.connectionResult, "not_attempted_missing_env");
|
||||
assert.equal(health.body.db.status, "blocked");
|
||||
assert.equal(health.body.runtime.durable, false);
|
||||
assert.deepEqual(health.body.db.missingEnv, ["HWLAB_CLOUD_DB_URL", "HWLAB_CLOUD_DB_SSL_MODE"]);
|
||||
|
||||
@@ -5,7 +5,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
|
||||
import { summarizeDbContract } from "../../internal/cloud/db-contract.mjs";
|
||||
import { DEV_DB_ENV_CONTRACT, summarizeDbContract } from "../../internal/cloud/db-contract.mjs";
|
||||
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
@@ -33,7 +33,8 @@ export async function runDevEdgeHealthSmoke(argv) {
|
||||
status: "not_run",
|
||||
classification: "not_run",
|
||||
blocker: "live network probes require --live",
|
||||
contracts: await inspectContracts()
|
||||
contracts: await inspectContracts(),
|
||||
runtimeDbReadiness: notRunDbReadiness("live network probes require --live")
|
||||
};
|
||||
const report = await createDevGateReport(edgeHealth);
|
||||
await maybeWriteReport(report, args);
|
||||
@@ -65,6 +66,7 @@ export async function runDevEdgeHealthSmoke(argv) {
|
||||
kubernetes,
|
||||
clusterDns
|
||||
};
|
||||
edgeHealth.runtimeDbReadiness = inspectRuntimeDbReadiness(edgeHealth);
|
||||
Object.assign(edgeHealth, classify(edgeHealth));
|
||||
const report = await createDevGateReport(edgeHealth);
|
||||
await maybeWriteReport(report, args);
|
||||
@@ -250,22 +252,39 @@ async function inspectContracts() {
|
||||
function inspectCloudApiDbContract(deploy) {
|
||||
const env = deploy.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {};
|
||||
const configReady = Boolean(env.HWLAB_CLOUD_DB_URL && env.HWLAB_CLOUD_DB_SSL_MODE);
|
||||
const connection = {
|
||||
attempted: false,
|
||||
networkAttempted: false,
|
||||
result: "manifest_only_not_attempted",
|
||||
classification: "manifest_only",
|
||||
probeType: "tcp-connect",
|
||||
endpointRedacted: true,
|
||||
valueRedacted: true,
|
||||
timeoutMs: null,
|
||||
durationMs: 0,
|
||||
errorCode: null
|
||||
};
|
||||
const db = {
|
||||
status: configReady ? "degraded" : "blocked",
|
||||
ready: false,
|
||||
configReady,
|
||||
connected: false,
|
||||
liveConnected: false,
|
||||
connectionChecked: false,
|
||||
connectionAttempted: false,
|
||||
connectionResult: connection.result,
|
||||
fields: [
|
||||
{
|
||||
name: "HWLAB_CLOUD_DB_URL",
|
||||
present: Boolean(env.HWLAB_CLOUD_DB_URL),
|
||||
redacted: true
|
||||
redacted: true,
|
||||
source: "k8s-secret-ref"
|
||||
},
|
||||
{
|
||||
name: "HWLAB_CLOUD_DB_SSL_MODE",
|
||||
present: Boolean(env.HWLAB_CLOUD_DB_SSL_MODE),
|
||||
redacted: false
|
||||
redacted: false,
|
||||
source: "runtime-env"
|
||||
}
|
||||
],
|
||||
missingEnv: ["HWLAB_CLOUD_DB_URL", "HWLAB_CLOUD_DB_SSL_MODE"].filter((name) => !env[name]),
|
||||
@@ -275,18 +294,143 @@ function inspectCloudApiDbContract(deploy) {
|
||||
secretName: "hwlab-cloud-api-dev-db",
|
||||
secretKey: "database-url",
|
||||
present: env.HWLAB_CLOUD_DB_URL === "secretRef:hwlab-cloud-api-dev-db/database-url",
|
||||
envInjected: false,
|
||||
secretPresent: "not_observed_by_manifest",
|
||||
secretKeyPresent: "declared",
|
||||
redacted: true
|
||||
}
|
||||
],
|
||||
connection,
|
||||
safety: {
|
||||
secretsRead: false,
|
||||
valuesRedacted: true,
|
||||
liveDbEvidence: false
|
||||
}
|
||||
},
|
||||
blocker: configReady
|
||||
? "Manifest declares DB env/Secret references but does not prove runtime env injection or live DB connection"
|
||||
: "Manifest is missing DB env/Secret references"
|
||||
};
|
||||
return {
|
||||
...summarizeDbContract(db),
|
||||
manifestConfigReady: configReady,
|
||||
envInjected: false,
|
||||
secretMaterialRead: false,
|
||||
valuesRedacted: true,
|
||||
note: "Manifest-only DB env placeholder; this is not live DB evidence."
|
||||
note: "Manifest-only DB env placeholder; this is not runtime env injection or live DB evidence."
|
||||
};
|
||||
}
|
||||
|
||||
function notRunDbReadiness(blockerMessage) {
|
||||
return {
|
||||
status: "not_run",
|
||||
configReady: false,
|
||||
envInjected: false,
|
||||
secret: unknownDbSecretPresence("not_observed"),
|
||||
connectionAttempted: false,
|
||||
connectionResult: "not_run",
|
||||
liveConnected: false,
|
||||
valuesRedacted: true,
|
||||
secretMaterialRead: false,
|
||||
liveDbEvidence: false,
|
||||
blocker: blockerMessage,
|
||||
evidence: []
|
||||
};
|
||||
}
|
||||
|
||||
function inspectRuntimeDbReadiness(edgeHealth) {
|
||||
const db = selectRuntimeDb(edgeHealth.publicHttp);
|
||||
return summarizeRuntimeDbReadiness(db, edgeHealth.kubernetes?.dbSecret ?? unknownDbSecretPresence("not_observed"));
|
||||
}
|
||||
|
||||
function selectRuntimeDb(publicHttp) {
|
||||
const live = publicHttp.find((probe) => probe.url.endsWith("/health/live") && probe.ok && probe.json?.db);
|
||||
const health = publicHttp.find((probe) => probe.url.endsWith("/health") && probe.ok && probe.json?.db);
|
||||
return live?.json?.db ?? health?.json?.db ?? null;
|
||||
}
|
||||
|
||||
function summarizeRuntimeDbReadiness(db, secretPresence = unknownDbSecretPresence("not_observed")) {
|
||||
if (!db) {
|
||||
return {
|
||||
status: "blocked",
|
||||
configReady: false,
|
||||
envInjected: false,
|
||||
secret: secretPresence,
|
||||
connectionAttempted: false,
|
||||
connectionResult: "health_db_missing",
|
||||
liveConnected: false,
|
||||
valuesRedacted: true,
|
||||
secretMaterialRead: false,
|
||||
liveDbEvidence: false,
|
||||
blocker: "cloud-api health is reachable but does not expose redacted DB readiness fields",
|
||||
evidence: []
|
||||
};
|
||||
}
|
||||
|
||||
const requiredEnv = Array.isArray(db.fields) ? db.fields : [];
|
||||
const envInjected = DEV_DB_ENV_CONTRACT.requiredEnv.every((name) =>
|
||||
requiredEnv.some((field) => field.name === name && field.present === true)
|
||||
);
|
||||
const connectionAttempted = Boolean(db.connectionAttempted ?? db.connectionChecked);
|
||||
const connectionResult = db.connectionResult ?? db.connection?.result ?? (connectionAttempted ? "unknown" : "not_attempted");
|
||||
const liveConnected = Boolean(db.liveConnected ?? db.connected ?? db.ready);
|
||||
const blockerMessage = liveConnected ? null : db.blocker ?? blockerForRuntimeDb(db, connectionAttempted, connectionResult);
|
||||
const status = liveConnected ? "pass" : "blocked";
|
||||
const secretRef = Array.isArray(db.secretRefs) ? db.secretRefs[0] : null;
|
||||
|
||||
return {
|
||||
status,
|
||||
configReady: Boolean(db.configReady),
|
||||
envInjected,
|
||||
secret: {
|
||||
...secretPresence,
|
||||
secretName: secretRef?.secretName ?? DEV_DB_ENV_CONTRACT.secretRefs[0].secretName,
|
||||
secretKey: secretRef?.secretKey ?? DEV_DB_ENV_CONTRACT.secretRefs[0].secretKey,
|
||||
secretRefPresent: Boolean(secretRef?.present),
|
||||
envInjected: Boolean(secretRef?.envInjected ?? secretRef?.present),
|
||||
redacted: true
|
||||
},
|
||||
connectionAttempted,
|
||||
connectionResult,
|
||||
liveConnected,
|
||||
valuesRedacted: db.safety?.valuesRedacted === true,
|
||||
secretMaterialRead: false,
|
||||
liveDbEvidence: Boolean(db.safety?.liveDbEvidence),
|
||||
blocker: blockerMessage,
|
||||
evidence: [
|
||||
{
|
||||
configReady: Boolean(db.configReady),
|
||||
missingEnv: Array.isArray(db.missingEnv) ? db.missingEnv : [],
|
||||
connectionAttempted,
|
||||
connectionResult,
|
||||
liveConnected,
|
||||
endpointRedacted: db.connection?.endpointRedacted !== false,
|
||||
valueRedacted: true
|
||||
}
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function blockerForRuntimeDb(db, connectionAttempted, connectionResult) {
|
||||
if (db.configReady !== true) {
|
||||
const missing = Array.isArray(db.missingEnv) && db.missingEnv.length > 0 ? db.missingEnv.join(", ") : "required DB env";
|
||||
return `cloud-api DB runtime env is not ready; missing ${missing}`;
|
||||
}
|
||||
if (!connectionAttempted) {
|
||||
return "cloud-api DB env is injected, but runtime health has not attempted a live DB connection";
|
||||
}
|
||||
return `cloud-api DB live connection is blocked; connectionResult=${connectionResult}`;
|
||||
}
|
||||
|
||||
function unknownDbSecretPresence(state) {
|
||||
const ref = DEV_DB_ENV_CONTRACT.secretRefs[0];
|
||||
return {
|
||||
observable: false,
|
||||
secretName: ref.secretName,
|
||||
secretKey: ref.secretKey,
|
||||
secretPresent: state,
|
||||
secretKeyPresent: state,
|
||||
secretValueRead: false,
|
||||
redacted: true
|
||||
};
|
||||
}
|
||||
|
||||
@@ -299,6 +443,7 @@ async function inspectKubernetes() {
|
||||
services: [],
|
||||
endpoints: [],
|
||||
pods: [],
|
||||
dbSecret: unknownDbSecretPresence("not_observed"),
|
||||
notes: []
|
||||
};
|
||||
if (!hasKubectl) {
|
||||
@@ -316,6 +461,7 @@ async function inspectKubernetes() {
|
||||
await collectKubectlJson(result, "pods", [
|
||||
"-n", namespace, "get", "pods", "-l", "hwlab.pikastech.local/profile=dev", "-o", "json"
|
||||
]);
|
||||
await collectDbSecretPresence(result);
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -363,6 +509,46 @@ async function collectKubectlJson(result, kind, args) {
|
||||
}
|
||||
}
|
||||
|
||||
async function collectDbSecretPresence(result) {
|
||||
const ref = DEV_DB_ENV_CONTRACT.secretRefs[0];
|
||||
const base = {
|
||||
observable: true,
|
||||
secretName: ref.secretName,
|
||||
secretKey: ref.secretKey,
|
||||
secretPresent: false,
|
||||
secretKeyPresent: false,
|
||||
secretValueRead: false,
|
||||
redacted: true
|
||||
};
|
||||
const exists = await runCommand("kubectl", ["-n", namespace, "get", "secret", ref.secretName, "-o", "name"], {
|
||||
timeoutMs: 5000
|
||||
});
|
||||
if (exists.exitCode !== 0) {
|
||||
result.dbSecret = {
|
||||
...base,
|
||||
command: "kubectl get secret -o name",
|
||||
error: exists.stderr.trim() || "secret not observed"
|
||||
};
|
||||
result.notes.push(`kubectl DB Secret presence probe failed: ${exists.stderr.trim()}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const keyPresence = await runCommand("kubectl", ["-n", namespace, "describe", "secret", ref.secretName], {
|
||||
timeoutMs: 5000
|
||||
});
|
||||
const keyLinePattern = new RegExp(`^\\s*${escapeRegExp(ref.secretKey)}:\\s+\\d+\\s+bytes\\s*$`, "mu");
|
||||
result.dbSecret = {
|
||||
...base,
|
||||
secretPresent: true,
|
||||
secretKeyPresent: keyPresence.exitCode === 0 && keyLinePattern.test(keyPresence.stdout),
|
||||
command: "kubectl describe secret <name> (key-presence-only)",
|
||||
error: keyPresence.exitCode === 0 ? null : keyPresence.stderr.trim()
|
||||
};
|
||||
if (keyPresence.exitCode !== 0) {
|
||||
result.notes.push(`kubectl DB Secret key presence probe failed: ${keyPresence.stderr.trim()}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function inspectClusterDns() {
|
||||
const results = [];
|
||||
for (const service of runtimeServices) {
|
||||
@@ -386,7 +572,7 @@ function classify(report) {
|
||||
const publicHealth = report.publicHttp.find((probe) => probe.url.endsWith("/health"));
|
||||
|
||||
if (publicHealth?.ok && publicHealth.json?.serviceId && publicHealth.json?.environment === ENVIRONMENT_DEV) {
|
||||
return classifyReachableHealth(publicHealth.json);
|
||||
return classifyReachableHealth(publicHealth.json, selectRuntimeDb(report.publicHttp));
|
||||
}
|
||||
if (publicEdgePort?.status === "error" && publicEdgePort.code === "ECONNREFUSED") {
|
||||
return classifyRefusedPort(publicEdgePort, frpsControl, tunnelHealth);
|
||||
@@ -403,7 +589,7 @@ function classify(report) {
|
||||
return blocker("dns_port_firewall_blocker", publicHealth?.error?.message ?? "public DEV health did not return a usable response");
|
||||
}
|
||||
|
||||
function classifyReachableHealth(json) {
|
||||
function classifyReachableHealth(json, runtimeDb = null) {
|
||||
const missingEvidence = [];
|
||||
if (!json.commit) missingEvidence.push("commit");
|
||||
if (!json.image) missingEvidence.push("image");
|
||||
@@ -411,6 +597,19 @@ function classifyReachableHealth(json) {
|
||||
if (missingEvidence.length > 0) {
|
||||
return blocker("app_health_blocker", `public /health is reachable but missing ${missingEvidence.join(", ")} evidence`);
|
||||
}
|
||||
const dbReadiness = summarizeRuntimeDbReadiness(runtimeDb ?? json.db);
|
||||
if (dbReadiness.status !== "pass") {
|
||||
return blocker("app_health_blocker", dbReadiness.blocker, {
|
||||
likelyLayer: "cloud-api-db",
|
||||
confidence: dbReadiness.configReady ? "high" : "medium",
|
||||
likelyCause: dbReadiness.blocker,
|
||||
evidence: dbReadiness.evidence,
|
||||
notProven: dbReadiness.connectionAttempted
|
||||
? ["authenticated SQL query readiness"]
|
||||
: ["live DB endpoint reachability", "authenticated SQL query readiness"],
|
||||
nextTask: "Deploy cloud-api DB runtime readiness probe and/or repair DEV DB connectivity, then rerun the read-only health smoke without reading or printing the DB secret value."
|
||||
});
|
||||
}
|
||||
return { status: "pass", classification: "none", blocker: null };
|
||||
}
|
||||
|
||||
@@ -491,6 +690,10 @@ function summarizeTcpProbe(probe) {
|
||||
};
|
||||
}
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
|
||||
}
|
||||
|
||||
async function commandExists(command) {
|
||||
const result = await runCommand("command", ["-v", command], { timeoutMs: 3000 });
|
||||
return result.exitCode === 0;
|
||||
|
||||
@@ -242,6 +242,40 @@ function assertTimestamp(value, label) {
|
||||
assert.ok(!Number.isNaN(Date.parse(value)), `${label} must be an RFC 3339 timestamp`);
|
||||
}
|
||||
|
||||
function assertDbLayerBooleans(value, label) {
|
||||
assertObject(value, label);
|
||||
for (const field of ["configReady", "liveConnected", "connectionAttempted"]) {
|
||||
assertBoolean(value[field], `${label}.${field}`);
|
||||
}
|
||||
assertString(value.connectionResult, `${label}.connectionResult`);
|
||||
assertNoDbSecretMaterial(value, label);
|
||||
}
|
||||
|
||||
function assertDbRuntimeReadiness(value, label) {
|
||||
assertDbLayerBooleans(value, label);
|
||||
for (const field of ["envInjected", "valuesRedacted", "secretMaterialRead", "liveDbEvidence"]) {
|
||||
assertBoolean(value[field], `${label}.${field}`);
|
||||
}
|
||||
assert.equal(value.valuesRedacted, true, `${label}.valuesRedacted`);
|
||||
assert.equal(value.secretMaterialRead, false, `${label}.secretMaterialRead`);
|
||||
assertObject(value.secret, `${label}.secret`);
|
||||
assertString(value.secret.secretName, `${label}.secret.secretName`);
|
||||
assertString(value.secret.secretKey, `${label}.secret.secretKey`);
|
||||
assert.equal(value.secret.redacted, true, `${label}.secret.redacted`);
|
||||
assert.equal(value.secret.secretValueRead, false, `${label}.secret.secretValueRead`);
|
||||
assertArray(value.evidence, `${label}.evidence`);
|
||||
if (value.status === "blocked") {
|
||||
assertString(value.blocker, `${label}.blocker`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertNoDbSecretMaterial(value, label) {
|
||||
const serialized = JSON.stringify(value);
|
||||
for (const forbidden of ["postgres://", "postgresql://", "password=", "://user:"]) {
|
||||
assert.equal(serialized.includes(forbidden), false, `${label} must not include ${forbidden}`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertRepoRelativePath(value, label) {
|
||||
assertString(value, label);
|
||||
assert.ok(!path.isAbsolute(value), `${label} must be repo-relative`);
|
||||
@@ -274,6 +308,9 @@ async function readJsonFile(relativePath) {
|
||||
}
|
||||
|
||||
async function collectReportFiles() {
|
||||
if (process.argv.length > 2) {
|
||||
return process.argv.slice(2).map((entry) => path.relative(repoRoot, path.resolve(process.cwd(), entry)));
|
||||
}
|
||||
const entries = await readdir(reportsDir, { withFileTypes: true });
|
||||
return entries
|
||||
.filter((entry) => entry.isFile() && entry.name.endsWith(".json"))
|
||||
@@ -824,6 +861,14 @@ async function validateDevEdgeReport(report, label) {
|
||||
);
|
||||
assertString(report.edgeHealth.classification, `${label}.edgeHealth.classification`);
|
||||
assertObject(report.edgeHealth.contracts, `${label}.edgeHealth.contracts`);
|
||||
assertDbLayerBooleans(
|
||||
report.edgeHealth.contracts?.deploy?.cloudApiDb,
|
||||
`${label}.edgeHealth.contracts.deploy.cloudApiDb`
|
||||
);
|
||||
assertDbRuntimeReadiness(
|
||||
report.edgeHealth.runtimeDbReadiness,
|
||||
`${label}.edgeHealth.runtimeDbReadiness`
|
||||
);
|
||||
assertArray(report.edgeHealth.publicTcp ?? [], `${label}.edgeHealth.publicTcp`);
|
||||
assertArray(report.edgeHealth.publicHttp ?? [], `${label}.edgeHealth.publicHttp`);
|
||||
assertObject(report.edgeHealth.kubernetes ?? {}, `${label}.edgeHealth.kubernetes`);
|
||||
|
||||
Reference in New Issue
Block a user