fix: layer dev DB readiness evidence

Refs #49

Merged by commander after reviewing Code Queue task codex_1779422762669_1. Adds redacted DB readiness layering and refreshes DEV endpoint/report evidence. Follow-up: deploy/verify the new readiness probe in DEV via standard image CI/CD.

internal/cloud/db-contract.mjs

@@ -1,3 +1,5 @@
+import net from "node:net";
+
 import { ENVIRONMENT_DEV } from "../protocol/index.mjs";
 
 export const DEV_DB_ENV_CONTRACT = Object.freeze({
@@ -25,6 +27,8 @@ export const DEV_DB_ENV_CONTRACT = Object.freeze({
 });
 
 const secretEnvNames = new Set(DEV_DB_ENV_CONTRACT.secretRefs.map((item) => item.env));
+const supportedDbProtocols = new Set(["postgres:", "postgresql:"]);
+const defaultProbeTimeoutMs = 1200;
 
 export function buildDbHealthContract(env = process.env) {
   const fields = DEV_DB_ENV_CONTRACT.requiredEnv.map((name) => {
@@ -39,6 +43,7 @@ export function buildDbHealthContract(env = process.env) {
   });
   const missingEnv = fields.filter((field) => !field.present).map((field) => field.name);
   const configReady = missingEnv.length === 0;
+  const connection = buildNotAttemptedConnection(configReady, missingEnv);
   const connected = false;
   const connectionChecked = false;
   const status = configReady ? "degraded" : "blocked";
@@ -47,11 +52,14 @@ export function buildDbHealthContract(env = process.env) {
     contractVersion: DEV_DB_ENV_CONTRACT.contractVersion,
     environment: DEV_DB_ENV_CONTRACT.environment,
     connected,
+    liveConnected: connected,
     connectionChecked,
+    connectionAttempted: connection.attempted,
+    connectionResult: connection.result,
     configReady,
     ready: connected,
     status,
-    mode: configReady ? "configured_without_live_connection" : "not_configured",
+    mode: configReady ? "configured_without_live_connection_attempt" : "not_configured",
     fields,
     missingEnv,
     secretRefs: DEV_DB_ENV_CONTRACT.secretRefs.map(({ env: envName, secretName, secretKey }) => ({
@@ -59,8 +67,12 @@ export function buildDbHealthContract(env = process.env) {
       secretName,
       secretKey,
       present: hasEnvValue(env, envName),
+      envInjected: hasEnvValue(env, envName),
+      secretPresent: "not_observed_by_runtime",
+      secretKeyPresent: "not_observed_by_runtime",
       redacted: true
     })),
+    connection,
     safety: {
       devOnly: true,
       prodAllowed: DEV_DB_ENV_CONTRACT.forbidden.prodAllowed,
@@ -68,10 +80,29 @@ export function buildDbHealthContract(env = process.env) {
       valuesRedacted: true,
       liveDbEvidence: false
     },
+    blocker: configReady ? "live DB connection has not been attempted by this health contract" : missingEnvBlocker(missingEnv),
     evidence: configReady ? "env_presence_only_no_live_db" : "env_contract_blocked"
   };
 }
 
+export async function buildDbRuntimeReadiness(env = process.env, options = {}) {
+  const base = buildDbHealthContract(env);
+  if (!base.configReady) {
+    return base;
+  }
+
+  if (probeDisabled(env, options)) {
+    return {
+      ...base,
+      blocker: "live DB probe is disabled for this runtime",
+      evidence: "env_presence_only_probe_disabled"
+    };
+  }
+
+  const connection = await probeDbConnection(env, options);
+  return applyConnectionResult(base, connection);
+}
+
 export function buildDbEnvManifestPlaceholder() {
   return {
     requiredEnv: [...DEV_DB_ENV_CONTRACT.requiredEnv],
@@ -90,20 +121,40 @@ export function summarizeDbContract(db = buildDbHealthContract()) {
     ready: db.ready,
     configReady: db.configReady,
     connected: db.connected,
+    liveConnected: db.liveConnected ?? db.connected,
     connectionChecked: db.connectionChecked,
+    connectionAttempted: db.connectionAttempted ?? db.connectionChecked,
+    connectionResult: db.connectionResult ?? db.connection?.result ?? "unknown",
     requiredEnv: db.fields.map((field) => ({
       name: field.name,
       present: field.present,
-      redacted: field.redacted
+      redacted: field.redacted,
+      source: field.source
     })),
     missingEnv: [...db.missingEnv],
-    secretRefs: db.secretRefs.map(({ env: envName, secretName, secretKey, present, redacted }) => ({
-      env: envName,
-      secretName,
-      secretKey,
-      present,
-      redacted
-    })),
+    secretRefs: db.secretRefs.map(
+      ({
+        env: envName,
+        secretName,
+        secretKey,
+        present,
+        envInjected,
+        secretPresent,
+        secretKeyPresent,
+        redacted
+      }) => ({
+        env: envName,
+        secretName,
+        secretKey,
+        present,
+        envInjected: envInjected ?? present,
+        secretPresent: secretPresent ?? "not_observed",
+        secretKeyPresent: secretKeyPresent ?? "not_observed",
+        redacted
+      })
+    ),
+    connection: summarizeConnection(db.connection),
+    blocker: db.blocker ?? null,
     liveDbEvidence: db.safety.liveDbEvidence,
     fixtureEvidence: false
   };
@@ -112,3 +163,252 @@ export function summarizeDbContract(db = buildDbHealthContract()) {
 function hasEnvValue(env, name) {
   return typeof env?.[name] === "string" && env[name].trim().length > 0;
 }
+
+function probeDisabled(env, options) {
+  return options.probe === false || env?.HWLAB_CLOUD_DB_PROBE_DISABLED === "1";
+}
+
+function buildNotAttemptedConnection(configReady, missingEnv) {
+  return {
+    attempted: false,
+    networkAttempted: false,
+    result: configReady ? "not_attempted" : "not_attempted_missing_env",
+    classification: configReady ? "configured_no_live_attempt" : "missing_runtime_env",
+    probeType: "tcp-connect",
+    endpointRedacted: true,
+    valueRedacted: true,
+    timeoutMs: null,
+    durationMs: 0,
+    errorCode: null,
+    missingEnv: [...missingEnv]
+  };
+}
+
+async function probeDbConnection(env, options) {
+  const timeoutMs = normalizeTimeoutMs(options.timeoutMs ?? env?.HWLAB_CLOUD_DB_PROBE_TIMEOUT_MS);
+  const parsed = parseDbTarget(env?.HWLAB_CLOUD_DB_URL);
+  if (!parsed.ok) {
+    return {
+      attempted: true,
+      networkAttempted: false,
+      result: parsed.result,
+      classification: parsed.classification,
+      probeType: "tcp-connect",
+      endpointRedacted: true,
+      valueRedacted: true,
+      timeoutMs,
+      durationMs: 0,
+      errorCode: parsed.errorCode,
+      missingEnv: []
+    };
+  }
+
+  return tcpConnect({
+    host: parsed.host,
+    port: parsed.port,
+    timeoutMs
+  });
+}
+
+function parseDbTarget(rawUrl) {
+  try {
+    const url = new URL(rawUrl);
+    if (!supportedDbProtocols.has(url.protocol)) {
+      return {
+        ok: false,
+        result: "unsupported_protocol",
+        classification: "db_url_unsupported_protocol",
+        errorCode: "UNSUPPORTED_PROTOCOL"
+      };
+    }
+    if (!url.hostname) {
+      return {
+        ok: false,
+        result: "invalid_url",
+        classification: "db_url_missing_host",
+        errorCode: "MISSING_HOST"
+      };
+    }
+    const port = url.port ? Number.parseInt(url.port, 10) : 5432;
+    if (!Number.isInteger(port) || port <= 0 || port > 65535) {
+      return {
+        ok: false,
+        result: "invalid_url",
+        classification: "db_url_invalid_port",
+        errorCode: "INVALID_PORT"
+      };
+    }
+    return {
+      ok: true,
+      host: url.hostname,
+      port
+    };
+  } catch {
+    return {
+      ok: false,
+      result: "invalid_url",
+      classification: "db_url_parse_error",
+      errorCode: "INVALID_URL"
+    };
+  }
+}
+
+function tcpConnect({ host, port, timeoutMs }) {
+  return new Promise((resolve) => {
+    const startedAt = Date.now();
+    const socket = net.connect({ host, port, timeout: timeoutMs });
+    let settled = false;
+
+    function finish(result) {
+      if (settled) return;
+      settled = true;
+      socket.destroy();
+      resolve({
+        attempted: true,
+        networkAttempted: true,
+        probeType: "tcp-connect",
+        endpointRedacted: true,
+        valueRedacted: true,
+        timeoutMs,
+        durationMs: Date.now() - startedAt,
+        missingEnv: [],
+        ...result
+      });
+    }
+
+    socket.on("connect", () => {
+      finish({
+        result: "connected",
+        classification: "tcp_connected",
+        errorCode: null
+      });
+    });
+    socket.on("timeout", () => {
+      finish({
+        result: "timeout",
+        classification: "tcp_timeout",
+        errorCode: "ETIMEDOUT"
+      });
+    });
+    socket.on("error", (error) => {
+      finish(classifySocketError(error));
+    });
+  });
+}
+
+function classifySocketError(error) {
+  const code = typeof error?.code === "string" ? error.code : "UNKNOWN";
+  if (code === "ECONNREFUSED") {
+    return {
+      result: "refused",
+      classification: "tcp_refused",
+      errorCode: code
+    };
+  }
+  if (code === "ENOTFOUND" || code === "EAI_AGAIN") {
+    return {
+      result: "dns_error",
+      classification: "dns_resolution_failed",
+      errorCode: code
+    };
+  }
+  if (code === "ETIMEDOUT") {
+    return {
+      result: "timeout",
+      classification: "tcp_timeout",
+      errorCode: code
+    };
+  }
+  if (code === "EHOSTUNREACH" || code === "ENETUNREACH") {
+    return {
+      result: "network_unreachable",
+      classification: "network_unreachable",
+      errorCode: code
+    };
+  }
+  return {
+    result: "error",
+    classification: "tcp_error",
+    errorCode: code
+  };
+}
+
+function applyConnectionResult(base, connection) {
+  const liveConnected = connection.result === "connected";
+  return {
+    ...base,
+    connected: liveConnected,
+    liveConnected,
+    connectionChecked: connection.attempted,
+    connectionAttempted: connection.attempted,
+    connectionResult: connection.result,
+    ready: liveConnected,
+    status: liveConnected ? "ready" : "degraded",
+    mode: liveConnected ? "live_connection_ready" : "live_connection_blocked",
+    connection,
+    safety: {
+      ...base.safety,
+      liveDbEvidence: connection.attempted,
+      liveDbConnectedEvidence: liveConnected
+    },
+    blocker: liveConnected ? null : blockerForConnection(connection),
+    evidence: liveConnected ? "live_db_tcp_connection_ready" : "live_db_tcp_connection_blocked"
+  };
+}
+
+function blockerForConnection(connection) {
+  if (connection.result === "invalid_url") {
+    return "DB URL is injected but cannot be parsed as a supported postgres URL";
+  }
+  if (connection.result === "unsupported_protocol") {
+    return "DB URL is injected but does not use a supported postgres protocol";
+  }
+  if (connection.result === "refused") {
+    return "DB endpoint refused a TCP connection from the cloud-api runtime";
+  }
+  if (connection.result === "timeout") {
+    return "DB endpoint TCP connection timed out from the cloud-api runtime";
+  }
+  if (connection.result === "dns_error") {
+    return "DB endpoint DNS resolution failed from the cloud-api runtime";
+  }
+  if (connection.result === "network_unreachable") {
+    return "DB endpoint network is unreachable from the cloud-api runtime";
+  }
+  return "DB endpoint TCP connection did not complete from the cloud-api runtime";
+}
+
+function missingEnvBlocker(missingEnv) {
+  return `DB runtime config is missing ${missingEnv.join(", ")}`;
+}
+
+function normalizeTimeoutMs(value) {
+  const parsed = Number.parseInt(String(value ?? ""), 10);
+  if (!Number.isInteger(parsed) || parsed <= 0) {
+    return defaultProbeTimeoutMs;
+  }
+  return Math.min(parsed, 5000);
+}
+
+function summarizeConnection(connection) {
+  if (!connection) {
+    return {
+      attempted: false,
+      result: "unknown",
+      endpointRedacted: true,
+      valueRedacted: true
+    };
+  }
+  return {
+    attempted: Boolean(connection.attempted),
+    networkAttempted: Boolean(connection.networkAttempted),
+    result: connection.result,
+    classification: connection.classification,
+    probeType: connection.probeType,
+    endpointRedacted: true,
+    valueRedacted: true,
+    timeoutMs: connection.timeoutMs,
+    durationMs: connection.durationMs,
+    errorCode: connection.errorCode ?? null
+  };
+}

internal/cloud/json-rpc.mjs

@@ -14,7 +14,7 @@ import {
   createAuditRecord,
   deriveActorFromMeta
 } from "../audit/index.mjs";
-import { buildDbHealthContract } from "./db-contract.mjs";
+import { buildDbRuntimeReadiness } from "./db-contract.mjs";
 import { createCloudRuntimeStore } from "../db/runtime-store.mjs";
 
 export const SUPPORTED_RPC_METHODS = Object.freeze([
@@ -217,24 +217,24 @@ export function createResponseMeta(requestMeta = {}) {
   return meta;
 }
 
-function handleSystemHealth(params, envelope, context) {
+async function handleSystemHealth(params, envelope, context) {
   return {
     status: healthStatus(),
     serviceId: CLOUD_API_SERVICE_ID,
     environment: ENVIRONMENT_DEV,
-    db: buildDbHealthContract(),
+    db: await buildDbRuntimeReadiness(process.env, context.dbProbe),
     runtime: getRuntimeStore(context).summary()
   };
 }
 
-function handleAdapterDescribe(params, envelope, context) {
+async function handleAdapterDescribe(params, envelope, context) {
   return {
     serviceId: CLOUD_API_SERVICE_ID,
     rpcEndpoint: "POST /rpc",
     restEndpoint: "POST /v1/rpc/{method}",
     methods: SUPPORTED_RPC_METHODS,
     auditFields: AUDIT_FIELD_NAMES,
-    db: buildDbHealthContract(),
+    db: await buildDbRuntimeReadiness(process.env, context.dbProbe),
     runtime: getRuntimeStore(context).summary()
   };
 }

internal/cloud/server.mjs

@@ -12,7 +12,7 @@ import {
   createErrorEnvelope,
   handleJsonRpcRequest
 } from "./json-rpc.mjs";
-import { buildDbHealthContract } from "./db-contract.mjs";
+import { buildDbRuntimeReadiness } from "./db-contract.mjs";
 import { createCloudRuntimeStore } from "../db/runtime-store.mjs";
 
 const DEFAULT_BODY_LIMIT_BYTES = 1024 * 1024;
@@ -34,12 +34,12 @@ export function createCloudApiServer(options = {}) {
   });
 }
 
-export function buildHealthPayload(options = {}) {
+export async function buildHealthPayload(options = {}) {
   const serviceId = CLOUD_API_SERVICE_ID;
   const commitId = process.env.HWLAB_COMMIT_ID || process.env.HWLAB_GIT_SHA || "unknown";
   const imageReference = process.env.HWLAB_IMAGE || "unknown";
   const imageTag = process.env.HWLAB_IMAGE_TAG || commitId.slice(0, 7) || "unknown";
-  const db = buildDbHealthContract();
+  const db = await buildDbRuntimeReadiness(process.env, options.dbProbe);
 
   return {
     serviceId,
@@ -71,7 +71,7 @@ async function routeRequest(request, response, options) {
   const url = new URL(request.url || "/", "http://hwlab-cloud-api.local");
 
   if (request.method === "GET" && (url.pathname === "/health" || url.pathname === "/health/live")) {
-    sendJson(response, 200, buildHealthPayload(options));
+    sendJson(response, 200, await buildHealthPayload(options));
     return;
   }
 
@@ -148,7 +148,7 @@ async function handleRestAdapter(request, response, url, options) {
       rpcBridge: "POST /v1/rpc/{method}",
       methods: SUPPORTED_RPC_METHODS,
       auditFields: AUDIT_FIELD_NAMES,
-      db: buildDbHealthContract(),
+      db: await buildDbRuntimeReadiness(process.env, options.dbProbe),
       runtime: options.runtimeStore.summary()
     });
     return;

internal/cloud/server.test.mjs

@@ -1,4 +1,5 @@
 import assert from "node:assert/strict";
+import { createServer as createTcpServer } from "node:net";
 import test from "node:test";
 
 import { createCloudApiServer } from "./server.mjs";
@@ -26,6 +27,9 @@ test("cloud api exposes /health, /health/live, and /live probes", async () => {
     assert.equal(healthPayload.service.id, "hwlab-cloud-api");
     assert.equal(healthPayload.db.status, "blocked");
     assert.equal(healthPayload.db.connected, false);
+    assert.equal(healthPayload.db.liveConnected, false);
+    assert.equal(healthPayload.db.connectionAttempted, false);
+    assert.equal(healthPayload.db.connectionResult, "not_attempted_missing_env");
     assert.equal(healthPayload.runtime.durable, false);
     assert.equal(healthPayload.runtime.status, "degraded");
     assert.deepEqual(healthPayload.db.missingEnv, ["HWLAB_CLOUD_DB_URL", "HWLAB_CLOUD_DB_SSL_MODE"]);
@@ -64,10 +68,13 @@ test("cloud api exposes /health, /health/live, and /live probes", async () => {
   }
 });
 
-test("cloud api health reports DB env presence without leaking values", async () => {
+test("cloud api health reports DB env presence and live connection classification without leaking values", async () => {
   const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
   const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
-  process.env.HWLAB_CLOUD_DB_URL = "postgres://user:password@db.example.invalid/hwlab";
+  const fakeDb = createTcpServer((socket) => socket.end());
+  await new Promise((resolve) => fakeDb.listen(0, "127.0.0.1", resolve));
+  const dbPort = fakeDb.address().port;
+  process.env.HWLAB_CLOUD_DB_URL = `postgres://user:password@127.0.0.1:${dbPort}/hwlab`;
   process.env.HWLAB_CLOUD_DB_SSL_MODE = "require";
 
   const server = createCloudApiServer();
@@ -78,16 +85,21 @@ test("cloud api health reports DB env presence without leaking values", async ()
     const response = await fetch(`http://127.0.0.1:${port}/health`);
     const payload = await response.json();
     assert.equal(payload.status, "degraded");
-    assert.equal(payload.db.status, "degraded");
-    assert.equal(payload.db.connected, false);
-    assert.equal(payload.db.connectionChecked, false);
+    assert.equal(payload.db.status, "ready");
+    assert.equal(payload.db.connected, true);
+    assert.equal(payload.db.liveConnected, true);
+    assert.equal(payload.db.connectionChecked, true);
+    assert.equal(payload.db.connectionAttempted, true);
+    assert.equal(payload.db.connectionResult, "connected");
     assert.equal(payload.db.configReady, true);
-    assert.equal(payload.db.ready, false);
-    assert.equal(payload.db.evidence, "env_presence_only_no_live_db");
+    assert.equal(payload.db.ready, true);
+    assert.equal(payload.db.evidence, "live_db_tcp_connection_ready");
     assert.deepEqual(payload.db.missingEnv, []);
     assert.equal(JSON.stringify(payload.db).includes("password"), false);
-    assert.equal(JSON.stringify(payload.db).includes("db.example.invalid"), false);
+    assert.equal(JSON.stringify(payload.db).includes("127.0.0.1"), false);
+    assert.equal(JSON.stringify(payload.db).includes(String(dbPort)), false);
     assert.equal(payload.db.fields.find((field) => field.name === "HWLAB_CLOUD_DB_URL").redacted, true);
+    assert.equal(payload.db.connection.endpointRedacted, true);
   } finally {
     if (originalUrl === undefined) {
       delete process.env.HWLAB_CLOUD_DB_URL;
@@ -102,5 +114,8 @@ test("cloud api health reports DB env presence without leaking values", async ()
     await new Promise((resolve, reject) => {
       server.close((error) => (error ? reject(error) : resolve()));
     });
+    await new Promise((resolve, reject) => {
+      fakeDb.close((error) => (error ? reject(error) : resolve()));
+    });
   }
 });