fix: layer dev DB readiness evidence

Refs #49 Merged by commander after reviewing Code Queue task codex_1779422762669_1. Adds redacted DB readiness layering and refreshes DEV endpoint/report evidence. Follow-up: deploy/verify the new readiness probe in DEV via standard image CI/CD.
2026-05-22 12:27:01 +08:00
parent 4ebb1e4aa6
commit 357a68bb95
29 changed files with 877 additions and 196 deletions
@@ -5,7 +5,7 @@ import { mkdir, readFile, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { fileURLToPath } from "node:url";

-import { summarizeDbContract } from "../../internal/cloud/db-contract.mjs";
+import { DEV_DB_ENV_CONTRACT, summarizeDbContract } from "../../internal/cloud/db-contract.mjs";
 import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";

 const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
@@ -33,7 +33,8 @@ export async function runDevEdgeHealthSmoke(argv) {
      status: "not_run",
      classification: "not_run",
      blocker: "live network probes require --live",
-      contracts: await inspectContracts()
+      contracts: await inspectContracts(),
+      runtimeDbReadiness: notRunDbReadiness("live network probes require --live")
    };
    const report = await createDevGateReport(edgeHealth);
    await maybeWriteReport(report, args);
@@ -65,6 +66,7 @@ export async function runDevEdgeHealthSmoke(argv) {
    kubernetes,
    clusterDns
  };
+  edgeHealth.runtimeDbReadiness = inspectRuntimeDbReadiness(edgeHealth);
  Object.assign(edgeHealth, classify(edgeHealth));
  const report = await createDevGateReport(edgeHealth);
  await maybeWriteReport(report, args);
@@ -250,22 +252,39 @@ async function inspectContracts() {
 function inspectCloudApiDbContract(deploy) {
  const env = deploy.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {};
  const configReady = Boolean(env.HWLAB_CLOUD_DB_URL && env.HWLAB_CLOUD_DB_SSL_MODE);
+  const connection = {
+    attempted: false,
+    networkAttempted: false,
+    result: "manifest_only_not_attempted",
+    classification: "manifest_only",
+    probeType: "tcp-connect",
+    endpointRedacted: true,
+    valueRedacted: true,
+    timeoutMs: null,
+    durationMs: 0,
+    errorCode: null
+  };
  const db = {
    status: configReady ? "degraded" : "blocked",
    ready: false,
    configReady,
    connected: false,
+    liveConnected: false,
    connectionChecked: false,
+    connectionAttempted: false,
+    connectionResult: connection.result,
    fields: [
      {
        name: "HWLAB_CLOUD_DB_URL",
        present: Boolean(env.HWLAB_CLOUD_DB_URL),
-        redacted: true
+        redacted: true,
+        source: "k8s-secret-ref"
      },
      {
        name: "HWLAB_CLOUD_DB_SSL_MODE",
        present: Boolean(env.HWLAB_CLOUD_DB_SSL_MODE),
-        redacted: false
+        redacted: false,
+        source: "runtime-env"
      }
    ],
    missingEnv: ["HWLAB_CLOUD_DB_URL", "HWLAB_CLOUD_DB_SSL_MODE"].filter((name) => !env[name]),
@@ -275,18 +294,143 @@ function inspectCloudApiDbContract(deploy) {
        secretName: "hwlab-cloud-api-dev-db",
        secretKey: "database-url",
        present: env.HWLAB_CLOUD_DB_URL === "secretRef:hwlab-cloud-api-dev-db/database-url",
+        envInjected: false,
+        secretPresent: "not_observed_by_manifest",
+        secretKeyPresent: "declared",
        redacted: true
      }
    ],
+    connection,
    safety: {
+      secretsRead: false,
+      valuesRedacted: true,
      liveDbEvidence: false
-    }
+    },
+    blocker: configReady
+      ? "Manifest declares DB env/Secret references but does not prove runtime env injection or live DB connection"
+      : "Manifest is missing DB env/Secret references"
  };
  return {
    ...summarizeDbContract(db),
+    manifestConfigReady: configReady,
+    envInjected: false,
    secretMaterialRead: false,
    valuesRedacted: true,
-    note: "Manifest-only DB env placeholder; this is not live DB evidence."
+    note: "Manifest-only DB env placeholder; this is not runtime env injection or live DB evidence."
+  };
+}
+
+function notRunDbReadiness(blockerMessage) {
+  return {
+    status: "not_run",
+    configReady: false,
+    envInjected: false,
+    secret: unknownDbSecretPresence("not_observed"),
+    connectionAttempted: false,
+    connectionResult: "not_run",
+    liveConnected: false,
+    valuesRedacted: true,
+    secretMaterialRead: false,
+    liveDbEvidence: false,
+    blocker: blockerMessage,
+    evidence: []
+  };
+}
+
+function inspectRuntimeDbReadiness(edgeHealth) {
+  const db = selectRuntimeDb(edgeHealth.publicHttp);
+  return summarizeRuntimeDbReadiness(db, edgeHealth.kubernetes?.dbSecret ?? unknownDbSecretPresence("not_observed"));
+}
+
+function selectRuntimeDb(publicHttp) {
+  const live = publicHttp.find((probe) => probe.url.endsWith("/health/live") && probe.ok && probe.json?.db);
+  const health = publicHttp.find((probe) => probe.url.endsWith("/health") && probe.ok && probe.json?.db);
+  return live?.json?.db ?? health?.json?.db ?? null;
+}
+
+function summarizeRuntimeDbReadiness(db, secretPresence = unknownDbSecretPresence("not_observed")) {
+  if (!db) {
+    return {
+      status: "blocked",
+      configReady: false,
+      envInjected: false,
+      secret: secretPresence,
+      connectionAttempted: false,
+      connectionResult: "health_db_missing",
+      liveConnected: false,
+      valuesRedacted: true,
+      secretMaterialRead: false,
+      liveDbEvidence: false,
+      blocker: "cloud-api health is reachable but does not expose redacted DB readiness fields",
+      evidence: []
+    };
+  }
+
+  const requiredEnv = Array.isArray(db.fields) ? db.fields : [];
+  const envInjected = DEV_DB_ENV_CONTRACT.requiredEnv.every((name) =>
+    requiredEnv.some((field) => field.name === name && field.present === true)
+  );
+  const connectionAttempted = Boolean(db.connectionAttempted ?? db.connectionChecked);
+  const connectionResult = db.connectionResult ?? db.connection?.result ?? (connectionAttempted ? "unknown" : "not_attempted");
+  const liveConnected = Boolean(db.liveConnected ?? db.connected ?? db.ready);
+  const blockerMessage = liveConnected ? null : db.blocker ?? blockerForRuntimeDb(db, connectionAttempted, connectionResult);
+  const status = liveConnected ? "pass" : "blocked";
+  const secretRef = Array.isArray(db.secretRefs) ? db.secretRefs[0] : null;
+
+  return {
+    status,
+    configReady: Boolean(db.configReady),
+    envInjected,
+    secret: {
+      ...secretPresence,
+      secretName: secretRef?.secretName ?? DEV_DB_ENV_CONTRACT.secretRefs[0].secretName,
+      secretKey: secretRef?.secretKey ?? DEV_DB_ENV_CONTRACT.secretRefs[0].secretKey,
+      secretRefPresent: Boolean(secretRef?.present),
+      envInjected: Boolean(secretRef?.envInjected ?? secretRef?.present),
+      redacted: true
+    },
+    connectionAttempted,
+    connectionResult,
+    liveConnected,
+    valuesRedacted: db.safety?.valuesRedacted === true,
+    secretMaterialRead: false,
+    liveDbEvidence: Boolean(db.safety?.liveDbEvidence),
+    blocker: blockerMessage,
+    evidence: [
+      {
+        configReady: Boolean(db.configReady),
+        missingEnv: Array.isArray(db.missingEnv) ? db.missingEnv : [],
+        connectionAttempted,
+        connectionResult,
+        liveConnected,
+        endpointRedacted: db.connection?.endpointRedacted !== false,
+        valueRedacted: true
+      }
+    ]
+  };
+}
+
+function blockerForRuntimeDb(db, connectionAttempted, connectionResult) {
+  if (db.configReady !== true) {
+    const missing = Array.isArray(db.missingEnv) && db.missingEnv.length > 0 ? db.missingEnv.join(", ") : "required DB env";
+    return `cloud-api DB runtime env is not ready; missing ${missing}`;
+  }
+  if (!connectionAttempted) {
+    return "cloud-api DB env is injected, but runtime health has not attempted a live DB connection";
+  }
+  return `cloud-api DB live connection is blocked; connectionResult=${connectionResult}`;
+}
+
+function unknownDbSecretPresence(state) {
+  const ref = DEV_DB_ENV_CONTRACT.secretRefs[0];
+  return {
+    observable: false,
+    secretName: ref.secretName,
+    secretKey: ref.secretKey,
+    secretPresent: state,
+    secretKeyPresent: state,
+    secretValueRead: false,
+    redacted: true
  };
 }

@@ -299,6 +443,7 @@ async function inspectKubernetes() {
    services: [],
    endpoints: [],
    pods: [],
+    dbSecret: unknownDbSecretPresence("not_observed"),
    notes: []
  };
  if (!hasKubectl) {
@@ -316,6 +461,7 @@ async function inspectKubernetes() {
  await collectKubectlJson(result, "pods", [
    "-n", namespace, "get", "pods", "-l", "hwlab.pikastech.local/profile=dev", "-o", "json"
  ]);
+  await collectDbSecretPresence(result);
  return result;
 }

@@ -363,6 +509,46 @@ async function collectKubectlJson(result, kind, args) {
  }
 }

+async function collectDbSecretPresence(result) {
+  const ref = DEV_DB_ENV_CONTRACT.secretRefs[0];
+  const base = {
+    observable: true,
+    secretName: ref.secretName,
+    secretKey: ref.secretKey,
+    secretPresent: false,
+    secretKeyPresent: false,
+    secretValueRead: false,
+    redacted: true
+  };
+  const exists = await runCommand("kubectl", ["-n", namespace, "get", "secret", ref.secretName, "-o", "name"], {
+    timeoutMs: 5000
+  });
+  if (exists.exitCode !== 0) {
+    result.dbSecret = {
+      ...base,
+      command: "kubectl get secret -o name",
+      error: exists.stderr.trim() || "secret not observed"
+    };
+    result.notes.push(`kubectl DB Secret presence probe failed: ${exists.stderr.trim()}`);
+    return;
+  }
+
+  const keyPresence = await runCommand("kubectl", ["-n", namespace, "describe", "secret", ref.secretName], {
+    timeoutMs: 5000
+  });
+  const keyLinePattern = new RegExp(`^\\s*${escapeRegExp(ref.secretKey)}:\\s+\\d+\\s+bytes\\s*$`, "mu");
+  result.dbSecret = {
+    ...base,
+    secretPresent: true,
+    secretKeyPresent: keyPresence.exitCode === 0 && keyLinePattern.test(keyPresence.stdout),
+    command: "kubectl describe secret <name> (key-presence-only)",
+    error: keyPresence.exitCode === 0 ? null : keyPresence.stderr.trim()
+  };
+  if (keyPresence.exitCode !== 0) {
+    result.notes.push(`kubectl DB Secret key presence probe failed: ${keyPresence.stderr.trim()}`);
+  }
+}
+
 async function inspectClusterDns() {
  const results = [];
  for (const service of runtimeServices) {
@@ -386,7 +572,7 @@ function classify(report) {
  const publicHealth = report.publicHttp.find((probe) => probe.url.endsWith("/health"));

  if (publicHealth?.ok && publicHealth.json?.serviceId && publicHealth.json?.environment === ENVIRONMENT_DEV) {
-    return classifyReachableHealth(publicHealth.json);
+    return classifyReachableHealth(publicHealth.json, selectRuntimeDb(report.publicHttp));
  }
  if (publicEdgePort?.status === "error" && publicEdgePort.code === "ECONNREFUSED") {
    return classifyRefusedPort(publicEdgePort, frpsControl, tunnelHealth);
@@ -403,7 +589,7 @@ function classify(report) {
  return blocker("dns_port_firewall_blocker", publicHealth?.error?.message ?? "public DEV health did not return a usable response");
 }

-function classifyReachableHealth(json) {
+function classifyReachableHealth(json, runtimeDb = null) {
  const missingEvidence = [];
  if (!json.commit) missingEvidence.push("commit");
  if (!json.image) missingEvidence.push("image");
@@ -411,6 +597,19 @@ function classifyReachableHealth(json) {
  if (missingEvidence.length > 0) {
    return blocker("app_health_blocker", `public /health is reachable but missing ${missingEvidence.join(", ")} evidence`);
  }
+  const dbReadiness = summarizeRuntimeDbReadiness(runtimeDb ?? json.db);
+  if (dbReadiness.status !== "pass") {
+    return blocker("app_health_blocker", dbReadiness.blocker, {
+      likelyLayer: "cloud-api-db",
+      confidence: dbReadiness.configReady ? "high" : "medium",
+      likelyCause: dbReadiness.blocker,
+      evidence: dbReadiness.evidence,
+      notProven: dbReadiness.connectionAttempted
+        ? ["authenticated SQL query readiness"]
+        : ["live DB endpoint reachability", "authenticated SQL query readiness"],
+      nextTask: "Deploy cloud-api DB runtime readiness probe and/or repair DEV DB connectivity, then rerun the read-only health smoke without reading or printing the DB secret value."
+    });
+  }
  return { status: "pass", classification: "none", blocker: null };
 }

@@ -491,6 +690,10 @@ function summarizeTcpProbe(probe) {
  };
 }

+function escapeRegExp(value) {
+  return value.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&");
+}
+
 async function commandExists(command) {
  const result = await runCommand("command", ["-v", command], { timeoutMs: 3000 });
  return result.exitCode === 0;