pikasTech-HWLAB/internal/cloud/server-health.test.ts

import assert from "node:assert/strict";
import { createServer as createTcpServer } from "node:net";
import { mkdtemp, rm, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import { test } from "bun:test";

import { createCloudApiServer, ensureCodeAgentRuntimeBase } from "./server.ts";
import { createCodexStdioSessionManager } from "./codex-stdio-session.ts";
import {
  RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
  RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED,
  RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
  RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
  RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED
} from "../db/runtime-store.ts";
import {
  createFakeCodexCommand,
  durableBlockedRuntime,
  durableReadyRuntime,
  layerForBlockedCase,
  layerStatus,
  prepareFakeCodexHome
} from "./server-test-helpers.ts";

test("cloud api health reports DB env presence and live connection classification without leaking values", async () => {
  const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
  const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
  const originalAliasEnv = {
    HWLAB_CLOUD_DB_SERVICE_NAME: process.env.HWLAB_CLOUD_DB_SERVICE_NAME,
    HWLAB_CLOUD_DB_SERVICE_NAMESPACE: process.env.HWLAB_CLOUD_DB_SERVICE_NAMESPACE,
    HWLAB_CLOUD_DB_HOST: process.env.HWLAB_CLOUD_DB_HOST,
    HWLAB_CLOUD_DB_PORT: process.env.HWLAB_CLOUD_DB_PORT
  };
  const fakeDb = createTcpServer((socket) => socket.end());
  await new Promise((resolve) => fakeDb.listen(0, "127.0.0.1", resolve));
  const dbPort = fakeDb.address().port;
  process.env.HWLAB_CLOUD_DB_URL = `postgres://hwlab_test@127.0.0.1:${dbPort}/hwlab`;
  process.env.HWLAB_CLOUD_DB_SSL_MODE = "disable";
  delete process.env.HWLAB_CLOUD_DB_SERVICE_NAME;
  delete process.env.HWLAB_CLOUD_DB_SERVICE_NAMESPACE;
  delete process.env.HWLAB_CLOUD_DB_HOST;
  delete process.env.HWLAB_CLOUD_DB_PORT;

  const server = createCloudApiServer();
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/health`);
    const payload = await response.json();
    assert.equal(payload.status, "degraded");
    assert.equal(payload.db.status, "ready");
    assert.equal(payload.db.connected, true);
    assert.equal(payload.db.liveConnected, true);
    assert.equal(payload.db.liveDbEvidence, true);
    assert.equal(payload.db.connectionChecked, true);
    assert.equal(payload.db.connectionAttempted, true);
    assert.equal(payload.db.connectionResult, "connected");
    assert.equal(payload.db.endpointSource, "secret-url-host");
    assert.equal(payload.db.connection.endpointSource, "secret-url-host");
    assert.equal(payload.db.endpoint.authoritative.source, "secret-url-host");
    assert.equal(payload.db.endpoint.authoritative.env, "HWLAB_CLOUD_DB_URL");
    assert.equal(payload.db.optionalPublicDnsAlias.source, "optional-public-dns-alias");
    assert.equal(payload.db.optionalPublicDnsAlias.requiredForReadiness, false);
    assert.equal(payload.db.optionalPublicDnsAlias.usedForProbe, false);
    assert.equal(payload.db.optionalPublicDnsAlias.envPresent, false);
    assert.equal(payload.db.configReady, true);
    assert.equal(payload.db.ready, true);
    assert.equal(payload.db.evidence, "live_db_tcp_connection_ready");
    assert.equal(payload.db.safety.liveDbEvidence, true);
    assert.equal(payload.db.safety.liveDbConnectedEvidence, true);
    assert.equal(payload.runtime.adapter, "memory");
    assert.equal(payload.runtime.durable, false);
    assert.equal(payload.runtime.status, "degraded");
    assert.match(payload.runtime.reason, /process-local/u);
    assert.equal(payload.runtime.durabilityContract.blockedLayer, "adapter");
    assert.equal(payload.runtime.durabilityContract.dbLiveEvidenceIsDurabilityEvidence, false);
    assert.equal(payload.readiness.components.db, "ready");
    assert.equal(payload.readiness.components.runtime, "blocked");
    assert.equal(payload.readiness.durability.status, "blocked");
    assert.equal(payload.readiness.durability.dbLiveEvidenceObserved, true);
    assert.equal(payload.readiness.durability.dbLiveEvidenceIsDurabilityEvidence, false);
    assert.equal(payload.readiness.durability.blockedLayer, "adapter");
    assert.equal(payload.readiness.durability.liveRuntimeEvidence, false);
    assert.equal(payload.readiness.provider.status, "blocked");
    assert.equal(payload.readiness.dbDurable.status, "blocked");
    assert.equal(payload.readiness.dbDurable.dbLiveEvidenceObserved, true);
    assert.equal(payload.readiness.dbDurable.blocker, "runtime_durable_adapter_missing");
    assert.equal(payload.readiness.sessionRunner.status, "blocked");
    assert.equal(payload.readiness.codexStdio.status, "blocked");
    assert.equal(payload.db.redaction.valuesRedacted, true);
    assert.equal(payload.db.redaction.endpointRedacted, true);
    assert.equal(payload.db.redaction.secretMaterialRead, false);
    assert.deepEqual(payload.db.missingEnv, []);
    assert.equal(JSON.stringify(payload.db).includes("password"), false);
    assert.equal(JSON.stringify(payload.db).includes("127.0.0.1"), false);
    assert.equal(JSON.stringify(payload.db).includes(String(dbPort)), false);
    assert.equal(JSON.stringify(payload.db).includes("cloud-api-db.hwlab-dev.svc.cluster.local"), false);
    assert.equal(payload.db.fields.find((field) => field.name === "HWLAB_CLOUD_DB_URL").redacted, true);
    assert.equal(payload.db.connection.endpointRedacted, true);
  } finally {
    if (originalUrl === undefined) {
      delete process.env.HWLAB_CLOUD_DB_URL;
    } else {
      process.env.HWLAB_CLOUD_DB_URL = originalUrl;
    }
    if (originalSslMode === undefined) {
      delete process.env.HWLAB_CLOUD_DB_SSL_MODE;
    } else {
      process.env.HWLAB_CLOUD_DB_SSL_MODE = originalSslMode;
    }
    for (const [name, value] of Object.entries(originalAliasEnv)) {
      if (value === undefined) {
        delete process.env[name];
      } else {
        process.env[name] = value;
      }
    }
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    await new Promise((resolve, reject) => {
      fakeDb.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api health reports the v02 DB contract under the v02 profile", async () => {
  const originalEnv = {
    HWLAB_ENVIRONMENT: process.env.HWLAB_ENVIRONMENT,
    HWLAB_GITOPS_PROFILE: process.env.HWLAB_GITOPS_PROFILE,
    HWLAB_CLOUD_DB_URL: process.env.HWLAB_CLOUD_DB_URL,
    HWLAB_CLOUD_DB_SSL_MODE: process.env.HWLAB_CLOUD_DB_SSL_MODE,
    HWLAB_CLOUD_DB_SERVICE_NAME: process.env.HWLAB_CLOUD_DB_SERVICE_NAME,
    HWLAB_CLOUD_DB_SERVICE_NAMESPACE: process.env.HWLAB_CLOUD_DB_SERVICE_NAMESPACE,
    HWLAB_CLOUD_DB_HOST: process.env.HWLAB_CLOUD_DB_HOST,
    HWLAB_CLOUD_DB_PORT: process.env.HWLAB_CLOUD_DB_PORT
  };
  const fakeDb = createTcpServer((socket) => socket.end());
  await new Promise((resolve) => fakeDb.listen(0, "127.0.0.1", resolve));
  const dbPort = fakeDb.address().port;
  process.env.HWLAB_ENVIRONMENT = "v02";
  process.env.HWLAB_GITOPS_PROFILE = "v02";
  process.env.HWLAB_CLOUD_DB_URL = `postgres://hwlab_test@127.0.0.1:${dbPort}/hwlab_v02`;
  process.env.HWLAB_CLOUD_DB_SSL_MODE = "disable";
  delete process.env.HWLAB_CLOUD_DB_SERVICE_NAME;
  delete process.env.HWLAB_CLOUD_DB_SERVICE_NAMESPACE;
  delete process.env.HWLAB_CLOUD_DB_HOST;
  delete process.env.HWLAB_CLOUD_DB_PORT;

  const server = createCloudApiServer();
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/health`);
    const payload = await response.json();
    const dbJson = JSON.stringify(payload.db);
    assert.equal(payload.environment, "v02");
    assert.equal(payload.db.environment, "v02");
    assert.equal(payload.db.secretRefs[0].secretName, "hwlab-cloud-api-v02-db");
    assert.equal(payload.db.secretRefs[0].secretKey, "database-url");
    assert.equal(payload.db.optionalPublicDnsAlias.serviceName, "hwlab-v02-postgres");
    assert.equal(payload.db.optionalPublicDnsAlias.namespace, "hwlab-v02");
    assert.equal(payload.db.optionalPublicDnsAlias.requiredForReadiness, false);
    assert.equal(payload.db.optionalPublicDnsAlias.usedForProbe, false);
    assert.equal(payload.db.safety.environment, "v02");
    assert.equal(payload.db.safety.devOnly, false);
    assert.equal(payload.db.redaction.secretMaterialRead, false);
    assert.equal(dbJson.includes("hwlab-cloud-api-dev-db"), false);
    assert.equal(dbJson.includes("hwlab-dev"), false);
    assert.equal(dbJson.includes("127.0.0.1"), false);
    assert.equal(dbJson.includes(String(dbPort)), false);
  } finally {
    for (const [name, value] of Object.entries(originalEnv)) {
      if (value === undefined) {
        delete process.env[name];
      } else {
        process.env[name] = value;
      }
    }
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    await new Promise((resolve, reject) => {
      fakeDb.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api health separates DB connected from durable runtime schema readiness", async () => {
  const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
  const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
  const fakeDb = createTcpServer((socket) => socket.end());
  await new Promise((resolve) => fakeDb.listen(0, "127.0.0.1", resolve));
  const dbPort = fakeDb.address().port;
  process.env.HWLAB_CLOUD_DB_URL = `postgres://hwlab_test@127.0.0.1:${dbPort}/hwlab`;
  process.env.HWLAB_CLOUD_DB_SSL_MODE = "disable";

  const server = createCloudApiServer({
    runtimeStore: {
      async readiness() {
        return {
          adapter: "postgres",
          durable: false,
          durableRequested: true,
          durableCapable: false,
          ready: false,
          status: "blocked",
          blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
          reason: "test runtime schema is missing",
          liveRuntimeEvidence: false,
          fixtureEvidence: false,
          connection: {
            queryAttempted: true,
            queryResult: "schema_blocked",
            endpointRedacted: true,
            valueRedacted: true
          },
          schema: {
            checked: true,
            ready: false,
            missingTables: ["gateway_sessions"],
            missingColumns: ["gateway_sessions.gateway_session_json"]
          },
          migration: {
            checked: false,
            ready: false,
            missing: true
          },
          counts: {}
        };
      }
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/health/live`);
    const payload = await response.json();
    assert.equal(response.status, 200);
    assert.equal(payload.status, "degraded");
    assert.equal(payload.db.connected, true);
    assert.equal(payload.db.liveDbEvidence, true);
    assert.equal(payload.runtime.durable, false);
    assert.equal(payload.runtime.durableRequested, true);
    assert.equal(payload.runtime.ready, false);
    assert.equal(payload.runtime.blocker, RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED);
    assert.equal(payload.readiness.components.db, "ready");
    assert.equal(payload.readiness.components.runtime, "blocked");
    assert.equal(payload.readiness.durability.status, "blocked");
    assert.equal(payload.readiness.durability.dbLiveEvidenceObserved, true);
    assert.equal(payload.readiness.durability.dbLiveEvidenceIsDurabilityEvidence, false);
    assert.equal(payload.readiness.durability.blockedLayer, "schema");
    assert.equal(payload.readiness.durability.queryAttempted, true);
    assert.equal(payload.readiness.durability.queryResult, "schema_blocked");
    assert.ok(payload.blockerCodes.includes(RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED));
    assert.equal(JSON.stringify(payload).includes("password"), false);
  } finally {
    if (originalUrl === undefined) {
      delete process.env.HWLAB_CLOUD_DB_URL;
    } else {
      process.env.HWLAB_CLOUD_DB_URL = originalUrl;
    }
    if (originalSslMode === undefined) {
      delete process.env.HWLAB_CLOUD_DB_SSL_MODE;
    } else {
      process.env.HWLAB_CLOUD_DB_SSL_MODE = originalSslMode;
    }
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    await new Promise((resolve, reject) => {
      fakeDb.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api health keeps DB live evidence separate when durable adapter query is blocked", async () => {
  const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
  const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
  const fakeDb = createTcpServer((socket) => socket.end());
  await new Promise((resolve) => fakeDb.listen(0, "127.0.0.1", resolve));
  const dbPort = fakeDb.address().port;
  process.env.HWLAB_CLOUD_DB_URL = `postgres://hwlab_test:password@127.0.0.1:${dbPort}/hwlab`;
  process.env.HWLAB_CLOUD_DB_SSL_MODE = "disable";

  const server = createCloudApiServer({
    runtimeStore: {
      async readiness() {
        return {
          adapter: "postgres",
          durable: false,
          durableRequested: true,
          durableCapable: false,
          ready: false,
          status: "blocked",
          blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
          reason: "test runtime durable read query is blocked",
          liveRuntimeEvidence: false,
          fixtureEvidence: false,
          connection: {
            queryAttempted: true,
            queryResult: "query_blocked",
            endpointRedacted: true,
            valueRedacted: true,
            errorCode: "57014"
          },
          schema: {
            checked: true,
            ready: true,
            missingTables: [],
            missingColumns: []
          },
          migration: {
            checked: true,
            ready: true,
            missing: false
          },
          gates: {
            auth: { checked: true, ready: true, status: "ready", blocker: null },
            schema: { checked: true, ready: true, status: "ready", blocker: null },
            migration: { checked: true, ready: true, status: "ready", blocker: null },
            durability: {
              checked: true,
              ready: false,
              status: "blocked",
              blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED
            }
          },
          counts: {}
        };
      }
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/health/live`);
    const payload = await response.json();
    assert.equal(response.status, 200);
    assert.equal(payload.status, "degraded");
    assert.equal(payload.db.ready, true);
    assert.equal(payload.db.connected, true);
    assert.equal(payload.db.liveDbEvidence, true);
    assert.equal(payload.runtime.durable, false);
    assert.equal(payload.runtime.ready, false);
    assert.equal(payload.runtime.blocker, RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED);
    assert.equal(payload.readiness.components.db, "ready");
    assert.equal(payload.readiness.components.runtime, "blocked");
    assert.equal(payload.readiness.durability.status, "blocked");
    assert.equal(payload.readiness.durability.dbLiveEvidenceObserved, true);
    assert.equal(payload.readiness.durability.dbLiveEvidenceIsDurabilityEvidence, false);
    assert.equal(payload.readiness.durability.blockedLayer, "durability_query");
    assert.equal(payload.readiness.durability.queryAttempted, true);
    assert.equal(payload.readiness.durability.queryResult, "query_blocked");
    assert.equal(payload.readiness.durability.liveRuntimeEvidence, false);
    assert.ok(payload.blockerCodes.includes(RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED));
    assert.equal(payload.readiness.dbDurable.status, "blocked");
    assert.equal(payload.readiness.dbDurable.blocker, RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED);
    assert.equal(payload.readiness.codeAgent.currentBlockers.includes(RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED), true);
    assert.equal(JSON.stringify(payload.db).includes("password"), false);
    assert.equal(JSON.stringify(payload.db).includes("127.0.0.1"), false);
    assert.equal(JSON.stringify(payload.db).includes(String(dbPort)), false);
  } finally {
    if (originalUrl === undefined) {
      delete process.env.HWLAB_CLOUD_DB_URL;
    } else {
      process.env.HWLAB_CLOUD_DB_URL = originalUrl;
    }
    if (originalSslMode === undefined) {
      delete process.env.HWLAB_CLOUD_DB_SSL_MODE;
    } else {
      process.env.HWLAB_CLOUD_DB_SSL_MODE = originalSslMode;
    }
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    await new Promise((resolve, reject) => {
      fakeDb.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api health contract distinguishes durable SSL, auth, schema, migration, and query blockers", async () => {
  const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
  const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
  process.env.HWLAB_CLOUD_DB_URL = "redacted";
  process.env.HWLAB_CLOUD_DB_SSL_MODE = "require";

  const cases = [
    {
      blocker: RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED,
      blockedLayer: "ssl",
      queryResult: "ssl_negotiation_blocked",
      gates: {
        ssl: "blocked",
        auth: "not_checked",
        schema: "not_checked",
        migration: "not_checked",
        durability: "blocked"
      }
    },
    {
      blocker: RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
      blockedLayer: "auth",
      queryResult: "auth_blocked",
      gates: {
        ssl: "ready",
        auth: "blocked",
        schema: "not_checked",
        migration: "not_checked",
        durability: "blocked"
      }
    },
    {
      blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
      blockedLayer: "schema",
      queryResult: "schema_blocked",
      gates: {
        ssl: "ready",
        auth: "ready",
        schema: "blocked",
        migration: "not_checked",
        durability: "blocked"
      }
    },
    {
      blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED,
      blockedLayer: "migration",
      queryResult: "migration_blocked",
      gates: {
        ssl: "ready",
        auth: "ready",
        schema: "ready",
        migration: "blocked",
        durability: "blocked"
      }
    },
    {
      blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
      blockedLayer: "durability_query",
      queryResult: "query_blocked",
      gates: {
        ssl: "ready",
        auth: "ready",
        schema: "ready",
        migration: "ready",
        durability: "blocked"
      }
    }
  ];

  try {
    for (const testCase of cases) {
      const server = createCloudApiServer({
        dbProbe: {
          probe: async ({ endpointSource, timeoutMs }) => ({
            attempted: true,
            networkAttempted: true,
            endpointSource,
            probeType: "tcp-connect",
            endpointRedacted: true,
            valueRedacted: true,
            timeoutMs,
            durationMs: 0,
            result: "connected",
            classification: "tcp_connected",
            errorCode: null,
            missingEnv: []
          })
        },
        runtimeStore: {
          async readiness() {
            return durableBlockedRuntime(testCase);
          }
        }
      });
      await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

      try {
        const { port } = server.address();
        const response = await fetch(`http://127.0.0.1:${port}/health/live`);
        const payload = await response.json();
        assert.equal(response.status, 200);
        assert.equal(payload.status, "degraded");
        assert.equal(payload.db.ready, true);
        assert.equal(payload.db.liveDbEvidence, true);
        assert.equal(payload.runtime.blocker, testCase.blocker);
        assert.equal(payload.runtime.connection.queryResult, testCase.queryResult);
        assert.equal(payload.runtime.gates.ssl.status, testCase.gates.ssl);
        assert.equal(payload.runtime.gates.auth.status, testCase.gates.auth);
        assert.equal(payload.runtime.gates.schema.status, testCase.gates.schema);
        assert.equal(payload.runtime.gates.migration.status, testCase.gates.migration);
        assert.equal(payload.runtime.gates.durability.status, testCase.gates.durability);
        assert.equal(payload.runtime.durabilityContract.blockedLayer, testCase.blockedLayer);
        assert.equal(payload.db.runtimeReadiness.blockedLayer, testCase.blockedLayer);
        assert.equal(payload.db.runtimeReadiness.queryResult, testCase.queryResult);
        assert.equal(payload.db.runtimeReadiness.dbLiveEvidenceIsDurabilityEvidence, false);
        assert.equal(payload.db.readinessLayers.ssl.status, layerStatus(testCase.gates.ssl));
        assert.equal(payload.db.readinessLayers.auth.status, layerStatus(testCase.gates.auth));
        assert.equal(payload.db.readinessLayers.schema.status, layerStatus(testCase.gates.schema));
        assert.equal(payload.db.readinessLayers.migration.status, layerStatus(testCase.gates.migration));
        assert.equal(payload.db.readinessLayers.durability.status, layerStatus(testCase.gates.durability));
        assert.equal(payload.db.readinessLayers[layerForBlockedCase(testCase.blockedLayer)].result, testCase.queryResult);
        assert.equal(payload.readiness.durability.blockedLayer, testCase.blockedLayer);
        assert.equal(payload.readiness.durability.queryResult, testCase.queryResult);
        assert.equal(payload.readiness.durability.secretMaterialRead, false);
        assert.ok(payload.blockerCodes.includes(testCase.blocker));
      } finally {
        await new Promise((resolve, reject) => {
          server.close((error) => (error ? reject(error) : resolve()));
        });
      }
    }
  } finally {
    if (originalUrl === undefined) {
      delete process.env.HWLAB_CLOUD_DB_URL;
    } else {
      process.env.HWLAB_CLOUD_DB_URL = originalUrl;
    }
    if (originalSslMode === undefined) {
      delete process.env.HWLAB_CLOUD_DB_SSL_MODE;
    } else {
      process.env.HWLAB_CLOUD_DB_SSL_MODE = originalSslMode;
    }
  }
});

test("cloud api health does not treat DB env presence-only as live readiness", async () => {
  const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
  const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
  const originalProbeDisabled = process.env.HWLAB_CLOUD_DB_PROBE_DISABLED;
  process.env.HWLAB_CLOUD_DB_URL = "postgres://user:password@db.example.invalid:5432/hwlab";
  process.env.HWLAB_CLOUD_DB_SSL_MODE = "disable";
  delete process.env.HWLAB_CLOUD_DB_PROBE_DISABLED;

  const server = createCloudApiServer();
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/health/live`);
    const payload = await response.json();
    assert.equal(response.status, 200);
    assert.equal(payload.db.configReady, true);
    assert.equal(payload.db.connected, false);
    assert.equal(payload.db.liveConnected, false);
    assert.equal(payload.db.liveDbEvidence, false);
    assert.equal(payload.db.safety.liveDbEvidence, false);
    assert.equal(payload.db.ready, false);
    assert.equal(payload.db.connectionAttempted, true);
    assert.equal(payload.db.connectionResult, "forbidden_runtime_host");
    assert.equal(payload.db.evidence, "live_db_tcp_connection_blocked");
    assert.equal(payload.db.endpointSource, "secret-url-host");
    assert.equal(payload.db.connection.endpointSource, "secret-url-host");
    assert.equal(payload.db.connection.networkAttempted, false);
    assert.equal(payload.db.connection.classification, "db_url_forbidden_invalid_host");
    assert.equal(payload.db.readinessLayers.dns.status, "not_proven");
    assert.equal(JSON.stringify(payload.db).includes("password"), false);
    assert.equal(JSON.stringify(payload.db).includes("db.example.invalid"), false);
  } finally {
    if (originalUrl === undefined) {
      delete process.env.HWLAB_CLOUD_DB_URL;
    } else {
      process.env.HWLAB_CLOUD_DB_URL = originalUrl;
    }
    if (originalSslMode === undefined) {
      delete process.env.HWLAB_CLOUD_DB_SSL_MODE;
    } else {
      process.env.HWLAB_CLOUD_DB_SSL_MODE = originalSslMode;
    }
    if (originalProbeDisabled === undefined) {
      delete process.env.HWLAB_CLOUD_DB_PROBE_DISABLED;
    } else {
      process.env.HWLAB_CLOUD_DB_PROBE_DISABLED = originalProbeDisabled;
    }
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1 describes Code Agent egress blocker without leaking API key", async () => {
  const server = createCloudApiServer({
    env: {
      OPENAI_API_KEY: "test-openai-key-material",
      HWLAB_CODE_AGENT_PROVIDER: "codex-stdio",
      HWLAB_CODE_AGENT_MODEL: "gpt-test",
      HWLAB_CODE_AGENT_OPENAI_BASE_URL: "https://api.openai.com/v1/responses"
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1`);
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.codeAgent.status, "blocked");
    assert.equal(payload.codeAgent.egress.directPublicOpenAi, true);
    assert.match(payload.codeAgent.blocker, /Codex stdio|Codex app-server stdio|long-lived|Codex CLI command|stdio protocol/u);
    assert.equal(payload.codeAgent.runner.ready, false);
    assert.equal(payload.codeAgent.safety.secretMaterialRead, false);
    assert.equal(JSON.stringify(payload).includes("test-openai-key-material"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api health reports provider, durable DB, and codex stdio ready when runtime contract passes", async () => {
  const workspace = await mkdtemp(path.join(os.tmpdir(), "hwlab-health-codex-stdio-"));
  const codexHome = path.join(workspace, "codex-home");
  const fakeCodex = await createFakeCodexCommand();
  await prepareFakeCodexHome(codexHome);
  const manager = createCodexStdioSessionManager({
    createRpcClient: async () => ({
      async initialize() {
        return { tools: ["codex", "codex-reply"] };
      },
      async listTools() {
        return ["codex", "codex-reply"];
      },
      close() {}
    })
  });
  const server = createCloudApiServer({
    env: {
      PATH: process.env.PATH,
      OPENAI_API_KEY: "test-openai-key-material",
      CODEX_HOME: codexHome,
      HWLAB_CODE_AGENT_PROVIDER: "codex-stdio",
      HWLAB_CODE_AGENT_MODEL: "gpt-test",
      HWLAB_CODE_AGENT_CODEX_COMMAND: fakeCodex.command,
      HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED: "1",
      HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR: "repo-owned",
      HWLAB_CODE_AGENT_CODEX_WORKSPACE: workspace,
      HWLAB_CODE_AGENT_WORKSPACE: workspace,
      HWLAB_CODE_AGENT_OPENAI_BASE_URL: "http://127.0.0.1:17680/v1/responses",
      HWLAB_CLOUD_DB_URL: "postgres://hwlab_test:password@db.internal.local:5432/hwlab",
      HWLAB_CLOUD_DB_SSL_MODE: "disable"
    },
    dbProbe: {
      probe: async ({ endpointSource, timeoutMs }) => ({
        attempted: true,
        networkAttempted: true,
        endpointSource,
        probeType: "tcp-connect",
        endpointRedacted: true,
        valueRedacted: true,
        timeoutMs,
        durationMs: 1,
        result: "connected",
        classification: "tcp_connected",
        errorCode: null,
        missingEnv: []
      })
    },
    runtimeStore: {
      async readiness() {
        return durableReadyRuntime();
      }
    },
    codexStdioManager: manager
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/health/live`);
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.status, "ok");
    assert.equal(payload.db.ready, true);
    assert.equal(payload.db.liveDbEvidence, true);
    assert.equal(payload.runtime.ready, true);
    assert.equal(payload.runtime.durable, true);
    assert.equal(payload.readiness.provider.status, "ready");
    assert.equal(payload.readiness.provider.ready, true);
    assert.equal(payload.readiness.provider.blocker, null);
    assert.equal(payload.readiness.dbDurable.status, "ready");
    assert.equal(payload.readiness.dbDurable.ready, true);
    assert.equal(payload.readiness.sessionRunner.status, "codex_stdio_ready");
    assert.equal(payload.readiness.sessionRunner.ready, true);
    assert.equal(payload.readiness.sessionRunner.codexStdio, true);
    assert.equal(payload.readiness.sessionRunner.writeCapable, true);
    assert.equal(payload.readiness.codexStdio.status, "ready");
    assert.equal(payload.readiness.codexStdio.ready, true);
    assert.deepEqual(payload.readiness.codeAgent.currentBlockers, []);
    assert.equal(payload.readiness.codeAgent.providerReady, true);
    assert.equal(payload.readiness.codeAgent.durableDbReady, true);
    assert.equal(payload.readiness.codeAgent.sessionRunnerReady, true);
    assert.equal(payload.readiness.codeAgent.codexStdioFeasible, true);
    assert.equal(payload.codeAgent.ready, true);
    assert.equal(payload.codeAgent.status, "codex-stdio-feasible");
    assert.equal(payload.codeAgent.longLivedSessionGate.status, "pass");
    assert.equal(payload.codeAgent.codexStdio.runtimeContract.binary.status, "present");
    assert.deepEqual(payload.codeAgent.codexStdio.protocol.toolsObserved, []);
    assert.equal(payload.codeAgent.codexStdio.commandProbe.ready, true);
    assert.equal(payload.codeAgent.codexStdio.runtimeContract.commandProbe.ready, true);
    assert.equal(payload.readiness.codexStdio.commandProbeReady, true);
    assert.equal(JSON.stringify(payload).includes("provider_unavailable"), false);
    assert.equal(JSON.stringify(payload).includes("dns_resolution_failed"), false);
    assert.equal(JSON.stringify(payload).includes("runtime_durable_adapter_auth_blocked"), false);
    assert.equal(JSON.stringify(payload).includes("test-openai-key-material"), false);
    assert.equal(JSON.stringify(payload).includes("password"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    await rm(workspace, { recursive: true, force: true });
    await rm(fakeCodex.root, { recursive: true, force: true });
  }
});

test("cloud api health blocks full Code Agent readiness when Codex stdio command probe fails", async () => {
  const workspace = await mkdtemp(path.join(os.tmpdir(), "hwlab-health-codex-stdio-probe-fail-"));
  const codexHome = path.join(workspace, "codex-home");
  const workspaceFile = path.join(workspace, "workspace-file");
  const fakeCodex = await createFakeCodexCommand();
  await prepareFakeCodexHome(codexHome);
  await writeFile(workspaceFile, "not a directory\n", "utf8");
  const manager = createCodexStdioSessionManager({
    createRpcClient: async () => ({
      async initialize() {
        return { tools: ["codex", "codex-reply"] };
      },
      async listTools() {
        return ["codex", "codex-reply"];
      },
      close() {}
    })
  });
  const server = createCloudApiServer({
    env: {
      PATH: process.env.PATH,
      OPENAI_API_KEY: "test-openai-key-material",
      CODEX_HOME: codexHome,
      HWLAB_CODE_AGENT_PROVIDER: "codex-stdio",
      HWLAB_CODE_AGENT_MODEL: "gpt-test",
      HWLAB_CODE_AGENT_CODEX_COMMAND: fakeCodex.command,
      HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED: "1",
      HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR: "repo-owned",
      HWLAB_CODE_AGENT_CODEX_WORKSPACE: workspaceFile,
      HWLAB_CODE_AGENT_WORKSPACE: workspaceFile,
      HWLAB_CODE_AGENT_OPENAI_BASE_URL: "http://127.0.0.1:17680/v1/responses",
      HWLAB_CLOUD_DB_URL: "postgres://hwlab_test:password@db.internal.local:5432/hwlab",
      HWLAB_CLOUD_DB_SSL_MODE: "disable"
    },
    dbProbe: {
      probe: async ({ endpointSource, timeoutMs }) => ({
        attempted: true,
        networkAttempted: true,
        endpointSource,
        probeType: "tcp-connect",
        endpointRedacted: true,
        valueRedacted: true,
        timeoutMs,
        durationMs: 1,
        result: "connected",
        classification: "tcp_connected",
        errorCode: null,
        missingEnv: []
      })
    },
    runtimeStore: {
      async readiness() {
        return durableReadyRuntime();
      }
    },
    codexStdioManager: manager
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/health/live`);
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.status, "degraded");
    assert.equal(payload.ready, false);
    assert.equal(payload.readiness.components.codeAgent, "blocked");
    assert.equal(payload.readiness.codexStdio.status, "blocked");
    assert.equal(payload.readiness.codexStdio.ready, false);
    assert.equal(payload.readiness.codexStdio.commandProbeReady, false);
    assert.ok(payload.readiness.codexStdio.blockerCodes.includes("codex_stdio_command_probe_failed"));
    assert.ok(payload.blockerCodes.includes("codex_stdio_command_probe_failed"));
    assert.equal(payload.codeAgent.ready, false);
    assert.equal(payload.codeAgent.codexStdio.commandProbe.ready, false);
    assert.equal(payload.codeAgent.codexStdio.commandProbe.status, "blocked");
    assert.equal(JSON.stringify(payload).includes("test-openai-key-material"), false);
    assert.equal(JSON.stringify(payload).includes("password"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    await rm(workspace, { recursive: true, force: true });
    await rm(fakeCodex.root, { recursive: true, force: true });
  }
});

test("cloud api REST JSON-RPC bridge exposes unknown serviceId reason for Code Agent internal calls", async () => {
  const server = createCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_CODE_AGENT_PROVIDER: "codex-stdio"
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/rpc/hardware.invoke.shell`, {
      method: "POST",
      headers: {
        "content-type": "application/json",
        "x-trace-id": "trc_server-test-invalid-service-id",
        "x-source-service-id": "hwlab-code-agent-hotfix"
      },
      body: JSON.stringify({
        projectId: "prj_mvp_topology",
        gatewaySessionId: "gws_DESKTOP-1MHOD9I",
        resourceId: "res_windows_host",
        capabilityId: "cap_windows_cmd_exec",
        input: {
          command: "cmd /c echo should-not-dispatch"
        }
      })
    });
    assert.equal(response.status, 400);
    const payload = await response.json();
    assert.equal(payload.error.code, -32600);
    assert.equal(payload.error.message, "Invalid JSON-RPC request");
    assert.match(payload.error.data.reason, /unknown serviceId "hwlab-code-agent-hotfix"/u);
    assert.equal(JSON.stringify(payload).includes("should-not-dispatch"), false);
    assert.equal(JSON.stringify(payload).includes("sk-"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("AgentRun adapter does not initialize repo-owned Codex stdio runtime defaults", () => {
  const env = {
    HWLAB_CODE_AGENT_ADAPTER: "agentrun-v01"
  };
  ensureCodeAgentRuntimeBase(env);
  assert.equal(env.HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED, undefined);
  assert.equal(env.HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR, undefined);
  assert.equal(env.HWLAB_CODE_AGENT_CODEX_WORKSPACE, undefined);
  assert.equal(env.CODEX_HOME, undefined);
  assert.equal(env.HWLAB_PREINSTALLED_SKILLS_DIR, "/app/skills");
  assert.equal(env.HWLAB_USER_SKILLS_DIR, "/data/user-skills");
});

test("explicit Codex stdio mode keeps local runtime defaults available", () => {
  const env = {
    HWLAB_CODE_AGENT_PROVIDER: "codex-stdio"
  };
  ensureCodeAgentRuntimeBase(env);
  assert.equal(env.HWLAB_CODE_AGENT_CODEX_STDIO_ENABLED, "1");
  assert.equal(env.HWLAB_CODE_AGENT_CODEX_STDIO_SUPERVISOR, "repo-owned");
  assert.equal(env.HWLAB_CODE_AGENT_CODEX_WORKSPACE, "/workspace/hwlab");
  assert.equal(env.CODEX_HOME, "/codex-home");
});