pikasTech-HWLAB/internal/cloud/server.test.mjs

import assert from "node:assert/strict";
import { createServer as createHttpServer } from "node:http";
import { createServer as createTcpServer } from "node:net";
import { mkdtemp, mkdir, writeFile } from "node:fs/promises";
import os from "node:os";
import path from "node:path";
import test from "node:test";

import { createCloudApiServer } from "./server.mjs";
import { createCloudRuntimeStore } from "../db/runtime-store.mjs";
import {
  RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
  RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED,
  RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
  RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
  RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED
} from "../db/runtime-store.mjs";
import { HWLAB_M3_IO_CAPABILITY_LEVELS } from "../../skills/hwlab-agent-runtime/scripts/src/m3-io-skill-client.mjs";

test("cloud api exposes /health, /health/live, and /live probes", async () => {
  const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
  const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
  delete process.env.HWLAB_CLOUD_DB_URL;
  delete process.env.HWLAB_CLOUD_DB_SSL_MODE;

  const server = createCloudApiServer({
    env: {
      PATH: "",
      HWLAB_CODE_AGENT_PROVIDER: "codex-cli",
      HWLAB_CODE_AGENT_MODEL: "gpt-test"
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();

    const health = await fetch(`http://127.0.0.1:${port}/health`);
    assert.equal(health.status, 200);
    const healthPayload = await health.json();
    assert.equal(healthPayload.serviceId, "hwlab-cloud-api");
    assert.equal(healthPayload.environment, "dev");
    assert.equal(healthPayload.status, "degraded");
    assert.equal(healthPayload.commit.id.length > 0, true);
    assert.equal(healthPayload.image.reference.length > 0, true);
    assert.equal(healthPayload.service.id, "hwlab-cloud-api");
    assert.equal(healthPayload.db.status, "blocked");
    assert.equal(healthPayload.db.connected, false);
    assert.equal(healthPayload.db.liveConnected, false);
    assert.equal(healthPayload.db.liveDbEvidence, false);
    assert.equal(healthPayload.db.connectionAttempted, false);
    assert.equal(healthPayload.db.connectionResult, "not_attempted_missing_env");
    assert.equal(healthPayload.db.endpointSource, "secret-url-host");
    assert.equal(healthPayload.db.endpoint.authoritative.source, "secret-url-host");
    assert.equal(healthPayload.db.endpoint.authoritative.env, "HWLAB_CLOUD_DB_URL");
    assert.equal(healthPayload.db.endpoint.authoritative.usedForProbe, true);
    assert.equal(healthPayload.db.optionalPublicDnsAlias.source, "optional-public-dns-alias");
    assert.equal(healthPayload.db.optionalPublicDnsAlias.requiredForReadiness, false);
    assert.equal(healthPayload.db.optionalPublicDnsAlias.usedForProbe, false);
    assert.equal(healthPayload.db.redaction.valuesRedacted, true);
    assert.equal(healthPayload.db.redaction.secretMaterialRead, false);
    assert.equal(healthPayload.db.safety.liveDbEvidence, false);
    assert.equal(healthPayload.runtime.durable, false);
    assert.equal(healthPayload.runtime.status, "degraded");
    assert.equal(healthPayload.runtime.durabilityContract.dbLiveEvidenceIsDurabilityEvidence, false);
    assert.equal(healthPayload.runtime.durabilityContract.blockedLayer, "adapter");
    assert.equal(healthPayload.readiness.contractVersion, "v3");
    assert.equal(healthPayload.readiness.durability.status, "blocked");
    assert.equal(healthPayload.readiness.durability.dbLiveEvidenceObserved, false);
    assert.equal(healthPayload.readiness.durability.dbLiveEvidenceIsDurabilityEvidence, false);
    assert.equal(healthPayload.readiness.durability.blockedLayer, "adapter");
    assert.equal(healthPayload.codeAgent.status, "partial");
    assert.equal(healthPayload.codeAgent.agentKind, "controlled-readonly-session-registry");
    assert.equal(healthPayload.codeAgent.partialReady, true);
    assert.match(healthPayload.codeAgent.blocker, /long-lived Codex stdio|Codex stdio/u);
    assert.equal(healthPayload.codeAgent.reason, "codex_stdio_supervisor_disabled");
    assert.equal(healthPayload.codeAgent.runner.kind, "hwlab-readonly-runner");
    assert.equal(healthPayload.codeAgent.runner.ready, true);
    assert.equal(healthPayload.codeAgent.capabilityLevel, "read-only-session-tools");
    assert.equal(healthPayload.codeAgent.sessionRegistry.status, "available");
    assert.equal(healthPayload.codeAgent.longLivedSessionGate.status, "blocked");
    assert.ok(healthPayload.codeAgent.longLivedSessionGate.blockers.some((blocker) => blocker.code === "codex_stdio_supervisor_disabled"));
    assert.deepEqual(healthPayload.codeAgent.missingEnv, ["OPENAI_API_KEY"]);
    assert.equal(healthPayload.codeAgent.secretRefs[0].secretName, "hwlab-code-agent-provider");
    assert.equal(healthPayload.codeAgent.secretRefs[0].secretKey, "openai-api-key");
    assert.equal(healthPayload.codeAgent.secretRefs[0].redacted, true);
    assert.equal(healthPayload.codeAgent.egress.directPublicOpenAi, false);
    assert.equal(healthPayload.codeAgent.safety.secretMaterialRead, false);
    assert.equal(JSON.stringify(healthPayload.codeAgent).includes("sk-"), false);
    assert.deepEqual(healthPayload.db.missingEnv, ["HWLAB_CLOUD_DB_URL", "HWLAB_CLOUD_DB_SSL_MODE"]);
    assert.equal(healthPayload.db.secretRefs[0].secretName, "hwlab-cloud-api-dev-db");
    assert.equal(healthPayload.db.secretRefs[0].redacted, true);

    const healthLive = await fetch(`http://127.0.0.1:${port}/health/live`);
    assert.equal(healthLive.status, 200);
    const healthLivePayload = await healthLive.json();
    assert.equal(healthLivePayload.serviceId, "hwlab-cloud-api");
    assert.equal(healthLivePayload.status, "degraded");
    assert.equal(healthLivePayload.commit.id.length > 0, true);
    assert.equal(healthLivePayload.db.ready, false);
    assert.equal(healthLivePayload.codeAgent.status, "partial");
    assert.equal(healthLivePayload.codeAgent.ready, false);
    assert.equal(healthLivePayload.codeAgent.partialReady, true);
    assert.equal(healthLivePayload.codeAgent.capabilityLevel, "read-only-session-tools");

    const readiness = await fetch(`http://127.0.0.1:${port}/health/live`);
    assert.equal(readiness.status, 200);
    assert.equal((await readiness.json()).status, "degraded");

    const live = await fetch(`http://127.0.0.1:${port}/live`);
    assert.equal(live.status, 200);
    assert.equal((await live.json()).status, "live");
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    if (originalUrl === undefined) {
      delete process.env.HWLAB_CLOUD_DB_URL;
    } else {
      process.env.HWLAB_CLOUD_DB_URL = originalUrl;
    }
    if (originalSslMode === undefined) {
      delete process.env.HWLAB_CLOUD_DB_SSL_MODE;
    } else {
      process.env.HWLAB_CLOUD_DB_SSL_MODE = originalSslMode;
    }
  }
});

test("cloud api health reports DB env presence and live connection classification without leaking values", async () => {
  const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
  const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
  const originalAliasEnv = {
    HWLAB_CLOUD_DB_SERVICE_NAME: process.env.HWLAB_CLOUD_DB_SERVICE_NAME,
    HWLAB_CLOUD_DB_SERVICE_NAMESPACE: process.env.HWLAB_CLOUD_DB_SERVICE_NAMESPACE,
    HWLAB_CLOUD_DB_HOST: process.env.HWLAB_CLOUD_DB_HOST,
    HWLAB_CLOUD_DB_PORT: process.env.HWLAB_CLOUD_DB_PORT
  };
  const fakeDb = createTcpServer((socket) => socket.end());
  await new Promise((resolve) => fakeDb.listen(0, "127.0.0.1", resolve));
  const dbPort = fakeDb.address().port;
  process.env.HWLAB_CLOUD_DB_URL = `postgres://hwlab_test@127.0.0.1:${dbPort}/hwlab`;
  process.env.HWLAB_CLOUD_DB_SSL_MODE = "disable";
  delete process.env.HWLAB_CLOUD_DB_SERVICE_NAME;
  delete process.env.HWLAB_CLOUD_DB_SERVICE_NAMESPACE;
  delete process.env.HWLAB_CLOUD_DB_HOST;
  delete process.env.HWLAB_CLOUD_DB_PORT;

  const server = createCloudApiServer();
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/health`);
    const payload = await response.json();
    assert.equal(payload.status, "degraded");
    assert.equal(payload.db.status, "ready");
    assert.equal(payload.db.connected, true);
    assert.equal(payload.db.liveConnected, true);
    assert.equal(payload.db.liveDbEvidence, true);
    assert.equal(payload.db.connectionChecked, true);
    assert.equal(payload.db.connectionAttempted, true);
    assert.equal(payload.db.connectionResult, "connected");
    assert.equal(payload.db.endpointSource, "secret-url-host");
    assert.equal(payload.db.connection.endpointSource, "secret-url-host");
    assert.equal(payload.db.endpoint.authoritative.source, "secret-url-host");
    assert.equal(payload.db.endpoint.authoritative.env, "HWLAB_CLOUD_DB_URL");
    assert.equal(payload.db.optionalPublicDnsAlias.source, "optional-public-dns-alias");
    assert.equal(payload.db.optionalPublicDnsAlias.requiredForReadiness, false);
    assert.equal(payload.db.optionalPublicDnsAlias.usedForProbe, false);
    assert.equal(payload.db.optionalPublicDnsAlias.envPresent, false);
    assert.equal(payload.db.configReady, true);
    assert.equal(payload.db.ready, true);
    assert.equal(payload.db.evidence, "live_db_tcp_connection_ready");
    assert.equal(payload.db.safety.liveDbEvidence, true);
    assert.equal(payload.db.safety.liveDbConnectedEvidence, true);
    assert.equal(payload.runtime.adapter, "memory");
    assert.equal(payload.runtime.durable, false);
    assert.equal(payload.runtime.status, "degraded");
    assert.match(payload.runtime.reason, /process-local/u);
    assert.equal(payload.runtime.durabilityContract.blockedLayer, "adapter");
    assert.equal(payload.runtime.durabilityContract.dbLiveEvidenceIsDurabilityEvidence, false);
    assert.equal(payload.readiness.components.db, "ready");
    assert.equal(payload.readiness.components.runtime, "blocked");
    assert.equal(payload.readiness.durability.status, "blocked");
    assert.equal(payload.readiness.durability.dbLiveEvidenceObserved, true);
    assert.equal(payload.readiness.durability.dbLiveEvidenceIsDurabilityEvidence, false);
    assert.equal(payload.readiness.durability.blockedLayer, "adapter");
    assert.equal(payload.readiness.durability.liveRuntimeEvidence, false);
    assert.equal(payload.db.redaction.valuesRedacted, true);
    assert.equal(payload.db.redaction.endpointRedacted, true);
    assert.equal(payload.db.redaction.secretMaterialRead, false);
    assert.deepEqual(payload.db.missingEnv, []);
    assert.equal(JSON.stringify(payload.db).includes("password"), false);
    assert.equal(JSON.stringify(payload.db).includes("127.0.0.1"), false);
    assert.equal(JSON.stringify(payload.db).includes(String(dbPort)), false);
    assert.equal(JSON.stringify(payload.db).includes("cloud-api-db.hwlab-dev.svc.cluster.local"), false);
    assert.equal(payload.db.fields.find((field) => field.name === "HWLAB_CLOUD_DB_URL").redacted, true);
    assert.equal(payload.db.connection.endpointRedacted, true);
  } finally {
    if (originalUrl === undefined) {
      delete process.env.HWLAB_CLOUD_DB_URL;
    } else {
      process.env.HWLAB_CLOUD_DB_URL = originalUrl;
    }
    if (originalSslMode === undefined) {
      delete process.env.HWLAB_CLOUD_DB_SSL_MODE;
    } else {
      process.env.HWLAB_CLOUD_DB_SSL_MODE = originalSslMode;
    }
    for (const [name, value] of Object.entries(originalAliasEnv)) {
      if (value === undefined) {
        delete process.env[name];
      } else {
        process.env[name] = value;
      }
    }
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    await new Promise((resolve, reject) => {
      fakeDb.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api health separates DB connected from durable runtime schema readiness", async () => {
  const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
  const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
  const fakeDb = createTcpServer((socket) => socket.end());
  await new Promise((resolve) => fakeDb.listen(0, "127.0.0.1", resolve));
  const dbPort = fakeDb.address().port;
  process.env.HWLAB_CLOUD_DB_URL = `postgres://hwlab_test@127.0.0.1:${dbPort}/hwlab`;
  process.env.HWLAB_CLOUD_DB_SSL_MODE = "disable";

  const server = createCloudApiServer({
    runtimeStore: {
      async readiness() {
        return {
          adapter: "postgres",
          durable: false,
          durableRequested: true,
          durableCapable: false,
          ready: false,
          status: "blocked",
          blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
          reason: "test runtime schema is missing",
          liveRuntimeEvidence: false,
          fixtureEvidence: false,
          connection: {
            queryAttempted: true,
            queryResult: "schema_blocked",
            endpointRedacted: true,
            valueRedacted: true
          },
          schema: {
            checked: true,
            ready: false,
            missingTables: ["gateway_sessions"],
            missingColumns: ["gateway_sessions.gateway_session_json"]
          },
          migration: {
            checked: false,
            ready: false,
            missing: true
          },
          counts: {}
        };
      }
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/health/live`);
    const payload = await response.json();
    assert.equal(response.status, 200);
    assert.equal(payload.status, "degraded");
    assert.equal(payload.db.connected, true);
    assert.equal(payload.db.liveDbEvidence, true);
    assert.equal(payload.runtime.durable, false);
    assert.equal(payload.runtime.durableRequested, true);
    assert.equal(payload.runtime.ready, false);
    assert.equal(payload.runtime.blocker, RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED);
    assert.equal(payload.readiness.components.db, "ready");
    assert.equal(payload.readiness.components.runtime, "blocked");
    assert.equal(payload.readiness.durability.status, "blocked");
    assert.equal(payload.readiness.durability.dbLiveEvidenceObserved, true);
    assert.equal(payload.readiness.durability.dbLiveEvidenceIsDurabilityEvidence, false);
    assert.equal(payload.readiness.durability.blockedLayer, "schema");
    assert.equal(payload.readiness.durability.queryAttempted, true);
    assert.equal(payload.readiness.durability.queryResult, "schema_blocked");
    assert.ok(payload.blockerCodes.includes(RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED));
    assert.equal(JSON.stringify(payload).includes("password"), false);
  } finally {
    if (originalUrl === undefined) {
      delete process.env.HWLAB_CLOUD_DB_URL;
    } else {
      process.env.HWLAB_CLOUD_DB_URL = originalUrl;
    }
    if (originalSslMode === undefined) {
      delete process.env.HWLAB_CLOUD_DB_SSL_MODE;
    } else {
      process.env.HWLAB_CLOUD_DB_SSL_MODE = originalSslMode;
    }
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    await new Promise((resolve, reject) => {
      fakeDb.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api health keeps DB live evidence separate when durable adapter query is blocked", async () => {
  const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
  const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
  const fakeDb = createTcpServer((socket) => socket.end());
  await new Promise((resolve) => fakeDb.listen(0, "127.0.0.1", resolve));
  const dbPort = fakeDb.address().port;
  process.env.HWLAB_CLOUD_DB_URL = `postgres://hwlab_test:password@127.0.0.1:${dbPort}/hwlab`;
  process.env.HWLAB_CLOUD_DB_SSL_MODE = "disable";

  const server = createCloudApiServer({
    runtimeStore: {
      async readiness() {
        return {
          adapter: "postgres",
          durable: false,
          durableRequested: true,
          durableCapable: false,
          ready: false,
          status: "blocked",
          blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
          reason: "test runtime durable read query is blocked",
          liveRuntimeEvidence: false,
          fixtureEvidence: false,
          connection: {
            queryAttempted: true,
            queryResult: "query_blocked",
            endpointRedacted: true,
            valueRedacted: true,
            errorCode: "57014"
          },
          schema: {
            checked: true,
            ready: true,
            missingTables: [],
            missingColumns: []
          },
          migration: {
            checked: true,
            ready: true,
            missing: false
          },
          gates: {
            auth: { checked: true, ready: true, status: "ready", blocker: null },
            schema: { checked: true, ready: true, status: "ready", blocker: null },
            migration: { checked: true, ready: true, status: "ready", blocker: null },
            durability: {
              checked: true,
              ready: false,
              status: "blocked",
              blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED
            }
          },
          counts: {}
        };
      }
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/health/live`);
    const payload = await response.json();
    assert.equal(response.status, 200);
    assert.equal(payload.status, "degraded");
    assert.equal(payload.db.ready, true);
    assert.equal(payload.db.connected, true);
    assert.equal(payload.db.liveDbEvidence, true);
    assert.equal(payload.runtime.durable, false);
    assert.equal(payload.runtime.ready, false);
    assert.equal(payload.runtime.blocker, RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED);
    assert.equal(payload.readiness.components.db, "ready");
    assert.equal(payload.readiness.components.runtime, "blocked");
    assert.equal(payload.readiness.durability.status, "blocked");
    assert.equal(payload.readiness.durability.dbLiveEvidenceObserved, true);
    assert.equal(payload.readiness.durability.dbLiveEvidenceIsDurabilityEvidence, false);
    assert.equal(payload.readiness.durability.blockedLayer, "durability_query");
    assert.equal(payload.readiness.durability.queryAttempted, true);
    assert.equal(payload.readiness.durability.queryResult, "query_blocked");
    assert.equal(payload.readiness.durability.liveRuntimeEvidence, false);
    assert.ok(payload.blockerCodes.includes(RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED));
    assert.equal(JSON.stringify(payload).includes("password"), false);
    assert.equal(JSON.stringify(payload).includes("127.0.0.1"), false);
    assert.equal(JSON.stringify(payload).includes(String(dbPort)), false);
  } finally {
    if (originalUrl === undefined) {
      delete process.env.HWLAB_CLOUD_DB_URL;
    } else {
      process.env.HWLAB_CLOUD_DB_URL = originalUrl;
    }
    if (originalSslMode === undefined) {
      delete process.env.HWLAB_CLOUD_DB_SSL_MODE;
    } else {
      process.env.HWLAB_CLOUD_DB_SSL_MODE = originalSslMode;
    }
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    await new Promise((resolve, reject) => {
      fakeDb.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api health contract distinguishes durable SSL, auth, schema, migration, and query blockers", async () => {
  const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
  const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
  process.env.HWLAB_CLOUD_DB_URL = "redacted";
  process.env.HWLAB_CLOUD_DB_SSL_MODE = "require";

  const cases = [
    {
      blocker: RUNTIME_DURABLE_ADAPTER_SSL_BLOCKED,
      blockedLayer: "ssl",
      queryResult: "ssl_negotiation_blocked",
      gates: {
        ssl: "blocked",
        auth: "not_checked",
        schema: "not_checked",
        migration: "not_checked",
        durability: "blocked"
      }
    },
    {
      blocker: RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
      blockedLayer: "auth",
      queryResult: "auth_blocked",
      gates: {
        ssl: "ready",
        auth: "blocked",
        schema: "not_checked",
        migration: "not_checked",
        durability: "blocked"
      }
    },
    {
      blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
      blockedLayer: "schema",
      queryResult: "schema_blocked",
      gates: {
        ssl: "ready",
        auth: "ready",
        schema: "blocked",
        migration: "not_checked",
        durability: "blocked"
      }
    },
    {
      blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED,
      blockedLayer: "migration",
      queryResult: "migration_blocked",
      gates: {
        ssl: "ready",
        auth: "ready",
        schema: "ready",
        migration: "blocked",
        durability: "blocked"
      }
    },
    {
      blocker: RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
      blockedLayer: "durability_query",
      queryResult: "query_blocked",
      gates: {
        ssl: "ready",
        auth: "ready",
        schema: "ready",
        migration: "ready",
        durability: "blocked"
      }
    }
  ];

  try {
    for (const testCase of cases) {
      const server = createCloudApiServer({
        dbProbe: {
          probe: async ({ endpointSource, timeoutMs }) => ({
            attempted: true,
            networkAttempted: true,
            endpointSource,
            probeType: "tcp-connect",
            endpointRedacted: true,
            valueRedacted: true,
            timeoutMs,
            durationMs: 0,
            result: "connected",
            classification: "tcp_connected",
            errorCode: null,
            missingEnv: []
          })
        },
        runtimeStore: {
          async readiness() {
            return durableBlockedRuntime(testCase);
          }
        }
      });
      await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

      try {
        const { port } = server.address();
        const response = await fetch(`http://127.0.0.1:${port}/health/live`);
        const payload = await response.json();
        assert.equal(response.status, 200);
        assert.equal(payload.status, "degraded");
        assert.equal(payload.db.ready, true);
        assert.equal(payload.db.liveDbEvidence, true);
        assert.equal(payload.runtime.blocker, testCase.blocker);
        assert.equal(payload.runtime.connection.queryResult, testCase.queryResult);
        assert.equal(payload.runtime.gates.ssl.status, testCase.gates.ssl);
        assert.equal(payload.runtime.gates.auth.status, testCase.gates.auth);
        assert.equal(payload.runtime.gates.schema.status, testCase.gates.schema);
        assert.equal(payload.runtime.gates.migration.status, testCase.gates.migration);
        assert.equal(payload.runtime.gates.durability.status, testCase.gates.durability);
        assert.equal(payload.runtime.durabilityContract.blockedLayer, testCase.blockedLayer);
        assert.equal(payload.db.runtimeReadiness.blockedLayer, testCase.blockedLayer);
        assert.equal(payload.db.runtimeReadiness.queryResult, testCase.queryResult);
        assert.equal(payload.db.runtimeReadiness.dbLiveEvidenceIsDurabilityEvidence, false);
        assert.equal(payload.db.readinessLayers.ssl.status, layerStatus(testCase.gates.ssl));
        assert.equal(payload.db.readinessLayers.auth.status, layerStatus(testCase.gates.auth));
        assert.equal(payload.db.readinessLayers.schema.status, layerStatus(testCase.gates.schema));
        assert.equal(payload.db.readinessLayers.migration.status, layerStatus(testCase.gates.migration));
        assert.equal(payload.db.readinessLayers.durability.status, layerStatus(testCase.gates.durability));
        assert.equal(payload.db.readinessLayers[layerForBlockedCase(testCase.blockedLayer)].result, testCase.queryResult);
        assert.equal(payload.readiness.durability.blockedLayer, testCase.blockedLayer);
        assert.equal(payload.readiness.durability.queryResult, testCase.queryResult);
        assert.equal(payload.readiness.durability.secretMaterialRead, false);
        assert.ok(payload.blockerCodes.includes(testCase.blocker));
      } finally {
        await new Promise((resolve, reject) => {
          server.close((error) => (error ? reject(error) : resolve()));
        });
      }
    }
  } finally {
    if (originalUrl === undefined) {
      delete process.env.HWLAB_CLOUD_DB_URL;
    } else {
      process.env.HWLAB_CLOUD_DB_URL = originalUrl;
    }
    if (originalSslMode === undefined) {
      delete process.env.HWLAB_CLOUD_DB_SSL_MODE;
    } else {
      process.env.HWLAB_CLOUD_DB_SSL_MODE = originalSslMode;
    }
  }
});

test("cloud api health does not treat DB env presence-only as live readiness", async () => {
  const originalUrl = process.env.HWLAB_CLOUD_DB_URL;
  const originalSslMode = process.env.HWLAB_CLOUD_DB_SSL_MODE;
  const originalProbeDisabled = process.env.HWLAB_CLOUD_DB_PROBE_DISABLED;
  process.env.HWLAB_CLOUD_DB_URL = "postgres://user:password@db.example.invalid:5432/hwlab";
  process.env.HWLAB_CLOUD_DB_SSL_MODE = "disable";
  delete process.env.HWLAB_CLOUD_DB_PROBE_DISABLED;

  const server = createCloudApiServer();
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/health/live`);
    const payload = await response.json();
    assert.equal(response.status, 200);
    assert.equal(payload.db.configReady, true);
    assert.equal(payload.db.connected, false);
    assert.equal(payload.db.liveConnected, false);
    assert.equal(payload.db.liveDbEvidence, false);
    assert.equal(payload.db.safety.liveDbEvidence, false);
    assert.equal(payload.db.ready, false);
    assert.equal(payload.db.connectionAttempted, true);
    assert.equal(payload.db.connectionResult, "forbidden_runtime_host");
    assert.equal(payload.db.evidence, "live_db_tcp_connection_blocked");
    assert.equal(payload.db.endpointSource, "secret-url-host");
    assert.equal(payload.db.connection.endpointSource, "secret-url-host");
    assert.equal(payload.db.connection.networkAttempted, false);
    assert.equal(payload.db.connection.classification, "db_url_forbidden_invalid_host");
    assert.equal(payload.db.readinessLayers.dns.status, "not_proven");
    assert.equal(JSON.stringify(payload.db).includes("password"), false);
    assert.equal(JSON.stringify(payload.db).includes("db.example.invalid"), false);
  } finally {
    if (originalUrl === undefined) {
      delete process.env.HWLAB_CLOUD_DB_URL;
    } else {
      process.env.HWLAB_CLOUD_DB_URL = originalUrl;
    }
    if (originalSslMode === undefined) {
      delete process.env.HWLAB_CLOUD_DB_SSL_MODE;
    } else {
      process.env.HWLAB_CLOUD_DB_SSL_MODE = originalSslMode;
    }
    if (originalProbeDisabled === undefined) {
      delete process.env.HWLAB_CLOUD_DB_PROBE_DISABLED;
    } else {
      process.env.HWLAB_CLOUD_DB_PROBE_DISABLED = originalProbeDisabled;
    }
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

function durableBlockedRuntime({ blocker, blockedLayer, queryResult, gates }) {
  return {
    adapter: "postgres",
    durable: false,
    durableRequested: true,
    durableCapable: false,
    ready: false,
    status: "blocked",
    blocker,
    reason: `test durable runtime blocker ${blocker}`,
    liveRuntimeEvidence: false,
    fixtureEvidence: false,
    connection: {
      queryAttempted: true,
      queryResult,
      endpointRedacted: true,
      valueRedacted: true,
      errorCode: "TEST_BLOCKER"
    },
    schema: {
      checked: gates.schema !== "not_checked",
      ready: gates.schema === "ready",
      missingTables: gates.schema === "blocked" ? ["gateway_sessions"] : [],
      missingColumns: gates.schema === "blocked" ? ["gateway_sessions.gateway_session_json"] : []
    },
    migration: {
      checked: gates.migration !== "not_checked",
      ready: gates.migration === "ready",
      missing: gates.migration !== "ready"
    },
    gates: Object.fromEntries(Object.entries(gates).map(([name, status]) => [
      name,
      {
        checked: status !== "not_checked",
        ready: status === "ready",
        status,
        blocker: status === "blocked" ? blocker : null
      }
    ])),
    durabilityContract: {
      ready: false,
      status: "blocked",
      requiredEvidence: "runtime_adapter_schema_migration_read_query",
      dbLiveEvidenceIsDurabilityEvidence: false,
      liveRuntimeEvidence: false,
      adapterQueryRequired: true,
      blockedLayer,
      blocker,
      secretMaterialRead: false
    },
    safety: {
      secretMaterialRead: false,
      valuesRedacted: true,
      endpointRedacted: true
    },
    counts: {}
  };
}

function layerStatus(gateStatus) {
  if (gateStatus === "ready") return "pass";
  if (gateStatus === "blocked") return "blocked";
  return "not_proven";
}

function layerForBlockedCase(blockedLayer) {
  if (blockedLayer === "durability_query") return "durability";
  return blockedLayer;
}

async function m3ReadinessRequestJson(url) {
  const parsed = new URL(url);
  if (url.startsWith("http://gateway-1") && parsed.pathname === "/status") {
    return {
      ok: true,
      status: 200,
      body: m3GatewayStatus({
        gatewayId: "gwsimu_1",
        gatewaySessionId: "gws_gwsimu_1",
        boxId: "boxsimu_1",
        resourceId: "res_boxsimu_1"
      })
    };
  }
  if (url.startsWith("http://gateway-1") && parsed.pathname === "/invoke") {
    return {
      ok: true,
      status: 200,
      body: {
        accepted: true,
        operationId: "op_m3_status_do1",
        traceId: "trc_m3_status_do1",
        boxId: "boxsimu_1",
        resourceId: "res_boxsimu_1",
        port: "DO1",
        value: false,
        state: {
          port: "DO1",
          direction: "output",
          value: false,
          source: "gateway-simu",
          updatedAt: "2026-05-23T00:00:00.000Z"
        },
        auditId: "aud_m3_status_do1",
        evidenceId: "evd_m3_status_do1"
      }
    };
  }
  if (url.startsWith("http://gateway-2") && parsed.pathname === "/status") {
    return {
      ok: true,
      status: 200,
      body: m3GatewayStatus({
        gatewayId: "gwsimu_2",
        gatewaySessionId: "gws_gwsimu_2",
        boxId: "boxsimu_2",
        resourceId: "res_boxsimu_2"
      })
    };
  }
  if (url.startsWith("http://gateway-2") && parsed.pathname === "/invoke") {
    return {
      ok: true,
      status: 200,
      body: {
        accepted: true,
        operationId: "op_m3_status_di1",
        traceId: "trc_m3_status_di1",
        boxId: "boxsimu_2",
        resourceId: "res_boxsimu_2",
        port: "DI1",
        value: false,
        state: {
          port: "DI1",
          direction: "input",
          value: false,
          source: "patch-panel",
          sourceResourceId: "res_boxsimu_1",
          sourcePort: "DO1",
          propagatedBy: "hwlab-patch-panel",
          updatedAt: "2026-05-23T00:00:00.000Z"
        },
        auditId: "aud_m3_status_di1",
        evidenceId: "evd_m3_status_di1"
      }
    };
  }
  if (url.startsWith("http://patch-panel") && parsed.pathname === "/status") {
    return {
      ok: true,
      status: 200,
      body: {
        serviceId: "hwlab-patch-panel",
        state: "active",
        activeConnections: [
          {
            fromResourceId: "res_boxsimu_1",
            fromPort: "DO1",
            toResourceId: "res_boxsimu_2",
            toPort: "DI1"
          }
        ]
      }
    };
  }
  if (url.startsWith("http://patch-panel") && parsed.pathname === "/wiring") {
    return {
      ok: true,
      status: 200,
      body: {
        wiringConfigId: "wir_m3_do1_di1",
        status: "active",
        connections: [
          {
            from: {
              resourceId: "res_boxsimu_1",
              port: "DO1"
            },
            to: {
              resourceId: "res_boxsimu_2",
              port: "DI1"
            },
            mode: "exclusive"
          }
        ]
      }
    };
  }
  throw new Error(`unexpected M3 readiness request ${url}`);
}

function m3GatewayStatus({ gatewayId, gatewaySessionId, boxId, resourceId }) {
  return {
    serviceId: "hwlab-gateway-simu",
    gatewayId,
    gatewaySessionId,
    session: {
      gatewayId,
      gatewaySessionId
    },
    registry: {
      gatewayId,
      gatewaySessionId,
      boxes: [
        {
          boxId,
          resourceId,
          state: "registered"
        }
      ]
    }
  };
}

test("cloud api /v1 describes Code Agent provider blocker without leaking secret values", async () => {
  const server = createCloudApiServer({
    env: {
      PATH: "",
      HWLAB_CODE_AGENT_PROVIDER: "openai",
      HWLAB_CODE_AGENT_MODEL: "gpt-test"
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1`);
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.codeAgent.endpoint, "POST /v1/agent/chat");
    assert.equal(payload.codeAgent.provider, "openai-responses");
    assert.equal(payload.codeAgent.model, "gpt-test");
    assert.equal(payload.codeAgent.backend, "hwlab-cloud-api/openai-responses");
    assert.equal(payload.codeAgent.mode, "openai");
    assert.equal(payload.codeAgent.status, "partial");
    assert.match(payload.codeAgent.blocker, /Codex stdio|long-lived/u);
    assert.equal(payload.codeAgent.reason, "codex_stdio_supervisor_disabled");
    assert.equal(payload.codeAgent.runner.kind, "hwlab-readonly-runner");
    assert.equal(payload.codeAgent.runner.ready, true);
    assert.equal(payload.codeAgent.capabilityLevel, "read-only-session-tools");
    assert.equal(payload.codeAgent.sessionRegistry.status, "available");
    assert.equal(payload.codeAgent.longLivedSessionGate.status, "blocked");
    assert.deepEqual(payload.codeAgent.missingEnv, ["OPENAI_API_KEY", "HWLAB_CODE_AGENT_OPENAI_BASE_URL"]);
    assert.equal(payload.codeAgent.secretRefs[0].env, "OPENAI_API_KEY");
    assert.equal(payload.codeAgent.secretRefs[0].secretName, "hwlab-code-agent-provider");
    assert.equal(payload.codeAgent.secretRefs[0].secretKey, "openai-api-key");
    assert.equal(payload.codeAgent.secretRefs[0].redacted, true);
    assert.equal(payload.codeAgent.ready, false);
    assert.equal(payload.codeAgent.partialReady, true);
    assert.equal(payload.codeAgent.egress.present, false);
    assert.equal(payload.codeAgent.egress.valueRedacted, true);
    assert.equal(payload.codeAgent.safety.secretMaterialRead, false);
    assert.match(payload.codeAgent.summary, /受控只读 runner/u);
    assert.match(payload.codeAgent.summary, /hwlab-code-agent-provider\/openai-api-key/u);
    assert.equal(JSON.stringify(payload).includes("sk-"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1 describes Code Agent egress blocker without leaking API key", async () => {
  const server = createCloudApiServer({
    env: {
      OPENAI_API_KEY: "test-openai-key-material",
      HWLAB_CODE_AGENT_PROVIDER: "openai",
      HWLAB_CODE_AGENT_MODEL: "gpt-test",
      HWLAB_CODE_AGENT_OPENAI_BASE_URL: "https://api.openai.com/v1/responses"
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1`);
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.codeAgent.status, "partial");
    assert.equal(payload.codeAgent.egress.directPublicOpenAi, true);
    assert.match(payload.codeAgent.blocker, /Codex stdio|long-lived/u);
    assert.equal(payload.codeAgent.runner.ready, true);
    assert.equal(payload.codeAgent.safety.secretMaterialRead, false);
    assert.equal(JSON.stringify(payload).includes("test-openai-key-material"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat returns structured completed Code Agent payload", async () => {
  const server = createCloudApiServer({
    callCodeAgentProvider: async ({ providerPlan, message, traceId }) => ({
      provider: providerPlan.provider,
      model: providerPlan.model,
      backend: providerPlan.backend,
      content: `真实 provider stub: ${message} / ${traceId}`,
      usage: null,
      providerTrace: {
        source: "test-provider"
      }
    })
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json",
        "x-trace-id": "trc_server-test-agent-chat"
      },
      body: JSON.stringify({
        conversationId: "cnv_server-test-agent-chat",
        message: "请用一句话说明当前 HWLAB 工作台可以做什么。"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.conversationId, "cnv_server-test-agent-chat");
    assert.equal(payload.sessionId, "cnv_server-test-agent-chat");
    assert.match(payload.messageId, /^msg_/);
    assert.equal(payload.status, "completed");
    assert.equal(payload.traceId, "trc_server-test-agent-chat");
    assert.equal(payload.provider.length > 0, true);
    assert.equal(payload.model.length > 0, true);
    assert.equal(payload.backend.length > 0, true);
    assert.equal(Number.isNaN(Date.parse(payload.createdAt)), false);
    assert.equal(Number.isNaN(Date.parse(payload.updatedAt)), false);
    assert.match(payload.reply.content, /HWLAB 工作台/);
    assert.equal(Object.hasOwn(payload, "error"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat parse and params errors use structured blocker envelope", async () => {
  const server = createCloudApiServer({
    env: {
      HWLAB_CODE_AGENT_MODEL: "gpt-test"
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const parse = await postAgentRaw(port, "{", { "x-trace-id": "trc_parse_error" });
    assert.equal(parse.status, 400);
    assert.equal(parse.body.status, "failed");
    assert.equal(parse.body.error.code, "parse_error");
    assert.equal(parse.body.error.layer, "api");
    assert.equal(parse.body.error.retryable, true);
    assert.equal(parse.body.error.traceId, "trc_parse_error");
    assert.equal(parse.body.error.route, "/v1/agent/chat");
    assert.match(parse.body.error.userMessage, /JSON/u);

    const invalid = await postAgentRaw(port, JSON.stringify([]), { "x-trace-id": "trc_invalid_params" });
    assert.equal(invalid.status, 400);
    assert.equal(invalid.body.error.code, "invalid_params");
    assert.equal(invalid.body.error.layer, "api");
    assert.equal(invalid.body.error.retryable, true);
    assert.equal(invalid.body.error.blocker.code, "invalid_params");
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat runs read-only runner pwd with workspace evidence", async () => {
  const workspace = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-workspace-"));
  const server = createCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_CODE_AGENT_WORKSPACE: workspace,
      HWLAB_CODE_AGENT_PROVIDER: "openai",
      HWLAB_CODE_AGENT_MODEL: "gpt-test",
      OPENAI_API_KEY: "test-openai-key-material",
      HWLAB_CODE_AGENT_OPENAI_BASE_URL: "http://127.0.0.1:65535/v1/responses"
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json",
        "x-trace-id": "trc_server-test-runner-pwd"
      },
      body: JSON.stringify({
        conversationId: "cnv_server-test-runner-pwd",
        message: "用pwd列出你当前的工作目录"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.status, "completed");
    assert.equal(payload.provider, "codex-readonly-runner");
    assert.equal(payload.backend, "hwlab-cloud-api/codex-readonly-runner");
    assert.equal(payload.capabilityLevel, "read-only-session-tools");
    assert.equal(payload.workspace, workspace);
    assert.equal(payload.sandbox, "read-only");
    assert.equal(payload.session.status, "idle");
    assert.equal(payload.session.workspace, workspace);
    assert.equal(payload.session.sandbox, "read-only");
    assert.equal(payload.session.lastTraceId, "trc_server-test-runner-pwd");
    assert.equal(typeof payload.session.idleTimeoutMs, "number");
    assert.equal(payload.runner.kind, "hwlab-readonly-runner");
    assert.equal(payload.runner.longLivedSession, false);
    assert.equal(payload.runner.codexStdio, false);
    assert.equal(payload.runner.writeCapable, false);
    assert.equal(payload.runner.durableSession, false);
    assert.equal(payload.runner.session, "controlled-readonly-session-registry");
    assert.equal(payload.sessionMode, "controlled-readonly-session-registry");
    assert.equal(payload.implementationType, "controlled-readonly-session-registry");
    assert.equal(payload.sessionReuse.reused, false);
    assert.equal(payload.sessionReuse.turn, 1);
    assert.equal(payload.sessionReuse.status, "idle");
    assert.ok(payload.runnerLimitations.includes("not-codex-stdio"));
    assert.ok(payload.codexStdioFeasibility.blockers.some((blocker) => blocker.code === "codex_stdio_supervisor_disabled"));
    assert.equal(payload.longLivedSessionGate.status, "blocked");
    assert.ok(payload.longLivedSessionGate.blockers.some((blocker) => blocker.code === "controlled_readonly_not_long_lived_stdio"));
    assert.equal(payload.toolCalls[0].name, "pwd");
    assert.equal(payload.toolCalls[0].status, "completed");
    assert.equal(payload.toolCalls[0].stdout, workspace);
    assert.equal(payload.skills.status, "not_requested");
    assert.equal(payload.runnerTrace.runnerKind, "hwlab-readonly-runner");
    assert.equal(payload.runnerTrace.sessionMode, "controlled-readonly-session-registry");
    assert.match(payload.reply.content, new RegExp(workspace.replace(/[.*+?^${}()|[\]\\]/gu, "\\$&")));
    assert.equal(JSON.stringify(payload).includes("test-openai-key-material"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat routes M3 IO through Skill CLI to /v1/m3/io only", async () => {
  const workspace = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-m3-skill-"));
  const calls = [];
  const server = createCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_CODE_AGENT_WORKSPACE: workspace,
      HWLAB_CODE_AGENT_HWLAB_API_BASE_URL: "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667",
      OPENAI_API_KEY: "must-not-be-used"
    },
    callCodeAgentProvider: async () => {
      throw new Error("OpenAI fallback must not handle M3 IO");
    },
    m3IoSkillRequestJson: async (url, request) => {
      calls.push({ url, request });
      return {
        ok: true,
        status: 200,
        body: {
          serviceId: "hwlab-cloud-api",
          contractVersion: "m3-io-control-v1",
          status: "succeeded",
          accepted: true,
          action: "do.write",
          traceId: "trc_server-test-m3-skill",
          operationId: "op_m3_do_write_server_skill",
          auditId: "aud_m3_do_write_server_skill_succeeded",
          evidenceId: "evd_m3_do_write_server_skill_succeeded",
          auditState: {
            status: "written_non_durable"
          },
          evidenceState: {
            status: "blocked",
            sourceKind: "BLOCKED",
            blocker: "runtime_durable_not_green",
            writeStatus: "written_non_durable"
          },
          durableStatus: {
            status: "degraded",
            durable: false,
            blocker: "runtime_durable_not_green"
          },
          result: {
            value: false,
            targetReadback: {
              status: "succeeded",
              value: false
            }
          },
          controlPath: {
            status: "succeeded",
            cloudApi: true,
            gatewaySimu: true,
            boxSimu: true,
            patchPanel: true,
            frontendBypass: false
          }
        }
      };
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json",
        "x-trace-id": "trc_server-test-m3-skill"
      },
      body: JSON.stringify({
        conversationId: "cnv_server-test-m3-skill",
        message: "请通过 M3 DO1 写入 false，并回读 DI1"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.status, "completed");
    assert.equal(payload.provider, "hwlab-skill-cli");
    assert.equal(payload.backend, "hwlab-cloud-api/hwlab-agent-runtime-skill-cli");
    assert.equal(payload.toolCalls[0].name, "hwlab-agent-runtime.m3-io");
    assert.equal(payload.toolCalls[0].route, "/v1/m3/io");
    assert.equal(payload.toolCalls[0].method, "POST");
    assert.equal(payload.toolCalls[0].accepted, true);
    assert.equal(payload.toolCalls[0].operationId, "op_m3_do_write_server_skill");
    assert.equal(payload.toolCalls[0].traceId, "trc_server-test-m3-skill");
    assert.equal(payload.toolCalls[0].auditId, "aud_m3_do_write_server_skill_succeeded");
    assert.equal(payload.toolCalls[0].evidenceId, "evd_m3_do_write_server_skill_succeeded");
    assert.equal(payload.toolCalls[0].readback.value, false);
    assert.equal(payload.providerTrace.fallbackUsed, false);
    assert.equal(payload.providerTrace.method, "POST");
    assert.equal(payload.providerTrace.readback.value, false);
    assert.equal(payload.runner.safety.directGatewayCallsAllowed, false);
    assert.equal(payload.runner.safety.directBoxSimuCallsAllowed, false);
    assert.equal(payload.runner.safety.directPatchPanelCallsAllowed, false);
    assert.equal(calls.length, 1);
    assert.equal(calls[0].url, "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667/v1/m3/io");
    assert.equal(calls[0].request.body.action, "do.write");
    assert.equal(calls[0].request.body.value, false);
    assert.equal(calls[0].url.includes("gateway-simu"), false);
    assert.equal(calls[0].url.includes("box-simu"), false);
    assert.equal(calls[0].url.includes("patch-panel"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat reports M3 Skill CLI missing API base as structured blocker", async () => {
  const workspace = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-m3-api-base-missing-"));
  const server = createCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_CODE_AGENT_WORKSPACE: workspace,
      HWLAB_CODE_AGENT_REQUIRE_HWLAB_API_BASE_URL: "1"
    },
    m3IoSkillRequestJson: async () => {
      throw new Error("missing API base must not call requestJson");
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const payload = await postAgent(port, {
      conversationId: "cnv_server-test-m3-api-base-missing",
      traceId: "trc_server-test-m3-api-base-missing",
      message: "读取 M3 DI1"
    });
    assert.equal(payload.status, "completed");
    assert.equal(payload.provider, "hwlab-skill-cli");
    assert.equal(payload.capabilityLevel, HWLAB_M3_IO_CAPABILITY_LEVELS.blocked);
    assert.equal(payload.blocker.code, "m3_readiness_blocked");
    assert.equal(payload.toolCalls[0].blocker.code, "skill_cli_api_base_missing");
    assert.deepEqual(payload.toolCalls[0].blocker.missingConfig, [
      "HWLAB_CODE_AGENT_HWLAB_API_BASE_URL",
      "HWLAB_API_BASE_URL",
      "HWLAB_CLOUD_API_BASE_URL",
      "contract:hwlab-agent-runtime.m3-io.apiBaseUrl"
    ]);
    assert.equal(JSON.stringify(payload).includes("test-openai-key-material"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat reuses controlled read-only session and exposes bounded file tools", async () => {
  const workspace = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-file-tools-"));
  await writeFile(path.join(workspace, "alpha.txt"), "alpha\nbeta\n", "utf8");
  await writeFile(path.join(workspace, "package.json"), "{\"name\":\"sample\"}\n", "utf8");
  await mkdir(path.join(workspace, "src"), { recursive: true });
  await writeFile(path.join(workspace, "src", "main.mjs"), "export const value = 1;\n", "utf8");
  const env = {
    PATH: process.env.PATH,
    HWLAB_CODE_AGENT_WORKSPACE: workspace
  };
  const server = createCloudApiServer({ env });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const first = await postAgent(port, {
      conversationId: "cnv_server-test-session-reuse",
      traceId: "trc_server-test-session-reuse-ls",
      message: "ls ."
    });
    assert.equal(first.status, "completed");
    assert.equal(first.sessionReuse.reused, false);
    assert.equal(first.sessionReuse.turn, 1);
    assert.equal(first.toolCalls[0].name, "ls");
    assert.match(first.toolCalls[0].stdout, /alpha\.txt/u);

    const second = await postAgent(port, {
      conversationId: "cnv_server-test-session-reuse",
      traceId: "trc_server-test-session-reuse-cat",
      message: "cat alpha.txt"
    });
    assert.equal(second.status, "completed");
    assert.equal(second.sessionId, first.sessionId);
    assert.equal(second.sessionReuse.reused, true);
    assert.equal(second.sessionReuse.turn, 2);
    assert.equal(second.toolCalls[0].name, "cat");
    assert.equal(second.toolCalls[0].stdout, "alpha\nbeta\n");

    const third = await postAgent(port, {
      conversationId: "cnv_server-test-session-reuse",
      traceId: "trc_server-test-session-reuse-rg",
      message: "rg --files ."
    });
    assert.equal(third.status, "completed");
    assert.equal(third.sessionReuse.reused, true);
    assert.equal(third.sessionReuse.turn, 3);
    assert.equal(third.toolCalls[0].name, "rg --files");
    assert.match(third.toolCalls[0].stdout, /src\/main\.mjs/u);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat blocks forbidden file paths without leaking values", async () => {
  const workspace = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-file-block-"));
  const server = createCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_CODE_AGENT_WORKSPACE: workspace
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const payload = await postAgent(port, {
      conversationId: "cnv_server-test-security-block",
      traceId: "trc_server-test-security-block",
      message: "cat ../outside.txt"
    });
    assert.equal(payload.status, "failed");
    assert.equal(payload.error.code, "security_blocked");
    assert.equal(payload.toolCalls[0].name, "cat");
    assert.equal(payload.toolCalls[0].status, "blocked");
    assert.equal(payload.capabilityLevel, "blocked");
    assert.equal(Object.hasOwn(payload, "reply"), false);
    assert.equal(JSON.stringify(payload).includes("sk-"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat discovers skills manifest with source and version evidence", async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-skills-"));
  const skillsDir = path.join(root, "skills");
  await mkdir(path.join(skillsDir, "alpha"), { recursive: true });
  await writeFile(path.join(skillsDir, "alpha", "SKILL.md"), [
    "---",
    "name: alpha-skill",
    "description: Alpha skill summary.",
    "version: 1.2.3",
    "commit: abc123def456",
    "---",
    "",
    "# Alpha"
  ].join("\n"));

  const server = createCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_CODE_AGENT_WORKSPACE: root
    },
    skillsDirs: [skillsDir],
    skillsDirsExact: true
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json"
      },
      body: JSON.stringify({
        conversationId: "cnv_server-test-skills",
        message: "列出你可用的skills"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.status, "completed");
    assert.equal(payload.provider, "codex-readonly-runner");
    assert.equal(payload.capabilityLevel, "read-only-session-tools");
    assert.equal(payload.skills.status, "ready");
    assert.equal(payload.skills.items[0].name, "alpha-skill");
    assert.equal(payload.skills.items[0].summary, "Alpha skill summary.");
    assert.equal(payload.skills.items[0].version, "1.2.3");
    assert.equal(payload.skills.items[0].commit, "abc123def456");
    assert.equal(payload.toolCalls[0].name, "skills.discover");
    assert.match(payload.reply.content, /alpha-skill/u);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat keeps slow skills discovery running past legacy 4500ms timeout", async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-slow-skills-"));
  const workspace = path.join(root, "workspace");
  const skillsDir = path.join(root, "skills");
  await mkdir(workspace, { recursive: true });
  await mkdir(path.join(skillsDir, "slow-skill"), { recursive: true });
  await writeFile(path.join(skillsDir, "slow-skill", "SKILL.md"), [
    "---",
    "name: slow-skill",
    "description: Slow skill discovery summary.",
    "version: 2026.05.23",
    "commit: slow12345678",
    "---",
    "",
    "# Slow"
  ].join("\n"));

  const server = createCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_CODE_AGENT_WORKSPACE: workspace
    },
    skillsDirs: [skillsDir],
    skillsDirsExact: true,
    skillsDiscoveryDelayMs: 4600
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const startedAt = Date.now();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json",
        "x-trace-id": "trc_server-test-slow-skills"
      },
      body: JSON.stringify({
        conversationId: "cnv_server-test-slow-skills",
        message: "请列出你可用的所有 skills"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(Date.now() - startedAt >= 4500, true);
    assert.equal(payload.status, "completed");
    assert.equal(payload.provider, "codex-readonly-runner");
    assert.equal(payload.backend, "hwlab-cloud-api/codex-readonly-runner");
    assert.equal(payload.traceId, "trc_server-test-slow-skills");
    assert.equal(payload.capabilityLevel, "read-only-session-tools");
    assert.equal(payload.sessionMode, "controlled-readonly-session-registry");
    assert.equal(payload.toolCalls[0].name, "skills.discover");
    assert.equal(payload.toolCalls[0].status, "completed");
    assert.equal(payload.skills.status, "ready");
    assert.equal(payload.skills.items[0].name, "slow-skill");
    assert.equal(payload.skills.items[0].traceId, "trc_server-test-slow-skills");
    assert.equal(payload.runner.kind, "hwlab-readonly-runner");
    assert.equal(payload.runnerTrace.traceId, "trc_server-test-slow-skills");
    assert.ok(payload.runnerTrace.events.includes("tool:skills.discover:completed"));
    assert.equal(Object.hasOwn(payload, "error"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat reports structured skills_unavailable blocker", async () => {
  const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-no-skills-"));
  const skillsDir = path.join(root, "missing-skills");
  const server = createCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_CODE_AGENT_WORKSPACE: root
    },
    skillsDirs: [skillsDir],
    skillsDirsExact: true
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json"
      },
      body: JSON.stringify({
        conversationId: "cnv_server-test-skills-missing",
        message: "列出你可用的skills"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.status, "failed");
    assert.equal(payload.provider, "codex-readonly-runner");
    assert.equal(payload.error.code, "skills_unavailable");
    assert.equal(payload.capabilityLevel, "blocked");
    assert.equal(payload.skills.status, "blocked");
    assert.equal(payload.skills.blockers[0].sourceIssue, "pikasTech/HWLAB#136");
    assert.ok(payload.skills.blockers[0].linkedIssues.includes("pikasTech/HWLAB#237"));
    assert.equal(payload.toolCalls[0].status, "blocked");
    assert.equal(Object.hasOwn(payload, "reply"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat reports unsupported read-only tool as blocked", async () => {
  const workspace = await mkdtemp(path.join(os.tmpdir(), "hwlab-agent-tool-blocked-"));
  const server = createCloudApiServer({
    env: {
      PATH: process.env.PATH,
      HWLAB_CODE_AGENT_WORKSPACE: workspace
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json"
      },
      body: JSON.stringify({
        conversationId: "cnv_server-test-tool-unavailable",
        message: "请用grep搜索 package.json"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.status, "failed");
    assert.equal(payload.error.code, "tool_unavailable");
    assert.equal(payload.capabilityLevel, "blocked");
    assert.equal(payload.toolCalls[0].name, "grep");
    assert.equal(payload.toolCalls[0].status, "blocked");
    assert.equal(Object.hasOwn(payload, "reply"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat uses streaming OpenAI Responses and parses SSE text", async () => {
  const providerRequests = [];
  const providerServer = createHttpServer((request, response) => {
    const chunks = [];
    request.on("data", (chunk) => chunks.push(chunk));
    request.on("end", () => {
      const bodyText = Buffer.concat(chunks).toString("utf8");
      const body = JSON.parse(bodyText);
      providerRequests.push({
        method: request.method,
        url: request.url,
        authorizationPresent: Boolean(request.headers.authorization),
        accept: request.headers.accept,
        contentType: request.headers["content-type"],
        body
      });
      response.writeHead(200, {
        "content-type": "text/event-stream"
      });
      response.end([
        `data: ${JSON.stringify({
          type: "response.created",
          response: {
            id: "resp_server_test_stream",
            model: body.model,
            usage: null
          }
        })}`,
        `data: ${JSON.stringify({
          type: "response.output_text.delta",
          response_id: "resp_server_test_stream",
          delta: "HWLAB Code Agent "
        })}`,
        `data: ${JSON.stringify({
          type: "response.output_text.delta",
          response_id: "resp_server_test_stream",
          delta: "streaming ready."
        })}`,
        `data: ${JSON.stringify({
          type: "response.completed",
          response: {
            id: "resp_server_test_stream",
            model: body.model,
            usage: null
          }
        })}`,
        "data: [DONE]",
        ""
      ].join("\n\n"));
    });
  });
  await new Promise((resolve) => providerServer.listen(0, "127.0.0.1", resolve));
  const providerPort = providerServer.address().port;

  const server = createCloudApiServer({
    env: {
      OPENAI_API_KEY: "test-openai-key-material",
      HWLAB_CODE_AGENT_PROVIDER: "openai",
      HWLAB_CODE_AGENT_MODEL: "gpt-test",
      HWLAB_CODE_AGENT_OPENAI_BASE_URL: `http://127.0.0.1:${providerPort}/v1/responses`
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json",
        "x-trace-id": "trc_server-test-agent-chat-stream"
      },
      body: JSON.stringify({
        conversationId: "cnv_server-test-agent-chat-stream",
        message: "请用一句话说明当前 HWLAB 工作台可以做什么。"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.status, "completed");
    assert.equal(payload.conversationId, "cnv_server-test-agent-chat-stream");
    assert.equal(payload.traceId, "trc_server-test-agent-chat-stream");
    assert.equal(payload.provider, "openai-responses");
    assert.equal(payload.model, "gpt-test");
    assert.equal(payload.providerTrace.responseId, "resp_server_test_stream");
    assert.equal(payload.reply.content, "HWLAB Code Agent streaming ready.");
    assert.equal(JSON.stringify(payload).includes("test-openai-key-material"), false);

    assert.equal(providerRequests.length, 1);
    assert.equal(providerRequests[0].method, "POST");
    assert.equal(providerRequests[0].url, "/v1/responses");
    assert.equal(providerRequests[0].authorizationPresent, true);
    assert.match(providerRequests[0].accept, /text\/event-stream/u);
    assert.equal(providerRequests[0].contentType, "application/json");
    assert.equal(providerRequests[0].body.model, "gpt-test");
    assert.equal(providerRequests[0].body.store, false);
    assert.equal(providerRequests[0].body.stream, true);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    await new Promise((resolve, reject) => {
      providerServer.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat accepts provider success delayed beyond legacy 4500ms UI timeout", async () => {
  const server = createCloudApiServer({
    callCodeAgentProvider: async ({ providerPlan, message, traceId }) => {
      await delay(4700);
      return {
        provider: providerPlan.provider,
        model: providerPlan.model,
        backend: providerPlan.backend,
        content: `延迟真实 provider stub: ${message} / ${traceId}`,
        usage: null,
        providerTrace: {
          source: "delayed-test-provider"
        }
      };
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const startedAt = Date.now();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json",
        "x-trace-id": "trc_server-test-agent-chat-delayed"
      },
      body: JSON.stringify({
        conversationId: "cnv_server-test-agent-chat-delayed",
        message: "请用一句话说明当前 HWLAB 工作台可以做什么，稍后回答"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(Date.now() - startedAt >= 4500, true);
    assert.equal(payload.status, "completed");
    assert.equal(payload.conversationId, "cnv_server-test-agent-chat-delayed");
    assert.equal(payload.traceId, "trc_server-test-agent-chat-delayed");
    assert.match(payload.reply.content, /稍后回答/u);
    assert.equal(Object.hasOwn(payload, "error"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat keeps delayed provider failure structured beyond legacy 4500ms UI timeout", async () => {
  const server = createCloudApiServer({
    env: {
      OPENAI_API_KEY: "test-openai-key-material",
      HWLAB_CODE_AGENT_PROVIDER: "openai",
      HWLAB_CODE_AGENT_MODEL: "gpt-test",
      HWLAB_CODE_AGENT_OPENAI_BASE_URL: "http://127.0.0.1/provider-fixture"
    },
    callCodeAgentProvider: async ({ providerPlan }) => {
      await delay(4600);
      const error = new Error("OpenAI Responses returned HTTP 503: slow upstream rejected");
      error.code = "provider_unavailable";
      error.provider = providerPlan.provider;
      error.model = providerPlan.model;
      error.backend = providerPlan.backend;
      error.providerStatus = 503;
      throw error;
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const startedAt = Date.now();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json",
        "x-trace-id": "trc_server-test-agent-chat-delayed-failure"
      },
      body: JSON.stringify({
        conversationId: "cnv_server-test-agent-chat-delayed-failure",
        message: "请等待后返回 provider 失败分类"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(Date.now() - startedAt >= 4500, true);
    assert.equal(payload.status, "failed");
    assert.equal(payload.traceId, "trc_server-test-agent-chat-delayed-failure");
    assert.equal(payload.provider, "openai-responses");
    assert.equal(payload.error.code, "provider_unavailable");
    assert.equal(payload.error.providerStatus, 503);
    assert.match(payload.error.message, /HTTP 503/u);
    assert.equal(payload.backend, "hwlab-cloud-api/openai-responses");
    assert.equal(payload.availability.fallback.backend, "hwlab-cloud-api/openai-responses");
    assert.equal(Object.hasOwn(payload, "reply"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat reports provider timeout as failed without a reply", async () => {
  const providerServer = createHttpServer((request, response) => {
    request.resume();
    setTimeout(() => {
      response.writeHead(200, { "content-type": "application/json" });
      response.end(JSON.stringify({ output_text: "too late" }));
    }, 250);
  });
  await new Promise((resolve) => providerServer.listen(0, "127.0.0.1", resolve));
  const providerPort = providerServer.address().port;

  const server = createCloudApiServer({
    env: {
      OPENAI_API_KEY: "test-openai-key-material",
      HWLAB_CODE_AGENT_PROVIDER: "openai",
      HWLAB_CODE_AGENT_MODEL: "gpt-test",
      HWLAB_CODE_AGENT_OPENAI_BASE_URL: `http://127.0.0.1:${providerPort}/v1/responses`
    },
    codeAgentTimeoutMs: 50
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json",
        "x-trace-id": "trc_server-test-agent-chat-provider-timeout"
      },
      body: JSON.stringify({
        conversationId: "cnv_server-test-agent-chat-provider-timeout",
        message: "请等待后回答"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.status, "failed");
    assert.equal(payload.traceId, "trc_server-test-agent-chat-provider-timeout");
    assert.equal(payload.error.code, "provider_timeout");
    assert.equal(payload.error.layer, "provider");
    assert.equal(payload.error.retryable, true);
    assert.match(payload.error.userMessage, /超时/u);
    assert.match(payload.error.message, /timed out after 50ms/u);
    assert.equal(payload.provider, "openai-responses");
    assert.equal(payload.backend, "hwlab-cloud-api/openai-responses");
    assert.equal(payload.availability.endpoint, "POST /v1/agent/chat");
    assert.equal(payload.availability.runner.kind, "hwlab-readonly-runner");
    assert.equal(Object.hasOwn(payload, "reply"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
    await new Promise((resolve, reject) => {
      providerServer.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/agent/chat reports OpenAI provider 502 and 503 as failed blocked payloads", async () => {
  for (const status of [502, 503]) {
    const providerServer = createHttpServer((request, response) => {
      request.resume();
      response.writeHead(status, { "content-type": "application/json" });
      response.end(JSON.stringify({
        error: {
          message: `upstream ${status}`
        }
      }));
    });
    await new Promise((resolve) => providerServer.listen(0, "127.0.0.1", resolve));
    const providerPort = providerServer.address().port;

    const server = createCloudApiServer({
      env: {
        OPENAI_API_KEY: "test-openai-key-material",
        HWLAB_CODE_AGENT_PROVIDER: "openai",
        HWLAB_CODE_AGENT_MODEL: "gpt-test",
        HWLAB_CODE_AGENT_OPENAI_BASE_URL: `http://127.0.0.1:${providerPort}/v1/responses`
      }
    });
    await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

    try {
      const { port } = server.address();
      const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
        method: "POST",
        headers: {
          "content-type": "application/json",
          "x-trace-id": `trc_server-test-agent-chat-provider-${status}`
        },
        body: JSON.stringify({
          conversationId: `cnv_server-test-agent-chat-provider-${status}`,
          message: `provider ${status}`
        })
      });
      assert.equal(response.status, 200);
      const payload = await response.json();
      assert.equal(payload.status, "failed");
      assert.equal(payload.traceId, `trc_server-test-agent-chat-provider-${status}`);
      assert.equal(payload.error.code, "provider_unavailable");
      assert.equal(payload.error.layer, "provider");
      assert.equal(payload.error.retryable, true);
      assert.match(payload.error.userMessage, /provider/u);
      assert.equal(payload.error.providerStatus, status);
      assert.match(payload.error.message, new RegExp(`HTTP ${status}`));
      assert.equal(Object.hasOwn(payload, "reply"), false);
    } finally {
      await new Promise((resolve, reject) => {
        server.close((error) => (error ? reject(error) : resolve()));
      });
      await new Promise((resolve, reject) => {
        providerServer.close((error) => (error ? reject(error) : resolve()));
      });
    }
  }
});

test("cloud api /v1/agent/chat reports provider gaps without faking a reply", async () => {
  const server = createCloudApiServer({
    env: {
      PATH: "",
      HWLAB_CODE_AGENT_PROVIDER: "codex-cli",
      HWLAB_CODE_AGENT_MODEL: "gpt-test"
    }
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json"
      },
      body: JSON.stringify({
        message: "你好"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.match(payload.conversationId, /^cnv_/);
    assert.match(payload.messageId, /^msg_/);
    assert.equal(payload.status, "failed");
    assert.equal(payload.provider, "codex-cli");
    assert.equal(payload.model, "gpt-test");
    assert.equal(payload.backend, "hwlab-cloud-api/codex-cli");
    assert.equal(Number.isNaN(Date.parse(payload.createdAt)), false);
    assert.equal(Number.isNaN(Date.parse(payload.updatedAt)), false);
    assert.equal(payload.error.code, "provider_unavailable");
    assert.equal(payload.error.layer, "provider-config");
    assert.equal(payload.error.retryable, false);
    assert.match(payload.error.userMessage, /配置缺失/u);
    assert.match(payload.error.message, /Codex CLI command is not available/);
    assert.deepEqual(payload.error.missingCommands, ["codex"]);
    assert.ok(payload.error.missingEnv.includes("OPENAI_API_KEY"));
    assert.ok(payload.error.missingConfig.includes("OPENAI_API_KEY"));
    assert.ok(payload.error.missingConfig.includes("command:codex"));
    assert.equal(payload.availability.status, "partial");
    assert.match(payload.availability.blocker, /Codex stdio|long-lived/u);
    assert.equal(payload.availability.reason, "codex_stdio_supervisor_disabled");
    assert.equal(payload.availability.runner.ready, true);
    assert.equal(payload.availability.secretRefs[0].secretName, "hwlab-code-agent-provider");
    assert.equal(payload.availability.secretRefs[0].secretKey, "openai-api-key");
    assert.equal(payload.availability.secretRefs[0].redacted, true);
    assert.match(payload.availability.summary, /受控只读 runner/u);
    assert.match(payload.availability.summary, /hwlab-code-agent-provider\/openai-api-key/u);
    assert.equal(JSON.stringify(payload).includes("sk-"), false);
    assert.equal(Object.hasOwn(payload, "reply"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

function delay(ms) {
  return new Promise((resolve) => setTimeout(resolve, ms));
}

test("cloud api /v1/agent/chat does not complete on empty provider text", async () => {
  const server = createCloudApiServer({
    callCodeAgentProvider: async ({ providerPlan }) => ({
      provider: providerPlan.provider,
      model: providerPlan.model,
      backend: providerPlan.backend,
      content: "   ",
      usage: null,
      providerTrace: {
        source: "empty-test-provider"
      }
    })
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
      method: "POST",
      headers: {
        "content-type": "application/json"
      },
      body: JSON.stringify({
        conversationId: "cnv_empty-provider-text",
        message: "你好"
      })
    });
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.conversationId, "cnv_empty-provider-text");
    assert.equal(payload.status, "failed");
    assert.equal(payload.error.code, "provider_unavailable");
    assert.match(payload.error.message, /no assistant text/);
    assert.equal(Object.hasOwn(payload, "reply"), false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1 exposes M3 IO control contract without generic frontend hardware RPC", async () => {
  const server = createCloudApiServer({
    env: {
      HWLAB_M3_IO_CONTROL_ENABLED: "true",
      HWLAB_M3_GATEWAY_SIMU_1_URL: "http://gateway-1",
      HWLAB_M3_GATEWAY_SIMU_2_URL: "http://gateway-2",
      HWLAB_M3_PATCH_PANEL_URL: "http://patch-panel"
    },
    m3IoRequestJson: m3ReadinessRequestJson
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const index = await fetch(`http://127.0.0.1:${port}/v1`);
    assert.equal(index.status, 200);
    const indexPayload = await index.json();
    assert.equal(indexPayload.m3IoControl.route, "/v1/m3/io");
    assert.equal(indexPayload.m3IoControl.boundaries.directFrontendGatewayOrBoxAccess, false);
    assert.ok(indexPayload.methods.includes("m3.io.do.write"));
    assert.ok(indexPayload.methods.includes("m3.io.di.read"));

    const contract = await fetch(`http://127.0.0.1:${port}/v1/m3/io`);
    assert.equal(contract.status, 200);
    const payload = await contract.json();
    assert.equal(payload.status, "available");
    assert.equal(payload.controlReady, true);
    assert.equal(payload.readiness.status, "ready");
    assert.equal(payload.chain.sourceResourceId, "res_boxsimu_1");
    assert.equal(payload.chain.targetResourceId, "res_boxsimu_2");
    assert.equal(payload.boundaries.frontendCallsOnly, "/v1/m3/io");
    assert.equal(payload.boundaries.patchPanelOwnsPropagation, true);
    assert.equal(payload.boundaries.genericHardwareRpcExposedToFrontend, false);

    const statusResponse = await fetch(`http://127.0.0.1:${port}/v1/m3/status`);
    assert.equal(statusResponse.status, 200);
    const statusPayload = await statusResponse.json();
    assert.equal(statusPayload.route, "/v1/m3/status");
    assert.equal(statusPayload.chain.sourceResourceId, "res_boxsimu_1");
    assert.equal(statusPayload.chain.targetResourceId, "res_boxsimu_2");
    assert.equal(statusPayload.gateways.length, 2);
    assert.equal(statusPayload.gateways.every((gateway) => gateway.online === true), true);
    assert.equal(statusPayload.boxes.find((box) => box.resourceId === "res_boxsimu_1").ports.DO1.value, false);
    assert.equal(statusPayload.boxes.find((box) => box.resourceId === "res_boxsimu_2").ports.DI1.value, false);
    assert.equal(statusPayload.patchPanel.connectionActive, true);
    assert.equal(statusPayload.trust.durableStatus, "blocked");
    assert.notEqual(statusPayload.trust.durableStatus, "green");
    assert.equal(statusPayload.trust.runtime.blocker, "runtime_durable_adapter_missing");
    assert.equal(statusPayload.boundaries.frontendCallsOnly, "/v1/m3/status");
    assert.equal(statusPayload.boundaries.directFrontendGatewayOrBoxAccess, false);
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/m3/status clears nested runtime blocker when trusted durable evidence is green", async () => {
  const server = createCloudApiServer({
    runtimeStore: createGreenM3StatusRuntimeStore({ blocker: "runtime_durable_not_green" }),
    env: {
      HWLAB_M3_GATEWAY_SIMU_1_URL: "http://gateway-1",
      HWLAB_M3_GATEWAY_SIMU_2_URL: "http://gateway-2",
      HWLAB_M3_PATCH_PANEL_URL: "http://patch-panel"
    },
    m3IoRequestJson: m3ReadinessRequestJson
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const statusResponse = await fetch(`http://127.0.0.1:${port}/v1/m3/status`);
    assert.equal(statusResponse.status, 200);
    const statusPayload = await statusResponse.json();

    assert.equal(statusPayload.contractVersion, "m3-status-v1");
    assert.equal(statusPayload.status, "live");
    assert.equal(statusPayload.sourceKind, "DEV-LIVE");
    assert.equal(statusPayload.patchPanel.connectionActive, true);
    assert.equal(statusPayload.trust.durableStatus, "green");
    assert.equal(statusPayload.trust.blocker, null);
    assert.equal(statusPayload.trust.runtime.status, "ready");
    assert.equal(statusPayload.trust.runtime.blocker, null);
    assert.equal(statusPayload.trust.operationId, "op_m3_di_read_75fc664f-e94f-41e8-816e-aff9fdb3666f");
    assert.equal(statusPayload.trust.traceId, "trc_hotfix_307_verify_1779527747043_read_false");
    assert.equal(statusPayload.trust.auditId, "aud_m3_di_read_d5afd84b-3a9d-475f-894f-dff8ea5f49ff_succeeded");
    assert.equal(statusPayload.trust.evidenceId, "evd_m3_di_read_75fc664f-e94f-41e8-816e-aff9fdb3666f_succeeded");
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/diagnostics/gate exposes Chinese single-table rows from live backend aggregation", async () => {
  const runtimeStore = createCloudRuntimeStore({
    now: () => "2026-05-23T00:00:00.000Z"
  });
  const server = createCloudApiServer({
    runtimeStore,
    env: {
      HWLAB_M3_GATEWAY_SIMU_1_URL: "http://gateway-1",
      HWLAB_M3_GATEWAY_SIMU_2_URL: "http://gateway-2",
      HWLAB_M3_PATCH_PANEL_URL: "http://patch-panel",
      HWLAB_COMMIT_ID: "abc123456789",
      HWLAB_IMAGE_TAG: "abc1234"
    },
    m3IoRequestJson: async () => ({
      ok: false,
      status: 503,
      body: {
        error: {
          message: "gateway offline"
        }
      }
    })
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/diagnostics/gate`);
    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.route, "/v1/diagnostics/gate");
    assert.equal(payload.contractVersion, "gate-diagnostics-table-v1");
    assert.equal(payload.sourceKind, "LIVE-BACKEND");
    assert.equal(payload.liveBackend, true);
    assert.deepEqual(payload.columns, ["类别", "检查项", "状态", "归因对象", "关键细节", "证据/trace", "更新时间", "下一步"]);
    assert.equal(payload.safety.frontendHardcodedRows, false);
    assert.equal(payload.safety.sourceFixturePromoted, false);
    assert.equal(payload.safety.blockedIsGreen, false);
    assert.equal(payload.safety.hardwareControlSurface, false);
    assert.equal(payload.safety.codeAgentConversationSurface, false);
    assert.ok(payload.rows.length >= 6);
    assert.ok(payload.rows.every((row) => row.sourceKind === "LIVE-BACKEND"));
    assert.ok(payload.rows.every((row) => ["通过", "阻塞", "失败", "待验证", "信息"].includes(row.status)));
    assert.ok(payload.rows.some((row) => row.category === "M3" && row.status === "阻塞"));
    assert.ok(payload.rows.some((row) => row.check === "/health/live readiness"));
    assert.ok(payload.rows.some((row) => row.check === "audit.event.query"));
    assert.ok(payload.rows.some((row) => row.check === "evidence.record.query"));
    assert.ok(payload.rows.some((row) => row.check === "Cloud API 实况身份" && row.status === "通过"));
    assert.equal(
      payload.rows.some((row) => row.status === "阻塞" && row.statusKey === "pass"),
      false,
      "blocked rows must not be mapped to pass"
    );
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

test("cloud api /v1/m3/io returns structured blocked payload when gateway is unavailable", async () => {
  const runtimeStore = createCloudRuntimeStore({
    now: () => "2026-05-23T00:00:00.000Z"
  });
  const server = createCloudApiServer({
    runtimeStore,
    env: {
      HWLAB_M3_GATEWAY_SIMU_1_URL: "http://gateway-1",
      HWLAB_M3_GATEWAY_SIMU_2_URL: "http://gateway-2",
      HWLAB_M3_PATCH_PANEL_URL: "http://patch-panel"
    },
    m3IoRequestJson: async () => ({
      ok: false,
      status: 503,
      body: {
        error: {
          message: "gateway offline"
        }
      }
    })
  });
  await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve));

  try {
    const { port } = server.address();
    const response = await fetch(`http://127.0.0.1:${port}/v1/m3/io`, {
      method: "POST",
      headers: {
        "content-type": "application/json",
        "x-trace-id": "trc_http_m3_gateway_missing",
        "x-actor-id": "usr_http_m3"
      },
      body: JSON.stringify({
        action: "do.write",
        value: true
      })
    });

    assert.equal(response.status, 200);
    const payload = await response.json();
    assert.equal(payload.status, "blocked");
    assert.equal(payload.accepted, false);
    assert.equal(payload.traceId, "trc_http_m3_gateway_missing");
    assert.equal(payload.operationId, null);
    assert.equal(payload.auditId, null);
    assert.equal(payload.evidenceId, null);
    assert.match(payload.blocker.zh, /gateway 未注册\/不可用/u);
    assert.equal(payload.controlPath.cloudApi, true);
    assert.equal(payload.controlPath.frontendBypass, false);
    assert.equal(payload.evidenceState.sourceKind, "BLOCKED");
  } finally {
    await new Promise((resolve, reject) => {
      server.close((error) => (error ? reject(error) : resolve()));
    });
  }
});

async function postAgent(port, body) {
  const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
    method: "POST",
    headers: {
      "content-type": "application/json",
      ...(body.traceId ? { "x-trace-id": body.traceId } : {})
    },
    body: JSON.stringify(body)
  });
  assert.equal(response.status, 200);
  return response.json();
}

async function postAgentRaw(port, body, headers = {}) {
  const response = await fetch(`http://127.0.0.1:${port}/v1/agent/chat`, {
    method: "POST",
    headers: {
      "content-type": "application/json",
      ...headers
    },
    body
  });
  return {
    status: response.status,
    body: await response.json()
  };
}

function createGreenM3StatusRuntimeStore({ blocker = null } = {}) {
  return {
    async readiness() {
      return {
        adapter: "postgres",
        durable: true,
        durableRequested: true,
        durableCapable: true,
        ready: true,
        status: "ready",
        blocker,
        reason: "Postgres durable runtime adapter completed live schema, migration, and read readiness queries",
        liveRuntimeEvidence: true,
        fixtureEvidence: false
      };
    },
    async queryAuditEvents() {
      return {
        events: [
          {
            auditId: "aud_m3_di_read_d5afd84b-3a9d-475f-894f-dff8ea5f49ff_succeeded",
            traceId: "trc_hotfix_307_verify_1779527747043_read_false",
            actorType: "user",
            actorId: "usr_m3_operator",
            action: "m3.io.di.read",
            targetType: "hardware_operation",
            targetId: "op_m3_di_read_75fc664f-e94f-41e8-816e-aff9fdb3666f",
            projectId: "prj_mvp_topology",
            gatewaySessionId: "gws_gwsimu_2",
            operationId: "op_m3_di_read_75fc664f-e94f-41e8-816e-aff9fdb3666f",
            serviceId: "hwlab-cloud-api",
            outcome: "succeeded",
            metadata: {
              route: "res_boxsimu_1:DO1->hwlab-patch-panel->res_boxsimu_2:DI1"
            },
            environment: "dev",
            occurredAt: "2026-05-23T00:00:00.000Z"
          }
        ],
        count: 1
      };
    },
    async queryEvidenceRecords() {
      return {
        records: [
          {
            evidenceId: "evd_m3_di_read_75fc664f-e94f-41e8-816e-aff9fdb3666f_succeeded",
            projectId: "prj_mvp_topology",
            operationId: "op_m3_di_read_75fc664f-e94f-41e8-816e-aff9fdb3666f",
            kind: "trace",
            uri: "memory://hwlab-cloud-api/op_m3_di_read_75fc664f-e94f-41e8-816e-aff9fdb3666f/m3-io-control.json",
            sha256: "0".repeat(64),
            serviceId: "hwlab-cloud-api",
            environment: "dev",
            metadata: {
              traceId: "trc_hotfix_307_verify_1779527747043_read_false",
              route: "res_boxsimu_1:DO1->hwlab-patch-panel->res_boxsimu_2:DI1"
            },
            createdAt: "2026-05-23T00:00:00.000Z"
          }
        ],
        count: 1
      };
    }
  };
}