fix: return errors for blocked durable queries

2026-05-23 02:53:10 +00:00
parent 892a0276bd
commit 6d474d10d7
3 changed files with 256 additions and 4 deletions
@@ -66,6 +66,17 @@ and a runtime blocker such as `runtime_durable_adapter_auth_blocked`,
 `runtime_durable_adapter_migration_blocked`. Fixture data and env presence are
 never promoted to `runtime.liveRuntimeEvidence`.

+When the Postgres durable adapter is selected or requested, JSON-RPC
+`audit.event.query` and `evidence.record.query` must also honor that durability
+gate. If auth, schema, migration, durability-query, or table read-query evidence
+is blocked, the method returns a JSON-RPC error with redacted blocker data such
+as `adapter: "postgres"`, `durableRequested: true`,
+`liveRuntimeEvidence: false`, `secretMaterialRead: false`, and the classified
+blocker. It must not
+return `events: []`, `records: []`, or `count: 0` on a blocked durable read.
+Empty successful result sets are valid only after the durable adapter has
+completed the read and the matching durable table rows are actually absent.
+
 If `HWLAB_CLOUD_DB_URL` and `HWLAB_CLOUD_DB_SSL_MODE` are absent, health
 reports `db.status: "blocked"`.
 If both env vars are present, cloud-api attempts a short read-only TCP
@@ -809,7 +809,9 @@ export class PostgresCloudRuntimeStore {
  }

  async queryAuditEvents(params = {}) {
-    const result = await this.query(
+    await this.assertReadyForDurableReads("audit.event.query");
+    const result = await this.queryDurableReadRows(
+      "audit.event.query",
      "SELECT event_json FROM audit_events ORDER BY timestamp ASC",
      []
    );
@@ -842,7 +844,9 @@ export class PostgresCloudRuntimeStore {
  }

  async queryEvidenceRecords(params = {}) {
-    const result = await this.query(
+    await this.assertReadyForDurableReads("evidence.record.query");
+    const result = await this.queryDurableReadRows(
+      "evidence.record.query",
      "SELECT metadata_json FROM evidence_records ORDER BY created_at ASC",
      []
    );
@@ -944,6 +948,25 @@ export class PostgresCloudRuntimeStore {
    }
  }

+  async assertReadyForDurableReads(method) {
+    const readiness = await this.readiness();
+    if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) {
+      throw durableRuntimeReadBlockedError(method, readiness);
+    }
+  }
+
+  async queryDurableReadRows(method, sql, params = []) {
+    try {
+      return await this.query(sql, params);
+    } catch (error) {
+      this.lastReadiness = this.blockedSummary({
+        ...classifyRuntimeDbError(error),
+        reason: `Postgres durable runtime adapter could not complete ${method} read query`
+      });
+      throw durableRuntimeReadBlockedError(method, this.summary());
+    }
+  }
+
  async persistGatewaySession(record) {
    await this.query(
      "INSERT INTO gateway_sessions (id, project_id, gateway_service_id, status, started_at, ended_at, gateway_session_json) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO UPDATE SET project_id = EXCLUDED.project_id, gateway_service_id = EXCLUDED.gateway_service_id, status = EXCLUDED.status, started_at = EXCLUDED.started_at, ended_at = EXCLUDED.ended_at, gateway_session_json = EXCLUDED.gateway_session_json",
@@ -1383,6 +1406,47 @@ function classifyRuntimeDbError(error) {
  };
 }

+function durableRuntimeReadBlockedError(method, readiness = {}) {
+  return new HwlabProtocolError("Postgres durable runtime adapter is blocked for runtime evidence queries", {
+    code: ERROR_CODES.internalError,
+    data: durableRuntimeReadBlockedData(method, readiness),
+    context: {
+      result: "failed"
+    }
+  });
+}
+
+function durableRuntimeReadBlockedData(method, readiness = {}) {
+  const durability = readiness.durabilityContract ?? {};
+  const connection = readiness.connection ?? {};
+  const blockedLayer = durability.blockedLayer ?? durabilityBlockedLayer(readiness);
+  const queryResult = connection.queryResult ?? (blockedLayer === "durability_query" ? "query_blocked" : "not_ready");
+
+  return pruneUndefined({
+    method,
+    adapter: readiness.adapter ?? RUNTIME_STORE_KIND_POSTGRES,
+    status: "blocked",
+    blocker: readiness.blocker ?? RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
+    durable: false,
+    durableRequested: true,
+    liveRuntimeEvidence: false,
+    queryResult,
+    blockedLayer,
+    requiredEvidence: durability.requiredEvidence ?? RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
+    dbLiveEvidenceIsDurabilityEvidence: false,
+    secretMaterialRead: false,
+    valueRedacted: true,
+    endpointRedacted: true,
+    errorCode: connection.errorCode ?? undefined,
+    redaction: {
+      valueRedacted: true,
+      valuesRedacted: true,
+      endpointRedacted: true,
+      secretMaterialRead: false
+    }
+  });
+}
+
 function summarizeRuntimeSchema(rows) {
  const columnsByTable = new Map();
  for (const row of rows) {
@@ -2,11 +2,12 @@ import assert from "node:assert/strict";
 import test from "node:test";

 import { handleJsonRpcRequest } from "../cloud/json-rpc.mjs";
-import { validateResponse } from "../protocol/index.mjs";
+import { ERROR_CODES, validateResponse } from "../protocol/index.mjs";
 import {
  RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED,
  RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED,
  RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED,
+  RUNTIME_DURABILITY_REQUIRED_EVIDENCE,
  RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED,
  RUNTIME_STORE_KIND_POSTGRES,
  createCloudRuntimeStore,
@@ -161,6 +162,172 @@ test("configured postgres runtime classifies durable read query blocker after sc
  assert.equal(readiness.durabilityContract.dbLiveEvidenceIsDurabilityEvidence, false);
 });

+test("JSON-RPC audit/evidence durable queries return blocked errors instead of empty success", async () => {
+  const secretDbUrl = "postgres://hwlab_user:super-secret-password@db.example.invalid:5432/hwlab";
+  const store = createConfiguredCloudRuntimeStore({
+    env: {
+      HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres",
+      HWLAB_CLOUD_RUNTIME_DURABLE: "true"
+    },
+    dbUrl: secretDbUrl,
+    queryClient: createFakePostgresClient({ migrationReady: true, countErrorCode: "57014" })
+  });
+  const context = { runtimeStore: store, dbProbe: { probe: false } };
+  const meta = {
+    traceId: "trc_01J00000000000000000002000",
+    actorId: "usr_01J00000000000000000002000",
+    serviceId: "hwlab-cloud-web",
+    environment: "dev"
+  };
+
+  for (const [id, method] of [
+    ["req_01J00000000000000000002001", "audit.event.query"],
+    ["req_01J00000000000000000002002", "evidence.record.query"]
+  ]) {
+    const response = await handleJsonRpcRequest(
+      {
+        jsonrpc: "2.0",
+        id,
+        method,
+        params: {
+          projectId: "prj_01J00000000000000000002000"
+        },
+        meta
+      },
+      context
+    );
+
+    validateResponse(response);
+    assert.equal(Object.hasOwn(response, "result"), false);
+    assert.equal(response.error.code, ERROR_CODES.internalError);
+    assert.equal(response.error.data.method, method);
+    assert.equal(response.error.data.adapter, "postgres");
+    assert.equal(response.error.data.status, "blocked");
+    assert.equal(response.error.data.blocker, RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED);
+    assert.equal(response.error.data.durable, false);
+    assert.equal(response.error.data.durableRequested, true);
+    assert.equal(response.error.data.liveRuntimeEvidence, false);
+    assert.equal(response.error.data.queryResult, "query_blocked");
+    assert.equal(response.error.data.blockedLayer, "durability_query");
+    assert.equal(response.error.data.requiredEvidence, RUNTIME_DURABILITY_REQUIRED_EVIDENCE);
+    assert.equal(response.error.data.dbLiveEvidenceIsDurabilityEvidence, false);
+    assert.equal(response.error.data.secretMaterialRead, false);
+    assert.equal(response.error.data.valueRedacted, true);
+    assert.equal(response.error.data.endpointRedacted, true);
+    assert.equal(Object.hasOwn(response.error.data, "events"), false);
+    assert.equal(Object.hasOwn(response.error.data, "records"), false);
+    assert.equal(Object.hasOwn(response.error.data, "count"), false);
+
+    const serialized = JSON.stringify(response);
+    assert.equal(serialized.includes("super-secret-password"), false);
+    assert.equal(serialized.includes(secretDbUrl), false);
+  }
+});
+
+test("JSON-RPC audit/evidence table read failures return blocked errors", async () => {
+  const queryClient = createFakePostgresClient({ migrationReady: true, readErrorCode: "57014" });
+  const store = createConfiguredCloudRuntimeStore({
+    env: {
+      HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres",
+      HWLAB_CLOUD_RUNTIME_DURABLE: "true"
+    },
+    dbUrl: "postgres://hwlab_user:another-secret@db.example.invalid:5432/hwlab",
+    queryClient
+  });
+  const context = { runtimeStore: store, dbProbe: { probe: false } };
+  const meta = {
+    traceId: "trc_01J00000000000000000002200",
+    actorId: "usr_01J00000000000000000002200",
+    serviceId: "hwlab-cloud-web",
+    environment: "dev"
+  };
+
+  for (const [id, method] of [
+    ["req_01J00000000000000000002201", "audit.event.query"],
+    ["req_01J00000000000000000002202", "evidence.record.query"]
+  ]) {
+    const response = await handleJsonRpcRequest(
+      {
+        jsonrpc: "2.0",
+        id,
+        method,
+        params: {
+          projectId: "prj_01J00000000000000000002200"
+        },
+        meta
+      },
+      context
+    );
+
+    validateResponse(response);
+    assert.equal(Object.hasOwn(response, "result"), false);
+    assert.equal(response.error.code, ERROR_CODES.internalError);
+    assert.equal(response.error.data.method, method);
+    assert.equal(response.error.data.blocker, RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED);
+    assert.equal(response.error.data.queryResult, "query_blocked");
+    assert.equal(response.error.data.blockedLayer, "durability_query");
+
+    const serialized = JSON.stringify(response);
+    assert.equal(serialized.includes("another-secret"), false);
+    assert.equal(serialized.includes("events"), false);
+    assert.equal(serialized.includes("records"), false);
+    assert.equal(serialized.includes("\"count\":0"), false);
+  }
+});
+
+test("JSON-RPC durable queries return successful empty result sets for true empty tables", async () => {
+  const queryClient = createFakePostgresClient({ migrationReady: true });
+  const store = createConfiguredCloudRuntimeStore({
+    env: {
+      HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres",
+      HWLAB_CLOUD_RUNTIME_DURABLE: "true"
+    },
+    dbUrl: "postgres://hwlab_redacted@db.example.invalid:5432/hwlab",
+    queryClient
+  });
+  const context = { runtimeStore: store, dbProbe: { probe: false } };
+  const meta = {
+    traceId: "trc_01J00000000000000000002100",
+    actorId: "usr_01J00000000000000000002100",
+    serviceId: "hwlab-cloud-web",
+    environment: "dev"
+  };
+
+  async function rpc(id, method, params = {}) {
+    const response = await handleJsonRpcRequest(
+      {
+        jsonrpc: "2.0",
+        id,
+        method,
+        params,
+        meta
+      },
+      context
+    );
+    validateResponse(response);
+    assert.equal(Object.hasOwn(response, "error"), false, response.error?.message);
+    return response.result;
+  }
+
+  const audit = await rpc("req_01J00000000000000000002101", "audit.event.query", {
+    projectId: "prj_01J00000000000000000002100"
+  });
+  assert.deepEqual(audit.events, []);
+  assert.equal(audit.count, 0);
+  assert.equal(audit.persistence.adapter, "postgres");
+  assert.equal(audit.persistence.durable, true);
+  assert.equal(audit.persistence.ready, true);
+
+  const evidence = await rpc("req_01J00000000000000000002102", "evidence.record.query", {
+    projectId: "prj_01J00000000000000000002100"
+  });
+  assert.deepEqual(evidence.records, []);
+  assert.equal(evidence.count, 0);
+  assert.equal(evidence.persistence.adapter, "postgres");
+  assert.equal(evidence.persistence.durable, true);
+  assert.equal(evidence.persistence.ready, true);
+});
+
 test("configured postgres runtime persists and queries records through query client", async () => {
  const queryClient = createFakePostgresClient({ migrationReady: true });
  const store = createConfiguredCloudRuntimeStore({
@@ -263,7 +430,7 @@ test("configured postgres runtime persists and queries records through query cli
  assert.equal(evidence.records[0].operationId, invoke.operationId);
 });

-function createFakePostgresClient({ migrationReady = true, countErrorCode = null } = {}) {
+function createFakePostgresClient({ migrationReady = true, countErrorCode = null, readErrorCode = null } = {}) {
  const state = {
    gateway_sessions: new Map(),
    box_resources: new Map(),
@@ -311,9 +478,19 @@ function createFakePostgresClient({ migrationReady = true, countErrorCode = null
        return jsonSelect(state.box_capabilities, params[0], "capability_json");
      }
      if (sql.startsWith("SELECT event_json FROM audit_events")) {
+        if (readErrorCode) {
+          const error = new Error("audit read query failed");
+          error.code = readErrorCode;
+          throw error;
+        }
        return { rows: [...state.audit_events.values()].map((record) => ({ event_json: record.event_json })) };
      }
      if (sql.startsWith("SELECT metadata_json FROM evidence_records")) {
+        if (readErrorCode) {
+          const error = new Error("evidence read query failed");
+          error.code = readErrorCode;
+          throw error;
+        }
        return { rows: [...state.evidence_records.values()].map((record) => ({ metadata_json: record.metadata_json })) };
      }
      if (sql.startsWith("INSERT INTO gateway_sessions")) {