diff --git a/docs/cloud-api-runtime.md b/docs/cloud-api-runtime.md index 9bc8618e..41be2b71 100644 --- a/docs/cloud-api-runtime.md +++ b/docs/cloud-api-runtime.md @@ -66,6 +66,17 @@ and a runtime blocker such as `runtime_durable_adapter_auth_blocked`, `runtime_durable_adapter_migration_blocked`. Fixture data and env presence are never promoted to `runtime.liveRuntimeEvidence`. +When the Postgres durable adapter is selected or requested, JSON-RPC +`audit.event.query` and `evidence.record.query` must also honor that durability +gate. If auth, schema, migration, durability-query, or table read-query evidence +is blocked, the method returns a JSON-RPC error with redacted blocker data such +as `adapter: "postgres"`, `durableRequested: true`, +`liveRuntimeEvidence: false`, `secretMaterialRead: false`, and the classified +blocker. It must not +return `events: []`, `records: []`, or `count: 0` on a blocked durable read. +Empty successful result sets are valid only after the durable adapter has +completed the read and the matching durable table rows are actually absent. + If `HWLAB_CLOUD_DB_URL` and `HWLAB_CLOUD_DB_SSL_MODE` are absent, health reports `db.status: "blocked"`. If both env vars are present, cloud-api attempts a short read-only TCP diff --git a/internal/db/runtime-store.mjs b/internal/db/runtime-store.mjs index 86d4c335..c9defe73 100644 --- a/internal/db/runtime-store.mjs +++ b/internal/db/runtime-store.mjs @@ -809,7 +809,9 @@ export class PostgresCloudRuntimeStore { } async queryAuditEvents(params = {}) { - const result = await this.query( + await this.assertReadyForDurableReads("audit.event.query"); + const result = await this.queryDurableReadRows( + "audit.event.query", "SELECT event_json FROM audit_events ORDER BY timestamp ASC", [] ); @@ -842,7 +844,9 @@ export class PostgresCloudRuntimeStore { } async queryEvidenceRecords(params = {}) { - const result = await this.query( + await this.assertReadyForDurableReads("evidence.record.query"); + const result = await this.queryDurableReadRows( + "evidence.record.query", "SELECT metadata_json FROM evidence_records ORDER BY created_at ASC", [] ); @@ -944,6 +948,25 @@ export class PostgresCloudRuntimeStore { } } + async assertReadyForDurableReads(method) { + const readiness = await this.readiness(); + if (readiness.ready !== true || readiness.durable !== true || readiness.liveRuntimeEvidence !== true) { + throw durableRuntimeReadBlockedError(method, readiness); + } + } + + async queryDurableReadRows(method, sql, params = []) { + try { + return await this.query(sql, params); + } catch (error) { + this.lastReadiness = this.blockedSummary({ + ...classifyRuntimeDbError(error), + reason: `Postgres durable runtime adapter could not complete ${method} read query` + }); + throw durableRuntimeReadBlockedError(method, this.summary()); + } + } + async persistGatewaySession(record) { await this.query( "INSERT INTO gateway_sessions (id, project_id, gateway_service_id, status, started_at, ended_at, gateway_session_json) VALUES ($1, $2, $3, $4, $5, $6, $7) ON CONFLICT (id) DO UPDATE SET project_id = EXCLUDED.project_id, gateway_service_id = EXCLUDED.gateway_service_id, status = EXCLUDED.status, started_at = EXCLUDED.started_at, ended_at = EXCLUDED.ended_at, gateway_session_json = EXCLUDED.gateway_session_json", @@ -1383,6 +1406,47 @@ function classifyRuntimeDbError(error) { }; } +function durableRuntimeReadBlockedError(method, readiness = {}) { + return new HwlabProtocolError("Postgres durable runtime adapter is blocked for runtime evidence queries", { + code: ERROR_CODES.internalError, + data: durableRuntimeReadBlockedData(method, readiness), + context: { + result: "failed" + } + }); +} + +function durableRuntimeReadBlockedData(method, readiness = {}) { + const durability = readiness.durabilityContract ?? {}; + const connection = readiness.connection ?? {}; + const blockedLayer = durability.blockedLayer ?? durabilityBlockedLayer(readiness); + const queryResult = connection.queryResult ?? (blockedLayer === "durability_query" ? "query_blocked" : "not_ready"); + + return pruneUndefined({ + method, + adapter: readiness.adapter ?? RUNTIME_STORE_KIND_POSTGRES, + status: "blocked", + blocker: readiness.blocker ?? RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + durable: false, + durableRequested: true, + liveRuntimeEvidence: false, + queryResult, + blockedLayer, + requiredEvidence: durability.requiredEvidence ?? RUNTIME_DURABILITY_REQUIRED_EVIDENCE, + dbLiveEvidenceIsDurabilityEvidence: false, + secretMaterialRead: false, + valueRedacted: true, + endpointRedacted: true, + errorCode: connection.errorCode ?? undefined, + redaction: { + valueRedacted: true, + valuesRedacted: true, + endpointRedacted: true, + secretMaterialRead: false + } + }); +} + function summarizeRuntimeSchema(rows) { const columnsByTable = new Map(); for (const row of rows) { diff --git a/internal/db/runtime-store.test.mjs b/internal/db/runtime-store.test.mjs index 80670e95..b780e221 100644 --- a/internal/db/runtime-store.test.mjs +++ b/internal/db/runtime-store.test.mjs @@ -2,11 +2,12 @@ import assert from "node:assert/strict"; import test from "node:test"; import { handleJsonRpcRequest } from "../cloud/json-rpc.mjs"; -import { validateResponse } from "../protocol/index.mjs"; +import { ERROR_CODES, validateResponse } from "../protocol/index.mjs"; import { RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED, RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED, + RUNTIME_DURABILITY_REQUIRED_EVIDENCE, RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, RUNTIME_STORE_KIND_POSTGRES, createCloudRuntimeStore, @@ -161,6 +162,172 @@ test("configured postgres runtime classifies durable read query blocker after sc assert.equal(readiness.durabilityContract.dbLiveEvidenceIsDurabilityEvidence, false); }); +test("JSON-RPC audit/evidence durable queries return blocked errors instead of empty success", async () => { + const secretDbUrl = "postgres://hwlab_user:super-secret-password@db.example.invalid:5432/hwlab"; + const store = createConfiguredCloudRuntimeStore({ + env: { + HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres", + HWLAB_CLOUD_RUNTIME_DURABLE: "true" + }, + dbUrl: secretDbUrl, + queryClient: createFakePostgresClient({ migrationReady: true, countErrorCode: "57014" }) + }); + const context = { runtimeStore: store, dbProbe: { probe: false } }; + const meta = { + traceId: "trc_01J00000000000000000002000", + actorId: "usr_01J00000000000000000002000", + serviceId: "hwlab-cloud-web", + environment: "dev" + }; + + for (const [id, method] of [ + ["req_01J00000000000000000002001", "audit.event.query"], + ["req_01J00000000000000000002002", "evidence.record.query"] + ]) { + const response = await handleJsonRpcRequest( + { + jsonrpc: "2.0", + id, + method, + params: { + projectId: "prj_01J00000000000000000002000" + }, + meta + }, + context + ); + + validateResponse(response); + assert.equal(Object.hasOwn(response, "result"), false); + assert.equal(response.error.code, ERROR_CODES.internalError); + assert.equal(response.error.data.method, method); + assert.equal(response.error.data.adapter, "postgres"); + assert.equal(response.error.data.status, "blocked"); + assert.equal(response.error.data.blocker, RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED); + assert.equal(response.error.data.durable, false); + assert.equal(response.error.data.durableRequested, true); + assert.equal(response.error.data.liveRuntimeEvidence, false); + assert.equal(response.error.data.queryResult, "query_blocked"); + assert.equal(response.error.data.blockedLayer, "durability_query"); + assert.equal(response.error.data.requiredEvidence, RUNTIME_DURABILITY_REQUIRED_EVIDENCE); + assert.equal(response.error.data.dbLiveEvidenceIsDurabilityEvidence, false); + assert.equal(response.error.data.secretMaterialRead, false); + assert.equal(response.error.data.valueRedacted, true); + assert.equal(response.error.data.endpointRedacted, true); + assert.equal(Object.hasOwn(response.error.data, "events"), false); + assert.equal(Object.hasOwn(response.error.data, "records"), false); + assert.equal(Object.hasOwn(response.error.data, "count"), false); + + const serialized = JSON.stringify(response); + assert.equal(serialized.includes("super-secret-password"), false); + assert.equal(serialized.includes(secretDbUrl), false); + } +}); + +test("JSON-RPC audit/evidence table read failures return blocked errors", async () => { + const queryClient = createFakePostgresClient({ migrationReady: true, readErrorCode: "57014" }); + const store = createConfiguredCloudRuntimeStore({ + env: { + HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres", + HWLAB_CLOUD_RUNTIME_DURABLE: "true" + }, + dbUrl: "postgres://hwlab_user:another-secret@db.example.invalid:5432/hwlab", + queryClient + }); + const context = { runtimeStore: store, dbProbe: { probe: false } }; + const meta = { + traceId: "trc_01J00000000000000000002200", + actorId: "usr_01J00000000000000000002200", + serviceId: "hwlab-cloud-web", + environment: "dev" + }; + + for (const [id, method] of [ + ["req_01J00000000000000000002201", "audit.event.query"], + ["req_01J00000000000000000002202", "evidence.record.query"] + ]) { + const response = await handleJsonRpcRequest( + { + jsonrpc: "2.0", + id, + method, + params: { + projectId: "prj_01J00000000000000000002200" + }, + meta + }, + context + ); + + validateResponse(response); + assert.equal(Object.hasOwn(response, "result"), false); + assert.equal(response.error.code, ERROR_CODES.internalError); + assert.equal(response.error.data.method, method); + assert.equal(response.error.data.blocker, RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED); + assert.equal(response.error.data.queryResult, "query_blocked"); + assert.equal(response.error.data.blockedLayer, "durability_query"); + + const serialized = JSON.stringify(response); + assert.equal(serialized.includes("another-secret"), false); + assert.equal(serialized.includes("events"), false); + assert.equal(serialized.includes("records"), false); + assert.equal(serialized.includes("\"count\":0"), false); + } +}); + +test("JSON-RPC durable queries return successful empty result sets for true empty tables", async () => { + const queryClient = createFakePostgresClient({ migrationReady: true }); + const store = createConfiguredCloudRuntimeStore({ + env: { + HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres", + HWLAB_CLOUD_RUNTIME_DURABLE: "true" + }, + dbUrl: "postgres://hwlab_redacted@db.example.invalid:5432/hwlab", + queryClient + }); + const context = { runtimeStore: store, dbProbe: { probe: false } }; + const meta = { + traceId: "trc_01J00000000000000000002100", + actorId: "usr_01J00000000000000000002100", + serviceId: "hwlab-cloud-web", + environment: "dev" + }; + + async function rpc(id, method, params = {}) { + const response = await handleJsonRpcRequest( + { + jsonrpc: "2.0", + id, + method, + params, + meta + }, + context + ); + validateResponse(response); + assert.equal(Object.hasOwn(response, "error"), false, response.error?.message); + return response.result; + } + + const audit = await rpc("req_01J00000000000000000002101", "audit.event.query", { + projectId: "prj_01J00000000000000000002100" + }); + assert.deepEqual(audit.events, []); + assert.equal(audit.count, 0); + assert.equal(audit.persistence.adapter, "postgres"); + assert.equal(audit.persistence.durable, true); + assert.equal(audit.persistence.ready, true); + + const evidence = await rpc("req_01J00000000000000000002102", "evidence.record.query", { + projectId: "prj_01J00000000000000000002100" + }); + assert.deepEqual(evidence.records, []); + assert.equal(evidence.count, 0); + assert.equal(evidence.persistence.adapter, "postgres"); + assert.equal(evidence.persistence.durable, true); + assert.equal(evidence.persistence.ready, true); +}); + test("configured postgres runtime persists and queries records through query client", async () => { const queryClient = createFakePostgresClient({ migrationReady: true }); const store = createConfiguredCloudRuntimeStore({ @@ -263,7 +430,7 @@ test("configured postgres runtime persists and queries records through query cli assert.equal(evidence.records[0].operationId, invoke.operationId); }); -function createFakePostgresClient({ migrationReady = true, countErrorCode = null } = {}) { +function createFakePostgresClient({ migrationReady = true, countErrorCode = null, readErrorCode = null } = {}) { const state = { gateway_sessions: new Map(), box_resources: new Map(), @@ -311,9 +478,19 @@ function createFakePostgresClient({ migrationReady = true, countErrorCode = null return jsonSelect(state.box_capabilities, params[0], "capability_json"); } if (sql.startsWith("SELECT event_json FROM audit_events")) { + if (readErrorCode) { + const error = new Error("audit read query failed"); + error.code = readErrorCode; + throw error; + } return { rows: [...state.audit_events.values()].map((record) => ({ event_json: record.event_json })) }; } if (sql.startsWith("SELECT metadata_json FROM evidence_records")) { + if (readErrorCode) { + const error = new Error("evidence read query failed"); + error.code = readErrorCode; + throw error; + } return { rows: [...state.evidence_records.values()].map((record) => ({ metadata_json: record.metadata_json })) }; } if (sql.startsWith("INSERT INTO gateway_sessions")) {