From 4714e788bc5ccf386e7c75a9489f47841fbed2a4 Mon Sep 17 00:00:00 2001 From: Code Queue Review Date: Fri, 22 May 2026 23:37:47 +0000 Subject: [PATCH] fix: gate durable runtime readiness --- docs/cloud-api-runtime.md | 21 +- docs/reference/dev-runtime-boundary.md | 10 +- internal/cloud/health-contract.mjs | 22 +- internal/cloud/server.test.mjs | 12 +- .../migrations/0001_cloud_core_skeleton.sql | 18 ++ internal/db/runtime-store.mjs | 295 +++++++++++++----- internal/db/runtime-store.test.mjs | 62 +++- internal/db/schema.mjs | 11 +- internal/db/schema.test.mjs | 15 + web/hwlab-cloud-web/app.mjs | 3 + .../workbench-hardware-panel-contract.mjs | 20 +- .../workbench-hardware-panel.mjs | 39 ++- 12 files changed, 430 insertions(+), 98 deletions(-) diff --git a/docs/cloud-api-runtime.md b/docs/cloud-api-runtime.md index 42166b71..9bc8618e 100644 --- a/docs/cloud-api-runtime.md +++ b/docs/cloud-api-runtime.md @@ -48,11 +48,22 @@ non-secret env flags: The Postgres adapter uses `HWLAB_CLOUD_DB_URL` from `hwlab-cloud-api-dev-db/database-url`; source records only the Secret reference -and non-secret DNS contract. The adapter reports `runtime.ready: true` only -after a live schema query proves the required runtime tables and columns exist. -If DB auth or schema is missing, `/health/live` stays degraded and reports a -runtime blocker such as `runtime_durable_adapter_auth_blocked` or -`runtime_durable_adapter_schema_blocked`. Fixture data and env presence are +and non-secret DNS contract. The adapter reports `runtime.durable: true` only +after the configured adapter proves all durable gates through read-only runtime +checks: + +- auth/query access to the configured DB path; +- required runtime table and column schema; +- the source-controlled migration ledger row for + `0001_cloud_core_skeleton` / `runtime-durable-postgres-v1`; +- read readiness against the runtime tables used by audit and evidence + queries. + +If DB auth, schema, migration, or read readiness is missing, `/health/live` +stays degraded with `runtime.durable: false`, `runtime.durableRequested: true`, +and a runtime blocker such as `runtime_durable_adapter_auth_blocked`, +`runtime_durable_adapter_schema_blocked`, or +`runtime_durable_adapter_migration_blocked`. Fixture data and env presence are never promoted to `runtime.liveRuntimeEvidence`. If `HWLAB_CLOUD_DB_URL` and `HWLAB_CLOUD_DB_SSL_MODE` are absent, health diff --git a/docs/reference/dev-runtime-boundary.md b/docs/reference/dev-runtime-boundary.md index ef3e7ecb..b3651ea5 100644 --- a/docs/reference/dev-runtime-boundary.md +++ b/docs/reference/dev-runtime-boundary.md @@ -168,12 +168,14 @@ Readiness reports must keep these dimensions distinct: | Dimension | Green condition | Degraded/blocking condition | | --- | --- | --- | | DB live | `db.ready=true`, `db.connected=true`, `db.liveDbEvidence=true` | Missing env, failed connection, disabled probe, or no `liveDbEvidence` | -| Runtime durability | `runtime.durable=true` with a durable adapter | `runtime.adapter="memory"` or `runtime.durable=false` | +| Runtime durability | `runtime.adapter="postgres"`, `runtime.durable=true`, `runtime.ready=true`, `runtime.liveRuntimeEvidence=true`, and runtime auth/schema/migration/read gates are all ready | `runtime.adapter="memory"`, `runtime.durable=false`, missing migration ledger, failed schema/auth/readiness, or no `liveRuntimeEvidence` | Do not declare DEV-LIVE complete while `runtime.adapter="memory"` or -`runtime.durable=false`, even when DB live readiness is connected. The next -fix boundary is a durable runtime store and migration path; documenting or -testing this contract must not mutate runtime state or PROD. +`runtime.durable=false`, even when DB live readiness is connected. A selected +Postgres adapter may report `runtime.durableRequested=true`, but it still stays +non-durable until the schema, migration ledger, and read readiness gates are +proven through the configured adapter. Documenting or testing this contract +must not mutate runtime state or PROD. ## Runtime Substitution Ban diff --git a/internal/cloud/health-contract.mjs b/internal/cloud/health-contract.mjs index e0939c79..fd27512d 100644 --- a/internal/cloud/health-contract.mjs +++ b/internal/cloud/health-contract.mjs @@ -1,4 +1,7 @@ -import { RUNTIME_DURABLE_ADAPTER_MISSING } from "../db/runtime-store.mjs"; +import { + RUNTIME_DURABLE_ADAPTER_MISSING, + RUNTIME_STORE_KIND_POSTGRES +} from "../db/runtime-store.mjs"; export const CLOUD_API_HEALTH_CONTRACT_VERSION = "v2"; @@ -46,6 +49,9 @@ function isRuntimeReady(runtime) { if (runtime?.durable !== true) { return false; } + if (runtime?.ready !== true || runtime?.liveRuntimeEvidence !== true) { + return false; + } return !["blocked", "degraded", "failed"].includes(runtime.status); } @@ -87,31 +93,37 @@ function codeAgentBlocker(codeAgent) { function runtimeBlocker(runtime) { const durable = runtime?.durable === true; - const unhealthy = durable && ["blocked", "degraded", "failed"].includes(runtime?.status); + const durableRequested = runtime?.durableRequested === true || runtime?.adapter === RUNTIME_STORE_KIND_POSTGRES; + const unhealthy = (durable || durableRequested) && ["blocked", "degraded", "failed"].includes(runtime?.status); const blocker = runtime?.blocker; return { - code: durable && unhealthy ? blocker ?? "runtime_durable_adapter_unhealthy" : RUNTIME_DURABLE_ADAPTER_MISSING, + code: durableRequested && unhealthy ? blocker ?? "runtime_durable_adapter_unhealthy" : RUNTIME_DURABLE_ADAPTER_MISSING, type: "runtime_blocker", scope: "runtime-durable-adapter", status: "open", sourceIssue: "pikasTech/HWLAB#164", - summary: durable && unhealthy + summary: durableRequested && unhealthy ? runtime?.reason ?? "cloud-api durable runtime adapter is configured but not ready" : "cloud-api DB is live, but L1 runtime writes still use process-local memory because no DB-backed durable adapter is configured", evidence: { adapter: runtime?.adapter ?? "unknown", durable: Boolean(runtime?.durable), + durableRequested: Boolean(runtime?.durableRequested), + durableCapable: Boolean(runtime?.durableCapable), runtimeStatus: runtime?.status ?? "unknown", runtimeReady: Boolean(runtime?.ready), liveRuntimeEvidence: Boolean(runtime?.liveRuntimeEvidence), schemaReady: Boolean(runtime?.schema?.ready), schemaChecked: Boolean(runtime?.schema?.checked), + migrationReady: Boolean(runtime?.migration?.ready), + migrationChecked: Boolean(runtime?.migration?.checked), + gates: runtime?.gates ?? null, queryAttempted: Boolean(runtime?.connection?.queryAttempted), queryResult: runtime?.connection?.queryResult ?? "unknown", dbBackedAdapterRequired: true, secretMaterialRead: false }, - nextTask: durable + nextTask: durableRequested ? "Apply/repair the runtime DB schema or DB auth, then rerun /health/live without printing DB secret material." : "Set HWLAB_CLOUD_RUNTIME_ADAPTER=postgres after the runtime DB schema is available; keep HWLAB_CLOUD_DB_URL injected from Secret." }; diff --git a/internal/cloud/server.test.mjs b/internal/cloud/server.test.mjs index fecbe7d2..0b7d2112 100644 --- a/internal/cloud/server.test.mjs +++ b/internal/cloud/server.test.mjs @@ -200,7 +200,9 @@ test("cloud api health separates DB connected from durable runtime schema readin async readiness() { return { adapter: "postgres", - durable: true, + durable: false, + durableRequested: true, + durableCapable: false, ready: false, status: "blocked", blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, @@ -219,6 +221,11 @@ test("cloud api health separates DB connected from durable runtime schema readin missingTables: ["gateway_sessions"], missingColumns: ["gateway_sessions.gateway_session_json"] }, + migration: { + checked: false, + ready: false, + missing: true + }, counts: {} }; } @@ -234,7 +241,8 @@ test("cloud api health separates DB connected from durable runtime schema readin assert.equal(payload.status, "degraded"); assert.equal(payload.db.connected, true); assert.equal(payload.db.liveDbEvidence, true); - assert.equal(payload.runtime.durable, true); + assert.equal(payload.runtime.durable, false); + assert.equal(payload.runtime.durableRequested, true); assert.equal(payload.runtime.ready, false); assert.equal(payload.runtime.blocker, RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED); assert.equal(payload.readiness.components.db, "ready"); diff --git a/internal/db/migrations/0001_cloud_core_skeleton.sql b/internal/db/migrations/0001_cloud_core_skeleton.sql index eea2cd78..10fb02f5 100644 --- a/internal/db/migrations/0001_cloud_core_skeleton.sql +++ b/internal/db/migrations/0001_cloud_core_skeleton.sql @@ -115,3 +115,21 @@ CREATE TABLE IF NOT EXISTS evidence_records ( metadata_json TEXT NOT NULL DEFAULT '{}', created_at TEXT NOT NULL ); + +CREATE TABLE IF NOT EXISTS hwlab_schema_migrations ( + id TEXT PRIMARY KEY, + schema_version TEXT NOT NULL, + applied_at TEXT NOT NULL DEFAULT CURRENT_TIMESTAMP, + migration_json TEXT NOT NULL DEFAULT '{}' +); + +INSERT INTO hwlab_schema_migrations (id, schema_version, applied_at, migration_json) +VALUES ( + '0001_cloud_core_skeleton', + 'runtime-durable-postgres-v1', + CURRENT_TIMESTAMP, + '{"path":"internal/db/migrations/0001_cloud_core_skeleton.sql","runtime":"cloud-api"}' +) +ON CONFLICT (id) DO UPDATE SET + schema_version = EXCLUDED.schema_version, + migration_json = EXCLUDED.migration_json; diff --git a/internal/db/runtime-store.mjs b/internal/db/runtime-store.mjs index d2514a1d..3c6df7fd 100644 --- a/internal/db/runtime-store.mjs +++ b/internal/db/runtime-store.mjs @@ -11,6 +11,12 @@ import { createProtocolAuditEvent, deriveProtocolActorFromMeta } from "../audit/index.mjs"; +import { + CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS +} from "./schema.mjs"; export const RUNTIME_STORE_KIND = "memory"; export const RUNTIME_STORE_KIND_POSTGRES = "postgres"; @@ -19,68 +25,13 @@ export const RUNTIME_DURABLE_ADAPTER_UNCONFIGURED = "runtime_durable_adapter_unc export const RUNTIME_DURABLE_ADAPTER_DRIVER_MISSING = "runtime_durable_adapter_driver_missing"; export const RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED = "runtime_durable_adapter_auth_blocked"; export const RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED = "runtime_durable_adapter_schema_blocked"; +export const RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED = "runtime_durable_adapter_migration_blocked"; export const RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED = "runtime_durable_adapter_query_blocked"; export const RUNTIME_ADAPTER_ENV = "HWLAB_CLOUD_RUNTIME_ADAPTER"; export const RUNTIME_DURABLE_ENV = "HWLAB_CLOUD_RUNTIME_DURABLE"; -const requiredPostgresSchema = Object.freeze({ - gateway_sessions: Object.freeze([ - "id", - "project_id", - "gateway_service_id", - "status", - "started_at", - "ended_at", - "gateway_session_json" - ]), - box_resources: Object.freeze([ - "id", - "project_id", - "gateway_session_id", - "resource_state", - "labels_json", - "resource_json", - "updated_at" - ]), - box_capabilities: Object.freeze([ - "id", - "box_resource_id", - "capability_type", - "capability_json", - "updated_at" - ]), - hardware_operations: Object.freeze([ - "id", - "project_id", - "requested_by", - "operation_type", - "operation_json", - "status", - "requested_at", - "updated_at" - ]), - audit_events: Object.freeze([ - "id", - "request_id", - "actor", - "source", - "operation", - "target", - "result", - "timestamp", - "event_json" - ]), - evidence_records: Object.freeze([ - "id", - "project_id", - "operation_id", - "evidence_type", - "uri", - "metadata_json", - "created_at" - ]) -}); +const requiredPostgresSchema = CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS; const postgresCountTables = Object.freeze([ "gateway_sessions", @@ -631,7 +582,7 @@ export class PostgresCloudRuntimeStore { this.pool = null; this.memory = new CloudRuntimeStore({ now }); this.lastReadiness = this.blockedSummary({ - blocker: dbUrl || queryClient ? "runtime durable adapter has not completed schema readiness" : RUNTIME_DURABLE_ADAPTER_UNCONFIGURED, + blocker: dbUrl || queryClient ? RUNTIME_DURABLE_ADAPTER_QUERY_BLOCKED : RUNTIME_DURABLE_ADAPTER_UNCONFIGURED, reason: dbUrl || queryClient ? "Postgres runtime adapter is configured but has not completed a live schema query" : "Postgres runtime adapter requires HWLAB_CLOUD_DB_URL or an injected query client" @@ -671,31 +622,131 @@ export class PostgresCloudRuntimeStore { this.lastReadiness = this.blockedSummary({ blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, reason: "Postgres runtime adapter connected, but required runtime tables or columns are missing", - schema + schema, + connection: { + queryAttempted: true, + queryResult: "schema_blocked" + }, + gates: runtimeGates({ + auth: readyGate(), + schema: blockedGate({ checked: true, blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED }), + migration: notCheckedGate(), + durability: blockedGate({ checked: false, blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED }) + }) + }); + return this.summary(); + } + + let migration; + try { + migration = await this.readMigrationReadiness(); + } catch (error) { + const classified = classifyRuntimeDbError(error); + this.lastReadiness = this.blockedSummary({ + blocker: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED + ? classified.blocker + : RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, + reason: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED + ? classified.reason + : "Postgres runtime schema is present, but the durable runtime migration ledger could not be proven", + schema, + migration: blockedRuntimeMigration({ errorCode: classified.connection?.errorCode }), + connection: { + queryAttempted: true, + queryResult: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED ? "auth_blocked" : "migration_blocked", + errorCode: classified.connection?.errorCode ?? null + }, + gates: runtimeGates({ + auth: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED + ? blockedGate({ checked: true, blocker: classified.blocker }) + : readyGate(), + schema: readyGate(), + migration: blockedGate({ + checked: true, + blocker: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED + ? classified.blocker + : RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED + }), + durability: blockedGate({ checked: false, blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED }) + }) + }); + return this.summary(); + } + + if (!migration.ready) { + this.lastReadiness = this.blockedSummary({ + blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, + reason: "Postgres runtime schema is present, but the required source migration is not recorded", + schema, + migration, + connection: { + queryAttempted: true, + queryResult: "migration_blocked" + }, + gates: runtimeGates({ + auth: readyGate(), + schema: readyGate(), + migration: blockedGate({ checked: true, blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED }), + durability: blockedGate({ checked: false, blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED }) + }) + }); + return this.summary(); + } + + let counts; + try { + counts = await this.readCounts(); + } catch (error) { + const classified = classifyRuntimeDbError(error); + this.lastReadiness = this.blockedSummary({ + ...classified, + schema, + migration, + connection: { + queryAttempted: true, + queryResult: classified.connection?.queryResult ?? "query_blocked", + errorCode: classified.connection?.errorCode ?? null + }, + gates: runtimeGates({ + auth: classified.blocker === RUNTIME_DURABLE_ADAPTER_AUTH_BLOCKED + ? blockedGate({ checked: true, blocker: classified.blocker }) + : readyGate(), + schema: readyGate(), + migration: readyGate(), + durability: blockedGate({ checked: true, blocker: classified.blocker }) + }) }); return this.summary(); } - const counts = await this.readCounts().catch(() => null); this.lastReadiness = { adapter: RUNTIME_STORE_KIND_POSTGRES, durable: true, + durableRequested: true, + durableCapable: true, ready: true, status: "ready", blocker: null, - reason: "Postgres durable runtime adapter completed a live schema readiness query", + reason: "Postgres durable runtime adapter completed live schema, migration, and read readiness queries", liveRuntimeEvidence: true, fixtureEvidence: false, connection: { queryAttempted: true, - queryResult: "schema_ready", + queryResult: "durable_readiness_ready", endpointRedacted: true, valueRedacted: true }, schema, + migration, + gates: runtimeGates({ + auth: readyGate(), + schema: readyGate(), + migration: readyGate(), + durability: readyGate() + }), safety: runtimeSafety(), adapterContract: postgresAdapterContract(), - counts: counts ?? this.memory.summary().counts + counts }; return this.summary(); } @@ -992,6 +1043,27 @@ export class PostgresCloudRuntimeStore { return counts; } + async readMigrationReadiness() { + const result = await this.query( + `SELECT id, schema_version FROM ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE} WHERE id = $1 LIMIT 1`, + [CLOUD_CORE_MIGRATION_ID] + ); + const row = result.rows?.[0] ?? null; + const ready = + row?.id === CLOUD_CORE_MIGRATION_ID && + row?.schema_version === CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION; + return { + checked: true, + ready, + table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + requiredMigrationId: CLOUD_CORE_MIGRATION_ID, + requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + appliedMigrationId: row?.id ?? null, + appliedSchemaVersion: row?.schema_version ?? null, + missing: !row + }; + } + async query(sql, params = []) { const client = await this.getQueryClient(); return client.query(sql, params); @@ -1033,10 +1105,19 @@ export class PostgresCloudRuntimeStore { return this.pool; } - blockedSummary({ blocker, reason, schema, connection }) { + blockedSummary({ blocker, reason, schema, migration, connection, gates }) { + const blockedSchema = schema ?? { + ready: false, + checked: false, + missingTables: [], + missingColumns: [] + }; + const blockedMigration = migration ?? notCheckedRuntimeMigration(); return { adapter: RUNTIME_STORE_KIND_POSTGRES, - durable: true, + durable: false, + durableRequested: true, + durableCapable: false, ready: false, status: "blocked", blocker, @@ -1050,12 +1131,18 @@ export class PostgresCloudRuntimeStore { valueRedacted: true, errorCode: connection?.errorCode ?? null }, - schema: schema ?? { - ready: false, - checked: false, - missingTables: [], - missingColumns: [] - }, + schema: blockedSchema, + migration: blockedMigration, + gates: gates ?? runtimeGates({ + auth: connection?.queryAttempted ? blockedGate({ checked: true, blocker }) : notCheckedGate(), + schema: blockedSchema.checked + ? gateFromReadiness(blockedSchema.ready, { blocker: RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED }) + : notCheckedGate(), + migration: blockedMigration.checked + ? gateFromReadiness(blockedMigration.ready, { blocker: RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED }) + : notCheckedGate(), + durability: blockedGate({ checked: false, blocker }) + }), safety: runtimeSafety(), adapterContract: postgresAdapterContract() }; @@ -1289,6 +1376,72 @@ function summarizeRuntimeSchema(rows) { }; } +function notCheckedRuntimeMigration() { + return { + checked: false, + ready: false, + table: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + requiredMigrationId: CLOUD_CORE_MIGRATION_ID, + requiredSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + appliedMigrationId: null, + appliedSchemaVersion: null, + missing: true + }; +} + +function blockedRuntimeMigration({ errorCode = null } = {}) { + return { + ...notCheckedRuntimeMigration(), + checked: true, + errorCode + }; +} + +function runtimeGates({ + auth = notCheckedGate(), + schema = notCheckedGate(), + migration = notCheckedGate(), + durability = notCheckedGate() +} = {}) { + return { + auth, + schema, + migration, + durability + }; +} + +function readyGate() { + return { + checked: true, + ready: true, + status: "ready", + blocker: null + }; +} + +function blockedGate({ checked = true, blocker = "runtime_durable_adapter_blocked" } = {}) { + return { + checked, + ready: false, + status: checked ? "blocked" : "not_checked", + blocker + }; +} + +function notCheckedGate() { + return { + checked: false, + ready: false, + status: "not_checked", + blocker: null + }; +} + +function gateFromReadiness(ready, { blocker } = {}) { + return ready ? readyGate() : blockedGate({ checked: true, blocker }); +} + function snapshotMemory(memory) { return { gatewaySessions: new Map(memory.gatewaySessions), diff --git a/internal/db/runtime-store.test.mjs b/internal/db/runtime-store.test.mjs index 8b085a4d..39686fd2 100644 --- a/internal/db/runtime-store.test.mjs +++ b/internal/db/runtime-store.test.mjs @@ -4,12 +4,17 @@ import test from "node:test"; import { handleJsonRpcRequest } from "../cloud/json-rpc.mjs"; import { validateResponse } from "../protocol/index.mjs"; import { + RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED, RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED, RUNTIME_STORE_KIND_POSTGRES, createCloudRuntimeStore, createConfiguredCloudRuntimeStore } from "./runtime-store.mjs"; -import { CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS } from "./schema.mjs"; +import { + CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS +} from "./schema.mjs"; test("memory runtime store is never marked durable", () => { const store = createCloudRuntimeStore(); @@ -39,7 +44,8 @@ test("configured postgres runtime reports schema blocker without green readiness const readiness = await store.readiness(); assert.equal(readiness.adapter, RUNTIME_STORE_KIND_POSTGRES); - assert.equal(readiness.durable, true); + assert.equal(readiness.durable, false); + assert.equal(readiness.durableRequested, true); assert.equal(readiness.ready, false); assert.equal(readiness.status, "blocked"); assert.equal(readiness.blocker, RUNTIME_DURABLE_ADAPTER_SCHEMA_BLOCKED); @@ -47,11 +53,40 @@ test("configured postgres runtime reports schema blocker without green readiness assert.equal(readiness.fixtureEvidence, false); assert.equal(readiness.schema.checked, true); assert.ok(readiness.schema.missingTables.includes("gateway_sessions")); + assert.equal(readiness.migration.checked, false); + assert.equal(readiness.gates.schema.status, "blocked"); + assert.equal(readiness.gates.migration.status, "not_checked"); assert.equal(JSON.stringify(readiness).includes("redacted@db.example"), false); }); +test("configured postgres runtime keeps durable false when migration ledger is missing", async () => { + const store = createConfiguredCloudRuntimeStore({ + env: { + HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres", + HWLAB_CLOUD_RUNTIME_DURABLE: "true" + }, + dbUrl: "postgres://hwlab_redacted@db.example.invalid:5432/hwlab", + queryClient: createFakePostgresClient({ migrationReady: false }) + }); + + const readiness = await store.readiness(); + assert.equal(readiness.adapter, RUNTIME_STORE_KIND_POSTGRES); + assert.equal(readiness.durable, false); + assert.equal(readiness.durableRequested, true); + assert.equal(readiness.ready, false); + assert.equal(readiness.status, "blocked"); + assert.equal(readiness.blocker, RUNTIME_DURABLE_ADAPTER_MIGRATION_BLOCKED); + assert.equal(readiness.schema.ready, true); + assert.equal(readiness.migration.checked, true); + assert.equal(readiness.migration.ready, false); + assert.equal(readiness.migration.missing, true); + assert.equal(readiness.gates.schema.status, "ready"); + assert.equal(readiness.gates.migration.status, "blocked"); + assert.equal(readiness.liveRuntimeEvidence, false); +}); + test("configured postgres runtime persists and queries records through query client", async () => { - const queryClient = createFakePostgresClient(); + const queryClient = createFakePostgresClient({ migrationReady: true }); const store = createConfiguredCloudRuntimeStore({ env: { HWLAB_CLOUD_RUNTIME_ADAPTER: "postgres", @@ -88,9 +123,15 @@ test("configured postgres runtime persists and queries records through query cli const health = await rpc("req_01J00000000000000000001000", "system.health"); assert.equal(health.runtime.adapter, "postgres"); assert.equal(health.runtime.durable, true); + assert.equal(health.runtime.durableRequested, true); + assert.equal(health.runtime.durableCapable, true); assert.equal(health.runtime.ready, true); assert.equal(health.runtime.liveRuntimeEvidence, true); assert.equal(health.runtime.fixtureEvidence, false); + assert.equal(health.runtime.schema.ready, true); + assert.equal(health.runtime.migration.ready, true); + assert.equal(health.runtime.gates.migration.status, "ready"); + assert.equal(health.runtime.gates.durability.status, "ready"); assert.equal(health.readiness.components.runtime, "ready"); await rpc("req_01J00000000000000000001001", "gateway.session.register", { @@ -141,15 +182,22 @@ test("configured postgres runtime persists and queries records through query cli assert.equal(evidence.records[0].operationId, invoke.operationId); }); -function createFakePostgresClient() { +function createFakePostgresClient({ migrationReady = true } = {}) { const state = { gateway_sessions: new Map(), box_resources: new Map(), box_capabilities: new Map(), hardware_operations: new Map(), audit_events: new Map(), - evidence_records: new Map() + evidence_records: new Map(), + hwlab_schema_migrations: new Map() }; + if (migrationReady) { + state.hwlab_schema_migrations.set(CLOUD_CORE_MIGRATION_ID, { + id: CLOUD_CORE_MIGRATION_ID, + schema_version: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION + }); + } const calls = []; return { @@ -163,6 +211,10 @@ function createFakePostgresClient() { const table = sql.match(/FROM ([a-z_]+)/u)?.[1]; return { rows: [{ count: state[table]?.size ?? 0 }] }; } + if (sql.startsWith("SELECT id, schema_version FROM hwlab_schema_migrations")) { + const row = state.hwlab_schema_migrations.get(params[0]); + return { rows: row ? [row] : [] }; + } if (sql.startsWith("SELECT gateway_session_json FROM gateway_sessions")) { return jsonSelect(state.gateway_sessions, params[0], "gateway_session_json"); } diff --git a/internal/db/schema.mjs b/internal/db/schema.mjs index 8fa8af47..f60d6d23 100644 --- a/internal/db/schema.mjs +++ b/internal/db/schema.mjs @@ -2,9 +2,17 @@ import { TABLES } from "../protocol/index.mjs"; export const CLOUD_CORE_MIGRATION_ID = "0001_cloud_core_skeleton"; export const CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION = "runtime-durable-postgres-v1"; +export const CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE = "hwlab_schema_migrations"; export const FROZEN_CLOUD_CORE_TABLES = Object.freeze([...TABLES]); +export const CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS = Object.freeze([ + "id", + "schema_version", + "applied_at", + "migration_json" +]); + export const CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS = Object.freeze({ gateway_sessions: Object.freeze([ "id", @@ -69,7 +77,8 @@ export const CLOUD_CORE_MIGRATIONS = Object.freeze([ path: "internal/db/migrations/0001_cloud_core_skeleton.sql", tables: FROZEN_CLOUD_CORE_TABLES, connectsToDatabase: false, - runtimeDurableAdapterSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION + runtimeDurableAdapterSchemaVersion: CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + runtimeDurableMigrationTable: CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE } ]); diff --git a/internal/db/schema.test.mjs b/internal/db/schema.test.mjs index 9f3aa5e5..92f719d7 100644 --- a/internal/db/schema.test.mjs +++ b/internal/db/schema.test.mjs @@ -7,6 +7,10 @@ import { fileURLToPath } from "node:url"; import { TABLES, assertProtocolRecord, assertProtocolRecords } from "../protocol/index.mjs"; import { CLOUD_CORE_MIGRATIONS, + CLOUD_CORE_MIGRATION_ID, + CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION, + CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE, + CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS, CLOUD_RUNTIME_DURABLE_TABLE_COLUMNS, FROZEN_CLOUD_CORE_TABLES, assertFrozenCloudCoreTables, @@ -48,6 +52,17 @@ test("initial migration exposes columns required by the durable runtime adapter" assert.match(tableMatch[1], new RegExp(`\\b${column}\\b`), `missing ${table}.${column}`); } } + + const migrationMatch = sql.match( + new RegExp(`CREATE TABLE IF NOT EXISTS ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE}\\s*\\(([\\s\\S]*?)\\);`, "u") + ); + assert.ok(migrationMatch, `missing ${CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE}`); + for (const column of CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE_COLUMNS) { + assert.match(migrationMatch[1], new RegExp(`\\b${column}\\b`), `missing migration ledger column ${column}`); + } + assert.match(sql, new RegExp(`'${CLOUD_CORE_MIGRATION_ID}'`, "u")); + assert.match(sql, new RegExp(`'${CLOUD_RUNTIME_DURABLE_ADAPTER_SCHEMA_VERSION}'`, "u")); + assert.equal(CLOUD_CORE_MIGRATIONS[0].runtimeDurableMigrationTable, CLOUD_RUNTIME_DURABLE_MIGRATION_TABLE); }); test("protocol record guards catch schema drift before runtime writes", () => { diff --git a/web/hwlab-cloud-web/app.mjs b/web/hwlab-cloud-web/app.mjs index 130c0068..39d7fb44 100644 --- a/web/hwlab-cloud-web/app.mjs +++ b/web/hwlab-cloud-web/app.mjs @@ -867,6 +867,9 @@ function liveRecordTone(record, live) { function liveRecordDurabilityBlocker(live) { const runtimeSummary = runtimeSummaryFrom(live); if (isDurableRuntimeReady(runtimeSummary)) return null; + if (runtimeSummary?.durableRequested === true || runtimeSummary?.adapter === "postgres") { + return `runtime durable adapter 未 ready（status=${runtimeSummary?.status ?? "unknown"}，durable=${runtimeSummary?.durable === true}）；可信记录保持 BLOCKED。`; + } if (runtimeSummary?.durable !== true) { return "runtime.durable=false；只读 audit/evidence 不能标为 durable ready 或 DEV-LIVE。"; } diff --git a/web/hwlab-cloud-web/scripts/workbench-hardware-panel-contract.mjs b/web/hwlab-cloud-web/scripts/workbench-hardware-panel-contract.mjs index 537967dd..2c5762fd 100644 --- a/web/hwlab-cloud-web/scripts/workbench-hardware-panel-contract.mjs +++ b/web/hwlab-cloud-web/scripts/workbench-hardware-panel-contract.mjs @@ -55,7 +55,8 @@ export function runWorkbenchHardwarePanelContract() { assert.equal(blockedModel.chain.toResourceId, "res_boxsimu_2"); assert.equal(blockedModel.chain.toPort, "DI1"); assert.match(blockedModel.chain.detail, /BLOCKED/u); - assert.match(blockedModel.chain.detail, /waiting for DEV-LIVE M3 loop/u); + assert.match(blockedModel.chain.detail, /runtime\.durable=false/u); + assert.equal(blockedModel.trustedLoop.runtimeDurability.ready, false); assert.ok( blockedModel.statuses.some((item) => item.kind === "BOX-SIMU" && item.id === "res_boxsimu_1"), @@ -110,7 +111,7 @@ export function runWorkbenchHardwarePanelContract() { assert.equal(misleadingGreenModel.chain.status, "blocked", "chain must be blocked without DEV-LIVE evidence"); assert.notEqual(misleadingGreenModel.statuses[0].status, "healthy", "healthy source-only status must be downgraded"); - const trustedModel = createHardwareEvidencePanelModel({ + const devLiveEvidenceWithMemoryRuntime = { chain: { ...M3_TRUSTED_CHAIN, status: "verified", @@ -142,6 +143,21 @@ export function runWorkbenchHardwarePanelContract() { summary: "Trusted M3 loop evidence is already recorded." } ] + } + }; + const nonDurableModel = createHardwareEvidencePanelModel(devLiveEvidenceWithMemoryRuntime); + assert.equal(nonDurableModel.trustedLoop.status, "blocked", "DEV-LIVE-looking records cannot promote without durable runtime"); + assert.equal(nonDurableModel.sourceKind, "BLOCKED"); + assert.match(nonDurableModel.trustedLoop.reason, /runtime\.durable=false/u); + + const trustedModel = createHardwareEvidencePanelModel({ + ...devLiveEvidenceWithMemoryRuntime, + runtime: { + adapter: "postgres", + durable: true, + ready: true, + status: "ready", + liveRuntimeEvidence: true }, controls: [ { diff --git a/web/hwlab-cloud-web/workbench-hardware-panel.mjs b/web/hwlab-cloud-web/workbench-hardware-panel.mjs index a6978fc2..8f4f8b26 100644 --- a/web/hwlab-cloud-web/workbench-hardware-panel.mjs +++ b/web/hwlab-cloud-web/workbench-hardware-panel.mjs @@ -127,15 +127,19 @@ export function createHardwareEvidencePanelModel(data = {}) { const records = normalizeRecords(data.records ?? defaultHardwareEvidencePanelData.records); const controls = normalizeControls(data.controls ?? defaultHardwareEvidencePanelData.controls); const chain = normalizeChain(data.chain ?? defaultHardwareEvidencePanelData.chain); + const runtimeDurability = normalizeRuntimeDurability(data.runtime ?? data.runtimeSummary); const hasTrustedDevLiveEvidence = records.evidence.some( (record) => record.sourceKind === "DEV-LIVE" && ["pass", "passed", "verified", "succeeded", "trusted"].includes(record.status) ); const chainMatches = isRequiredM3Chain(chain); - const trustStatus = hasTrustedDevLiveEvidence && chainMatches ? "verified" : "blocked"; + const trustStatus = runtimeDurability.ready && hasTrustedDevLiveEvidence && chainMatches ? "verified" : "blocked"; const trustTone = trustStatus === "verified" ? "trusted" : "degraded"; const sourceKind = trustStatus === "verified" ? "DEV-LIVE" : "BLOCKED"; + const blockedReason = runtimeDurability.ready + ? "Current data lacks real DEV-LIVE evidence; the workbench must render BLOCKED/degraded, never green." + : runtimeDurability.reason; const guardedChain = { ...chain, status: trustStatus === "verified" ? normalizeStatus(chain.status, "verified") : "blocked", @@ -143,7 +147,7 @@ export function createHardwareEvidencePanelModel(data = {}) { detail: trustStatus === "verified" ? chain.detail - : "BLOCKED: waiting for DEV-LIVE M3 loop evidence for res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1." + : `BLOCKED: ${blockedReason}` }; return { @@ -157,10 +161,11 @@ export function createHardwareEvidencePanelModel(data = {}) { status: trustStatus, tone: trustTone, sourceKind, + runtimeDurability, reason: trustStatus === "verified" ? "DEV-LIVE evidence is attached to the required M3 chain." - : "Current data lacks real DEV-LIVE evidence; the workbench must render BLOCKED/degraded, never green." + : blockedReason }, statuses: statuses.map((status) => ({ ...status, @@ -220,6 +225,34 @@ function normalizeRecords(records) { }; } +function normalizeRuntimeDurability(runtime) { + if (!runtime || typeof runtime !== "object" || Array.isArray(runtime)) { + return { + ready: false, + adapter: "memory", + durable: false, + liveRuntimeEvidence: false, + reason: "runtime.durable=false; memory/non-durable runtime cannot render DEV-LIVE or durable evidence." + }; + } + + const ready = + runtime.durable === true && + runtime.ready !== false && + runtime.liveRuntimeEvidence === true && + !["blocked", "degraded", "failed"].includes(String(runtime.status ?? "").toLowerCase()); + return { + ready, + adapter: stringOr(runtime.adapter, "unknown"), + durable: runtime.durable === true, + durableRequested: runtime.durableRequested === true, + liveRuntimeEvidence: runtime.liveRuntimeEvidence === true, + reason: ready + ? "Durable runtime readiness is proven." + : `runtime durable adapter is not ready (adapter=${stringOr(runtime.adapter, "unknown")}, durable=${runtime.durable === true}); trusted records stay BLOCKED.` + }; +} + function normalizeRecordList(records) { return arrayFrom(records).map((record) => ({ id: stringOr(record.id ?? record.operationId ?? record.auditId ?? record.evidenceId, "record"),