fix(workbench): retry transient runtime store reads (#1797)

deploy/deploy.yaml

@@ -266,6 +266,9 @@ lanes:
       postgres:
         poolMax: 16
         connectionTimeoutMs: 5000
+        queryRetryMaxAttempts: 5
+        queryRetryInitialDelayMs: 250
+        queryRetryMaxDelayMs: 5000
     envReuseServices:
       - hwlab-cloud-api
       - hwlab-user-billing

internal/db/runtime-store.ts

@@ -120,6 +120,22 @@ export function buildPostgresPoolConfig({ dbUrl, sslMode = "require", timeoutMs,
   };
 }
 
+const RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS";
+const RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_INITIAL_DELAY_MS";
+const RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV = "HWLAB_CLOUD_DB_QUERY_RETRY_MAX_DELAY_MS";
+const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS = 5;
+const DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS = 250;
+const DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS = 5_000;
+
+function normalizePositiveInteger(value, fallback) {
+  const number = Number(value);
+  return Number.isFinite(number) && number > 0 ? Math.trunc(number) : fallback;
+}
+
+function delayRuntimeDbQueryRetry(ms) {
+  return new Promise((resolve) => setTimeout(resolve, Math.max(0, Math.trunc(Number(ms) || 0))));
+}
+
 export class CloudRuntimeStore {
   constructor({ now = () => new Date().toISOString() } = {}) {
     this.now = now;
@@ -1775,7 +1791,7 @@ export class PostgresCloudRuntimeStore {
     }
   }
 
-  recordRuntimeDbOperationFailure(label, error) {
+  recordRuntimeDbOperationFailure(label, error, retry = {}) {
     const classified = classifyRuntimeDbError(error);
     this.lastReadiness = this.blockedSummary({
       ...classified,
@@ -1790,21 +1806,48 @@ export class PostgresCloudRuntimeStore {
       retryable: classified.retryable === true,
       transient: classified.transient === true,
       retryAfterMs: classified.retryAfterMs ?? null,
+      retryAttempt: Number.isFinite(Number(retry.attempt)) ? Number(retry.attempt) : null,
+      retryMaxAttempts: Number.isFinite(Number(retry.maxAttempts)) ? Number(retry.maxAttempts) : null,
+      retrying: retry.retrying === true,
+      retryDelayMs: Number.isFinite(Number(retry.delayMs)) ? Number(retry.delayMs) : null,
+      retryLabel: retry.label ?? null,
       valuesRedacted: true,
       endpointRedacted: true
     });
   }
 
   async query(sql, params = []) {
-    try {
-      const client = await this.getQueryClient();
-      return await client.query(sql, params);
-    } catch (error) {
-      this.recordRuntimeDbOperationFailure("query", error);
-      throw error;
+    const retry = this.runtimeQueryRetryPolicy();
+    for (let attempt = 1; attempt <= retry.maxAttempts; attempt += 1) {
+      try {
+        const client = await this.getQueryClient();
+        return await client.query(sql, params);
+      } catch (error) {
+        const classified = classifyRuntimeDbError(error);
+        const retrying = classified.retryable === true && classified.transient === true && attempt < retry.maxAttempts;
+        const delayMs = retrying ? Math.min(retry.maxDelayMs, retry.initialDelayMs * (2 ** Math.max(0, attempt - 1))) : null;
+        this.recordRuntimeDbOperationFailure("query", error, {
+          attempt,
+          maxAttempts: retry.maxAttempts,
+          retrying,
+          delayMs,
+          label: `${attempt}/${retry.maxAttempts}`
+        });
+        if (!retrying) throw error;
+        await delayRuntimeDbQueryRetry(delayMs);
+      }
     }
   }
 
+  runtimeQueryRetryPolicy() {
+    const env = this.env ?? {};
+    return {
+      maxAttempts: normalizePositiveInteger(env[RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS_ENV], DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_ATTEMPTS),
+      initialDelayMs: normalizePositiveInteger(env[RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS_ENV], DEFAULT_RUNTIME_DB_QUERY_RETRY_INITIAL_DELAY_MS),
+      maxDelayMs: normalizePositiveInteger(env[RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS_ENV], DEFAULT_RUNTIME_DB_QUERY_RETRY_MAX_DELAY_MS)
+    };
+  }
+
   async withDurableTransaction(label, fn) {
     try {
       const client = await this.getQueryClient();

scripts/gitops-render.mjs

@@ -994,6 +994,15 @@ function runtimeStoreEnvForProfile(deploy, profile, serviceId) {
   if (Number.isInteger(postgres.connectionTimeoutMs)) {
     env.HWLAB_CLOUD_DB_PROBE_TIMEOUT_MS = String(postgres.connectionTimeoutMs);
   }
+  if (Number.isInteger(postgres.queryRetryMaxAttempts)) {
+    env.HWLAB_CLOUD_DB_QUERY_RETRY_MAX_ATTEMPTS = String(postgres.queryRetryMaxAttempts);
+  }
+  if (Number.isInteger(postgres.queryRetryInitialDelayMs)) {
+    env.HWLAB_CLOUD_DB_QUERY_RETRY_INITIAL_DELAY_MS = String(postgres.queryRetryInitialDelayMs);
+  }
+  if (Number.isInteger(postgres.queryRetryMaxDelayMs)) {
+    env.HWLAB_CLOUD_DB_QUERY_RETRY_MAX_DELAY_MS = String(postgres.queryRetryMaxDelayMs);
+  }
   return env;
 }
 

web/hwlab-cloud-web/src/stores/workbench.ts

@@ -1199,9 +1199,10 @@ export const useWorkbenchStore = defineStore("workbench", () => {
   }
 
   async function clearActiveTrace(traceId: string, reason: string): Promise<void> {
-    void reason;
+    const shouldKeepGapRefresh = activeTurnGapTraceId() === traceId;
     if (currentRequest.value?.traceId === traceId) currentRequest.value = null;
     clearActiveTurnGapRefresh(traceId);
+    if (shouldKeepGapRefresh && activeTurnGapTraceId() === traceId) scheduleActiveTurnGapRefresh(`clear-active-trace:${reason}`);
   }
 
   function beginSessionSelection(): number {