From df487ec088909366461791d4f055c58799c7de9d Mon Sep 17 00:00:00 2001
From: Lyon <88232613+pikasTech@users.noreply.github.com>
Date: Mon, 22 Jun 2026 18:58:31 +0800
Subject: [PATCH] fix: bound Workbench read model latency (#1912)

---
 deploy/deploy.yaml                      |  7 +++++--
 internal/cloud/server-workbench-http.ts | 25 ++++++++++++++++++++++---
 2 files changed, 27 insertions(+), 5 deletions(-)

diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml
index 3cb7de17..5281198e 100644
--- a/deploy/deploy.yaml
+++ b/deploy/deploy.yaml
@@ -557,7 +557,10 @@ lanes:
           HWLAB_BOOTSTRAP_ADMIN_API_KEY: secretRef:hwlab-v03-master-server-admin-api-key/api-key
           HWLAB_USER_BILLING_URL: http://hwlab-user-billing.hwlab-v03.svc.cluster.local:6670
           HWLAB_WORKBENCH_RUNTIME_URL: http://hwlab-workbench-runtime.hwlab-v03.svc.cluster.local:6671
-          HWLAB_WORKBENCH_RUNTIME_TIMEOUT_MS: "10000"
+          HWLAB_WORKBENCH_RUNTIME_TIMEOUT_MS: "1800"
+          HWLAB_WORKBENCH_FACTS_QUERY_MAX_ATTEMPTS: "2"
+          HWLAB_WORKBENCH_FACTS_QUERY_BACKOFF_BASE_MS: "120"
+          HWLAB_WORKBENCH_FACTS_QUERY_BACKOFF_MAX_MS: "250"
           HWLAB_USER_BILLING_LOGIN_RETRY_ATTEMPTS: "2"
           HWLAB_USER_BILLING_LOGIN_RETRY_DELAY_MS: "250"
           HWLAB_USER_BILLING_CODE_AGENT_ENABLED: "1"
@@ -586,7 +589,7 @@ lanes:
           HWLAB_WORKBENCH_RUNTIME_DB_URL: secretRef:hwlab-cloud-api-v03-db/database-url
           HWLAB_WORKBENCH_RUNTIME_PORT: "6671"
           HWLAB_WORKBENCH_RUNTIME_DB_POOL_MAX: "32"
-          HWLAB_WORKBENCH_RUNTIME_QUERY_TIMEOUT_MS: "10000"
+          HWLAB_WORKBENCH_RUNTIME_QUERY_TIMEOUT_MS: "1500"
           HWLAB_WORKBENCH_RUNTIME_READY_TIMEOUT_MS: "2000"
           OTEL_EXPORTER_OTLP_TRACES_ENDPOINT: http://otel-collector.platform-infra.svc.cluster.local:4318/v1/traces
           OTEL_SERVICE_NAME: hwlab-workbench-runtime
diff --git a/internal/cloud/server-workbench-http.ts b/internal/cloud/server-workbench-http.ts
index 944e20c7..d1b1c4f6 100644
--- a/internal/cloud/server-workbench-http.ts
+++ b/internal/cloud/server-workbench-http.ts
@@ -136,7 +136,7 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option
   });
   emitWorkbenchRealtimeAcceptedOtelSpan(request, options.env);
   if (activeTraceId) {
-    await (perf ? perf.measure("workbench_initial_trace", () => writeTraceRealtimeSnapshot({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })) : writeTraceRealtimeSnapshot({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" }));
+    await (perf ? perf.measure("workbench_initial_trace", () => writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" })) : writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "initial" }));
     const runtimeStore = options.runtimeStore ?? null;
     const outboxPollMs = parsePositiveInteger(options.env?.HWLAB_WORKBENCH_SSE_OUTBOX_POLL_MS, 2000);
     let outboxCursor = 0;
@@ -158,7 +158,7 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option
                 cursor: { traceSeq: row.projectedSeq, outboxSeq: row.outboxSeq },
                 projectionStatus: { terminal: row.terminal, sealed: row.sealed }
               });
-              void writeTraceRealtimeSnapshot({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "terminal" });
+              void writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "terminal" });
             } else {
               writeEvent("workbench.trace.event", {
                 type: "trace.event",
@@ -203,7 +203,7 @@ export async function handleWorkbenchRealtimeHttp(request, response, url, option
           });
         }
         if (event?.terminal === true || TERMINAL_STATUSES.has(normalizeStatus(event?.status))) {
-          void writeTraceRealtimeSnapshot({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "terminal" });
+          void writeTraceRealtimeSnapshotSafe({ writeEvent, options, actor: auth.actor, traceId: activeTraceId, reason: "terminal" });
         }
       });
       cleanup.push(unsubscribe);
@@ -1826,6 +1826,25 @@ async function writeTraceRealtimeSnapshot({ writeEvent, options, actor, traceId,
   });
 }
 
+async function writeTraceRealtimeSnapshotSafe(input) {
+  try {
+    await writeTraceRealtimeSnapshot(input);
+  } catch (error) {
+    input.writeEvent("workbench.error", {
+      type: "error",
+      traceId: input.traceId,
+      reason: input.reason,
+      error: {
+        code: "workbench_trace_snapshot_failed",
+        causeCode: error?.code ?? null,
+        retryable: error?.data?.retryable === true,
+        message: error?.message ?? "Workbench trace snapshot failed.",
+        valuesRedacted: true
+      }
+    });
+  }
+}
+
 async function visibleTraceContext(options, actor, traceId) {
   const readModel = createWorkbenchReadModel(options, actor);
   const result = readModel.resultForTrace(traceId);