From 6d05f20f82f449de487bc1669e977a0158b0d085 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Mon, 22 Jun 2026 19:16:19 +0800 Subject: [PATCH] fix(web): cooldown transient workbench read hydration failures (#1913) --- web/hwlab-cloud-web/src/stores/workbench.ts | 80 +++++++++++++++++++-- 1 file changed, 74 insertions(+), 6 deletions(-) diff --git a/web/hwlab-cloud-web/src/stores/workbench.ts b/web/hwlab-cloud-web/src/stores/workbench.ts index b87fe400..8abc99dc 100644 --- a/web/hwlab-cloud-web/src/stores/workbench.ts +++ b/web/hwlab-cloud-web/src/stores/workbench.ts @@ -26,6 +26,7 @@ const ACTIVE_TURN_GAP_INITIAL_DELAY_MS = 1_000; const ACTIVE_TURN_GAP_MAX_DELAY_MS = 5_000; const SESSION_LIST_PAGE_LIMIT = 20; const WORKBENCH_READ_HYDRATION_CONCURRENCY = 3; +const WORKBENCH_READ_FAILURE_COOLDOWN_MS = 5_000; const SESSION_LIST_REALTIME_REFRESH_DELAY_MS = 1_200; const SESSION_LIST_TERMINAL_REFRESH_DELAY_MS = 400; @@ -81,6 +82,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { const sessionListRefreshTimers = new Map(); let workbenchReadHydrationActive = 0; const workbenchReadHydrationQueue: Array<() => void> = []; + const workbenchReadCooldownUntilByKey = new Map(); const activeSession = computed(() => selectActiveSession(serverState.value, explicitSessionId.value)); const activeSessionSelectionSource = computed(() => activeSelectionSource.value); @@ -422,13 +424,31 @@ export const useWorkbenchStore = defineStore("workbench", () => { void source; } - function runWorkbenchReadHydration(task: () => Promise): Promise { + function runWorkbenchReadHydration(task: () => Promise, cooldownKey?: string): Promise { return new Promise((resolve, reject) => { const run = () => { + if (cooldownKey) { + const cooldownUntilMs = workbenchReadCooldownUntilByKey.get(cooldownKey) ?? 0; + if (cooldownUntilMs > Date.now()) { + resolve(createWorkbenchReadCooldownResult(cooldownKey, cooldownUntilMs) as T); + const next = workbenchReadHydrationQueue.shift(); + if (next) next(); + return; + } + if (cooldownUntilMs > 0) workbenchReadCooldownUntilByKey.delete(cooldownKey); + } workbenchReadHydrationActive += 1; Promise.resolve() .then(task) - .then(resolve, reject) + .then( + (value) => { + if (cooldownKey && isApiResultLike(value) && shouldCooldownWorkbenchReadFailure(value)) { + workbenchReadCooldownUntilByKey.set(cooldownKey, Date.now() + WORKBENCH_READ_FAILURE_COOLDOWN_MS); + } + resolve(value); + }, + reject, + ) .finally(() => { workbenchReadHydrationActive = Math.max(0, workbenchReadHydrationActive - 1); const next = workbenchReadHydrationQueue.shift(); @@ -443,18 +463,65 @@ export const useWorkbenchStore = defineStore("workbench", () => { }); } + function workbenchReadCooldownKey(kind: string, id: string): string { + return `${kind}:${id}`; + } + + function createWorkbenchReadCooldownResult(key: string, cooldownUntilMs: number): ApiResult { + const retryAfterMs = Math.max(0, cooldownUntilMs - Date.now()); + const message = `Workbench read hydration is cooling down after a transient failure (${key}); retry after ${retryAfterMs}ms.`; + const diagnostic = { + code: "workbench_read_hydration_cooldown", + category: "network", + source: "workbench-web", + message, + retryable: true, + }; + return { + ok: false, + status: 503, + error: message, + apiError: { + code: "workbench_read_hydration_cooldown", + category: "network", + source: "workbench-web", + message, + retryable: true, + diagnostic, + }, + diagnostic, + } as ApiResult; + } + + function isApiResultLike(value: unknown): value is ApiResult { + return Boolean(value && typeof value === "object" && "ok" in value && "status" in value); + } + + function shouldCooldownWorkbenchReadFailure(result: ApiResult): boolean { + const diagnostic = normalizeErrorDiagnostic(result.diagnostic, result.apiError?.diagnostic); + const code = firstStringOrNumber(result.apiError?.code, diagnostic?.code); + const category = firstNonEmptyString(result.apiError?.category, diagnostic?.category); + const source = firstNonEmptyString(result.apiError?.source, diagnostic?.source); + if (result.status === 0 && source === "browser" && code === "browser_network_error" && category === "network") return true; + if (result.status === 503) return code === "projection_store_unavailable" || code === "workbench_read_model_store_unavailable"; + return false; + } + function fetchWorkbenchTurnStatus(traceId: string): Promise> { - return runWorkbenchReadHydration(() => api.workbench.turn(traceId, 8000, () => activityRef.value)); + return runWorkbenchReadHydration(() => api.workbench.turn(traceId, 8000, () => activityRef.value), workbenchReadCooldownKey("turn", traceId)); } function fetchWorkbenchTraceEvents(traceId: string, afterProjectedSeq: number): Promise> { - return runWorkbenchReadHydration(() => api.workbench.traceEvents(traceId, codeAgentTimeoutMs.value, () => activityRef.value, { afterProjectedSeq, limit: TRACE_HYDRATION_PAGE_LIMIT })); + return runWorkbenchReadHydration( + () => api.workbench.traceEvents(traceId, codeAgentTimeoutMs.value, () => activityRef.value, { afterProjectedSeq, limit: TRACE_HYDRATION_PAGE_LIMIT }), + workbenchReadCooldownKey("trace-events", traceId), + ); } async function refreshSessionMessageProjectionPage(sessionId: string | null | undefined): Promise { const id = normalizeWorkbenchSessionId(sessionId); if (!id) return; - const response = await runWorkbenchReadHydration(() => api.workbench.sessionMessages(id, { limit: 100 })); + const response = await runWorkbenchReadHydration(() => api.workbench.sessionMessages(id, { limit: 100 }), workbenchReadCooldownKey("session-messages", id)); if (!response.ok || !response.data) return; const pageMessages = Array.isArray(response.data.messages) ? response.data.messages.map((message) => normalizeChatMessage(message as ChatMessage)) : []; rememberSessionMessages(id, mergeMessageProjectionPage(id, pageMessages)); @@ -463,7 +530,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { async function refreshMessageProjectionForTrace(sessionId: string | null | undefined, traceId: string): Promise { const id = normalizeWorkbenchSessionId(sessionId); if (!id) return; - const response = await api.workbench.sessionMessages(id, { limit: 100 }); + const response = await runWorkbenchReadHydration(() => api.workbench.sessionMessages(id, { limit: 100 }), workbenchReadCooldownKey("session-messages", id)); if (!response.ok || !response.data) { if (shouldSuppressTransientWorkbenchReadFailure(response)) return; if (traceHasCompletedFinalResponse(traceId, messages.value)) return; @@ -1742,6 +1809,7 @@ function shouldSuppressTransientWorkbenchReadFailure(result: ApiResult) const code = firstStringOrNumber(result.apiError?.code, diagnostic?.code); const category = firstNonEmptyString(result.apiError?.category, diagnostic?.category); const source = firstNonEmptyString(result.apiError?.source, diagnostic?.source); + if (code === "workbench_read_hydration_cooldown") return true; return result.status === 0 && source === "browser" && code === "browser_network_error" && category === "network"; }