From 0c25d44eacb18bbf6815741a420c1194856c21d7 Mon Sep 17 00:00:00 2001
From: Lyon <88232613+pikasTech@users.noreply.github.com>
Date: Sun, 21 Jun 2026 09:51:15 +0800
Subject: [PATCH] fix(workbench): back off non-terminal projection resumes
 (#1803)

---
 internal/cloud/server-code-agent-http.ts | 49 ++++++++++++++++++++++--
 1 file changed, 46 insertions(+), 3 deletions(-)

diff --git a/internal/cloud/server-code-agent-http.ts b/internal/cloud/server-code-agent-http.ts
index 1f315c6e..aa9de56f 100644
--- a/internal/cloud/server-code-agent-http.ts
+++ b/internal/cloud/server-code-agent-http.ts
@@ -1273,6 +1273,7 @@ async function runAgentRunProjectionResumePass({ options = {}, traceStore = defa
   const candidates = stateCandidates.length > 0 ? stateCandidates : sessions.flatMap((session) => agentRunProjectionResumeCandidates(session, { minAgeMs }));
   let resumed = 0;
   let terminal = 0;
+  let nonTerminal = 0;
   let failed = 0;
   let skippedCooldown = 0;
   let skippedExhausted = 0;
@@ -1291,8 +1292,25 @@ async function runAgentRunProjectionResumePass({ options = {}, traceStore = defa
     try {
       const result = await resumeAgentRunProjectionCandidate({ candidate, options, traceStore });
       resumed += 1;
-      candidateRetryState.delete(retryKey);
-      if (isTraceCommandTerminalStatus(result?.status)) terminal += 1;
+      if (isTraceCommandTerminalStatus(result?.status)) {
+        terminal += 1;
+        candidateRetryState.delete(retryKey);
+      } else {
+        nonTerminal += 1;
+        const previousAttempts = Number.isFinite(retryState?.attempts) ? retryState.attempts : 0;
+        const retryAttempt = previousAttempts + 1;
+        const retryExhausted = retryAttempt >= candidateRetryMaxAttempts;
+        const retryDelayMs = retryExhausted ? 0 : projectionResumeCandidateRetryDelayMs(retryAttempt, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs);
+        const nextRetryAtMs = retryExhausted ? null : Date.now() + retryDelayMs;
+        const nextRetryAt = Number.isFinite(nextRetryAtMs) ? new Date(nextRetryAtMs).toISOString() : null;
+        candidateRetryState.set(retryKey, {
+          attempts: retryAttempt,
+          nextRetryAtMs,
+          exhausted: retryExhausted,
+          lastErrorCode: "projection_resume_non_terminal"
+        });
+        appendProjectionResumeRetry(traceStore, candidate, { retryAttempt, retryMaxAttempts: candidateRetryMaxAttempts, retryLabel: retryAttempt + "/" + candidateRetryMaxAttempts, retryDelayMs, nextRetryAt, retryExhausted, status: result?.status ?? null });
+      }
     } catch (error) {
       failed += 1;
       const previousAttempts = Number.isFinite(retryState?.attempts) ? retryState.attempts : 0;
@@ -1321,6 +1339,7 @@ async function runAgentRunProjectionResumePass({ options = {}, traceStore = defa
     candidates: candidates.length,
     resumed,
     terminal,
+    nonTerminal,
     failed,
     skippedCooldown,
     skippedExhausted,
@@ -1328,7 +1347,7 @@ async function runAgentRunProjectionResumePass({ options = {}, traceStore = defa
     candidateRetryMaxAttempts,
     valuesRedacted: true
   });
-  return { offset, scannedSessions: sessions.length, candidates: candidates.length, resumed, terminal, failed, skippedCooldown, skippedExhausted };
+  return { offset, scannedSessions: sessions.length, candidates: candidates.length, resumed, terminal, nonTerminal, failed, skippedCooldown, skippedExhausted };
 }
 
 async function listAgentRunProjectionResumeStateCandidates(options = {}, batchSize = DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE, offset = 0, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS } = {}) {
@@ -1570,6 +1589,30 @@ function appendProjectionResumeError(traceStore, candidate, error, retry = {}) {
   });
 }
 
+function appendProjectionResumeRetry(traceStore, candidate, retry = {}) {
+  const traceId = safeTraceId(candidate?.traceId);
+  if (!traceId) return;
+  traceStore.append(traceId, {
+    type: "turn-status",
+    status: "degraded",
+    label: "projection-resume:non-terminal-backoff",
+    errorCode: "projection_resume_non_terminal",
+    message: "AgentRun projection resume did not reach a terminal status; next resume is delayed.",
+    runId: candidate?.result?.agentRun?.runId ?? null,
+    commandId: candidate?.result?.agentRun?.commandId ?? null,
+    waitingFor: "agentrun-result-resume",
+    observedStatus: retry.status ?? null,
+    retryAttempt: retry.retryAttempt ?? null,
+    retryMaxAttempts: retry.retryMaxAttempts ?? null,
+    retryLabel: retry.retryLabel ?? null,
+    retryDelayMs: retry.retryDelayMs ?? null,
+    nextRetryAt: retry.nextRetryAt ?? null,
+    retryExhausted: retry.retryExhausted === true,
+    terminal: false,
+    valuesPrinted: false
+  });
+}
+
 function logAgentRunProjectionResume(logger, payload) {
   try {
     const line = JSON.stringify(payload);