diff --git a/internal/cloud/server-code-agent-http.ts b/internal/cloud/server-code-agent-http.ts index 0fbdfbda..1f315c6e 100644 --- a/internal/cloud/server-code-agent-http.ts +++ b/internal/cloud/server-code-agent-http.ts @@ -1184,7 +1184,11 @@ export function startAgentRunProjectionResume({ options = {}, traceStore = defau const candidateRetryMaxAttempts = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_MAX_ATTEMPTS, 5); const candidateRetryInitialDelayMs = Math.max(1000, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_INITIAL_DELAY_MS, 30000)); const candidateRetryMaxDelayMs = Math.max(candidateRetryInitialDelayMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_MAX_DELAY_MS, 600000)); + const passRetryMaxAttempts = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_MAX_ATTEMPTS, 5); + const passRetryInitialDelayMs = Math.max(failureBackoffMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_INITIAL_DELAY_MS, failureBackoffMs)); + const passRetryMaxDelayMs = Math.max(passRetryInitialDelayMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_MAX_DELAY_MS, 600000)); const candidateRetryState = new Map(); + let passFailureAttempt = 0; let stopped = false; let active = false; let timer = null; @@ -1197,28 +1201,42 @@ export function startAgentRunProjectionResume({ options = {}, traceStore = defau timer = null; const offset = nextOffset; active = true; - let failed = false; + let nextDelayMs = intervalMs; + let nextReason = "interval"; void runAgentRunProjectionResumePass({ options, traceStore, logger, batchSize, minAgeMs, offset, reason, candidateRetryState, candidateRetryMaxAttempts, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs }) .then((result) => { - failed = result?.failed > 0; - nextOffset = failed ? offset : (result?.scannedSessions >= batchSize ? offset + batchSize : 0); + passFailureAttempt = 0; + const candidateFailed = result?.failed > 0; + nextOffset = candidateFailed ? offset : (result?.scannedSessions >= batchSize ? offset + batchSize : 0); + nextDelayMs = candidateFailed ? failureBackoffMs : intervalMs; + nextReason = candidateFailed ? "failure_backoff" : "interval"; }) .catch((error) => { - failed = true; + passFailureAttempt += 1; + const retryDelayMs = projectionResumeCandidateRetryDelayMs(passFailureAttempt, passRetryInitialDelayMs, passRetryMaxDelayMs); + const retryExhausted = passFailureAttempt >= passRetryMaxAttempts; + nextDelayMs = retryDelayMs; + nextReason = retryExhausted ? "pass_retry_exhausted" : "pass_failure_backoff"; logAgentRunProjectionResume(logger, { event: "workbench_projection_resume", ok: false, - status: "failed", + status: retryExhausted ? "stopped" : "failed", reason, offset, errorCode: error?.code ?? "projection_resume_pass_failed", message: error?.message ?? "AgentRun projection resume pass failed.", + passRetryAttempt: passFailureAttempt, + passRetryMaxAttempts, + passRetryLabel: passFailureAttempt + "/" + passRetryMaxAttempts, + passRetryDelayMs: retryDelayMs, + passRetryExhausted: retryExhausted, valuesRedacted: true }); + if (retryExhausted) stopped = true; }) .finally(() => { active = false; - if (!stopped && intervalMs > 0) schedule(failed ? failureBackoffMs : intervalMs, failed ? "failure_backoff" : "interval"); + if (!stopped && intervalMs > 0) schedule(nextDelayMs, nextReason); }); }, scheduledDelayMs); timer.unref?.(); @@ -1236,6 +1254,9 @@ export function startAgentRunProjectionResume({ options = {}, traceStore = defau candidateRetryMaxAttempts, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs, + passRetryMaxAttempts, + passRetryInitialDelayMs, + passRetryMaxDelayMs, valuesPrinted: false, stop() { stopped = true;