mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(workbench): back off projection resume pass failures (#1802)

Browse Source
This commit is contained in:
Lyon
2026-06-21 09:45:11 +08:00
committed by GitHub
parent 5b66e9db10
commit bc8805baaf
1 changed files with 27 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/cloud/server-code-agent-http.ts
+27 -6
View File
@@ -1184,7 +1184,11 @@ export function startAgentRunProjectionResume({ options = {}, traceStore = defau
const candidateRetryMaxAttempts = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_MAX_ATTEMPTS, 5);
const candidateRetryInitialDelayMs = Math.max(1000, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_INITIAL_DELAY_MS, 30000));
const candidateRetryMaxDelayMs = Math.max(candidateRetryInitialDelayMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_CANDIDATE_RETRY_MAX_DELAY_MS, 600000));
const passRetryMaxAttempts = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_MAX_ATTEMPTS, 5);
const passRetryInitialDelayMs = Math.max(failureBackoffMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_INITIAL_DELAY_MS, failureBackoffMs));
const passRetryMaxDelayMs = Math.max(passRetryInitialDelayMs, parseNonNegativeInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_RESUME_PASS_RETRY_MAX_DELAY_MS, 600000));
const candidateRetryState = new Map();
let passFailureAttempt = 0;
let stopped = false;
let active = false;
let timer = null;
@@ -1197,28 +1201,42 @@ export function startAgentRunProjectionResume({ options = {}, traceStore = defau
timer = null;
const offset = nextOffset;
active = true;
let failed = false;
let nextDelayMs = intervalMs;
let nextReason = "interval";
void runAgentRunProjectionResumePass({ options, traceStore, logger, batchSize, minAgeMs, offset, reason, candidateRetryState, candidateRetryMaxAttempts, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs })
.then((result) => {
failed = result?.failed > 0;
nextOffset = failed ? offset : (result?.scannedSessions >= batchSize ? offset + batchSize : 0);
passFailureAttempt = 0;
const candidateFailed = result?.failed > 0;
nextOffset = candidateFailed ? offset : (result?.scannedSessions >= batchSize ? offset + batchSize : 0);
nextDelayMs = candidateFailed ? failureBackoffMs : intervalMs;
nextReason = candidateFailed ? "failure_backoff" : "interval";
})
.catch((error) => {
failed = true;
passFailureAttempt += 1;
const retryDelayMs = projectionResumeCandidateRetryDelayMs(passFailureAttempt, passRetryInitialDelayMs, passRetryMaxDelayMs);
const retryExhausted = passFailureAttempt >= passRetryMaxAttempts;
nextDelayMs = retryDelayMs;
nextReason = retryExhausted ? "pass_retry_exhausted" : "pass_failure_backoff";
logAgentRunProjectionResume(logger, {
event: "workbench_projection_resume",
ok: false,
status: "failed",
status: retryExhausted ? "stopped" : "failed",
reason,
offset,
errorCode: error?.code ?? "projection_resume_pass_failed",
message: error?.message ?? "AgentRun projection resume pass failed.",
passRetryAttempt: passFailureAttempt,
passRetryMaxAttempts,
passRetryLabel: passFailureAttempt + "/" + passRetryMaxAttempts,
passRetryDelayMs: retryDelayMs,
passRetryExhausted: retryExhausted,
valuesRedacted: true
});
if (retryExhausted) stopped = true;
})
.finally(() => {
active = false;
if (!stopped && intervalMs > 0) schedule(failed ? failureBackoffMs : intervalMs, failed ? "failure_backoff" : "interval");
if (!stopped && intervalMs > 0) schedule(nextDelayMs, nextReason);
});
}, scheduledDelayMs);
timer.unref?.();
@@ -1236,6 +1254,9 @@ export function startAgentRunProjectionResume({ options = {}, traceStore = defau
candidateRetryMaxAttempts,
candidateRetryInitialDelayMs,
candidateRetryMaxDelayMs,
passRetryMaxAttempts,
passRetryInitialDelayMs,
passRetryMaxDelayMs,
valuesPrinted: false,
stop() {
stopped = true;