mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(workbench): back off non-terminal projection resumes (#1803)

Browse Source
This commit is contained in:
Lyon
2026-06-21 09:51:15 +08:00
committed by GitHub
parent bc8805baaf
commit 0c25d44eac
1 changed files with 46 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/cloud/server-code-agent-http.ts
+46 -3
View File
@@ -1273,6 +1273,7 @@ async function runAgentRunProjectionResumePass({ options = {}, traceStore = defa
const candidates = stateCandidates.length > 0 ? stateCandidates : sessions.flatMap((session) => agentRunProjectionResumeCandidates(session, { minAgeMs }));
let resumed = 0;
let terminal = 0;
let nonTerminal = 0;
let failed = 0;
let skippedCooldown = 0;
let skippedExhausted = 0;
@@ -1291,8 +1292,25 @@ async function runAgentRunProjectionResumePass({ options = {}, traceStore = defa
try {
const result = await resumeAgentRunProjectionCandidate({ candidate, options, traceStore });
resumed += 1;
candidateRetryState.delete(retryKey);
if (isTraceCommandTerminalStatus(result?.status)) terminal += 1;
if (isTraceCommandTerminalStatus(result?.status)) {
terminal += 1;
candidateRetryState.delete(retryKey);
} else {
nonTerminal += 1;
const previousAttempts = Number.isFinite(retryState?.attempts) ? retryState.attempts : 0;
const retryAttempt = previousAttempts + 1;
const retryExhausted = retryAttempt >= candidateRetryMaxAttempts;
const retryDelayMs = retryExhausted ? 0 : projectionResumeCandidateRetryDelayMs(retryAttempt, candidateRetryInitialDelayMs, candidateRetryMaxDelayMs);
const nextRetryAtMs = retryExhausted ? null : Date.now() + retryDelayMs;
const nextRetryAt = Number.isFinite(nextRetryAtMs) ? new Date(nextRetryAtMs).toISOString() : null;
candidateRetryState.set(retryKey, {
attempts: retryAttempt,
nextRetryAtMs,
exhausted: retryExhausted,
lastErrorCode: "projection_resume_non_terminal"
});
appendProjectionResumeRetry(traceStore, candidate, { retryAttempt, retryMaxAttempts: candidateRetryMaxAttempts, retryLabel: retryAttempt + "/" + candidateRetryMaxAttempts, retryDelayMs, nextRetryAt, retryExhausted, status: result?.status ?? null });
}
} catch (error) {
failed += 1;
const previousAttempts = Number.isFinite(retryState?.attempts) ? retryState.attempts : 0;
@@ -1321,6 +1339,7 @@ async function runAgentRunProjectionResumePass({ options = {}, traceStore = defa
candidates: candidates.length,
resumed,
terminal,
nonTerminal,
failed,
skippedCooldown,
skippedExhausted,
@@ -1328,7 +1347,7 @@ async function runAgentRunProjectionResumePass({ options = {}, traceStore = defa
candidateRetryMaxAttempts,
valuesRedacted: true
});
return { offset, scannedSessions: sessions.length, candidates: candidates.length, resumed, terminal, failed, skippedCooldown, skippedExhausted };
return { offset, scannedSessions: sessions.length, candidates: candidates.length, resumed, terminal, nonTerminal, failed, skippedCooldown, skippedExhausted };
}
async function listAgentRunProjectionResumeStateCandidates(options = {}, batchSize = DEFAULT_AGENTRUN_PROJECTION_RESUME_BATCH_SIZE, offset = 0, { minAgeMs = DEFAULT_AGENTRUN_PROJECTION_RESUME_MIN_AGE_MS } = {}) {
@@ -1570,6 +1589,30 @@ function appendProjectionResumeError(traceStore, candidate, error, retry = {}) {
});
}
function appendProjectionResumeRetry(traceStore, candidate, retry = {}) {
const traceId = safeTraceId(candidate?.traceId);
if (!traceId) return;
traceStore.append(traceId, {
type: "turn-status",
status: "degraded",
label: "projection-resume:non-terminal-backoff",
errorCode: "projection_resume_non_terminal",
message: "AgentRun projection resume did not reach a terminal status; next resume is delayed.",
runId: candidate?.result?.agentRun?.runId ?? null,
commandId: candidate?.result?.agentRun?.commandId ?? null,
waitingFor: "agentrun-result-resume",
observedStatus: retry.status ?? null,
retryAttempt: retry.retryAttempt ?? null,
retryMaxAttempts: retry.retryMaxAttempts ?? null,
retryLabel: retry.retryLabel ?? null,
retryDelayMs: retry.retryDelayMs ?? null,
nextRetryAt: retry.nextRetryAt ?? null,
retryExhausted: retry.retryExhausted === true,
terminal: false,
valuesPrinted: false
});
}
function logAgentRunProjectionResume(logger, payload) {
try {
const line = JSON.stringify(payload);