From 437ea223bb4cee818ac4a92ff75d40765e396614 Mon Sep 17 00:00:00 2001 From: lyon Date: Mon, 22 Jun 2026 05:56:24 +0800 Subject: [PATCH] fix: stop stale agentrun unreachable terminal projection --- internal/cloud/code-agent-agentrun-adapter.ts | 17 ++++-- internal/cloud/server-code-agent-http.ts | 55 ++++++++++++++++--- internal/cloud/workbench-projection-writer.ts | 2 +- internal/cloud/workbench-turn-projection.ts | 23 +++++++- 4 files changed, 82 insertions(+), 15 deletions(-) diff --git a/internal/cloud/code-agent-agentrun-adapter.ts b/internal/cloud/code-agent-agentrun-adapter.ts index 8e8c34e2..995db6a5 100644 --- a/internal/cloud/code-agent-agentrun-adapter.ts +++ b/internal/cloud/code-agent-agentrun-adapter.ts @@ -853,13 +853,24 @@ export async function refreshAgentRunTrace({ traceId, result = null, options = { export async function cancelAgentRunChatTurn({ traceId, currentResult = null, options = {}, traceStore = defaultCodeAgentTraceStore }) { const mapped = currentResult ?? await loadPersistedAgentRunResult(traceId, options); if (!mapped?.agentRun?.commandId) return null; + const env = options.env ?? process.env; + const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); + const cancelTimeoutMs = Math.min(timeoutMs, parsePositiveInteger(env.HWLAB_CODE_AGENT_CANCEL_TIMEOUT_MS, 5_000)); + const cancelOptions = { + ...options, + env: { + ...env, + HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS: String(cancelTimeoutMs), + HWLAB_CODE_AGENT_CANCEL_TIMEOUT_MS: String(cancelTimeoutMs) + } + }; const localTerminal = agentRunCancelTerminalStatus(mapped); if (localTerminal) return agentRunAlreadyTerminalCancelPayload({ traceId, payload: mapped, terminalStatus: localTerminal, options, traceStore }); try { const guard = await syncAgentRunChatResult({ traceId, currentResult: mapped, - options: { ...options, deferAgentRunResultSync: false }, + options: { ...cancelOptions, deferAgentRunResultSync: false }, traceStore, forceResultSync: true, refreshEvents: true @@ -880,14 +891,12 @@ export async function cancelAgentRunChatTurn({ traceId, currentResult = null, op valuesPrinted: false }, mapped.agentRun)); } - const env = options.env ?? process.env; const fetchImpl = options.fetchImpl ?? globalThis.fetch; const managerUrl = resolveAgentRunManagerUrl(env, mapped.agentRun.managerUrl); - const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_HTTP_TIMEOUT_MS, 20_000); await agentRunJson(fetchImpl, managerUrl, `/api/v1/commands/${encodeURIComponent(mapped.agentRun.commandId)}/cancel`, { method: "POST", body: { reason: "hwlab-user-cancel", traceId }, - timeoutMs, + timeoutMs: cancelTimeoutMs, env }); traceStore.append(traceId, agentRunTraceEvent({ diff --git a/internal/cloud/server-code-agent-http.ts b/internal/cloud/server-code-agent-http.ts index 61008a79..b9182a97 100644 --- a/internal/cloud/server-code-agent-http.ts +++ b/internal/cloud/server-code-agent-http.ts @@ -2202,17 +2202,18 @@ function codeAgentTurnStatusPayload({ traceId, result, snapshot, resultPollError const events = Array.isArray(snapshotObject?.events) ? snapshotObject.events : Array.isArray(resultObject?.runnerTrace?.events) ? resultObject.runnerTrace.events : []; const lastEvent = events.at(-1) ?? null; const finalResponse = resultObject?.finalResponse ?? snapshotObject?.finalResponse ?? snapshotObject?.terminalEvidence?.finalResponse ?? codeAgentFinalResponseEvidence(resultObject ?? snapshotObject ?? {}, traceId); - const sealedFinalStatus = codeAgentPayloadHasSealedFinalResponse(resultObject ?? snapshotObject ?? {}) ? "completed" : null; + const terminalStatus = codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId); const status = normalizeTurnStatus( - sealedFinalStatus, - resultObject?.status, - resultObject?.agentRun?.terminalStatus, + terminalStatus, resultObject?.agentRun?.commandState, resultObject?.agentRun?.status, - snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus, - snapshotObject?.traceStatus, - snapshotObject?.status, - snapshotObject?.runnerTrace?.status + resultObject?.agentRun?.runStatus, + codeAgentRunningStatus(snapshotObject?.status), + codeAgentRunningStatus(snapshotObject?.traceStatus), + codeAgentRunningStatus(snapshotObject?.runnerTrace?.status), + codeAgentRunningStatus(resultObject?.status), + resultObject?.agentRun?.terminalStatus, + snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus ); const found = Boolean(resultObject || (snapshotObject && snapshotObject.status !== "missing") || snapshotObject?.persisted === true); const running = isTurnRunningStatus(status); @@ -2251,6 +2252,35 @@ function codeAgentTurnStatusPayload({ traceId, result, snapshot, resultPollError }; } +function codeAgentAuthoritativeTerminalStatus(resultObject, snapshotObject, traceId) { + const sealedPayload = codeAgentPayloadHasSealedFinalResponse(resultObject ?? {}) ? resultObject : codeAgentPayloadHasSealedFinalResponse(snapshotObject ?? {}) ? snapshotObject : null; + if (sealedPayload) return "completed"; + const snapshotStatus = normalizeTurnStatus(snapshotObject?.terminalEvidence?.traceSummary?.terminalStatus, snapshotObject?.terminalEvidence?.agentRun?.terminalStatus, snapshotObject?.terminalEvidence?.status, snapshotObject?.status); + if (snapshotStatus && isTurnTerminalStatus(snapshotStatus) && codeAgentSnapshotHasTerminalAuthority(snapshotObject)) return snapshotStatus; + const agentRunStatus = normalizeTurnStatus(resultObject?.agentRun?.terminalStatus, resultObject?.agentRun?.commandState, resultObject?.agentRun?.status, resultObject?.agentRun?.runStatus); + if (agentRunStatus && isTurnTerminalStatus(agentRunStatus)) return agentRunStatus; + const resultStatus = normalizeTurnStatus(resultObject?.status); + if (resultStatus && isTurnTerminalStatus(resultStatus) && codeAgentResultHasTerminalAuthority(resultObject, traceId)) return resultStatus; + return null; +} + +function codeAgentSnapshotHasTerminalAuthority(snapshot = null) { + if (!snapshot || typeof snapshot !== "object") return false; + if (snapshot.terminal === true || snapshot.sealed === true) return true; + if (snapshot.terminalEvidence?.available === true || snapshot.terminalEvidence?.source) return true; + const events = Array.isArray(snapshot.events) ? snapshot.events : []; + return events.some((event) => event?.terminal === true || event?.final === true || event?.replyAuthority === true); +} + +function codeAgentResultHasTerminalAuthority(result = null, traceId = null) { + if (!result || typeof result !== "object") return false; + if (result.terminal === true || result.sealed === true) return true; + if (result.error || result.blocker) return true; + if (codeAgentPayloadHasSealedFinalResponse(result)) return true; + if (agentRunTerminalTraceEvidence(result, traceId)) return true; + return Boolean(textValue(result.finishedAt ?? result.completedAt ?? result.endedAt)); +} + function codeAgentRefreshErrorPayload(error, traceId, agentRun, fallbackCode) { return { code: error?.code ?? fallbackCode, @@ -2288,6 +2318,11 @@ function isTurnRunningStatus(status) { return status === "running"; } +function codeAgentRunningStatus(value) { + const status = normalizeTurnStatus(value); + return isTurnRunningStatus(status) ? status : null; +} + function isTurnTerminalStatus(status) { return CODE_AGENT_TERMINAL_STATUSES.has(String(status ?? "").trim().toLowerCase().replace(/_/gu, "-")); } @@ -3706,6 +3741,8 @@ function agentRunTerminalTraceEvidence(result, traceId) { lastSeq: result.agentRun.lastSeq ?? null, valuesPrinted: false } : null; + const terminalStatus = normalizeTurnStatus(result.agentRun?.terminalStatus, result.agentRun?.commandState, result.agentRun?.status, result.agentRun?.runStatus); + if (!isTraceCommandTerminalStatus(terminalStatus)) return null; if (!agentRun?.runId || !agentRun?.commandId || (!storedSummary && !finalResponse)) return null; const traceSummary = { traceId, @@ -3714,7 +3751,7 @@ function agentRunTerminalTraceEvidence(result, traceId) { renderedRowSummary: storedSummary?.renderedRowSummary ?? null, noiseEventCount: numberOrNull(storedSummary?.noiseEventCount), omittedNoiseCount: numberOrNull(storedSummary?.omittedNoiseCount), - terminalStatus: storedSummary?.terminalStatus ?? result.agentRun?.terminalStatus ?? result.status ?? null, + terminalStatus: storedSummary?.terminalStatus ?? terminalStatus, updatedAt: storedSummary?.updatedAt ?? result.updatedAt ?? null, valuesPrinted: false }; diff --git a/internal/cloud/workbench-projection-writer.ts b/internal/cloud/workbench-projection-writer.ts index 046c2abe..0e74525d 100644 --- a/internal/cloud/workbench-projection-writer.ts +++ b/internal/cloud/workbench-projection-writer.ts @@ -242,7 +242,7 @@ function buildWorkbenchProjectionFacts({ traceId = null, ownerUserId = null, own const normalizedStatus = normalizeWorkbenchStatus(payload?.status ?? session?.sessionStatus ?? status); const projection = createWorkbenchTurnProjection({ traceId: safeId, result: payload, session: { id: resolvedSessionId, status: normalizedStatus, session }, trace: payload?.runnerTrace ?? null }); const projectedStatus = normalizeWorkbenchStatus(projection.status ?? normalizedStatus); - const terminal = projection.terminal === true || TERMINAL_STATUSES.has(projectedStatus) || TERMINAL_STATUSES.has(normalizedStatus); + const terminal = projection.terminal === true; const terminalStatus = terminal ? (TERMINAL_STATUSES.has(projectedStatus) ? projectedStatus : normalizedStatus) : projectedStatus; const timing = projectionTimingForStatus(projection.timing, terminal); const diagnostic = projectionDiagnostics({ traceId: safeId, result: payload, trace: payload?.runnerTrace ?? null, projection }); diff --git a/internal/cloud/workbench-turn-projection.ts b/internal/cloud/workbench-turn-projection.ts index 24e2fe03..cd30d2c6 100644 --- a/internal/cloud/workbench-turn-projection.ts +++ b/internal/cloud/workbench-turn-projection.ts @@ -174,15 +174,36 @@ export function normalizeWorkbenchStatus(value) { } function terminalTurnEvidence({ result = null, traceTerminal = null } = {}) { - const resultStatus = terminalStatusFromValue(result?.status ?? result?.terminalStatus ?? result?.agentRun?.terminalStatus); + const resultStatus = terminalStatusFromValue( + result?.terminalStatus + ?? result?.agentRun?.terminalStatus + ?? result?.agentRun?.commandState + ?? result?.agentRun?.status + ?? result?.agentRun?.runStatus + ); if (resultStatus) { if (resultStatus !== "completed" && retryableProviderInterruptionEvidence(result, result?.agentRun, result?.providerTrace, traceTerminal?.evidence)) return null; return { source: "result", status: resultStatus, finalResponse: traceTerminal?.finalResponse ?? null, valuesRedacted: true }; } + const statusOnly = terminalStatusFromValue(result?.status); + if (statusOnly && resultHasTerminalAuthority(result, traceTerminal)) { + if (statusOnly !== "completed" && retryableProviderInterruptionEvidence(result, result?.agentRun, result?.providerTrace, traceTerminal?.evidence)) return null; + return { source: "result", status: statusOnly, finalResponse: traceTerminal?.finalResponse ?? null, valuesRedacted: true }; + } if (traceTerminal && retryableProviderInterruptionEvidence(traceTerminal.evidence)) return null; return traceTerminal; } +function resultHasTerminalAuthority(result = null, traceTerminal = null) { + if (!result || typeof result !== "object") return false; + if (traceTerminal) return true; + if (result.terminal === true || result.sealed === true) return true; + if (result.error || result.blocker) return true; + if (firstTimestamp(result.finishedAt, result.completedAt, result.endedAt)) return true; + const finalText = projectionText(result.finalResponse, result.assistantText, result.reply, result.text, result.summary); + return Boolean(finalText && terminalStatusFromValue(result.status)); +} + function activeTurnEvidence({ result = null, session = null, trace = null } = {}) { const retryEvidence = retryableProviderInterruptionEvidence(result, result?.agentRun, result?.providerTrace, trace); if (retryEvidence) return { source: "provider-retry", status: "retrying", evidence: retryEvidence, valuesRedacted: true };