From edae2b40b42ac3a82fc66f3b0f7a4b794985347b Mon Sep 17 00:00:00 2001 From: lyon Date: Thu, 18 Jun 2026 22:36:30 +0800 Subject: [PATCH] fix: use idle-only AgentRun projection timeout --- deploy/deploy.yaml | 4 +- internal/cloud/server-code-agent-http.ts | 86 ++++++++++++++++++------ 2 files changed, 69 insertions(+), 21 deletions(-) diff --git a/deploy/deploy.yaml b/deploy/deploy.yaml index 9a96efcd..71e26f80 100644 --- a/deploy/deploy.yaml +++ b/deploy/deploy.yaml @@ -301,8 +301,10 @@ lanes: - skills/hwlab-agent-runtime/ env: HWLAB_METRICS_NAMESPACE: hwlab-v03 - HWLAB_OBSERVABILITY_CONFIG_REVISION: PJ2026-01060505-20260617-turn-status-budget + HWLAB_OBSERVABILITY_CONFIG_REVISION: PJ2026-01060505-20260618-no-response-timeout HWLAB_CODE_AGENT_TURN_STATUS_REFRESH_TIMEOUT_MS: "2500" + HWLAB_CODE_AGENT_AGENTRUN_NO_RESPONSE_TIMEOUT_MS: "90000" + HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS: "1000" observable: true hwlab-user-billing: runtimeKind: go-service diff --git a/internal/cloud/server-code-agent-http.ts b/internal/cloud/server-code-agent-http.ts index a9e49ff6..1a629278 100644 --- a/internal/cloud/server-code-agent-http.ts +++ b/internal/cloud/server-code-agent-http.ts @@ -791,16 +791,24 @@ function submitCodeAgentChatTurn({ params, options, traceId }) { function scheduleAgentRunProjectionSync({ traceId, params = {}, options = {}, traceStore = defaultCodeAgentTraceStore, currentResult = null } = {}) { if (!safeTraceId(traceId) || !currentResult?.agentRun?.runId || !currentResult?.agentRun?.commandId) return null; const env = options.env ?? process.env; - const timeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_TIMEOUT_MS, 90_000); + const noResponseTimeoutMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_NO_RESPONSE_TIMEOUT_MS, null); const pollIntervalMs = parsePositiveInteger(env.HWLAB_CODE_AGENT_AGENTRUN_PROJECTION_SYNC_POLL_INTERVAL_MS, 1_000); const refreshOptions = codeAgentTurnStatusRefreshOptions(options); - const startedAt = Date.now(); + let lastActivityAt = Date.now(); + let lastActivitySignature = agentRunProjectionActivitySignature(currentResult, traceStore.snapshot(traceId)); + let idleDegraded = false; setImmediate(() => { void (async () => { let result = currentResult; let lastError = null; - while (Date.now() - startedAt <= timeoutMs) { + while (true) { + const cached = options.codeAgentChatResults?.get?.(traceId); + if (isCodeAgentResultCanceled(cached)) return; + if (isTraceCommandTerminalStatus(cached?.status)) { + scheduleCodeAgentTerminalTurnStatusEffects({ payload: cached, params, options, preserveLastTraceId: true }); + return; + } try { const synced = await syncAgentRunChatResult({ traceId, currentResult: result, options: refreshOptions, traceStore }); result = synced.result ?? result; @@ -808,27 +816,38 @@ function scheduleAgentRunProjectionSync({ traceId, params = {}, options = {}, tr scheduleCodeAgentTerminalTurnStatusEffects({ payload: result, params, options, preserveLastTraceId: true }); return; } + const signature = agentRunProjectionActivitySignature(result, synced.runnerTrace ?? traceStore.snapshot(traceId)); + if (signature && signature !== lastActivitySignature) { + lastActivitySignature = signature; + lastActivityAt = Date.now(); + idleDegraded = false; + } lastError = null; } catch (error) { lastError = error; } - const elapsedMs = Date.now() - startedAt; - if (elapsedMs >= timeoutMs) break; - await sleepAgentRunProjectionSync(Math.min(pollIntervalMs, Math.max(0, timeoutMs - elapsedMs))); + const idleMs = Date.now() - lastActivityAt; + if (noResponseTimeoutMs && idleMs >= noResponseTimeoutMs && !idleDegraded) { + traceStore.append(traceId, { + type: "turn-status", + status: "degraded", + label: "agentrun:projection:no-response-idle-timeout", + errorCode: lastError?.code ?? "no_response_idle_timeout", + degradedReason: lastError ? "projection_activity_stale" : "no_response_idle_timeout", + message: lastError?.message ?? `AgentRun projection has no new activity for ${idleMs}ms; idle threshold ${noResponseTimeoutMs}ms.`, + runId: result?.agentRun?.runId ?? currentResult.agentRun.runId, + commandId: result?.agentRun?.commandId ?? currentResult.agentRun.commandId, + timeoutMs: noResponseTimeoutMs, + idleMs, + lastActivityAt: new Date(lastActivityAt).toISOString(), + waitingFor: "agentrun-result-activity", + terminal: false, + valuesPrinted: false + }); + idleDegraded = true; + } + await sleepAgentRunProjectionSync(pollIntervalMs); } - traceStore.append(traceId, { - type: "turn-status", - status: "degraded", - label: lastError ? "agentrun:projection:sync-failed" : "agentrun:projection:sync-timeout", - errorCode: lastError?.code ?? null, - message: lastError?.message ?? "AgentRun projection sync did not reach terminal within the configured budget.", - runId: result?.agentRun?.runId ?? currentResult.agentRun.runId, - commandId: result?.agentRun?.commandId ?? currentResult.agentRun.commandId, - timeoutMs, - waitingFor: "agentrun-result", - terminal: false, - valuesPrinted: false - }); })().catch((error) => { traceStore.append(traceId, { type: "turn-status", @@ -844,7 +863,34 @@ function scheduleAgentRunProjectionSync({ traceId, params = {}, options = {}, tr }); }); }); - return { scheduled: true, traceId, timeoutMs, pollIntervalMs, valuesPrinted: false }; + return { scheduled: true, traceId, noResponseTimeoutMs, pollIntervalMs, valuesPrinted: false }; +} + +function agentRunProjectionActivitySignature(result, runnerTrace) { + const agentRun = result?.agentRun && typeof result.agentRun === "object" ? result.agentRun : {}; + const trace = runnerTrace && typeof runnerTrace === "object" ? runnerTrace : result?.runnerTrace; + const events = Array.isArray(trace?.events) ? trace.events : []; + const lastEvent = trace?.lastEvent ?? events.at(-1) ?? null; + return [ + result?.status, + result?.updatedAt, + agentRun.status, + agentRun.runStatus, + agentRun.commandState, + agentRun.terminalStatus, + agentRun.lastSeq, + agentRun.updatedAt, + trace?.status, + trace?.eventCount ?? events.length, + trace?.updatedAt, + trace?.lastEventLabel, + lastEvent?.seq, + lastEvent?.sourceSeq, + lastEvent?.label, + lastEvent?.type, + lastEvent?.status, + lastEvent?.createdAt + ].map((value) => value ?? "").join("|"); } function sleepAgentRunProjectionSync(ms) {