diff --git a/internal/cloud/code-agent-agentrun-adapter.ts b/internal/cloud/code-agent-agentrun-adapter.ts index f0933420..c9cd5542 100644 --- a/internal/cloud/code-agent-agentrun-adapter.ts +++ b/internal/cloud/code-agent-agentrun-adapter.ts @@ -496,7 +496,31 @@ async function agentRunDispatchJson({ fetchImpl, managerUrl, path, method = "GET return await agentRunJson(fetchImpl, managerUrl, path, { method, body, timeoutMs, env }); } catch (error) { const failureKind = agentRunDispatchFailureKind(error); - if (!agentRunDispatchRetryable(error, failureKind) || retryAttempt >= policy.maxRetries) throw Object.assign(error, { retryAttempt, retryMax: policy.maxRetries, failureKind }); + const retryable = agentRunDispatchRetryable(error, failureKind); + if (!retryable || retryAttempt >= policy.maxRetries) { + traceStore.append(traceId, agentRunTraceEvent({ + type: "error", + eventType: "error", + status: "failed", + label: retryable ? `agentrun:dispatch-retry-exhausted:${failureKind}` : `agentrun:dispatch-failed:${failureKind}`, + errorCode: failureKind, + failureKind, + retryable, + willRetry: false, + retryAttempt, + retryMax: policy.maxRetries, + retryExhausted: retryable && retryAttempt >= policy.maxRetries, + runId, + commandId, + waitingFor: "agentrun-dispatch-failed", + message: retryable + ? `AgentRun dispatch transient ${failureKind}; retry ${retryAttempt}/${policy.maxRetries} exhausted at stage ${stage}. Code Agent stopped this turn.` + : `AgentRun dispatch failed with non-retryable ${failureKind} at stage ${stage}. Code Agent stopped this turn.`, + terminal: true, + valuesPrinted: false + }, backendProfile)); + throw Object.assign(error, { retryAttempt, retryMax: policy.maxRetries, failureKind, retryExhausted: retryable && retryAttempt >= policy.maxRetries }); + } retryAttempt += 1; const retryDelayMs = agentRunDispatchRetryDelayMs(policy, retryAttempt); const nextRetryAt = new Date(Date.now() + retryDelayMs).toISOString(); diff --git a/internal/cloud/workbench-turn-projection.ts b/internal/cloud/workbench-turn-projection.ts index 7d77b82f..e1a04135 100644 --- a/internal/cloud/workbench-turn-projection.ts +++ b/internal/cloud/workbench-turn-projection.ts @@ -348,11 +348,14 @@ function firstTimestamp(...values) { } function durationValue(...values) { + let max = null; for (const value of values) { const number = Number(value); - if (Number.isFinite(number) && number >= 0) return Math.trunc(number); + if (!Number.isFinite(number) || number < 0) continue; + const duration = Math.trunc(number); + max = max === null ? duration : Math.max(max, duration); } - return null; + return max; } function elapsedBetween(startedAt, endedAt) { diff --git a/web/hwlab-cloud-web/src/components/common/ApiErrorDiagnostic.vue b/web/hwlab-cloud-web/src/components/common/ApiErrorDiagnostic.vue index 11acb884..d0343267 100644 --- a/web/hwlab-cloud-web/src/components/common/ApiErrorDiagnostic.vue +++ b/web/hwlab-cloud-web/src/components/common/ApiErrorDiagnostic.vue @@ -31,6 +31,11 @@ const facts = computed(() => { fact("route", firstString(diagnostic?.route, props.apiError?.route)), fact("source", firstString(diagnostic?.source, props.apiError?.source)), fact("status", firstValue(diagnostic?.httpStatus)), + fact("retryable", firstBooleanText(diagnostic?.retryable, props.apiError?.retryable)), + fact("retry", retryProgressText(diagnostic, props.apiError)), + fact("retryDelayMs", firstValue(diagnostic?.retryDelayMs, props.apiError?.retryDelayMs)), + fact("nextRetryAt", firstString(diagnostic?.nextRetryAt, props.apiError?.nextRetryAt)), + fact("retryExhausted", firstBooleanText(diagnostic?.retryExhausted, props.apiError?.retryExhausted)), fact("valuesPrinted", valuesPrinted) ].filter((item): item is { label: string; value: string } => Boolean(item)); }); @@ -62,6 +67,20 @@ function firstValue(...values: unknown[]): string | number | null { return null; } +function firstBooleanText(...values: unknown[]): string | null { + for (const value of values) if (typeof value === "boolean") return value ? "true" : "false"; + return null; +} + +function retryProgressText(diagnostic: ErrorDiagnostic | null, apiError?: ApiError | null): string | null { + const attempt = firstValue(diagnostic?.retryAttempt, diagnostic?.retryAttempts, apiError?.retryAttempt, apiError?.retryAttempts); + const max = firstValue(diagnostic?.retryMax, apiError?.retryMax); + if (attempt !== null && max !== null) return `${attempt}/${max}`; + if (attempt !== null) return String(attempt); + if (max !== null) return `max ${max}`; + return null; +} + function layerText(diagnostic: ErrorDiagnostic | null, apiError?: ApiError | null): string | null { const layer = firstString(diagnostic?.layer, apiError?.layer); const category = firstString(diagnostic?.category, apiError?.category); diff --git a/web/hwlab-cloud-web/src/stores/workbench.ts b/web/hwlab-cloud-web/src/stores/workbench.ts index 36e95071..417b55b4 100644 --- a/web/hwlab-cloud-web/src/stores/workbench.ts +++ b/web/hwlab-cloud-web/src/stores/workbench.ts @@ -544,12 +544,13 @@ export const useWorkbenchStore = defineStore("workbench", () => { const resultError = normalizeAgentError((result as AgentChatResultResponse).error ?? null); const resultProjection = projectionFromResult(result as AgentChatResultResponse); const mergedRunnerTrace = mergeTerminalResultTrace(message.runnerTrace, result as AgentChatResultResponse); - const runnerTrace = terminal && !resultError && !resultProjection ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace; + const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(resultStatus, resultError); + const runnerTrace = clearCompletedDiagnostics ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace; rememberTraceAuthority(runnerTrace); - const error = resultError ?? (terminal ? null : normalizeAgentError(runnerTrace?.error ?? message.error)); + const error = resultError ?? (terminal || clearCompletedDiagnostics ? null : normalizeAgentError(runnerTrace?.error ?? message.error)); const agentRun = agentRunFromResult(result as AgentChatResultResponse, runnerTrace) ?? agentRunFromMessage(message); - const projection = resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null); - return { ...message, ...messageTimingPatch(result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() }; + const projection = clearCompletedDiagnostics ? nonBlockingProjection(resultProjection) : resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null); + return { ...message, ...messageTimingPatchForMerge(message, result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() }; }); } @@ -557,6 +558,19 @@ export const useWorkbenchStore = defineStore("workbench", () => { return { ...trace, error: undefined, projection: null, projectionStatus: null, projectionHealth: null, staleMs: null, blocker: null }; } +function shouldClearCompletedTurnDiagnostics(status: unknown, error: ChatMessage["error"] | null): boolean { + return normalizedStatusText(status) === "completed" && !error; +} + +function nonBlockingProjection(projection: ProjectionDiagnostic | null): ProjectionDiagnostic | null { + if (!projection) return null; + const status = firstNonEmptyString(projection.projectionStatus); + const health = firstNonEmptyString(projection.projectionHealth); + if (status === "caught-up" || health === "healthy") return projection; + if (!projection.blocker && !projection.apiError && !projection.diagnostic) return projection; + return null; +} + async function submitMessage(text: string): Promise { const value = text.trim(); if (!value) return false; @@ -709,7 +723,7 @@ export const useWorkbenchStore = defineStore("workbench", () => { const authority = traceId ? traceAuthorityById.value[traceId] ?? null : null; if (!authority) return message; const runnerTrace = mergeRunnerTrace(message.runnerTrace, authority); - return { ...message, ...messageTimingPatch(message), runnerTrace }; + return { ...message, ...messageTimingPatchForMerge(message, runnerTrace), runnerTrace }; }); } @@ -768,12 +782,13 @@ export const useWorkbenchStore = defineStore("workbench", () => { const resultError = normalizeAgentError(result.error ?? null); const resultProjection = projectionFromResult(result); const mergedRunnerTrace = mergeRunnerTrace(message.runnerTrace, nextTrace); - const runnerTrace = terminal && !resultError && !resultProjection ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace; - const error = resultError ?? (terminal ? null : normalizeAgentError(runnerTrace?.error ?? message.error)); - const projection = resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null); + const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(resultStatus, resultError); + const runnerTrace = clearCompletedDiagnostics ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace; + const error = resultError ?? (terminal || clearCompletedDiagnostics ? null : normalizeAgentError(runnerTrace?.error ?? message.error)); + const projection = clearCompletedDiagnostics ? nonBlockingProjection(resultProjection) : resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null); const agentRun = agentRunFromResult(result, runnerTrace) ?? agentRunFromMessage(message); rememberTraceAuthority(runnerTrace); - return { ...message, ...messageTimingPatch(result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() }; + return { ...message, ...messageTimingPatchForMerge(message, result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() }; })); markWorkbenchTraceProjected(traceId); if (result.terminal === true || isTerminalMessageStatus(result.status)) { @@ -1084,11 +1099,14 @@ export const useWorkbenchStore = defineStore("workbench", () => { if (!ownerSessionId) return; updateSessionMessages(ownerSessionId, (source) => source.map((message) => { if (!messageMatchesTraceAuthority(message, traceId, authoritySessionId, ownerSessionId)) return message; - const runnerTrace = mergeRunnerTrace(message.runnerTrace, trace); + const mergedRunnerTrace = mergeRunnerTrace(message.runnerTrace, trace); + const traceStatus = normalizedStatusText(trace.status ?? snapshot.status) ?? null; + const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(traceStatus, null); + const runnerTrace = clearCompletedDiagnostics ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace; rememberTraceAuthority(runnerTrace); - const error = message.role === "agent" ? normalizeAgentError(runnerTrace.error ?? message.error) : normalizeAgentError(message.error); - const projection = trace.projection ?? runnerTrace.projection ?? message.projection ?? null; - return { ...message, ...messageTimingPatch(trace), runnerTrace, error: error ?? message.error ?? null, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, updatedAt: new Date().toISOString() }; + const error = clearCompletedDiagnostics ? null : message.role === "agent" ? normalizeAgentError(runnerTrace.error ?? message.error) : normalizeAgentError(message.error); + const projection = clearCompletedDiagnostics ? nonBlockingProjection(trace.projection ?? null) : trace.projection ?? runnerTrace.projection ?? message.projection ?? null; + return { ...message, ...messageTimingPatchForMerge(message, trace), runnerTrace, error: clearCompletedDiagnostics ? null : error ?? message.error ?? null, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, updatedAt: new Date().toISOString() }; })); markWorkbenchTraceProjected(traceId); void refreshSessions(ownerSessionId); @@ -1106,12 +1124,13 @@ export const useWorkbenchStore = defineStore("workbench", () => { const resultError = normalizeAgentError(result.error ?? null); const resultProjection = projectionFromResult(result); const mergedRunnerTrace = mergeTerminalResultTrace(message.runnerTrace, result); - const runnerTrace = terminal && !resultError && !resultProjection ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace; + const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(resultStatus, resultError); + const runnerTrace = clearCompletedDiagnostics ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace; rememberTraceAuthority(runnerTrace); - const error = resultError ?? (terminal ? null : normalizeAgentError(runnerTrace?.error ?? message.error)); - const projection = resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null); + const error = resultError ?? (terminal || clearCompletedDiagnostics ? null : normalizeAgentError(runnerTrace?.error ?? message.error)); + const projection = clearCompletedDiagnostics ? nonBlockingProjection(resultProjection) : resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null); const agentRun = agentRunFromResult(result, runnerTrace) ?? agentRunFromMessage(message); - return { ...message, ...messageTimingPatch(result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() }; + return { ...message, ...messageTimingPatchForMerge(message, result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() }; })); rememberTurnStatus(traceId, result); markWorkbenchTraceProjected(traceId); @@ -1150,12 +1169,13 @@ export const useWorkbenchStore = defineStore("workbench", () => { const resultError = normalizeAgentError(result.error ?? null); const resultProjection = projectionFromResult(result); const mergedRunnerTrace = mergeTerminalResultTrace(message.runnerTrace, result); - const runnerTrace = terminal && !resultError && !resultProjection ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace; + const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(resultStatus, resultError); + const runnerTrace = clearCompletedDiagnostics ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace; rememberTraceAuthority(runnerTrace); - const error = resultError ?? (terminal ? null : normalizeAgentError(runnerTrace?.error ?? message.error)); + const error = resultError ?? (terminal || clearCompletedDiagnostics ? null : normalizeAgentError(runnerTrace?.error ?? message.error)); const agentRun = agentRunFromResult(result, runnerTrace) ?? agentRunFromMessage(message); - const projection = resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null); - return { ...message, ...messageTimingPatch(result), title: normalizeWorkbenchMessageTitle(message.role, message.title), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() }; + const projection = clearCompletedDiagnostics ? nonBlockingProjection(resultProjection) : resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null); + return { ...message, ...messageTimingPatchForMerge(message, result), title: normalizeWorkbenchMessageTitle(message.role, message.title), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() }; })); void hydrateTraceEvents(serverState.value.messagesBySessionId[ownerSessionId] ?? []); void refreshSessions(ownerSessionId); @@ -1546,6 +1566,25 @@ function messageTimingPatch(value: unknown): Partial { return { timing, startedAt: timing.startedAt ?? null, lastEventAt: timing.lastEventAt ?? null, finishedAt: timing.finishedAt ?? null, durationMs: timing.durationMs ?? null }; } +function messageTimingPatchForMerge(message: ChatMessage, value: unknown): Partial { + const patch = messageTimingPatch(value); + if (!patch.timing) return patch; + const durationMs = maxFiniteNumber(patch.timing.durationMs, message.durationMs, message.timing?.durationMs, message.runnerTrace?.durationMs, message.runnerTrace?.timing?.durationMs); + const timing = { ...patch.timing, durationMs } as WorkbenchTurnTimingProjection; + return { timing, startedAt: timing.startedAt ?? null, lastEventAt: timing.lastEventAt ?? null, finishedAt: timing.finishedAt ?? null, durationMs }; +} + +function maxFiniteNumber(...values: unknown[]): number | null { + let max: number | null = null; + for (const value of values) { + const number = Number(value); + if (!Number.isFinite(number) || number < 0) continue; + const normalized = Math.trunc(number); + max = max === null ? normalized : Math.max(max, normalized); + } + return max; +} + function projectionFromResult(result: AgentChatResultResponse | TraceSnapshot | Record | null | undefined): ProjectionDiagnostic | null { return normalizeProjectionDiagnostic(result) ?? normalizeProjectionDiagnostic(recordValue(result)?.runnerTrace) ?? null; }