fix: stabilize workbench turn diagnostics (#1828)

internal/cloud/code-agent-agentrun-adapter.ts

@@ -496,7 +496,31 @@ async function agentRunDispatchJson({ fetchImpl, managerUrl, path, method = "GET
       return await agentRunJson(fetchImpl, managerUrl, path, { method, body, timeoutMs, env });
     } catch (error) {
       const failureKind = agentRunDispatchFailureKind(error);
-      if (!agentRunDispatchRetryable(error, failureKind) || retryAttempt >= policy.maxRetries) throw Object.assign(error, { retryAttempt, retryMax: policy.maxRetries, failureKind });
+      const retryable = agentRunDispatchRetryable(error, failureKind);
+      if (!retryable || retryAttempt >= policy.maxRetries) {
+        traceStore.append(traceId, agentRunTraceEvent({
+          type: "error",
+          eventType: "error",
+          status: "failed",
+          label: retryable ? `agentrun:dispatch-retry-exhausted:${failureKind}` : `agentrun:dispatch-failed:${failureKind}`,
+          errorCode: failureKind,
+          failureKind,
+          retryable,
+          willRetry: false,
+          retryAttempt,
+          retryMax: policy.maxRetries,
+          retryExhausted: retryable && retryAttempt >= policy.maxRetries,
+          runId,
+          commandId,
+          waitingFor: "agentrun-dispatch-failed",
+          message: retryable
+            ? `AgentRun dispatch transient ${failureKind}; retry ${retryAttempt}/${policy.maxRetries} exhausted at stage ${stage}. Code Agent stopped this turn.`
+            : `AgentRun dispatch failed with non-retryable ${failureKind} at stage ${stage}. Code Agent stopped this turn.`,
+          terminal: true,
+          valuesPrinted: false
+        }, backendProfile));
+        throw Object.assign(error, { retryAttempt, retryMax: policy.maxRetries, failureKind, retryExhausted: retryable && retryAttempt >= policy.maxRetries });
+      }
       retryAttempt += 1;
       const retryDelayMs = agentRunDispatchRetryDelayMs(policy, retryAttempt);
       const nextRetryAt = new Date(Date.now() + retryDelayMs).toISOString();

internal/cloud/workbench-turn-projection.ts

@@ -348,11 +348,14 @@ function firstTimestamp(...values) {
 }
 
 function durationValue(...values) {
+  let max = null;
   for (const value of values) {
     const number = Number(value);
-    if (Number.isFinite(number) && number >= 0) return Math.trunc(number);
+    if (!Number.isFinite(number) || number < 0) continue;
+    const duration = Math.trunc(number);
+    max = max === null ? duration : Math.max(max, duration);
   }
-  return null;
+  return max;
 }
 
 function elapsedBetween(startedAt, endedAt) {

web/hwlab-cloud-web/src/components/common/ApiErrorDiagnostic.vue

@@ -31,6 +31,11 @@ const facts = computed(() => {
     fact("route", firstString(diagnostic?.route, props.apiError?.route)),
     fact("source", firstString(diagnostic?.source, props.apiError?.source)),
     fact("status", firstValue(diagnostic?.httpStatus)),
+    fact("retryable", firstBooleanText(diagnostic?.retryable, props.apiError?.retryable)),
+    fact("retry", retryProgressText(diagnostic, props.apiError)),
+    fact("retryDelayMs", firstValue(diagnostic?.retryDelayMs, props.apiError?.retryDelayMs)),
+    fact("nextRetryAt", firstString(diagnostic?.nextRetryAt, props.apiError?.nextRetryAt)),
+    fact("retryExhausted", firstBooleanText(diagnostic?.retryExhausted, props.apiError?.retryExhausted)),
     fact("valuesPrinted", valuesPrinted)
   ].filter((item): item is { label: string; value: string } => Boolean(item));
 });
@@ -62,6 +67,20 @@ function firstValue(...values: unknown[]): string | number | null {
   return null;
 }
 
+function firstBooleanText(...values: unknown[]): string | null {
+  for (const value of values) if (typeof value === "boolean") return value ? "true" : "false";
+  return null;
+}
+
+function retryProgressText(diagnostic: ErrorDiagnostic | null, apiError?: ApiError | null): string | null {
+  const attempt = firstValue(diagnostic?.retryAttempt, diagnostic?.retryAttempts, apiError?.retryAttempt, apiError?.retryAttempts);
+  const max = firstValue(diagnostic?.retryMax, apiError?.retryMax);
+  if (attempt !== null && max !== null) return `${attempt}/${max}`;
+  if (attempt !== null) return String(attempt);
+  if (max !== null) return `max ${max}`;
+  return null;
+}
+
 function layerText(diagnostic: ErrorDiagnostic | null, apiError?: ApiError | null): string | null {
   const layer = firstString(diagnostic?.layer, apiError?.layer);
   const category = firstString(diagnostic?.category, apiError?.category);

web/hwlab-cloud-web/src/stores/workbench.ts

@@ -544,12 +544,13 @@ export const useWorkbenchStore = defineStore("workbench", () => {
       const resultError = normalizeAgentError((result as AgentChatResultResponse).error ?? null);
       const resultProjection = projectionFromResult(result as AgentChatResultResponse);
       const mergedRunnerTrace = mergeTerminalResultTrace(message.runnerTrace, result as AgentChatResultResponse);
-      const runnerTrace = terminal && !resultError && !resultProjection ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace;
+      const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(resultStatus, resultError);
+      const runnerTrace = clearCompletedDiagnostics ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace;
       rememberTraceAuthority(runnerTrace);
-      const error = resultError ?? (terminal ? null : normalizeAgentError(runnerTrace?.error ?? message.error));
+      const error = resultError ?? (terminal || clearCompletedDiagnostics ? null : normalizeAgentError(runnerTrace?.error ?? message.error));
       const agentRun = agentRunFromResult(result as AgentChatResultResponse, runnerTrace) ?? agentRunFromMessage(message);
-      const projection = resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null);
-      return { ...message, ...messageTimingPatch(result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() };
+      const projection = clearCompletedDiagnostics ? nonBlockingProjection(resultProjection) : resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null);
+      return { ...message, ...messageTimingPatchForMerge(message, result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() };
     });
   }
 
@@ -557,6 +558,19 @@ export const useWorkbenchStore = defineStore("workbench", () => {
     return { ...trace, error: undefined, projection: null, projectionStatus: null, projectionHealth: null, staleMs: null, blocker: null };
   }
 
+function shouldClearCompletedTurnDiagnostics(status: unknown, error: ChatMessage["error"] | null): boolean {
+  return normalizedStatusText(status) === "completed" && !error;
+}
+
+function nonBlockingProjection(projection: ProjectionDiagnostic | null): ProjectionDiagnostic | null {
+  if (!projection) return null;
+  const status = firstNonEmptyString(projection.projectionStatus);
+  const health = firstNonEmptyString(projection.projectionHealth);
+  if (status === "caught-up" || health === "healthy") return projection;
+  if (!projection.blocker && !projection.apiError && !projection.diagnostic) return projection;
+  return null;
+}
+
   async function submitMessage(text: string): Promise<boolean> {
     const value = text.trim();
     if (!value) return false;
@@ -709,7 +723,7 @@ export const useWorkbenchStore = defineStore("workbench", () => {
       const authority = traceId ? traceAuthorityById.value[traceId] ?? null : null;
       if (!authority) return message;
       const runnerTrace = mergeRunnerTrace(message.runnerTrace, authority);
-      return { ...message, ...messageTimingPatch(message), runnerTrace };
+      return { ...message, ...messageTimingPatchForMerge(message, runnerTrace), runnerTrace };
     });
   }
 
@@ -768,12 +782,13 @@ export const useWorkbenchStore = defineStore("workbench", () => {
       const resultError = normalizeAgentError(result.error ?? null);
       const resultProjection = projectionFromResult(result);
       const mergedRunnerTrace = mergeRunnerTrace(message.runnerTrace, nextTrace);
-      const runnerTrace = terminal && !resultError && !resultProjection ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace;
-      const error = resultError ?? (terminal ? null : normalizeAgentError(runnerTrace?.error ?? message.error));
-      const projection = resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null);
+      const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(resultStatus, resultError);
+      const runnerTrace = clearCompletedDiagnostics ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace;
+      const error = resultError ?? (terminal || clearCompletedDiagnostics ? null : normalizeAgentError(runnerTrace?.error ?? message.error));
+      const projection = clearCompletedDiagnostics ? nonBlockingProjection(resultProjection) : resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null);
       const agentRun = agentRunFromResult(result, runnerTrace) ?? agentRunFromMessage(message);
       rememberTraceAuthority(runnerTrace);
-      return { ...message, ...messageTimingPatch(result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() };
+      return { ...message, ...messageTimingPatchForMerge(message, result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() };
     }));
     markWorkbenchTraceProjected(traceId);
     if (result.terminal === true || isTerminalMessageStatus(result.status)) {
@@ -1084,11 +1099,14 @@ export const useWorkbenchStore = defineStore("workbench", () => {
     if (!ownerSessionId) return;
     updateSessionMessages(ownerSessionId, (source) => source.map((message) => {
       if (!messageMatchesTraceAuthority(message, traceId, authoritySessionId, ownerSessionId)) return message;
-      const runnerTrace = mergeRunnerTrace(message.runnerTrace, trace);
+      const mergedRunnerTrace = mergeRunnerTrace(message.runnerTrace, trace);
+      const traceStatus = normalizedStatusText(trace.status ?? snapshot.status) ?? null;
+      const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(traceStatus, null);
+      const runnerTrace = clearCompletedDiagnostics ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace;
       rememberTraceAuthority(runnerTrace);
-      const error = message.role === "agent" ? normalizeAgentError(runnerTrace.error ?? message.error) : normalizeAgentError(message.error);
-      const projection = trace.projection ?? runnerTrace.projection ?? message.projection ?? null;
-      return { ...message, ...messageTimingPatch(trace), runnerTrace, error: error ?? message.error ?? null, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, updatedAt: new Date().toISOString() };
+      const error = clearCompletedDiagnostics ? null : message.role === "agent" ? normalizeAgentError(runnerTrace.error ?? message.error) : normalizeAgentError(message.error);
+      const projection = clearCompletedDiagnostics ? nonBlockingProjection(trace.projection ?? null) : trace.projection ?? runnerTrace.projection ?? message.projection ?? null;
+      return { ...message, ...messageTimingPatchForMerge(message, trace), runnerTrace, error: clearCompletedDiagnostics ? null : error ?? message.error ?? null, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, updatedAt: new Date().toISOString() };
     }));
     markWorkbenchTraceProjected(traceId);
     void refreshSessions(ownerSessionId);
@@ -1106,12 +1124,13 @@ export const useWorkbenchStore = defineStore("workbench", () => {
       const resultError = normalizeAgentError(result.error ?? null);
       const resultProjection = projectionFromResult(result);
       const mergedRunnerTrace = mergeTerminalResultTrace(message.runnerTrace, result);
-      const runnerTrace = terminal && !resultError && !resultProjection ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace;
+      const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(resultStatus, resultError);
+      const runnerTrace = clearCompletedDiagnostics ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace;
       rememberTraceAuthority(runnerTrace);
-      const error = resultError ?? (terminal ? null : normalizeAgentError(runnerTrace?.error ?? message.error));
-      const projection = resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null);
+      const error = resultError ?? (terminal || clearCompletedDiagnostics ? null : normalizeAgentError(runnerTrace?.error ?? message.error));
+      const projection = clearCompletedDiagnostics ? nonBlockingProjection(resultProjection) : resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null);
       const agentRun = agentRunFromResult(result, runnerTrace) ?? agentRunFromMessage(message);
-      return { ...message, ...messageTimingPatch(result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() };
+      return { ...message, ...messageTimingPatchForMerge(message, result), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() };
     }));
     rememberTurnStatus(traceId, result);
     markWorkbenchTraceProjected(traceId);
@@ -1150,12 +1169,13 @@ export const useWorkbenchStore = defineStore("workbench", () => {
       const resultError = normalizeAgentError(result.error ?? null);
       const resultProjection = projectionFromResult(result);
       const mergedRunnerTrace = mergeTerminalResultTrace(message.runnerTrace, result);
-      const runnerTrace = terminal && !resultError && !resultProjection ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace;
+      const clearCompletedDiagnostics = shouldClearCompletedTurnDiagnostics(resultStatus, resultError);
+      const runnerTrace = clearCompletedDiagnostics ? clearRunnerTraceTransientDiagnostics(mergedRunnerTrace) : mergedRunnerTrace;
       rememberTraceAuthority(runnerTrace);
-      const error = resultError ?? (terminal ? null : normalizeAgentError(runnerTrace?.error ?? message.error));
+      const error = resultError ?? (terminal || clearCompletedDiagnostics ? null : normalizeAgentError(runnerTrace?.error ?? message.error));
       const agentRun = agentRunFromResult(result, runnerTrace) ?? agentRunFromMessage(message);
-      const projection = resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null);
-      return { ...message, ...messageTimingPatch(result), title: normalizeWorkbenchMessageTitle(message.role, message.title), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() };
+      const projection = clearCompletedDiagnostics ? nonBlockingProjection(resultProjection) : resultProjection ?? (terminal ? null : runnerTrace.projection ?? message.projection ?? null);
+      return { ...message, ...messageTimingPatchForMerge(message, result), title: normalizeWorkbenchMessageTitle(message.role, message.title), runnerTrace, error, projection, projectionStatus: projection?.projectionStatus ?? null, projectionHealth: projection?.projectionHealth ?? null, blocker: projection?.blocker ?? null, agentRun: agentRun ?? undefined, updatedAt: new Date().toISOString() };
     }));
     void hydrateTraceEvents(serverState.value.messagesBySessionId[ownerSessionId] ?? []);
     void refreshSessions(ownerSessionId);
@@ -1546,6 +1566,25 @@ function messageTimingPatch(value: unknown): Partial<ChatMessage> {
   return { timing, startedAt: timing.startedAt ?? null, lastEventAt: timing.lastEventAt ?? null, finishedAt: timing.finishedAt ?? null, durationMs: timing.durationMs ?? null };
 }
 
+function messageTimingPatchForMerge(message: ChatMessage, value: unknown): Partial<ChatMessage> {
+  const patch = messageTimingPatch(value);
+  if (!patch.timing) return patch;
+  const durationMs = maxFiniteNumber(patch.timing.durationMs, message.durationMs, message.timing?.durationMs, message.runnerTrace?.durationMs, message.runnerTrace?.timing?.durationMs);
+  const timing = { ...patch.timing, durationMs } as WorkbenchTurnTimingProjection;
+  return { timing, startedAt: timing.startedAt ?? null, lastEventAt: timing.lastEventAt ?? null, finishedAt: timing.finishedAt ?? null, durationMs };
+}
+
+function maxFiniteNumber(...values: unknown[]): number | null {
+  let max: number | null = null;
+  for (const value of values) {
+    const number = Number(value);
+    if (!Number.isFinite(number) || number < 0) continue;
+    const normalized = Math.trunc(number);
+    max = max === null ? normalized : Math.max(max, normalized);
+  }
+  return max;
+}
+
 function projectionFromResult(result: AgentChatResultResponse | TraceSnapshot | Record<string, unknown> | null | undefined): ProjectionDiagnostic | null {
   return normalizeProjectionDiagnostic(result) ?? normalizeProjectionDiagnostic(recordValue(result)?.runnerTrace) ?? null;
 }