fix(v02): retry trace snapshot before failure persist (#873)

Co-authored-by: Codex Agent <codex@hwlab.local>
2026-06-04 23:32:08 +08:00
parent dae9c9b5f2
commit 0fdd659826
2 changed files with 78 additions and 7 deletions
@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import { test } from "bun:test";

-import { mergeTraceResults, replayFullTrace, subscribeToTrace, type TraceSnapshot } from "./runner-trace";
+import { fetchTraceSnapshot, mergeTraceResults, replayFullTrace, subscribeToTrace, type TraceSnapshot } from "./runner-trace";
 import type { AgentChatResultResponse, TraceEvent } from "../types/domain";

 test("mergeTraceResults derives final response from trace assistant event when trace terminal arrives first", () => {
@@ -333,6 +333,52 @@ test("replayFullTrace retries transient trace API failures before returning an e
  }
 });

+test("fetchTraceSnapshot retries until fast-fail trace events are readable", async () => {
+  const originalFetch = globalThis.fetch;
+  const originalSetTimeout = globalThis.window?.setTimeout;
+  const originalClearTimeout = globalThis.window?.clearTimeout;
+  const fetches: string[] = [];
+  globalThis.window = {
+    ...(globalThis.window ?? {}),
+    setTimeout: ((callback: () => void) => { callback(); return 1; }) as typeof window.setTimeout,
+    clearTimeout: (() => undefined) as typeof window.clearTimeout
+  } as typeof window;
+  globalThis.fetch = (async (input: RequestInfo | URL) => {
+    const url = String(input);
+    fetches.push(url);
+    if (fetches.length === 1) {
+      return jsonResponse({ status: "failed", traceId: "trc_fast_fail_retry", events: [], eventCount: 0 });
+    }
+    return jsonResponse({
+      status: "failed",
+      traceId: "trc_fast_fail_retry",
+      events: [
+        { seq: 1, label: "agentrun:request:accepted", status: "accepted" },
+        { seq: 35, label: "agentrun:result:failed", type: "result", status: "failed", message: "provider failed" }
+      ],
+      eventCount: 35
+    });
+  }) as typeof fetch;
+
+  try {
+    const snapshot = await fetchTraceSnapshot("trc_fast_fail_retry", "trc_fast_fail_retry", 1000);
+    assert.equal(snapshot?.status, "failed");
+    assert.equal(snapshot?.events?.length, 2);
+    assert.equal(snapshot?.eventCount, 35);
+    assert.equal(snapshot?.lastEventLabel, "agentrun:result:failed");
+    assert.deepEqual(fetches, [
+      "/v1/agent/chat/trace/trc_fast_fail_retry?projectId=prj_device_pod_workbench",
+      "/v1/agent/chat/trace/trc_fast_fail_retry?projectId=prj_device_pod_workbench"
+    ]);
+  } finally {
+    globalThis.fetch = originalFetch;
+    if (globalThis.window) {
+      globalThis.window.setTimeout = originalSetTimeout ?? globalThis.window.setTimeout;
+      globalThis.window.clearTimeout = originalClearTimeout ?? globalThis.window.clearTimeout;
+    }
+  }
+});
+
 function jsonResponse(payload: unknown): Response {
  return new Response(JSON.stringify(payload), { status: 200, headers: { "content-type": "application/json" } });
 }
@@ -24,6 +24,8 @@ export interface TraceSnapshot {

 const TRACE_POLL_INTERVAL_MS = 1500;
 const TRACE_REPLAY_RETRY_INTERVAL_MS = 500;
+const TRACE_SNAPSHOT_RETRY_INTERVAL_MS = 500;
+const TRACE_SNAPSHOT_MAX_TIMEOUT_MS = 30000;

 export function isTerminalStatus(status: string | undefined): boolean {
  if (!status) return false;
@@ -316,19 +318,42 @@ export async function subscribeToTrace(config: TraceSubscriptionConfig): Promise
 export async function fetchTraceSnapshot(traceId: string, traceUrlOrId: string, inactivityTimeoutMs: number, signal?: AbortSignal): Promise<TraceSnapshot | null> {
  if (signal?.aborted) return null;
  const url = traceUrlOrId.startsWith("/") ? traceUrlOrId : `/v1/agent/chat/trace/${encodeURIComponent(traceUrlOrId)}?projectId=prj_device_pod_workbench`;
-  const tracePolled = await api.getAgentChatResult(url, inactivityTimeoutMs, null);
-  if (signal?.aborted || !tracePolled.ok || !tracePolled.data) return null;
-  const events = Array.isArray(tracePolled.data.events) ? tracePolled.data.events : [];
+  const totalTimeoutMs = Math.max(1000, Math.min(inactivityTimeoutMs, TRACE_SNAPSHOT_MAX_TIMEOUT_MS));
+  const startedAt = Date.now();
+  const maxAttempts = Math.max(1, Math.ceil(totalTimeoutMs / TRACE_SNAPSHOT_RETRY_INTERVAL_MS));
+  let attempts = 0;
+
+  while (attempts < maxAttempts && Date.now() - startedAt <= totalTimeoutMs) {
+    attempts += 1;
+    const remainingMs = Math.max(1000, totalTimeoutMs - (Date.now() - startedAt));
+    const tracePolled = await api.getAgentChatResult(url, remainingMs, null);
+    if (signal?.aborted) return null;
+    if (tracePolled.ok && tracePolled.data) {
+      const snapshot = snapshotFromTracePoll(traceId, tracePolled.data);
+      if (hasTraceEvents(snapshot)) return snapshot;
+    }
+    if (attempts >= maxAttempts || Date.now() - startedAt >= totalTimeoutMs) break;
+    await new Promise<void>((resolve) => window.setTimeout(resolve, Math.min(TRACE_SNAPSHOT_RETRY_INTERVAL_MS, totalTimeoutMs - (Date.now() - startedAt))));
+  }
+  return null;
+}
+
+function snapshotFromTracePoll(traceId: string, result: AgentChatResultResponse): TraceSnapshot {
+  const events = Array.isArray(result.events) ? result.events : [];
  const last = events.at(-1);
  return snapshotToTraceSnapshot({
-    ...tracePolled.data,
+    ...result,
    traceId,
    events,
-    eventCount: Number(tracePolled.data.eventCount ?? events.length) || 0,
-    lastEventLabel: String(last?.label ?? last?.type ?? tracePolled.data.lastEventLabel ?? "")
+    eventCount: Number(result.eventCount ?? events.length) || 0,
+    lastEventLabel: String(last?.label ?? last?.type ?? result.lastEventLabel ?? "")
  } as AgentChatResultResponse);
 }

+function hasTraceEvents(snapshot: TraceSnapshot): boolean {
+  return Array.isArray(snapshot.events) && snapshot.events.length > 0;
+}
+
 function snapshotToTraceSnapshot(result: AgentChatResultResponse): TraceSnapshot {
  const events = Array.isArray((result as { events?: TraceEvent[] }).events) ? ((result as { events?: TraceEvent[] }).events ?? []) : [];
  return {