fix(v02): retry historical trace replay (#864)

Co-authored-by: Codex Agent <codex@hwlab.local>

web/hwlab-cloud-web/src/state/runner-trace.test.ts

@@ -1,7 +1,7 @@
 import assert from "node:assert/strict";
 import { test } from "bun:test";
 
-import { mergeTraceResults, subscribeToTrace, type TraceSnapshot } from "./runner-trace";
+import { mergeTraceResults, replayFullTrace, subscribeToTrace, type TraceSnapshot } from "./runner-trace";
 import type { AgentChatResultResponse, TraceEvent } from "../types/domain";
 
 test("mergeTraceResults derives final response from trace assistant event when trace terminal arrives first", () => {
@@ -250,6 +250,48 @@ test("subscribeToTrace merges terminal result without text with trace assistant
   }
 });
 
+test("replayFullTrace retries transient trace API failures before returning an empty replay", async () => {
+  const originalFetch = globalThis.fetch;
+  const originalSetTimeout = globalThis.window?.setTimeout;
+  const originalClearTimeout = globalThis.window?.clearTimeout;
+  const fetches: string[] = [];
+  globalThis.window = {
+    ...(globalThis.window ?? {}),
+    setTimeout: ((callback: () => void) => { callback(); return 1; }) as typeof window.setTimeout,
+    clearTimeout: (() => undefined) as typeof window.clearTimeout
+  } as typeof window;
+  globalThis.fetch = (async (input: RequestInfo | URL) => {
+    const url = String(input);
+    fetches.push(url);
+    if (fetches.length === 1) {
+      return new Response(JSON.stringify({ error: "trace store warming" }), { status: 500, headers: { "content-type": "application/json" } });
+    }
+    return jsonResponse({
+      status: "completed",
+      traceId: "trc_replay_retry",
+      events: [
+        { seq: 1, label: "agentrun:request:accepted", status: "accepted" },
+        { seq: 2, label: "agentrun:assistant:message", type: "assistant", status: "completed", message: "恢复后的 trace" }
+      ],
+      eventCount: 2
+    });
+  }) as typeof fetch;
+
+  try {
+    const snapshot = await replayFullTrace("trc_replay_retry", 2000);
+    assert.equal(snapshot?.events?.length, 2);
+    assert.equal(snapshot?.eventCount, 2);
+    assert.equal(snapshot?.lastEventLabel, "agentrun:assistant:message");
+    assert.equal(fetches.length, 2);
+  } finally {
+    globalThis.fetch = originalFetch;
+    if (globalThis.window) {
+      globalThis.window.setTimeout = originalSetTimeout ?? globalThis.window.setTimeout;
+      globalThis.window.clearTimeout = originalClearTimeout ?? globalThis.window.clearTimeout;
+    }
+  }
+});
+
 function jsonResponse(payload: unknown): Response {
   return new Response(JSON.stringify(payload), { status: 200, headers: { "content-type": "application/json" } });
 }

web/hwlab-cloud-web/src/state/runner-trace.ts

@@ -23,6 +23,7 @@ export interface TraceSnapshot {
 }
 
 const TRACE_POLL_INTERVAL_MS = 1500;
+const TRACE_REPLAY_RETRY_INTERVAL_MS = 500;
 
 export function isTerminalStatus(status: string | undefined): boolean {
   if (!status) return false;
@@ -324,7 +325,12 @@ function snapshotToTraceSnapshot(result: AgentChatResultResponse): TraceSnapshot
 
 export async function replayFullTrace(traceId: string, totalTimeoutMs = 15000): Promise<TraceSnapshot | null> {
   const url = `/v1/agent/chat/trace/${encodeURIComponent(traceId)}?projectId=prj_device_pod_workbench`;
-  const polled = await api.getAgentChatResult(url, totalTimeoutMs);
+  const startedAt = Date.now();
+  let polled = await api.getAgentChatResult(url, totalTimeoutMs);
+  while ((!polled.ok || !polled.data) && Date.now() - startedAt < totalTimeoutMs) {
+    await new Promise<void>((resolve) => window.setTimeout(resolve, TRACE_REPLAY_RETRY_INTERVAL_MS));
+    polled = await api.getAgentChatResult(url, Math.max(1000, totalTimeoutMs - (Date.now() - startedAt)));
+  }
   if (!polled.ok || !polled.data) return null;
   const events = Array.isArray(polled.data.events) ? polled.data.events : [];
   const lastEvent = events[events.length - 1];