diff --git a/web/hwlab-cloud-web/src/state/runner-trace.test.ts b/web/hwlab-cloud-web/src/state/runner-trace.test.ts index 097ea0ee..4de244bb 100644 --- a/web/hwlab-cloud-web/src/state/runner-trace.test.ts +++ b/web/hwlab-cloud-web/src/state/runner-trace.test.ts @@ -1,7 +1,7 @@ import assert from "node:assert/strict"; import { test } from "bun:test"; -import { mergeTraceResults, replayFullTrace, subscribeToTrace, type TraceSnapshot } from "./runner-trace"; +import { fetchTraceSnapshot, mergeTraceResults, replayFullTrace, subscribeToTrace, type TraceSnapshot } from "./runner-trace"; import type { AgentChatResultResponse, TraceEvent } from "../types/domain"; test("mergeTraceResults derives final response from trace assistant event when trace terminal arrives first", () => { @@ -333,6 +333,52 @@ test("replayFullTrace retries transient trace API failures before returning an e } }); +test("fetchTraceSnapshot retries until fast-fail trace events are readable", async () => { + const originalFetch = globalThis.fetch; + const originalSetTimeout = globalThis.window?.setTimeout; + const originalClearTimeout = globalThis.window?.clearTimeout; + const fetches: string[] = []; + globalThis.window = { + ...(globalThis.window ?? {}), + setTimeout: ((callback: () => void) => { callback(); return 1; }) as typeof window.setTimeout, + clearTimeout: (() => undefined) as typeof window.clearTimeout + } as typeof window; + globalThis.fetch = (async (input: RequestInfo | URL) => { + const url = String(input); + fetches.push(url); + if (fetches.length === 1) { + return jsonResponse({ status: "failed", traceId: "trc_fast_fail_retry", events: [], eventCount: 0 }); + } + return jsonResponse({ + status: "failed", + traceId: "trc_fast_fail_retry", + events: [ + { seq: 1, label: "agentrun:request:accepted", status: "accepted" }, + { seq: 35, label: "agentrun:result:failed", type: "result", status: "failed", message: "provider failed" } + ], + eventCount: 35 + }); + }) as typeof fetch; + + try { + const snapshot = await fetchTraceSnapshot("trc_fast_fail_retry", "trc_fast_fail_retry", 1000); + assert.equal(snapshot?.status, "failed"); + assert.equal(snapshot?.events?.length, 2); + assert.equal(snapshot?.eventCount, 35); + assert.equal(snapshot?.lastEventLabel, "agentrun:result:failed"); + assert.deepEqual(fetches, [ + "/v1/agent/chat/trace/trc_fast_fail_retry?projectId=prj_device_pod_workbench", + "/v1/agent/chat/trace/trc_fast_fail_retry?projectId=prj_device_pod_workbench" + ]); + } finally { + globalThis.fetch = originalFetch; + if (globalThis.window) { + globalThis.window.setTimeout = originalSetTimeout ?? globalThis.window.setTimeout; + globalThis.window.clearTimeout = originalClearTimeout ?? globalThis.window.clearTimeout; + } + } +}); + function jsonResponse(payload: unknown): Response { return new Response(JSON.stringify(payload), { status: 200, headers: { "content-type": "application/json" } }); } diff --git a/web/hwlab-cloud-web/src/state/runner-trace.ts b/web/hwlab-cloud-web/src/state/runner-trace.ts index 9e248b5b..f669dd2b 100644 --- a/web/hwlab-cloud-web/src/state/runner-trace.ts +++ b/web/hwlab-cloud-web/src/state/runner-trace.ts @@ -24,6 +24,8 @@ export interface TraceSnapshot { const TRACE_POLL_INTERVAL_MS = 1500; const TRACE_REPLAY_RETRY_INTERVAL_MS = 500; +const TRACE_SNAPSHOT_RETRY_INTERVAL_MS = 500; +const TRACE_SNAPSHOT_MAX_TIMEOUT_MS = 30000; export function isTerminalStatus(status: string | undefined): boolean { if (!status) return false; @@ -316,19 +318,42 @@ export async function subscribeToTrace(config: TraceSubscriptionConfig): Promise export async function fetchTraceSnapshot(traceId: string, traceUrlOrId: string, inactivityTimeoutMs: number, signal?: AbortSignal): Promise { if (signal?.aborted) return null; const url = traceUrlOrId.startsWith("/") ? traceUrlOrId : `/v1/agent/chat/trace/${encodeURIComponent(traceUrlOrId)}?projectId=prj_device_pod_workbench`; - const tracePolled = await api.getAgentChatResult(url, inactivityTimeoutMs, null); - if (signal?.aborted || !tracePolled.ok || !tracePolled.data) return null; - const events = Array.isArray(tracePolled.data.events) ? tracePolled.data.events : []; + const totalTimeoutMs = Math.max(1000, Math.min(inactivityTimeoutMs, TRACE_SNAPSHOT_MAX_TIMEOUT_MS)); + const startedAt = Date.now(); + const maxAttempts = Math.max(1, Math.ceil(totalTimeoutMs / TRACE_SNAPSHOT_RETRY_INTERVAL_MS)); + let attempts = 0; + + while (attempts < maxAttempts && Date.now() - startedAt <= totalTimeoutMs) { + attempts += 1; + const remainingMs = Math.max(1000, totalTimeoutMs - (Date.now() - startedAt)); + const tracePolled = await api.getAgentChatResult(url, remainingMs, null); + if (signal?.aborted) return null; + if (tracePolled.ok && tracePolled.data) { + const snapshot = snapshotFromTracePoll(traceId, tracePolled.data); + if (hasTraceEvents(snapshot)) return snapshot; + } + if (attempts >= maxAttempts || Date.now() - startedAt >= totalTimeoutMs) break; + await new Promise((resolve) => window.setTimeout(resolve, Math.min(TRACE_SNAPSHOT_RETRY_INTERVAL_MS, totalTimeoutMs - (Date.now() - startedAt)))); + } + return null; +} + +function snapshotFromTracePoll(traceId: string, result: AgentChatResultResponse): TraceSnapshot { + const events = Array.isArray(result.events) ? result.events : []; const last = events.at(-1); return snapshotToTraceSnapshot({ - ...tracePolled.data, + ...result, traceId, events, - eventCount: Number(tracePolled.data.eventCount ?? events.length) || 0, - lastEventLabel: String(last?.label ?? last?.type ?? tracePolled.data.lastEventLabel ?? "") + eventCount: Number(result.eventCount ?? events.length) || 0, + lastEventLabel: String(last?.label ?? last?.type ?? result.lastEventLabel ?? "") } as AgentChatResultResponse); } +function hasTraceEvents(snapshot: TraceSnapshot): boolean { + return Array.isArray(snapshot.events) && snapshot.events.length > 0; +} + function snapshotToTraceSnapshot(result: AgentChatResultResponse): TraceSnapshot { const events = Array.isArray((result as { events?: TraceEvent[] }).events) ? ((result as { events?: TraceEvent[] }).events ?? []) : []; return {