diff --git a/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts b/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts index 94b0e31c..e98a23f1 100644 --- a/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts +++ b/web/hwlab-cloud-web/scripts/workbench-e2e-server.ts @@ -691,8 +691,8 @@ async function createWorkbenchLaunch(request: IncomingMessage): Promise { const traceHydrationInFlight = new Set(); const traceHydrationQueued = new Set(); const traceHydrationQueue: ChatMessage[] = []; + const forcedTraceHydrationRetryTimers = new Map(); let traceHydrationPumpActive = false; const terminalRealtimeRefreshInFlight = new Set(); const realtimeSessionMessagesInFlight = new Set(); @@ -614,7 +615,9 @@ export const useWorkbenchStore = defineStore("workbench", () => { ); if (!response.ok || !response.data) return; const pageMessages = Array.isArray(response.data.messages) ? response.data.messages.map((message) => normalizeChatMessage(message as ChatMessage)) : []; - rememberSessionMessages(id, mergeMessageProjectionPage(id, pageMessages)); + const merged = mergeMessageProjectionPage(id, pageMessages); + rememberSessionMessages(id, merged); + hydrateTerminalTraceGaps(merged, "session-message-page"); } async function refreshRealtimeSessionMessages(sessionId: string | null | undefined, reason: string, options: { force?: boolean } = {}): Promise { @@ -647,7 +650,9 @@ export const useWorkbenchStore = defineStore("workbench", () => { return; } const pageMessages = Array.isArray(response.data.messages) ? response.data.messages.map((message) => normalizeChatMessage(message as ChatMessage)) : []; - rememberSessionMessages(id, mergeMessageProjectionPage(id, pageMessages)); + const merged = mergeMessageProjectionPage(id, pageMessages); + rememberSessionMessages(id, merged); + hydrateTerminalTraceGaps(merged, `trace-message-page:${traceId}`); } function mergeMessageProjectionPage(sessionId: string, pageMessages: ChatMessage[]): ChatMessage[] { @@ -914,7 +919,10 @@ function nonBlockingProjection(projection: ProjectionDiagnostic | null): Project async function hydrateTraceEventsForMessage(message: ChatMessage, options: { force?: boolean } = {}): Promise { const traceId = message.traceId ?? message.runnerTrace?.traceId; if (!traceId) return; - if (traceHydrationInFlight.has(traceId)) return; + if (traceHydrationInFlight.has(traceId)) { + if (options.force) scheduleForcedTraceHydrationRetry(message); + return; + } traceHydrationInFlight.add(traceId); try { let afterProjectedSeq = traceHydrationProjectedSeq(message.runnerTrace); @@ -951,6 +959,31 @@ function nonBlockingProjection(projection: ProjectionDiagnostic | null): Project return new Promise((resolve) => window.setTimeout(resolve, ms)); } + function hydrateTerminalTraceGaps(source: ChatMessage[], reason: string): void { + for (const message of terminalTraceHydrationGaps(source)) { + const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId); + if (traceId && activeSessionId.value) recordActivity(`${reason}:${traceId}`); + void hydrateTraceEventsForMessage(message, { force: true }); + } + } + + function terminalTraceHydrationGaps(source: ChatMessage[]): ChatMessage[] { + return source.filter(messageNeedsTerminalTraceGapHydration).slice(-TRACE_HYDRATION_AUTO_QUEUE_LIMIT).reverse(); + } + + function scheduleForcedTraceHydrationRetry(message: ChatMessage): void { + const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId); + if (!traceId || typeof window === "undefined" || forcedTraceHydrationRetryTimers.has(traceId)) return; + const timer = window.setTimeout(() => { + forcedTraceHydrationRetryTimers.delete(traceId); + const ownerSessionId = traceOwnerSessionId(traceId, messageSessionAuthority(message)); + const ownerMessages = ownerSessionId ? serverState.value.messagesBySessionId[ownerSessionId] ?? [] : messages.value; + const latest = latestMessageForTrace(traceId, ownerMessages) ?? message; + void hydrateTraceEventsForMessage(latest, { force: true }); + }, TRACE_HYDRATION_RETRY_DELAY_MS); + forcedTraceHydrationRetryTimers.set(traceId, timer); + } + async function hydrateTraceEvents(source: ChatMessage[] = messages.value): Promise { for (const message of traceHydrationCandidates(source)) queueTraceHydration(message); void pumpTraceHydrationQueue(); @@ -2453,6 +2486,19 @@ function messageNeedsTraceHydration(message: ChatMessage): boolean { return events.length === 0 || trace?.eventsCompacted === true || trace?.fullTraceLoaded !== true; } +function messageNeedsTerminalTraceGapHydration(message: ChatMessage): boolean { + if (!messageHasCompletedFinalResponse(message)) return false; + const traceId = firstNonEmptyString(message.traceId, message.runnerTrace?.traceId); + if (!traceId) return false; + const trace = message.runnerTrace; + if (!trace) return true; + const events = Array.isArray(trace.events) ? trace.events : []; + const eventCount = firstFiniteNumber(trace.eventCount) ?? events.length; + if (trace.eventsCompacted === true || trace.fullTraceLoaded !== true) return true; + if (events.length === 0 && eventCount > 0) return true; + return isTraceActiveStatus(trace.status) || isTraceActiveStatus(trace.traceStatus); +} + function traceHasCompletedFinalResponse(traceId: string | null | undefined, source: ChatMessage[]): boolean { const id = firstNonEmptyString(traceId); if (!id) return false;