diff --git a/internal/cloud/server-code-agent-http.ts b/internal/cloud/server-code-agent-http.ts index 6bb98706..b63fa877 100644 --- a/internal/cloud/server-code-agent-http.ts +++ b/internal/cloud/server-code-agent-http.ts @@ -2518,15 +2518,7 @@ export async function handleCodeAgentTraceHttp(request, response, url, options) await recordCodeAgentSessionOwner({ payload: agentRunResult, params: agentRunResult, options, status: codeAgentOwnerStatusForResult(agentRunResult), preserveLastTraceId: true }); } } catch (error) { - sendJson(response, Number(error?.statusCode ?? 502), { - error: { - code: error?.code ?? "agentrun_trace_command_result_unavailable", - message: error?.message ?? "AgentRun command result could not be resolved for this traceId", - traceId, - valuesPrinted: false - } - }); - return; + refreshError = error; } } diff --git a/scripts/web-live-dom-probe.mjs b/scripts/web-live-dom-probe.mjs index 40a039d6..4a280992 100644 --- a/scripts/web-live-dom-probe.mjs +++ b/scripts/web-live-dom-probe.mjs @@ -56,6 +56,8 @@ export function parseProbeArgs(argv) { conversationId: process.env.HWLAB_WEB_PROBE_CONVERSATION_ID ?? "", waitMessagesMs: 2500, waitAgentTerminalMs: Number(process.env.HWLAB_WEB_PROBE_WAIT_AGENT_TERMINAL_MS ?? "0"), + traceSampleCount: envNonNegativeInteger("HWLAB_WEB_PROBE_TRACE_SAMPLE_COUNT", 0), + traceSampleIntervalMs: envNonNegativeInteger("HWLAB_WEB_PROBE_TRACE_SAMPLE_INTERVAL_MS", 0), reportPath: tempReportPath("web-live-dom-probe.json"), screenshotPath: tempReportPath("web-live-dom-probe.png"), jobId: "" @@ -90,6 +92,8 @@ export function parseProbeArgs(argv) { else if (arg === "--wait-after-submit-ms" || arg.startsWith("--wait-after-submit-ms=")) args.waitAfterSubmitMs = positiveInteger(readValue("--wait-after-submit-ms"), "--wait-after-submit-ms"); else if (arg === "--wait-messages-ms" || arg.startsWith("--wait-messages-ms=")) args.waitMessagesMs = positiveInteger(readValue("--wait-messages-ms"), "--wait-messages-ms"); else if (arg === "--wait-agent-terminal-ms" || arg.startsWith("--wait-agent-terminal-ms=")) args.waitAgentTerminalMs = positiveInteger(readValue("--wait-agent-terminal-ms"), "--wait-agent-terminal-ms"); + else if (arg === "--trace-sample-count" || arg.startsWith("--trace-sample-count=")) args.traceSampleCount = nonNegativeInteger(readValue("--trace-sample-count"), "--trace-sample-count"); + else if (arg === "--trace-sample-interval-ms" || arg.startsWith("--trace-sample-interval-ms=")) args.traceSampleIntervalMs = nonNegativeInteger(readValue("--trace-sample-interval-ms"), "--trace-sample-interval-ms"); else if (arg === "--timeout-ms" || arg.startsWith("--timeout-ms=")) args.timeoutMs = positiveInteger(readValue("--timeout-ms"), "--timeout-ms"); else if (arg === "--fresh-session") args.freshSession = true; else if (arg === "--no-cancel-running") args.cancelRunning = false; @@ -219,6 +223,7 @@ export async function runProbe(args) { let browser; let page; let result; + let traceSamples = []; if (args.credentialError) { result = { ok: false, @@ -258,7 +263,9 @@ export async function runProbe(args) { if (args.conversationId) await selectConversationInBrowserSession(page, args, actions); if (args.freshSession) await createFreshSession(page, args, actions); if (args.message) await submitPrompt(page, args, actions); + if (args.message && args.traceSampleCount > 0) traceSamples = await sampleTraceTimeline(page, args, actions); if (args.waitAgentTerminalMs) await waitForAgentTerminal(page, args, actions); + if (args.message && args.cancelRunning) await maybeClick(page, ".message-action-cancel", actions, "cancel-running-message"); await waitForMessageCards(page, args, actions); await page.screenshot({ path: screenshotPath, fullPage: true }); const dom = await collectDomEvidence(page, args); @@ -276,6 +283,7 @@ export async function runProbe(args) { actions, credentials: args.credentials, bootstrapRoutes: summarizeBootstrapRoutes(bootstrapRoutes), + traceSamples, dom, promptValidation, artifacts: { reportPath, screenshotPath }, @@ -286,6 +294,7 @@ export async function runProbe(args) { conversationSelected: Boolean(args.conversationId), freshSessionRequested: args.freshSession, promptSubmitted: Boolean(args.message), + traceSamplingRequested: args.traceSampleCount > 0, cancelRunningRequested: args.cancelRunning } }; @@ -346,6 +355,7 @@ async function collectFailureDomEvidence(page) { function validatePromptFinalResponse(dom, args) { if (!args.message) return { ok: true, applicable: false, reason: "no-prompt-submitted" }; + if (args.traceSampleCount > 0 && !args.waitAgentTerminalMs) return { ok: true, applicable: false, reason: "trace-sampling-without-terminal-wait" }; const messages = Array.isArray(dom.messages) ? dom.messages : []; const latestAgent = [...messages].reverse().find((message) => message.role === "agent"); const finalResponse = latestAgent?.finalResponse ?? null; @@ -402,6 +412,7 @@ async function startProbe(args) { "--wait-messages-ms", String(args.waitMessagesMs) ]; if (args.waitAgentTerminalMs > 0) childArgs.push("--wait-agent-terminal-ms", String(args.waitAgentTerminalMs)); + if (args.traceSampleCount > 0) childArgs.push("--trace-sample-count", String(args.traceSampleCount), "--trace-sample-interval-ms", String(args.traceSampleIntervalMs)); if (args.freshSession) childArgs.push("--fresh-session"); if (!args.cancelRunning) childArgs.push("--no-cancel-running"); if (args.conversationId) childArgs.push("--conversation-id", args.conversationId); @@ -425,6 +436,7 @@ async function startProbe(args) { url: args.url, projectId: args.projectId, conversationId: args.conversationId || null, + traceSampling: args.traceSampleCount > 0 ? { count: args.traceSampleCount, intervalMs: args.traceSampleIntervalMs } : null, reportPath, screenshotPath, stdoutPath, @@ -659,7 +671,42 @@ async function submitPrompt(page, args, actions) { const afterClick = await collectCommandState(page); actions.push({ action: "submit-prompt", chars: args.message.length, readyBefore, afterFill, afterClick }); await page.waitForTimeout(args.waitAfterSubmitMs); - if (args.cancelRunning) await maybeClick(page, ".message-action-cancel", actions, "cancel-running-message"); +} + +async function sampleTraceTimeline(page, args, actions) { + const samples = []; + for (let index = 0; index < args.traceSampleCount; index += 1) { + if (index > 0 && args.traceSampleIntervalMs > 0) await page.waitForTimeout(args.traceSampleIntervalMs); + samples.push(await collectTraceTimelineSample(page, index)); + } + actions.push({ action: "trace-interval-samples", count: samples.length, intervalMs: args.traceSampleIntervalMs, samples }); + return samples; +} + +async function collectTraceTimelineSample(page, index) { + return page.evaluate((sampleIndex) => { + const cards = [...document.querySelectorAll(".message-card")]; + const latestAgent = [...cards].reverse().find((card) => card.getAttribute("data-message-role") === "agent" || card.classList.contains("message-agent")); + const trace = latestAgent?.querySelector(".trace-timeline") ?? null; + const details = trace?.querySelector("details") ?? null; + const rows = latestAgent ? [...latestAgent.querySelectorAll(".trace-render-row")] : []; + const latestRow = rows.at(-1) ?? null; + const empty = latestAgent?.querySelector(".trace-empty") ?? null; + const finalResponse = latestAgent?.querySelector(".message-final-response") ?? null; + return { + index: sampleIndex, + sampledAt: new Date().toISOString(), + messageCount: cards.length, + agentStatus: latestAgent?.getAttribute("data-message-status") ?? null, + tracePresent: Boolean(trace), + traceStatus: trace?.getAttribute("data-status") ?? null, + traceOpen: details instanceof HTMLDetailsElement ? details.open : null, + rowCount: rows.length, + emptyLabel: empty?.textContent?.replace(/\s+/gu, " ").trim() ?? null, + latestRowPreview: latestRow?.textContent?.replace(/\s+/gu, " ").trim().slice(0, 240) ?? null, + finalTextChars: finalResponse?.textContent?.trim().length ?? 0 + }; + }, index); } async function waitForCommandReady(page, args, actions) { @@ -930,6 +977,17 @@ function positiveInteger(value, label) { return parsed; } +function nonNegativeInteger(value, label) { + const parsed = Number(value); + if (!Number.isInteger(parsed) || parsed < 0) throw new Error(`${label} must be a non-negative integer`); + return parsed; +} + +function envNonNegativeInteger(name, fallback) { + const parsed = Number(process.env[name] ?? fallback); + return Number.isInteger(parsed) && parsed >= 0 ? parsed : fallback; +} + function processAlive(pid) { try { process.kill(pid, 0); @@ -953,6 +1011,7 @@ function helpPayload() { usage: [ "node scripts/web-live-dom-probe.mjs run --url http://74.48.78.17:19666/ --fresh-session --report /tmp/hwlab-dev-gate/web-live-dom-probe.json", "node scripts/web-live-dom-probe.mjs start --url http://74.48.78.17:19666/ --fresh-session --message '...' --no-cancel-running --wait-agent-terminal-ms 180000", + "node scripts/web-live-dom-probe.mjs run --url https://hwlab.pikapython.com/ --fresh-session --message '...' --no-cancel-running --trace-sample-count 6 --trace-sample-interval-ms 5000", "node scripts/web-live-dom-probe.mjs run --url http://74.48.78.17:19666/#/workspace --conversation-id cnv_...", "node scripts/web-live-dom-probe.mjs start --url http://74.48.78.17:19666/ --fresh-session", "node scripts/web-live-dom-probe.mjs status " diff --git a/web/hwlab-cloud-web/src/composables/useTraceSubscription.ts b/web/hwlab-cloud-web/src/composables/useTraceSubscription.ts index 2d1b7360..4d0296bf 100644 --- a/web/hwlab-cloud-web/src/composables/useTraceSubscription.ts +++ b/web/hwlab-cloud-web/src/composables/useTraceSubscription.ts @@ -42,6 +42,8 @@ export interface TraceSubscriptionConfig { } const TRACE_POLL_INTERVAL_MS = 1500; +const TRACE_LIVE_PAGE_LIMIT = 80; +const TRACE_LIVE_MAX_PAGES_PER_POLL = 4; export function isTerminalStatus(status: string | undefined): boolean { if (!status) return false; @@ -168,7 +170,7 @@ export async function subscribeToTrace(config: TraceSubscriptionConfig): Promise const { traceId, initial, onActivity, onSnapshot, onComplete, onInfrastructureError, signal, inactivityTimeoutMs, activityRef } = config; const projectId = normalizeWorkbenchProjectId(config.projectId) ?? DEFAULT_WORKBENCH_PROJECT_ID; if (isTerminalStatus(initial.status)) { - const snapshot = resultToTraceSnapshot(traceId, initial as AgentChatResultResponse); + const snapshot = resultToTraceSnapshot(traceId, initial as AgentChatResultResponse, "turn-api"); onSnapshot(snapshot); onComplete(mergeTraceResults(initial as AgentChatResultResponse, snapshot)); return; @@ -178,8 +180,25 @@ export async function subscribeToTrace(config: TraceSubscriptionConfig): Promise return; } - let lastEventCount = -1; - let lastStatus: string | null = null; + let lastSnapshotKey = ""; + let traceSinceSeq = 0; + let accumulatedTrace: TraceSnapshot | null = null; + + const fetchLiveTracePages = async (): Promise => { + for (let page = 0; page < TRACE_LIVE_MAX_PAGES_PER_POLL; page += 1) { + const previousSinceSeq = traceSinceSeq; + const tracePolled = await agentAPI.getAgentTrace(traceId, projectId, inactivityTimeoutMs, activityRef, { sinceSeq: traceSinceSeq, limit: TRACE_LIVE_PAGE_LIMIT }); + if (signal.aborted) return accumulatedTrace; + if (!tracePolled.ok || !tracePolled.data) return accumulatedTrace; + onActivity(); + const pageSnapshot = resultToTraceSnapshot(traceId, tracePolled.data, "trace-api"); + accumulatedTrace = mergeTraceSnapshots(accumulatedTrace, pageSnapshot); + const nextSinceSeq = traceSnapshotNextSinceSeq(pageSnapshot, traceSinceSeq); + if (nextSinceSeq > traceSinceSeq) traceSinceSeq = nextSinceSeq; + if (pageSnapshot.hasMore !== true || nextSinceSeq <= previousSinceSeq) break; + } + return accumulatedTrace; + }; for (;;) { if (signal.aborted) return; @@ -191,16 +210,17 @@ export async function subscribeToTrace(config: TraceSubscriptionConfig): Promise if (turnPolled.ok && turnPolled.data) { onActivity(); - const snapshot = resultToTraceSnapshot(traceId, turnPolled.data); - const eventCount = snapshot.eventCount ?? snapshot.events?.length ?? 0; - const status = snapshot.status ?? ""; - if (eventCount !== lastEventCount || status !== lastStatus) { - lastEventCount = eventCount; - lastStatus = status; + const turnSnapshot = resultToTraceSnapshot(traceId, turnPolled.data, "turn-api"); + const liveTrace = await fetchLiveTracePages(); + if (signal.aborted) return; + const snapshot = traceSnapshotWithTurnStatus(liveTrace ?? turnSnapshot, turnSnapshot); + const snapshotKey = traceSnapshotSignalKey(snapshot); + if (snapshotKey !== lastSnapshotKey) { + lastSnapshotKey = snapshotKey; onSnapshot(snapshot); } if (turnPolled.data.terminal === true || isTerminalStatus(turnPolled.data.status)) { - onComplete(mergeTraceResults(turnPolled.data, snapshot)); + onComplete(mergeTraceResults(turnPolled.data, accumulatedTrace ?? snapshot)); return; } } else if (!turnPolled.ok && turnPolled.status >= 500) { @@ -212,7 +232,7 @@ export async function subscribeToTrace(config: TraceSubscriptionConfig): Promise } } -function resultToTraceSnapshot(traceId: string, result: AgentChatResultResponse): TraceSnapshot { +function resultToTraceSnapshot(traceId: string, result: AgentChatResultResponse, eventSource = "trace-api"): TraceSnapshot { const events = Array.isArray(result.events) ? result.events : Array.isArray(result.traceEvents) ? result.traceEvents : []; const lastEvent = events.at(-1); return { @@ -236,11 +256,75 @@ function resultToTraceSnapshot(traceId: string, result: AgentChatResultResponse) traceSummary: result.traceSummary, error: result.error, lastEventLabel: result.lastEventLabel ?? lastEvent?.label ?? lastEvent?.type, - eventSource: "trace-api", + eventSource, updatedAt: new Date().toISOString() }; } +function mergeTraceSnapshots(previous: TraceSnapshot | null, next: TraceSnapshot): TraceSnapshot { + if (!previous) return next; + const previousEvents = Array.isArray(previous.events) ? previous.events : []; + const nextEvents = Array.isArray(next.events) ? next.events : []; + const events = mergeTraceEvents(previousEvents, nextEvents); + return { + ...previous, + ...next, + events, + eventCount: next.eventCount ?? previous.eventCount ?? events.length, + lastEventLabel: next.lastEventLabel ?? previous.lastEventLabel ?? events.at(-1)?.label ?? events.at(-1)?.type, + updatedAt: next.updatedAt ?? previous.updatedAt ?? new Date().toISOString() + }; +} + +function traceSnapshotWithTurnStatus(trace: TraceSnapshot, turn: TraceSnapshot): TraceSnapshot { + return { + ...trace, + traceId: trace.traceId ?? turn.traceId, + status: turn.status ?? trace.status, + sessionId: trace.sessionId ?? turn.sessionId, + threadId: trace.threadId ?? turn.threadId, + agentRun: trace.agentRun ?? turn.agentRun, + traceStatus: trace.traceStatus ?? turn.traceStatus, + terminalEvidence: trace.terminalEvidence ?? turn.terminalEvidence, + finalResponse: trace.finalResponse ?? turn.finalResponse, + traceSummary: trace.traceSummary ?? turn.traceSummary, + error: trace.error ?? turn.error, + waitingFor: trace.waitingFor ?? turn.waitingFor, + updatedAt: new Date().toISOString() + }; +} + +function traceSnapshotNextSinceSeq(snapshot: TraceSnapshot, fallback: number): number { + const direct = Number(snapshot.nextSinceSeq ?? snapshot.range?.toSeq); + if (Number.isFinite(direct) && direct >= 0) return Math.trunc(direct); + const events = Array.isArray(snapshot.events) ? snapshot.events : []; + return events.reduce((max, event) => { + const seq = Number(event.seq); + return Number.isFinite(seq) && seq > max ? Math.trunc(seq) : max; + }, fallback); +} + +function traceSnapshotSignalKey(snapshot: TraceSnapshot): string { + const events = Array.isArray(snapshot.events) ? snapshot.events : []; + return [ + snapshot.status ?? "", + snapshot.traceStatus ?? "", + snapshot.eventCount ?? events.length, + events.length, + snapshot.nextSinceSeq ?? snapshot.range?.toSeq ?? "", + snapshot.hasMore === true ? "more" : "caught-up", + snapshot.fullTraceLoaded === true ? "loaded" : "partial", + snapshot.lastEventLabel ?? events.at(-1)?.label ?? events.at(-1)?.type ?? "", + traceSnapshotErrorKey(snapshot.error) + ].join("|"); +} + +function traceSnapshotErrorKey(error: TraceSnapshot["error"]): string { + if (!error) return ""; + if (typeof error === "string") return error; + return firstNonEmptyString(error.code, error.message) ?? ""; +} + function sleep(ms: number): Promise { return new Promise((resolve) => window.setTimeout(resolve, ms)); }