From 5604cd0f8a8ee8ac94288dc6fb68f7c06cc5a716 Mon Sep 17 00:00:00 2001 From: lyon Date: Thu, 18 Jun 2026 18:02:05 +0800 Subject: [PATCH] fix: stabilize web probe fresh session evidence --- scripts/web-live-dom-probe.mjs | 162 ++++++++++++++++++++++++++++----- 1 file changed, 139 insertions(+), 23 deletions(-) diff --git a/scripts/web-live-dom-probe.mjs b/scripts/web-live-dom-probe.mjs index 12668904..fb1f52c2 100644 --- a/scripts/web-live-dom-probe.mjs +++ b/scripts/web-live-dom-probe.mjs @@ -277,7 +277,7 @@ export async function runProbe(args) { sessionSummary = summarizeSessionRun(actions, dom); traceSummary = summarizeTraceRun(traceSamples, dom, sessionSummary); performanceSummary = summarizeProbePerformance(actions, startedAt, traceSamples, traceSummary); - const degradedReason = runProbeDegradedReason({ traceSummary, sessionSummary, promptValidation }); + const degradedReason = runProbeDegradedReason({ traceSummary, sessionSummary, promptValidation, actions }); result = { ok: true, status: dom.requiredSelectors.workspace && dom.requiredSelectors.commandInput && dom.requiredSelectors.conversationList && promptValidation.ok && !degradedReason ? "pass" : "blocked", @@ -475,7 +475,7 @@ async function probeStatus(args) { if (!args.jobId) throw new Error("status requires a job id"); const metaPath = path.join(stateRoot, `${args.jobId}.json`); const meta = JSON.parse(await readFile(metaPath, "utf8")); - const report = fs.existsSync(meta.reportPath) ? JSON.parse(await readFile(meta.reportPath, "utf8")) : null; + const report = fs.existsSync(meta.reportPath) ? await attachReportArtifactFingerprints(JSON.parse(await readFile(meta.reportPath, "utf8")), meta) : null; const running = !report && typeof meta.pid === "number" && processAlive(meta.pid); return { ok: report ? report.ok === true : running, @@ -496,6 +496,22 @@ async function probeStatus(args) { }; } +async function attachReportArtifactFingerprints(report, meta) { + if (!report) return report; + const reportPath = report.artifacts?.reportPath ?? meta.reportPath ?? null; + const screenshotPath = report.artifacts?.screenshotPath ?? meta.screenshotPath ?? null; + return { + ...report, + artifacts: { + ...(report.artifacts ?? {}), + reportPath, + reportSha256: await fileSha256(reportPath), + screenshotPath, + screenshotSha256: await fileSha256(screenshotPath) + } + }; +} + async function maybeLogin(page, args, actions) { const username = page.locator("#login-username"); if (await username.isVisible({ timeout: Math.min(args.timeoutMs, 12000) }).catch(() => false)) { @@ -706,11 +722,42 @@ function freshSessionCandidate(before, after) { async function realignFreshSession(page, args, sessionOrConversationId) { const selected = await selectConversationViaApi(page, args, sessionOrConversationId, "web-live-dom-probe-fresh-realign").catch((error) => ({ ok: false, phase: "select-conversation", error: error instanceof Error ? error.message : String(error) })); - const target = new URL(`/workbench/sessions/${encodeURIComponent(sessionOrConversationId)}`, args.url).toString(); - await page.goto(target, { waitUntil: "domcontentloaded", timeout: args.timeoutMs }); - await page.locator("#workspace").waitFor({ state: "visible", timeout: args.timeoutMs }); - await page.locator("#command-input").waitFor({ state: "visible", timeout: args.timeoutMs }); - return { action: "fresh-session-realign", id: sessionOrConversationId, target, selected }; + const targetUrl = new URL(`/workbench/sessions/${encodeURIComponent(sessionOrConversationId)}`, args.url); + targetUrl.searchParams.set("projectId", args.projectId); + const target = targetUrl.toString(); + const attempts = []; + for (let attempt = 1; attempt <= 2; attempt += 1) { + const navigation = await page.goto(target, { waitUntil: "domcontentloaded", timeout: Math.min(args.timeoutMs, 30000) }) + .then((response) => ({ ok: true, status: response?.status() ?? null, url: page.url() })) + .catch((error) => ({ ok: false, error: error instanceof Error ? error.message : String(error), url: page.url() })); + const ready = await waitForWorkbenchSessionRouteReady(page, args, attempt === 1 ? 15000 : 30000); + attempts.push({ attempt, navigation, ready }); + if (ready.ok) return { action: "fresh-session-realign", id: sessionOrConversationId, target, selected, attempts }; + await page.waitForTimeout(1000); + } + return { action: "fresh-session-realign", id: sessionOrConversationId, target, selected, attempts, ok: false }; +} + +async function waitForWorkbenchSessionRouteReady(page, args, timeoutMs) { + await page.waitForFunction(() => { + const authState = document.body.getAttribute("data-auth-state"); + const workspace = document.querySelector("#workspace"); + const loginVisible = Boolean(document.querySelector("#login-username") || document.querySelector('input[autocomplete="username"]') || document.querySelector(".login-card input")); + return authState !== "checking" || workspace || loginVisible; + }, null, { timeout: Math.min(timeoutMs, 10000) }).catch(() => null); + const workspace = await page.locator("#workspace").waitFor({ state: "visible", timeout: timeoutMs }).then(() => ({ ok: true })).catch((error) => ({ ok: false, error: error instanceof Error ? error.message : String(error) })); + const commandInput = workspace.ok + ? await page.locator("#command-input").waitFor({ state: "visible", timeout: timeoutMs }).then(() => ({ ok: true })).catch((error) => ({ ok: false, error: error instanceof Error ? error.message : String(error) })) + : { ok: false, skipped: "workspace-not-visible" }; + const dom = await page.evaluate(() => ({ + href: window.location.href, + authState: document.body.getAttribute("data-auth-state"), + title: document.title, + workspaceVisible: Boolean(document.querySelector("#workspace")), + commandInputVisible: Boolean(document.querySelector("#command-input")), + bodyTextPreview: document.body.textContent?.replace(/\s+/gu, " ").trim().slice(0, 200) ?? "" + })).catch((error) => ({ error: error instanceof Error ? error.message : String(error) })); + return { ok: workspace.ok && commandInput.ok, timeoutMs, workspace, commandInput, dom }; } async function selectConversationViaApi(page, args, conversationId, updatedByClient) { @@ -771,6 +818,35 @@ function sessionAuthorityId(state) { return firstNonEmpty(state?.routeConversationId, state?.api?.workspace?.selectedConversationId, state?.activeTab?.conversationId, state?.activeTab?.sessionId); } +function sessionAuthorityAlignment(state, expectedId, options = {}) { + const routeConversationId = state?.routeConversationId ?? null; + const selectedConversationId = state?.api?.workspace?.selectedConversationId ?? null; + const selectedAgentSessionId = state?.api?.workspace?.selectedAgentSessionId ?? null; + const activeTabSessionId = state?.activeTab?.sessionId ?? null; + const activeTabConversationId = state?.activeTab?.conversationId ?? null; + const workspaceApiAvailable = state?.api?.workspace?.ok === true; + const conversationsApiAvailable = state?.api?.conversations?.ok === true; + const apiConversation = expectedId ? state?.api?.conversations?.items?.find((item) => item.conversationId === expectedId || item.sessionId === expectedId) ?? null : null; + const routeMatches = Boolean(routeConversationId && expectedId && routeConversationId === expectedId); + const workspaceMatches = workspaceApiAvailable + ? Boolean(expectedId && (selectedConversationId === expectedId || selectedAgentSessionId === expectedId)) + : true; + const apiConversationPresent = conversationsApiAvailable ? Boolean(apiConversation) : true; + const activeTabMatches = Boolean(!activeTabSessionId && !activeTabConversationId) || activeTabSessionId === expectedId || activeTabConversationId === expectedId; + const commandReady = state?.command?.inputDisabled === false; + const empty = options.requireEmpty === true ? state?.messageCount === 0 : true; + const ok = Boolean(expectedId && routeMatches && workspaceMatches && apiConversationPresent && activeTabMatches && commandReady && empty); + const reason = ok ? null + : !expectedId ? "session-id-missing" + : !routeMatches ? "session-route-not-aligned" + : !workspaceMatches ? "session-workspace-not-aligned" + : !apiConversationPresent ? "session-conversation-api-missing" + : !activeTabMatches ? "session-active-tab-not-aligned" + : !commandReady ? "session-command-not-ready" + : "session-not-empty"; + return { ok, reason, expectedId, routeConversationId, selectedConversationId, selectedAgentSessionId, activeTabConversationId, activeTabSessionId, workspaceApiAvailable, conversationsApiAvailable, apiConversationPresent, commandReady, messageCount: state?.messageCount ?? null }; +} + async function waitForSessionShellReady(page, args, actions) { const timeoutMs = Math.min(args.timeoutMs, 30000); const before = await collectSessionState(page, args); @@ -956,16 +1032,16 @@ async function submitPrompt(page, args, actions) { async function ensureConversationAligned(page, args, conversationId, actions, action) { const before = await collectSessionState(page, args); - const beforeAligned = before.routeConversationId === conversationId && before.api?.workspace?.selectedConversationId === conversationId; - if (beforeAligned) { - actions.push({ action, conversationId, aligned: true, repaired: false, before: compactSessionState(before) }); + const beforeAlignment = sessionAuthorityAlignment(before, conversationId); + if (beforeAlignment.ok) { + actions.push({ action, conversationId, aligned: true, repaired: false, before: compactSessionState(before), alignment: beforeAlignment }); return before; } const repair = await realignFreshSession(page, args, conversationId); const after = await collectSessionState(page, args); - const aligned = after.routeConversationId === conversationId && after.api?.workspace?.selectedConversationId === conversationId; - actions.push({ action, conversationId, aligned, repaired: true, before: compactSessionState(before), after: compactSessionState(after), repair }); - if (!aligned) throw new Error(`${action} failed: ${JSON.stringify({ conversationId, before: compactSessionState(before), after: compactSessionState(after), repair })}`); + const afterAlignment = sessionAuthorityAlignment(after, conversationId); + actions.push({ action, conversationId, aligned: afterAlignment.ok, repaired: true, before: compactSessionState(before), after: compactSessionState(after), beforeAlignment, afterAlignment, repair }); + if (!afterAlignment.ok) throw new Error(`${action} failed: ${JSON.stringify({ conversationId, reason: afterAlignment.reason, before: compactSessionState(before), after: compactSessionState(after), beforeAlignment, afterAlignment, repair })}`); return after; } @@ -981,6 +1057,13 @@ async function sampleTraceTimeline(page, args, actions) { async function collectTraceTimelineSample(page, args, index) { const dom = await page.evaluate((sampleIndex) => { + const firstTraceId = (...sources) => { + for (const source of sources) { + const match = String(source ?? "").match(/\btrc_[A-Za-z0-9_.:-]+\b/u); + if (match) return match[0]; + } + return null; + }; const cardRole = (card) => card.getAttribute("data-role") ?? card.getAttribute("data-message-role") ?? [...card.classList].find((item) => item === "message-agent" || item === "message-user" || item === "message-system")?.replace(/^message-/, "") ?? null; const cardStatus = (card) => card.getAttribute("data-status") ?? card.getAttribute("data-message-status") ?? null; const cards = [...document.querySelectorAll(".message-card")]; @@ -1011,11 +1094,15 @@ async function collectTraceTimelineSample(page, args, index) { const latestUserVisibleRow = userVisibleRows.at(-1) ?? null; const empty = latestAgent?.querySelector(".trace-empty") ?? null; const finalResponse = latestAgent?.querySelector(".message-final-response") ?? null; + const modeText = document.querySelector(".composer-mode")?.textContent?.trim() ?? null; + const traceId = firstTraceId(latestAgent?.getAttribute("data-trace-id"), trace?.getAttribute("data-trace-id"), trace?.getAttribute("data-traceid"), modeText, latestAgent?.textContent, trace?.textContent); return { index: sampleIndex, sampledAt: new Date().toISOString(), messageCount: cards.length, agentStatus: latestAgent ? cardStatus(latestAgent) : null, + traceId, + modeText, tracePresent: Boolean(trace), traceStatus: trace?.getAttribute("data-status") ?? null, traceOpen: details instanceof HTMLDetailsElement ? details.open : null, @@ -1031,8 +1118,8 @@ async function collectTraceTimelineSample(page, args, index) { }, index); const session = await collectSessionState(page, args); const conversationId = firstNonEmpty(session.routeConversationId, session.api?.workspace?.selectedConversationId, session.activeTab?.conversationId); - const conversation = conversationId ? session.api?.conversations?.items?.find((item) => item.conversationId === conversationId) ?? null : null; - const traceId = firstNonEmpty(conversation?.lastTraceId, session.api?.workspace?.activeTraceId); + const conversation = conversationId ? session.api?.conversations?.items?.find((item) => item.conversationId === conversationId || item.sessionId === conversationId) ?? null : null; + const traceId = firstNonEmpty(dom.traceId, conversation?.lastTraceId, session.api?.workspace?.activeTraceId); const traceProjectId = normalizeProbeProjectId(conversation?.projectId) ?? normalizeProbeProjectId(session.api?.workspace?.projectId) ?? normalizeProbeProjectId(args.projectId); const traceFetch = traceId ? await fetchTraceSummaryWithRetry(page, args, traceId, traceProjectId) @@ -1041,7 +1128,7 @@ async function collectTraceTimelineSample(page, args, index) { } async function fetchTraceSummaryWithRetry(page, args, traceId, projectId) { - const paths = [traceSummaryPath(traceId, projectId), ...(projectId ? [traceSummaryPath(traceId, null)] : [])]; + const paths = [traceSummaryPath(traceId, projectId)]; const attempts = []; for (const candidate of paths) { for (let attempt = 1; attempt <= 3; attempt += 1) { @@ -1059,8 +1146,7 @@ async function fetchTraceSummaryWithRetry(page, args, traceId, projectId) { function traceSummaryPath(traceId, projectId) { const params = new URLSearchParams({ sinceSeq: "0", limit: "20" }); - if (projectId) params.set("projectId", projectId); - return { path: `/v1/agent/traces/${encodeURIComponent(traceId)}?${params.toString()}`, projectId, scoped: Boolean(projectId) }; + return { path: `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?${params.toString()}`, projectId, scoped: Boolean(projectId) }; } function summarizeTraceFetch(fetchResult, expectedTraceId, attempt, request) { @@ -1078,7 +1164,7 @@ function summarizeTraceFetch(fetchResult, expectedTraceId, attempt, request) { httpStatus: fetchResult?.status ?? null, elapsedMs: fetchResult?.elapsedMs ?? null, traceId: trace?.traceId ?? body?.traceId ?? expectedTraceId, - traceStatus: trace?.status ?? body?.status ?? null, + traceStatus: trace?.traceStatus ?? trace?.status ?? body?.traceStatus ?? body?.status ?? null, running: trace?.running ?? body?.running ?? null, terminal: trace?.terminal ?? body?.terminal ?? null, eventCount: events.length, @@ -1178,9 +1264,11 @@ function summarizeProbePerformance(actions, startedAt, samples, traceSummary) { }; } -function runProbeDegradedReason({ traceSummary, sessionSummary, promptValidation }) { +function runProbeDegradedReason({ traceSummary, sessionSummary, promptValidation, actions }) { if (sessionSummary?.degradedReason) return sessionSummary.degradedReason; if (traceSummary?.degradedReason) return traceSummary.degradedReason; + const terminalWait = Array.isArray(actions) ? [...actions].reverse().find((item) => item.action === "wait-agent-terminal") : null; + if (terminalWait?.terminal === false) return terminalWait.degradedReason ?? "agent-terminal-timeout"; if (promptValidation?.ok === false) return "prompt-validation-failed"; return null; } @@ -1192,8 +1280,10 @@ function classifyRunProbeError(error, context = {}) { const login = failureDom.login && typeof failureDom.login === "object" ? failureDom.login : {}; if (/\/login(?:\?|$)/u.test(finalUrl) || failureDom.authState === "login" || login.cardVisible === true) return "auth-redirect-login"; if (/session_required|session-not-selected/u.test(message)) return "session-not-selected"; + if (/pre-submit-realign|session-(?:route|workspace|active-tab)-not-aligned|select-conversation/u.test(message)) return "session-not-selected"; if (/composer|command-send|command bar|disabledReason|button_disabled/u.test(message)) return "composer-disabled"; if (/fresh session/u.test(message)) return "fresh-session-not-aligned"; + if (/agent final response did not reach terminal DOM state/u.test(message)) return "agent-terminal-timeout"; if (/login|auth/u.test(message)) return "auth-login-failed"; if (/Timeout|timed out|timeout/u.test(message)) return "browser-timeout"; if (/fetch/u.test(message)) return "trace-fetch-failed"; @@ -1336,20 +1426,31 @@ async function waitForAgentTerminal(page, args, actions) { return status && status !== "running" && text.length > 0; }, null, { timeout: timeoutMs }).catch(() => null); const after = await collectLatestAgentMessageState(page); - actions.push({ action: "wait-agent-terminal", terminal: Boolean(terminal), timeoutMs, before, after }); - if (!terminal) throw new Error(`agent final response did not reach terminal DOM state: ${JSON.stringify({ before, after, timeoutMs })}`); + actions.push({ action: "wait-agent-terminal", terminal: Boolean(terminal), timeoutMs, before, after, degradedReason: terminal ? null : "agent-terminal-timeout" }); + return Boolean(terminal); } async function collectLatestAgentMessageState(page) { return page.evaluate(() => { + const firstTraceId = (...sources) => { + for (const source of sources) { + const match = String(source ?? "").match(/\btrc_[A-Za-z0-9_.:-]+\b/u); + if (match) return match[0]; + } + return null; + }; const cardRole = (card) => card.getAttribute("data-role") ?? card.getAttribute("data-message-role") ?? [...card.classList].find((item) => item === "message-agent" || item === "message-user" || item === "message-system")?.replace(/^message-/, "") ?? null; const cardStatus = (card) => card.getAttribute("data-status") ?? card.getAttribute("data-message-status") ?? null; const cards = [...document.querySelectorAll(".message-card")]; const latestAgent = [...cards].reverse().find((card) => cardRole(card) === "agent" || card.classList.contains("message-agent")); + const trace = latestAgent?.querySelector(".trace-timeline, .message-trace") ?? null; const finalResponse = latestAgent?.querySelector(".message-final-response"); + const modeText = document.querySelector(".composer-mode")?.textContent?.trim() ?? null; return { messageCount: cards.length, status: latestAgent ? cardStatus(latestAgent) : null, + traceId: firstTraceId(latestAgent?.getAttribute("data-trace-id"), trace?.getAttribute("data-trace-id"), trace?.getAttribute("data-traceid"), modeText, latestAgent?.textContent, trace?.textContent), + modeText, textChars: finalResponse?.textContent?.trim().length ?? 0, textPreview: (finalResponse?.textContent ?? "").replace(/\s+/gu, " ").trim().slice(0, 240) }; @@ -1358,6 +1459,13 @@ async function collectLatestAgentMessageState(page) { async function collectDomEvidence(page, args) { return page.evaluate(async ({ projectId }) => { + const firstTraceId = (...sources) => { + for (const source of sources) { + const match = String(source ?? "").match(/\btrc_[A-Za-z0-9_.:-]+\b/u); + if (match) return match[0]; + } + return null; + }; const cardRole = (card) => card.getAttribute("data-role") ?? card.getAttribute("data-message-role") ?? [...card.classList].find((item) => item === "message-user" || item === "message-agent" || item === "message-system")?.replace(/^message-/, "") ?? null; const cardStatus = (card) => card.getAttribute("data-status") ?? card.getAttribute("data-message-status") ?? null; const workspace = document.querySelector("#workspace"); @@ -1370,7 +1478,12 @@ async function collectDomEvidence(page, args) { const activeSessionTab = sessionTabs.find((item) => item.getAttribute("data-active") === "true") ?? null; const routeMatch = window.location.pathname.match(/\/workbench\/sessions\/([^/]+)/u) ?? window.location.pathname.match(/\/workspace\/sessions\/([^/]+)/u); const sessionIds = conversations.map((item) => item?.sessionId).filter((value) => typeof value === "string" && value).slice(0, 8); - const traceIds = conversations.map((item) => item?.lastTraceId).filter((value) => typeof value === "string" && value).slice(0, 8); + const conversationTraceIds = conversations.map((item) => item?.lastTraceId).filter((value) => typeof value === "string" && value); + const domTraceIds = cards.map((card) => { + const trace = card.querySelector(".trace-timeline, .message-trace"); + return firstTraceId(card.getAttribute("data-trace-id"), trace?.getAttribute("data-trace-id"), trace?.getAttribute("data-traceid"), card.textContent, trace?.textContent, document.querySelector(".composer-mode")?.textContent); + }).filter((value) => typeof value === "string" && value); + const traceIds = [...new Set([...conversationTraceIds, ...domTraceIds])].slice(0, 8); const scroll = workspace ? { scrollTop: Math.round(workspace.scrollTop), scrollHeight: Math.round(workspace.scrollHeight), @@ -1417,12 +1530,15 @@ async function collectDomEvidence(page, args) { const traceRect = traceEvents?.getBoundingClientRect(); const finalRect = finalResponse?.getBoundingClientRect(); const finalBody = finalResponse?.querySelector(".message-body"); + const traceId = firstTraceId(card.getAttribute("data-trace-id"), trace?.getAttribute("data-trace-id"), trace?.getAttribute("data-traceid"), card.textContent, trace?.textContent, document.querySelector(".composer-mode")?.textContent); return { status: cardStatus(card), role: cardRole(card), + traceId, className: card.className, trace: trace ? { present: true, + traceId, open: trace instanceof HTMLDetailsElement ? trace.open : null, status: trace.getAttribute("data-trace-status"), mode: trace.getAttribute("data-trace-mode"),