fix: stabilize web probe fresh session evidence

This commit is contained in:
lyon
2026-06-18 18:02:05 +08:00
parent e28ecdeda4
commit 5604cd0f8a
+139 -23
View File
@@ -277,7 +277,7 @@ export async function runProbe(args) {
sessionSummary = summarizeSessionRun(actions, dom);
traceSummary = summarizeTraceRun(traceSamples, dom, sessionSummary);
performanceSummary = summarizeProbePerformance(actions, startedAt, traceSamples, traceSummary);
const degradedReason = runProbeDegradedReason({ traceSummary, sessionSummary, promptValidation });
const degradedReason = runProbeDegradedReason({ traceSummary, sessionSummary, promptValidation, actions });
result = {
ok: true,
status: dom.requiredSelectors.workspace && dom.requiredSelectors.commandInput && dom.requiredSelectors.conversationList && promptValidation.ok && !degradedReason ? "pass" : "blocked",
@@ -475,7 +475,7 @@ async function probeStatus(args) {
if (!args.jobId) throw new Error("status requires a job id");
const metaPath = path.join(stateRoot, `${args.jobId}.json`);
const meta = JSON.parse(await readFile(metaPath, "utf8"));
const report = fs.existsSync(meta.reportPath) ? JSON.parse(await readFile(meta.reportPath, "utf8")) : null;
const report = fs.existsSync(meta.reportPath) ? await attachReportArtifactFingerprints(JSON.parse(await readFile(meta.reportPath, "utf8")), meta) : null;
const running = !report && typeof meta.pid === "number" && processAlive(meta.pid);
return {
ok: report ? report.ok === true : running,
@@ -496,6 +496,22 @@ async function probeStatus(args) {
};
}
async function attachReportArtifactFingerprints(report, meta) {
if (!report) return report;
const reportPath = report.artifacts?.reportPath ?? meta.reportPath ?? null;
const screenshotPath = report.artifacts?.screenshotPath ?? meta.screenshotPath ?? null;
return {
...report,
artifacts: {
...(report.artifacts ?? {}),
reportPath,
reportSha256: await fileSha256(reportPath),
screenshotPath,
screenshotSha256: await fileSha256(screenshotPath)
}
};
}
async function maybeLogin(page, args, actions) {
const username = page.locator("#login-username");
if (await username.isVisible({ timeout: Math.min(args.timeoutMs, 12000) }).catch(() => false)) {
@@ -706,11 +722,42 @@ function freshSessionCandidate(before, after) {
async function realignFreshSession(page, args, sessionOrConversationId) {
const selected = await selectConversationViaApi(page, args, sessionOrConversationId, "web-live-dom-probe-fresh-realign").catch((error) => ({ ok: false, phase: "select-conversation", error: error instanceof Error ? error.message : String(error) }));
const target = new URL(`/workbench/sessions/${encodeURIComponent(sessionOrConversationId)}`, args.url).toString();
await page.goto(target, { waitUntil: "domcontentloaded", timeout: args.timeoutMs });
await page.locator("#workspace").waitFor({ state: "visible", timeout: args.timeoutMs });
await page.locator("#command-input").waitFor({ state: "visible", timeout: args.timeoutMs });
return { action: "fresh-session-realign", id: sessionOrConversationId, target, selected };
const targetUrl = new URL(`/workbench/sessions/${encodeURIComponent(sessionOrConversationId)}`, args.url);
targetUrl.searchParams.set("projectId", args.projectId);
const target = targetUrl.toString();
const attempts = [];
for (let attempt = 1; attempt <= 2; attempt += 1) {
const navigation = await page.goto(target, { waitUntil: "domcontentloaded", timeout: Math.min(args.timeoutMs, 30000) })
.then((response) => ({ ok: true, status: response?.status() ?? null, url: page.url() }))
.catch((error) => ({ ok: false, error: error instanceof Error ? error.message : String(error), url: page.url() }));
const ready = await waitForWorkbenchSessionRouteReady(page, args, attempt === 1 ? 15000 : 30000);
attempts.push({ attempt, navigation, ready });
if (ready.ok) return { action: "fresh-session-realign", id: sessionOrConversationId, target, selected, attempts };
await page.waitForTimeout(1000);
}
return { action: "fresh-session-realign", id: sessionOrConversationId, target, selected, attempts, ok: false };
}
async function waitForWorkbenchSessionRouteReady(page, args, timeoutMs) {
await page.waitForFunction(() => {
const authState = document.body.getAttribute("data-auth-state");
const workspace = document.querySelector("#workspace");
const loginVisible = Boolean(document.querySelector("#login-username") || document.querySelector('input[autocomplete="username"]') || document.querySelector(".login-card input"));
return authState !== "checking" || workspace || loginVisible;
}, null, { timeout: Math.min(timeoutMs, 10000) }).catch(() => null);
const workspace = await page.locator("#workspace").waitFor({ state: "visible", timeout: timeoutMs }).then(() => ({ ok: true })).catch((error) => ({ ok: false, error: error instanceof Error ? error.message : String(error) }));
const commandInput = workspace.ok
? await page.locator("#command-input").waitFor({ state: "visible", timeout: timeoutMs }).then(() => ({ ok: true })).catch((error) => ({ ok: false, error: error instanceof Error ? error.message : String(error) }))
: { ok: false, skipped: "workspace-not-visible" };
const dom = await page.evaluate(() => ({
href: window.location.href,
authState: document.body.getAttribute("data-auth-state"),
title: document.title,
workspaceVisible: Boolean(document.querySelector("#workspace")),
commandInputVisible: Boolean(document.querySelector("#command-input")),
bodyTextPreview: document.body.textContent?.replace(/\s+/gu, " ").trim().slice(0, 200) ?? ""
})).catch((error) => ({ error: error instanceof Error ? error.message : String(error) }));
return { ok: workspace.ok && commandInput.ok, timeoutMs, workspace, commandInput, dom };
}
async function selectConversationViaApi(page, args, conversationId, updatedByClient) {
@@ -771,6 +818,35 @@ function sessionAuthorityId(state) {
return firstNonEmpty(state?.routeConversationId, state?.api?.workspace?.selectedConversationId, state?.activeTab?.conversationId, state?.activeTab?.sessionId);
}
function sessionAuthorityAlignment(state, expectedId, options = {}) {
const routeConversationId = state?.routeConversationId ?? null;
const selectedConversationId = state?.api?.workspace?.selectedConversationId ?? null;
const selectedAgentSessionId = state?.api?.workspace?.selectedAgentSessionId ?? null;
const activeTabSessionId = state?.activeTab?.sessionId ?? null;
const activeTabConversationId = state?.activeTab?.conversationId ?? null;
const workspaceApiAvailable = state?.api?.workspace?.ok === true;
const conversationsApiAvailable = state?.api?.conversations?.ok === true;
const apiConversation = expectedId ? state?.api?.conversations?.items?.find((item) => item.conversationId === expectedId || item.sessionId === expectedId) ?? null : null;
const routeMatches = Boolean(routeConversationId && expectedId && routeConversationId === expectedId);
const workspaceMatches = workspaceApiAvailable
? Boolean(expectedId && (selectedConversationId === expectedId || selectedAgentSessionId === expectedId))
: true;
const apiConversationPresent = conversationsApiAvailable ? Boolean(apiConversation) : true;
const activeTabMatches = Boolean(!activeTabSessionId && !activeTabConversationId) || activeTabSessionId === expectedId || activeTabConversationId === expectedId;
const commandReady = state?.command?.inputDisabled === false;
const empty = options.requireEmpty === true ? state?.messageCount === 0 : true;
const ok = Boolean(expectedId && routeMatches && workspaceMatches && apiConversationPresent && activeTabMatches && commandReady && empty);
const reason = ok ? null
: !expectedId ? "session-id-missing"
: !routeMatches ? "session-route-not-aligned"
: !workspaceMatches ? "session-workspace-not-aligned"
: !apiConversationPresent ? "session-conversation-api-missing"
: !activeTabMatches ? "session-active-tab-not-aligned"
: !commandReady ? "session-command-not-ready"
: "session-not-empty";
return { ok, reason, expectedId, routeConversationId, selectedConversationId, selectedAgentSessionId, activeTabConversationId, activeTabSessionId, workspaceApiAvailable, conversationsApiAvailable, apiConversationPresent, commandReady, messageCount: state?.messageCount ?? null };
}
async function waitForSessionShellReady(page, args, actions) {
const timeoutMs = Math.min(args.timeoutMs, 30000);
const before = await collectSessionState(page, args);
@@ -956,16 +1032,16 @@ async function submitPrompt(page, args, actions) {
async function ensureConversationAligned(page, args, conversationId, actions, action) {
const before = await collectSessionState(page, args);
const beforeAligned = before.routeConversationId === conversationId && before.api?.workspace?.selectedConversationId === conversationId;
if (beforeAligned) {
actions.push({ action, conversationId, aligned: true, repaired: false, before: compactSessionState(before) });
const beforeAlignment = sessionAuthorityAlignment(before, conversationId);
if (beforeAlignment.ok) {
actions.push({ action, conversationId, aligned: true, repaired: false, before: compactSessionState(before), alignment: beforeAlignment });
return before;
}
const repair = await realignFreshSession(page, args, conversationId);
const after = await collectSessionState(page, args);
const aligned = after.routeConversationId === conversationId && after.api?.workspace?.selectedConversationId === conversationId;
actions.push({ action, conversationId, aligned, repaired: true, before: compactSessionState(before), after: compactSessionState(after), repair });
if (!aligned) throw new Error(`${action} failed: ${JSON.stringify({ conversationId, before: compactSessionState(before), after: compactSessionState(after), repair })}`);
const afterAlignment = sessionAuthorityAlignment(after, conversationId);
actions.push({ action, conversationId, aligned: afterAlignment.ok, repaired: true, before: compactSessionState(before), after: compactSessionState(after), beforeAlignment, afterAlignment, repair });
if (!afterAlignment.ok) throw new Error(`${action} failed: ${JSON.stringify({ conversationId, reason: afterAlignment.reason, before: compactSessionState(before), after: compactSessionState(after), beforeAlignment, afterAlignment, repair })}`);
return after;
}
@@ -981,6 +1057,13 @@ async function sampleTraceTimeline(page, args, actions) {
async function collectTraceTimelineSample(page, args, index) {
const dom = await page.evaluate((sampleIndex) => {
const firstTraceId = (...sources) => {
for (const source of sources) {
const match = String(source ?? "").match(/\btrc_[A-Za-z0-9_.:-]+\b/u);
if (match) return match[0];
}
return null;
};
const cardRole = (card) => card.getAttribute("data-role") ?? card.getAttribute("data-message-role") ?? [...card.classList].find((item) => item === "message-agent" || item === "message-user" || item === "message-system")?.replace(/^message-/, "") ?? null;
const cardStatus = (card) => card.getAttribute("data-status") ?? card.getAttribute("data-message-status") ?? null;
const cards = [...document.querySelectorAll(".message-card")];
@@ -1011,11 +1094,15 @@ async function collectTraceTimelineSample(page, args, index) {
const latestUserVisibleRow = userVisibleRows.at(-1) ?? null;
const empty = latestAgent?.querySelector(".trace-empty") ?? null;
const finalResponse = latestAgent?.querySelector(".message-final-response") ?? null;
const modeText = document.querySelector(".composer-mode")?.textContent?.trim() ?? null;
const traceId = firstTraceId(latestAgent?.getAttribute("data-trace-id"), trace?.getAttribute("data-trace-id"), trace?.getAttribute("data-traceid"), modeText, latestAgent?.textContent, trace?.textContent);
return {
index: sampleIndex,
sampledAt: new Date().toISOString(),
messageCount: cards.length,
agentStatus: latestAgent ? cardStatus(latestAgent) : null,
traceId,
modeText,
tracePresent: Boolean(trace),
traceStatus: trace?.getAttribute("data-status") ?? null,
traceOpen: details instanceof HTMLDetailsElement ? details.open : null,
@@ -1031,8 +1118,8 @@ async function collectTraceTimelineSample(page, args, index) {
}, index);
const session = await collectSessionState(page, args);
const conversationId = firstNonEmpty(session.routeConversationId, session.api?.workspace?.selectedConversationId, session.activeTab?.conversationId);
const conversation = conversationId ? session.api?.conversations?.items?.find((item) => item.conversationId === conversationId) ?? null : null;
const traceId = firstNonEmpty(conversation?.lastTraceId, session.api?.workspace?.activeTraceId);
const conversation = conversationId ? session.api?.conversations?.items?.find((item) => item.conversationId === conversationId || item.sessionId === conversationId) ?? null : null;
const traceId = firstNonEmpty(dom.traceId, conversation?.lastTraceId, session.api?.workspace?.activeTraceId);
const traceProjectId = normalizeProbeProjectId(conversation?.projectId) ?? normalizeProbeProjectId(session.api?.workspace?.projectId) ?? normalizeProbeProjectId(args.projectId);
const traceFetch = traceId
? await fetchTraceSummaryWithRetry(page, args, traceId, traceProjectId)
@@ -1041,7 +1128,7 @@ async function collectTraceTimelineSample(page, args, index) {
}
async function fetchTraceSummaryWithRetry(page, args, traceId, projectId) {
const paths = [traceSummaryPath(traceId, projectId), ...(projectId ? [traceSummaryPath(traceId, null)] : [])];
const paths = [traceSummaryPath(traceId, projectId)];
const attempts = [];
for (const candidate of paths) {
for (let attempt = 1; attempt <= 3; attempt += 1) {
@@ -1059,8 +1146,7 @@ async function fetchTraceSummaryWithRetry(page, args, traceId, projectId) {
function traceSummaryPath(traceId, projectId) {
const params = new URLSearchParams({ sinceSeq: "0", limit: "20" });
if (projectId) params.set("projectId", projectId);
return { path: `/v1/agent/traces/${encodeURIComponent(traceId)}?${params.toString()}`, projectId, scoped: Boolean(projectId) };
return { path: `/v1/workbench/traces/${encodeURIComponent(traceId)}/events?${params.toString()}`, projectId, scoped: Boolean(projectId) };
}
function summarizeTraceFetch(fetchResult, expectedTraceId, attempt, request) {
@@ -1078,7 +1164,7 @@ function summarizeTraceFetch(fetchResult, expectedTraceId, attempt, request) {
httpStatus: fetchResult?.status ?? null,
elapsedMs: fetchResult?.elapsedMs ?? null,
traceId: trace?.traceId ?? body?.traceId ?? expectedTraceId,
traceStatus: trace?.status ?? body?.status ?? null,
traceStatus: trace?.traceStatus ?? trace?.status ?? body?.traceStatus ?? body?.status ?? null,
running: trace?.running ?? body?.running ?? null,
terminal: trace?.terminal ?? body?.terminal ?? null,
eventCount: events.length,
@@ -1178,9 +1264,11 @@ function summarizeProbePerformance(actions, startedAt, samples, traceSummary) {
};
}
function runProbeDegradedReason({ traceSummary, sessionSummary, promptValidation }) {
function runProbeDegradedReason({ traceSummary, sessionSummary, promptValidation, actions }) {
if (sessionSummary?.degradedReason) return sessionSummary.degradedReason;
if (traceSummary?.degradedReason) return traceSummary.degradedReason;
const terminalWait = Array.isArray(actions) ? [...actions].reverse().find((item) => item.action === "wait-agent-terminal") : null;
if (terminalWait?.terminal === false) return terminalWait.degradedReason ?? "agent-terminal-timeout";
if (promptValidation?.ok === false) return "prompt-validation-failed";
return null;
}
@@ -1192,8 +1280,10 @@ function classifyRunProbeError(error, context = {}) {
const login = failureDom.login && typeof failureDom.login === "object" ? failureDom.login : {};
if (/\/login(?:\?|$)/u.test(finalUrl) || failureDom.authState === "login" || login.cardVisible === true) return "auth-redirect-login";
if (/session_required|session-not-selected/u.test(message)) return "session-not-selected";
if (/pre-submit-realign|session-(?:route|workspace|active-tab)-not-aligned|select-conversation/u.test(message)) return "session-not-selected";
if (/composer|command-send|command bar|disabledReason|button_disabled/u.test(message)) return "composer-disabled";
if (/fresh session/u.test(message)) return "fresh-session-not-aligned";
if (/agent final response did not reach terminal DOM state/u.test(message)) return "agent-terminal-timeout";
if (/login|auth/u.test(message)) return "auth-login-failed";
if (/Timeout|timed out|timeout/u.test(message)) return "browser-timeout";
if (/fetch/u.test(message)) return "trace-fetch-failed";
@@ -1336,20 +1426,31 @@ async function waitForAgentTerminal(page, args, actions) {
return status && status !== "running" && text.length > 0;
}, null, { timeout: timeoutMs }).catch(() => null);
const after = await collectLatestAgentMessageState(page);
actions.push({ action: "wait-agent-terminal", terminal: Boolean(terminal), timeoutMs, before, after });
if (!terminal) throw new Error(`agent final response did not reach terminal DOM state: ${JSON.stringify({ before, after, timeoutMs })}`);
actions.push({ action: "wait-agent-terminal", terminal: Boolean(terminal), timeoutMs, before, after, degradedReason: terminal ? null : "agent-terminal-timeout" });
return Boolean(terminal);
}
async function collectLatestAgentMessageState(page) {
return page.evaluate(() => {
const firstTraceId = (...sources) => {
for (const source of sources) {
const match = String(source ?? "").match(/\btrc_[A-Za-z0-9_.:-]+\b/u);
if (match) return match[0];
}
return null;
};
const cardRole = (card) => card.getAttribute("data-role") ?? card.getAttribute("data-message-role") ?? [...card.classList].find((item) => item === "message-agent" || item === "message-user" || item === "message-system")?.replace(/^message-/, "") ?? null;
const cardStatus = (card) => card.getAttribute("data-status") ?? card.getAttribute("data-message-status") ?? null;
const cards = [...document.querySelectorAll(".message-card")];
const latestAgent = [...cards].reverse().find((card) => cardRole(card) === "agent" || card.classList.contains("message-agent"));
const trace = latestAgent?.querySelector(".trace-timeline, .message-trace") ?? null;
const finalResponse = latestAgent?.querySelector(".message-final-response");
const modeText = document.querySelector(".composer-mode")?.textContent?.trim() ?? null;
return {
messageCount: cards.length,
status: latestAgent ? cardStatus(latestAgent) : null,
traceId: firstTraceId(latestAgent?.getAttribute("data-trace-id"), trace?.getAttribute("data-trace-id"), trace?.getAttribute("data-traceid"), modeText, latestAgent?.textContent, trace?.textContent),
modeText,
textChars: finalResponse?.textContent?.trim().length ?? 0,
textPreview: (finalResponse?.textContent ?? "").replace(/\s+/gu, " ").trim().slice(0, 240)
};
@@ -1358,6 +1459,13 @@ async function collectLatestAgentMessageState(page) {
async function collectDomEvidence(page, args) {
return page.evaluate(async ({ projectId }) => {
const firstTraceId = (...sources) => {
for (const source of sources) {
const match = String(source ?? "").match(/\btrc_[A-Za-z0-9_.:-]+\b/u);
if (match) return match[0];
}
return null;
};
const cardRole = (card) => card.getAttribute("data-role") ?? card.getAttribute("data-message-role") ?? [...card.classList].find((item) => item === "message-user" || item === "message-agent" || item === "message-system")?.replace(/^message-/, "") ?? null;
const cardStatus = (card) => card.getAttribute("data-status") ?? card.getAttribute("data-message-status") ?? null;
const workspace = document.querySelector("#workspace");
@@ -1370,7 +1478,12 @@ async function collectDomEvidence(page, args) {
const activeSessionTab = sessionTabs.find((item) => item.getAttribute("data-active") === "true") ?? null;
const routeMatch = window.location.pathname.match(/\/workbench\/sessions\/([^/]+)/u) ?? window.location.pathname.match(/\/workspace\/sessions\/([^/]+)/u);
const sessionIds = conversations.map((item) => item?.sessionId).filter((value) => typeof value === "string" && value).slice(0, 8);
const traceIds = conversations.map((item) => item?.lastTraceId).filter((value) => typeof value === "string" && value).slice(0, 8);
const conversationTraceIds = conversations.map((item) => item?.lastTraceId).filter((value) => typeof value === "string" && value);
const domTraceIds = cards.map((card) => {
const trace = card.querySelector(".trace-timeline, .message-trace");
return firstTraceId(card.getAttribute("data-trace-id"), trace?.getAttribute("data-trace-id"), trace?.getAttribute("data-traceid"), card.textContent, trace?.textContent, document.querySelector(".composer-mode")?.textContent);
}).filter((value) => typeof value === "string" && value);
const traceIds = [...new Set([...conversationTraceIds, ...domTraceIds])].slice(0, 8);
const scroll = workspace ? {
scrollTop: Math.round(workspace.scrollTop),
scrollHeight: Math.round(workspace.scrollHeight),
@@ -1417,12 +1530,15 @@ async function collectDomEvidence(page, args) {
const traceRect = traceEvents?.getBoundingClientRect();
const finalRect = finalResponse?.getBoundingClientRect();
const finalBody = finalResponse?.querySelector(".message-body");
const traceId = firstTraceId(card.getAttribute("data-trace-id"), trace?.getAttribute("data-trace-id"), trace?.getAttribute("data-traceid"), card.textContent, trace?.textContent, document.querySelector(".composer-mode")?.textContent);
return {
status: cardStatus(card),
role: cardRole(card),
traceId,
className: card.className,
trace: trace ? {
present: true,
traceId,
open: trace instanceof HTMLDetailsElement ? trace.open : null,
status: trace.getAttribute("data-trace-status"),
mode: trace.getAttribute("data-trace-mode"),