|
|
|
@@ -1,4 +1,5 @@
|
|
|
|
|
#!/usr/bin/env node
|
|
|
|
|
import { createHash } from "node:crypto";
|
|
|
|
|
import { spawn } from "node:child_process";
|
|
|
|
|
import fs from "node:fs";
|
|
|
|
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
|
|
@@ -99,6 +100,7 @@ export async function runProbe(args) {
|
|
|
|
|
browser = await launchChromium(chromium);
|
|
|
|
|
const context = await browser.newContext({ viewport: { width, height } });
|
|
|
|
|
const page = await context.newPage();
|
|
|
|
|
const bootstrapRoutes = observeBootstrapRoutes(page);
|
|
|
|
|
await page.goto(args.url, { waitUntil: "domcontentloaded", timeout: args.timeoutMs });
|
|
|
|
|
await waitForAuthBootstrap(page, args, actions);
|
|
|
|
|
await maybeLogin(page, args, actions);
|
|
|
|
@@ -110,7 +112,7 @@ export async function runProbe(args) {
|
|
|
|
|
if (args.waitAgentTerminalMs) await waitForAgentTerminal(page, args, actions);
|
|
|
|
|
await waitForMessageCards(page, args, actions);
|
|
|
|
|
await page.screenshot({ path: screenshotPath, fullPage: true });
|
|
|
|
|
const dom = await collectDomEvidence(page);
|
|
|
|
|
const dom = await collectDomEvidence(page, args);
|
|
|
|
|
const promptValidation = validatePromptFinalResponse(dom, args);
|
|
|
|
|
result = {
|
|
|
|
|
ok: true,
|
|
|
|
@@ -123,14 +125,15 @@ export async function runProbe(args) {
|
|
|
|
|
viewport: { width, height },
|
|
|
|
|
browser: browser.hwlabLaunchPlan ?? null,
|
|
|
|
|
actions,
|
|
|
|
|
bootstrapRoutes: summarizeBootstrapRoutes(bootstrapRoutes),
|
|
|
|
|
dom,
|
|
|
|
|
promptValidation,
|
|
|
|
|
artifacts: { reportPath, screenshotPath },
|
|
|
|
|
safety: {
|
|
|
|
|
repoOwnedLauncher: true,
|
|
|
|
|
directChromiumLaunch: false,
|
|
|
|
|
projectId: args.projectId,
|
|
|
|
|
conversationSelected: Boolean(args.conversationId),
|
|
|
|
|
projectId: args.projectId,
|
|
|
|
|
conversationSelected: Boolean(args.conversationId),
|
|
|
|
|
freshSessionRequested: args.freshSession,
|
|
|
|
|
promptSubmitted: Boolean(args.message),
|
|
|
|
|
cancelRunningRequested: args.cancelRunning
|
|
|
|
@@ -154,6 +157,13 @@ export async function runProbe(args) {
|
|
|
|
|
}
|
|
|
|
|
await mkdir(path.dirname(reportPath), { recursive: true });
|
|
|
|
|
await writeFile(reportPath, `${JSON.stringify(result, null, 2)}\n`, "utf8");
|
|
|
|
|
result.artifacts = {
|
|
|
|
|
...(result.artifacts ?? {}),
|
|
|
|
|
reportPath,
|
|
|
|
|
reportSha256: await fileSha256(reportPath),
|
|
|
|
|
screenshotPath,
|
|
|
|
|
screenshotSha256: await fileSha256(screenshotPath)
|
|
|
|
|
};
|
|
|
|
|
return result;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -297,26 +307,14 @@ async function maybeLogin(page, args, actions) {
|
|
|
|
|
actions.push({ action: "login", skipped: "login-form-not-visible", authState: await page.locator("body").getAttribute("data-auth-state").catch(() => null) });
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
const cardInputs = page.locator(".login-card input");
|
|
|
|
|
if ((await cardInputs.count().catch(() => 0)) < 2) {
|
|
|
|
|
actions.push({ action: "login", skipped: "login-form-incomplete", selectorMode: "login-card", inputCount: await cardInputs.count().catch(() => 0), authState: await page.locator("body").getAttribute("data-auth-state").catch(() => null) });
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
await cardInputs.nth(0).fill(args.user);
|
|
|
|
|
await cardInputs.nth(1).fill(args.pass);
|
|
|
|
|
await page.locator(".login-card button[type=submit]").click();
|
|
|
|
|
actions.push({ action: "login", user: args.user, selectorMode: "login-card" });
|
|
|
|
|
await page.locator("#command-input").waitFor({ state: "visible", timeout: args.timeoutMs });
|
|
|
|
|
const finalUrl = new URL(page.url());
|
|
|
|
|
if (finalUrl.searchParams.has("username") || finalUrl.searchParams.has("password")) {
|
|
|
|
|
throw new Error("login credentials leaked into URL query");
|
|
|
|
|
}
|
|
|
|
|
await submitCurrentLoginForm(page, args, actions, "current-autocomplete");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function waitForLoginSurface(page, args) {
|
|
|
|
|
const handle = await page.waitForFunction(() => {
|
|
|
|
|
if (document.querySelector("#workspace")) return "workspace";
|
|
|
|
|
if (document.querySelector("#login-username")) return "legacy-id";
|
|
|
|
|
if (document.querySelector('input[autocomplete="username"]') && document.querySelector('input[autocomplete="current-password"]')) return "login-card";
|
|
|
|
|
if (document.querySelector(".login-card input")) return "login-card";
|
|
|
|
|
return "";
|
|
|
|
|
}, null, { timeout: args.timeoutMs }).catch(() => null);
|
|
|
|
@@ -324,11 +322,38 @@ async function waitForLoginSurface(page, args) {
|
|
|
|
|
return handle.jsonValue().catch(() => null);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function submitCurrentLoginForm(page, args, actions, selectorMode) {
|
|
|
|
|
const username = page.locator('input[autocomplete="username"], .login-card input').first();
|
|
|
|
|
const password = page.locator('input[autocomplete="current-password"], .login-card input[type="password"], input[type="password"]').first();
|
|
|
|
|
const submit = page.locator('.login-card button[type="submit"], button[type="submit"]').first();
|
|
|
|
|
const inputCount = await page.locator(".login-card input, input[autocomplete=\"username\"], input[autocomplete=\"current-password\"]").count().catch(() => 0);
|
|
|
|
|
if (!(await username.isVisible({ timeout: 5000 }).catch(() => false)) || !(await password.isVisible({ timeout: 5000 }).catch(() => false))) {
|
|
|
|
|
actions.push({ action: "login", skipped: "login-form-incomplete", selectorMode, inputCount, authState: await page.locator("body").getAttribute("data-auth-state").catch(() => null) });
|
|
|
|
|
return;
|
|
|
|
|
}
|
|
|
|
|
await username.fill(args.user);
|
|
|
|
|
await password.fill(args.pass);
|
|
|
|
|
await Promise.allSettled([
|
|
|
|
|
page.waitForURL(/\/workbench/u, { timeout: Math.min(args.timeoutMs, 15000) }),
|
|
|
|
|
submit.click()
|
|
|
|
|
]);
|
|
|
|
|
actions.push({ action: "login", user: args.user, selectorMode });
|
|
|
|
|
await page.locator("#command-input").waitFor({ state: "visible", timeout: args.timeoutMs });
|
|
|
|
|
assertNoCredentialLeak(page.url());
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function assertNoCredentialLeak(value) {
|
|
|
|
|
const finalUrl = new URL(value);
|
|
|
|
|
if (finalUrl.searchParams.has("username") || finalUrl.searchParams.has("password")) {
|
|
|
|
|
throw new Error("login credentials leaked into URL query");
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function waitForAuthBootstrap(page, args, actions) {
|
|
|
|
|
const timeout = Math.min(args.timeoutMs, 12000);
|
|
|
|
|
const settled = await page.waitForFunction(() => {
|
|
|
|
|
const authState = document.body.getAttribute("data-auth-state");
|
|
|
|
|
const loginVisible = Boolean(document.querySelector("#login-username"));
|
|
|
|
|
const loginVisible = Boolean(document.querySelector("#login-username") || document.querySelector('input[autocomplete="username"]') || document.querySelector(".login-card input"));
|
|
|
|
|
const workspaceVisible = Boolean(document.querySelector("#workspace"));
|
|
|
|
|
return authState !== "checking" || loginVisible || workspaceVisible;
|
|
|
|
|
}, null, { timeout }).catch(() => null);
|
|
|
|
@@ -560,11 +585,18 @@ async function collectLatestAgentMessageState(page) {
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function collectDomEvidence(page) {
|
|
|
|
|
return page.evaluate(() => {
|
|
|
|
|
async function collectDomEvidence(page, args) {
|
|
|
|
|
return page.evaluate(async ({ projectId }) => {
|
|
|
|
|
const workspace = document.querySelector("#workspace");
|
|
|
|
|
const list = document.querySelector("#conversation-list");
|
|
|
|
|
const cards = [...document.querySelectorAll(".message-card")];
|
|
|
|
|
const conversationResponse = await fetch(`/v1/agent/conversations?projectId=${encodeURIComponent(projectId)}`, { credentials: "same-origin" }).catch(() => null);
|
|
|
|
|
const conversationBody = conversationResponse ? await conversationResponse.json().catch(() => null) : null;
|
|
|
|
|
const conversations = Array.isArray(conversationBody?.conversations) ? conversationBody.conversations : [];
|
|
|
|
|
const sessionTabs = [...document.querySelectorAll("#session-tabs .session-tab")];
|
|
|
|
|
const activeSessionTab = sessionTabs.find((item) => item.getAttribute("data-active") === "true") ?? null;
|
|
|
|
|
const sessionIds = conversations.map((item) => item?.sessionId).filter((value) => typeof value === "string" && value).slice(0, 8);
|
|
|
|
|
const traceIds = conversations.map((item) => item?.lastTraceId).filter((value) => typeof value === "string" && value).slice(0, 8);
|
|
|
|
|
const scroll = workspace ? {
|
|
|
|
|
scrollTop: Math.round(workspace.scrollTop),
|
|
|
|
|
scrollHeight: Math.round(workspace.scrollHeight),
|
|
|
|
@@ -582,6 +614,16 @@ async function collectDomEvidence(page) {
|
|
|
|
|
conversationList: Boolean(list),
|
|
|
|
|
sessionCreate: Boolean(document.querySelector("#session-create"))
|
|
|
|
|
},
|
|
|
|
|
sessionRail: {
|
|
|
|
|
tabCount: sessionTabs.length,
|
|
|
|
|
activeTitle: activeSessionTab?.querySelector("strong")?.textContent?.trim() ?? null,
|
|
|
|
|
objectTitleCount: sessionTabs.map((item) => item.querySelector("strong")?.textContent?.trim() ?? "").filter((text) => text.includes("[object Object]")).length,
|
|
|
|
|
titles: sessionTabs.map((item) => item.querySelector("strong")?.textContent?.trim() ?? "").slice(0, 8),
|
|
|
|
|
sessionIds,
|
|
|
|
|
traceIds,
|
|
|
|
|
conversationsStatus: conversationResponse?.status ?? null,
|
|
|
|
|
conversationsCount: conversations.length
|
|
|
|
|
},
|
|
|
|
|
messageCount: cards.length,
|
|
|
|
|
messageRoles: cards.map((card) => card.getAttribute("data-message-role") ?? [...card.classList].find((item) => item === "message-user" || item === "message-agent" || item === "message-system") ?? "unknown"),
|
|
|
|
|
messages: cards.map((card) => {
|
|
|
|
@@ -630,7 +672,37 @@ async function collectDomEvidence(page) {
|
|
|
|
|
scroll,
|
|
|
|
|
textPreview: (document.body.textContent ?? "").replace(/\s+/gu, " ").trim().slice(0, 800)
|
|
|
|
|
};
|
|
|
|
|
}, { projectId: args.projectId });
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function observeBootstrapRoutes(page) {
|
|
|
|
|
const routes = [];
|
|
|
|
|
page.on("request", (request) => {
|
|
|
|
|
try {
|
|
|
|
|
const url = new URL(request.url());
|
|
|
|
|
if (!["/auth/bootstrap", "/auth/session", "/auth/workspace-bootstrap", "/v1/workbench/workspace"].includes(url.pathname)) return;
|
|
|
|
|
routes.push({ method: request.method(), path: url.pathname });
|
|
|
|
|
} catch {
|
|
|
|
|
// Ignore malformed browser-internal URLs.
|
|
|
|
|
}
|
|
|
|
|
});
|
|
|
|
|
return routes;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function summarizeBootstrapRoutes(routes) {
|
|
|
|
|
const paths = routes.map((route) => route.path);
|
|
|
|
|
return {
|
|
|
|
|
authBootstrapRequested: paths.includes("/auth/bootstrap"),
|
|
|
|
|
authSessionRequested: paths.includes("/auth/session"),
|
|
|
|
|
workbenchWorkspaceRequested: paths.includes("/v1/workbench/workspace"),
|
|
|
|
|
legacyWorkspaceBootstrapRequested: paths.includes("/auth/workspace-bootstrap"),
|
|
|
|
|
routes: routes.slice(0, 20)
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function fileSha256(file) {
|
|
|
|
|
if (!file || !fs.existsSync(file)) return null;
|
|
|
|
|
return createHash("sha256").update(await readFile(file)).digest("hex");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function parseViewport(value) {
|
|
|
|
|