test: add workbench dom-only live validation

This commit is contained in:
Code Queue Review
2026-05-23 03:21:34 +00:00
parent 33e4eeb7d0
commit 7018efc9d4
6 changed files with 531 additions and 24 deletions
+300 -9
View File
@@ -219,6 +219,7 @@ export async function runDevCloudWorkbenchSmoke(argv = []) {
const args = parseSmokeArgs(argv);
if (args.mode === "live") return runLiveSmoke(args);
if (args.mode === "local-agent-timeout-fixture") return runDevCloudWorkbenchTimeoutFixtureSmoke();
if (args.mode === "dom-only") return runLiveDomOnlySmoke(args);
if (args.mode === "local-agent-fixture") return runDevCloudWorkbenchLocalAgentFixtureSmoke();
if (args.mode === "mobile") return runDevCloudWorkbenchMobileSmoke();
return runStaticSmoke();
@@ -268,6 +269,8 @@ export function parseSmokeArgs(argv) {
args.mode = "live";
} else if (arg === "--confirm-dev-live") {
args.confirmDevLive = true;
} else if (arg === "--dom-only") {
args.mode = "dom-only";
} else if (arg === "--mobile") {
args.mode = "mobile";
args.mobile = true;
@@ -670,6 +673,162 @@ async function runLiveSmoke(args) {
});
}
async function runLiveDomOnlySmoke(args) {
const checks = [];
const blockers = [];
const sourceIdentity = observeSourceIdentity();
const runtimeIdentity = await observeLiveRuntimeIdentity(runtime.endpoints.api);
const apiRuntimeReadiness = classifyLiveApiRuntimeReadiness(runtimeIdentity);
addCheck(checks, blockers, "live-api-runtime-readiness", apiRuntimeReadiness.status, apiRuntimeReadiness.summary, {
blocker: "runtime_blocker",
evidence: apiRuntimeReadiness.evidence,
observations: apiRuntimeReadiness
});
const expectedRuntimeIdentity = observeExpectedRuntimeIdentity("hwlab-cloud-api");
const deploymentIdentity = classifyLiveDeploymentIdentity(sourceIdentity, runtimeIdentity, expectedRuntimeIdentity);
addCheck(checks, blockers, "live-runtime-current-main", deploymentIdentity.status, deploymentIdentity.summary, {
blocker: "observability_blocker",
evidence: deploymentIdentity.evidence,
observations: deploymentIdentity
});
const http = await fetchText(args.url);
addCheck(checks, blockers, "live-http-html", http.ok && http.status === 200 && /text\/html/u.test(http.contentType) && liveHtmlLooksLikeWorkbench(http.body), "Live URL serves the Cloud Workbench HTML.", {
blocker: "runtime_blocker",
evidence: [args.url, `HTTP ${http.status ?? "none"}`, http.contentType ?? "no content-type"]
});
const webAssetIdentity = http.ok
? await inspectLiveWebAssetIdentity(args.url, http.body)
: {
status: "blocked",
summary: "Live web asset identity could not be checked because the 16666 HTML fetch failed.",
assets: [],
mismatches: ["index.html"]
};
addCheck(checks, blockers, "live-web-assets-current-main", webAssetIdentity.status, webAssetIdentity.summary, {
blocker: "observability_blocker",
evidence: webAssetIdentity.assets.length > 0
? webAssetIdentity.assets.map((asset) => `${asset.path}: ${asset.status}`)
: ["web asset identity not observed"],
observations: webAssetIdentity
});
const dom = http.ok ? await inspectLiveDom(args.url, { noCodeAgentPost: true }) : { status: "skip", summary: "Browser DOM check skipped because HTTP fetch failed.", evidence: [] };
addCheck(checks, blockers, "live-browser-dom", dom.status, dom.summary, {
blocker: dom.status === "pass" ? null : "observability_blocker",
evidence: dom.evidence,
observations: dom.observations
});
if (dom.status === "skip") {
blockers.push({
type: "observability_blocker",
scope: "live-browser-dom",
status: "open",
summary: dom.summary
});
}
const gate = http.ok ? await inspectLiveGateRoute(args.url, { noCodeAgentPost: true }) : { status: "skip", summary: "Browser Gate route check skipped because HTTP fetch failed.", evidence: [] };
addCheck(checks, blockers, "live-gate-route", gate.status, gate.summary, {
blocker: gate.status === "pass" ? null : "runtime_blocker",
evidence: gate.evidence,
observations: gate.observations
});
if (gate.status === "skip") {
blockers.push({
type: "observability_blocker",
scope: "live-gate-route",
status: "open",
summary: gate.summary
});
}
const help = http.ok ? await inspectLiveHelpRoute(args.url, { noCodeAgentPost: true }) : { status: "skip", summary: "Browser Markdown help check skipped because HTTP fetch failed.", evidence: [] };
addCheck(checks, blockers, "live-help-markdown-route", help.status, help.summary, {
blocker: help.status === "pass" ? null : "runtime_blocker",
evidence: help.evidence,
observations: help.observations
});
if (help.status === "skip") {
blockers.push({
type: "observability_blocker",
scope: "live-help-markdown-route",
status: "open",
summary: help.summary
});
}
addDomOnlyCodeAgentCheck(checks);
const status = blockers.length === 0 ? "pass" : "blocked";
return baseReport({
mode: "dom-only",
url: args.url,
status,
checks,
blockers,
evidenceLevel: status === "pass" ? "DEV-LIVE-DOM-READONLY" : "BLOCKED",
devLive: false,
runtimeIdentity,
sourceIdentity,
deploymentIdentity,
expectedRuntimeIdentity,
webAssetIdentity,
safety: {
prodTouched: false,
servicesRestarted: false,
heavyE2E: false,
hardwareWriteApis: false,
sourceIsDevLive: false,
liveMode: "dom-only-readonly-no-code-agent-post",
codeAgentBrowserJourneySkipped: true,
codeAgentPostSent: false,
retainedApiFields: ["runtime.commitId", "runtime.imageTag", "web asset hash/status", "DOM route/control/wiring observations"],
statement: "DOM-only live smoke preserves runtime and web-asset identity preflight semantics, continues with read-only browser DOM/help inspection even when identity is blocked, and never posts /v1/agent/chat, reads Secret values, mutates live DEV, or claims M3/M4/M5 acceptance."
}
});
}
function addDomOnlyCodeAgentCheck(checks) {
checks.push({
id: "live-code-agent-browser-journey",
status: "not_applicable",
summary: "DOM-only mode never posts the browser Code Agent journey; /v1/agent/chat is not sent.",
evidence: [
"POST /v1/agent/chat not_applicable",
"POST /v1/agent/chat not sent",
"DOM-only read-only mode"
],
observations: {
request: {
method: "POST",
status: "not_applicable",
urlPath: "/v1/agent/chat"
},
response: {
status: "not_applicable",
provider: "not_observed",
model: "not_observed",
backend: "not_observed",
traceId: "not_observed",
hasReply: false,
error: null
},
classification: {
status: "not_applicable",
reason: "dom_only_no_code_agent_post",
codeAgentBrowserJourneySkipped: true
},
networkEvents: []
}
});
}
function baseReport({
mode,
status,
@@ -755,8 +914,11 @@ function baseReport({
"node scripts/dev-cloud-workbench-smoke.mjs --mobile",
"node scripts/dev-cloud-workbench-smoke.mjs --local-agent-fixture",
"node scripts/dev-cloud-workbench-smoke.mjs --local-agent-timeout-fixture",
"node scripts/dev-cloud-workbench-smoke.mjs --dom-only --url http://74.48.78.17:16666/ --report reports/dev-gate/dev-cloud-workbench-dom-only.json",
"node scripts/dev-cloud-workbench-smoke.mjs --live --confirm-dev-live --url http://74.48.78.17:16666/ --report reports/dev-gate/dev-cloud-workbench-live.json",
"node scripts/validate-dev-gate-report.mjs reports/dev-gate/dev-cloud-workbench-live.json",
mode === "dom-only"
? "node scripts/validate-dev-gate-report.mjs reports/dev-gate/dev-cloud-workbench-dom-only.json"
: "node scripts/validate-dev-gate-report.mjs reports/dev-gate/dev-cloud-workbench-live.json",
"node scripts/validate-dev-gate-report.mjs"
],
localSmoke: {
@@ -781,16 +943,20 @@ function baseReport({
requirements: [
"GET http://74.48.78.17:16666/ serves the Cloud Workbench HTML.",
"Browser DOM exposes the default workbench and core controls.",
"DOM-only mode may inspect read-only deployed DOM wiring after identity preflight blocks, but must not send Code Agent chat.",
"A controlled same-origin POST /v1/agent/chat returns completed UI-visible evidence or reports a blocker.",
"No hardware write API, service restart, PROD access, or secret read is performed."
],
commands: [
"node scripts/dev-cloud-workbench-smoke.mjs --dom-only --url http://74.48.78.17:16666/ --report reports/dev-gate/dev-cloud-workbench-dom-only.json",
"node scripts/dev-cloud-workbench-smoke.mjs --live --confirm-dev-live --url http://74.48.78.17:16666/ --report reports/dev-gate/dev-cloud-workbench-live.json"
],
summary: liveJourneyPassed
? "Deployed 16666 browser journey and same-origin Code Agent chat completed."
: liveJourneyDegraded
? "Deployed UI usable in degraded/read-only mode; API health/runtime durability is degraded, so this is not full DEV-LIVE acceptance."
: mode === "dom-only"
? "Deployed browser DOM was inspected read-only; Code Agent chat was not posted."
: mode === "live"
? "Deployed browser journey is blocked; see checks and blockers."
: "Static/local fixture evidence does not establish the deployed 16666 browser journey."
@@ -1886,7 +2052,7 @@ function liveHtmlLooksLikeWorkbench(html) {
);
}
async function inspectLiveDom(url) {
async function inspectLiveDom(url, options = {}) {
let chromium;
try {
({ chromium } = await importPlaywright());
@@ -1902,6 +2068,7 @@ async function inspectLiveDom(url) {
try {
browser = await chromium.launch({ headless: true });
const page = await browser.newPage({ viewport: { width: 1366, height: 768 } });
const postGuard = options.noCodeAgentPost ? await installNoCodeAgentPostGuard(page) : null;
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 8000 });
const dom = await page.evaluate(async (forbiddenTerms) => {
const workspace = document.querySelector('[data-view="workspace"]');
@@ -1964,6 +2131,15 @@ async function inspectLiveDom(url) {
firstViewportForbiddenPresent: forbiddenTerms.filter((term) => textHasForbiddenTerm(firstViewportText, term)),
firstViewportTextSample: firstViewportText.slice(0, 400),
labelsPresent: ["硬件资源", "Agent 对话", "工作清单", "可信记录", "使用说明"].every((label) => text.includes(label)),
navLabelsPresent: ["工作台", "内部复核", "使用说明", "资源树"].every((label) =>
[...document.querySelectorAll(".activity-rail button")].some((button) => button.textContent?.trim() === label)
),
defaultBackstageHidden: !/(Gate|诊断|验收|BLOCKED|M0-M5|执行轨迹)/u.test(document.querySelector(".center-workspace")?.innerText ?? ""),
gateRouteAvailable: [...document.querySelectorAll("[data-route='gate']")].some((button) => button.textContent?.trim() === "内部复核"),
wiringHeaders: [...document.querySelectorAll(".wiring-table thead th")].map((cell) => cell.textContent?.trim() ?? ""),
m3WiringRows: [...document.querySelectorAll("#wiring-body tr")].map((row) =>
[...row.querySelectorAll("td")].map((cell) => cell.textContent?.replace(/\s+/gu, " ").trim() ?? "")
),
coreControlsVisible: {
commandInput: visible("#command-input"),
commandSend: visible("#command-send"),
@@ -1989,6 +2165,18 @@ async function inspectLiveDom(url) {
dom.rootAfterScrollAttempt.bodyScrollTop === 0 &&
dom.firstViewportForbiddenPresent.length === 0 &&
dom.labelsPresent &&
dom.navLabelsPresent &&
dom.defaultBackstageHidden &&
dom.gateRouteAvailable &&
requiredWiringColumns.every((column) => dom.wiringHeaders.includes(column)) &&
dom.m3WiringRows.some((row) =>
row.includes("res_boxsimu_1") &&
row.includes("DO1") &&
row.includes("hwlab-patch-panel") &&
row.includes("res_boxsimu_2") &&
row.includes("DI1")
) &&
(postGuard ? postGuard.attempts.length === 0 : true) &&
Object.values(dom.coreControlsVisible).every(Boolean);
return {
status: pass ? "pass" : "blocked",
@@ -1997,9 +2185,15 @@ async function inspectLiveDom(url) {
`title=${dom.title}`,
`bodyOverflow=${dom.bodyOverflow}`,
`htmlOverflow=${dom.htmlOverflow}`,
`firstViewportForbidden=${dom.firstViewportForbiddenPresent.join(",") || "none"}`
`firstViewportForbidden=${dom.firstViewportForbiddenPresent.join(",") || "none"}`,
`gateRouteAvailable=${dom.gateRouteAvailable}`,
`m3WiringRows=${dom.m3WiringRows.length}`,
...(postGuard ? [`codeAgentPostAttempts=${postGuard.attempts.length}`] : [])
],
observations: dom
observations: {
...dom,
...(postGuard ? { codeAgentPostGuard: postGuard.observation() } : {})
}
};
} catch (error) {
return {
@@ -2012,7 +2206,7 @@ async function inspectLiveDom(url) {
}
}
async function inspectLiveHelpRoute(url) {
async function inspectLiveHelpRoute(url, options = {}) {
let chromium;
try {
({ chromium } = await importPlaywright());
@@ -2028,11 +2222,12 @@ async function inspectLiveHelpRoute(url) {
try {
browser = await chromium.launch({ headless: true });
const page = await browser.newPage({ viewport: { width: 1366, height: 768 } });
const postGuard = options.noCodeAgentPost ? await installNoCodeAgentPostGuard(page) : null;
await page.goto(url, { waitUntil: "domcontentloaded", timeout: 12000 });
await page.locator('[data-route="help"]').click();
await page.waitForFunction(() => document.querySelector("#help-content")?.dataset.helpState === "ready", null, { timeout: 8000 });
const help = await inspectHelpMarkdownRoute(page);
const pass = help.nonDefaultMarkdownHelp && help.termsPresent;
const pass = help.nonDefaultMarkdownHelp && help.termsPresent && (postGuard ? postGuard.attempts.length === 0 : true);
return {
status: pass ? "pass" : "blocked",
summary: pass
@@ -2042,9 +2237,13 @@ async function inspectLiveHelpRoute(url) {
`hash=${help.hash ?? "unknown"}`,
`heading=${help.heading ?? "unknown"}`,
`rootStillLocked=${help.rootStillLocked}`,
`helpPanelScrolls=${help.helpPanelScrolls}`
`helpPanelScrolls=${help.helpPanelScrolls}`,
...(postGuard ? [`codeAgentPostAttempts=${postGuard.attempts.length}`] : [])
],
observations: help
observations: {
...help,
...(postGuard ? { codeAgentPostGuard: postGuard.observation() } : {})
}
};
} catch (error) {
return {
@@ -2057,6 +2256,97 @@ async function inspectLiveHelpRoute(url) {
}
}
async function inspectLiveGateRoute(url, options = {}) {
let chromium;
try {
({ chromium } = await importPlaywright());
} catch (error) {
return {
status: "skip",
summary: `Browser Gate route check skipped because Playwright is unavailable: ${error.message}`,
evidence: ["playwright import failed"]
};
}
let browser;
try {
browser = await chromium.launch({ headless: true });
const page = await browser.newPage({ viewport: { width: 1366, height: 768 } });
const postGuard = options.noCodeAgentPost ? await installNoCodeAgentPostGuard(page) : null;
await page.goto(new URL("/gate", url).toString(), { waitUntil: "domcontentloaded", timeout: 12000 });
const gate = await page.evaluate(() => {
const workspace = document.querySelector('[data-view="workspace"]');
const gateView = document.querySelector('[data-view="gate"]');
const help = document.querySelector('[data-view="help"]');
const text = document.body.textContent ?? "";
return {
pathname: window.location.pathname,
hash: window.location.hash,
title: document.title,
workspaceHidden: workspace ? workspace.hidden : null,
gateHidden: gateView ? gateView.hidden : null,
helpHidden: help ? help.hidden : null,
diagnosticsTermsPresent: ["Gate / 诊断 / 验收", "M0-M5", "BLOCKED", "DB live readiness", "patch-panel DO1 -> DI1"].every((term) => text.includes(term))
};
});
const pass =
gate.pathname === "/gate" &&
gate.title === "HWLAB 云工作台" &&
gate.workspaceHidden === true &&
gate.gateHidden === false &&
(gate.helpHidden === true || gate.helpHidden === null) &&
gate.diagnosticsTermsPresent &&
(postGuard ? postGuard.attempts.length === 0 : true);
return {
status: pass ? "pass" : "blocked",
summary: pass
? "Direct /gate route opens the internal diagnostics view without becoming the default workbench."
: "Direct /gate route did not satisfy the internal diagnostics route contract.",
evidence: [
`pathname=${gate.pathname}`,
`gateHidden=${gate.gateHidden}`,
`diagnosticsTermsPresent=${gate.diagnosticsTermsPresent}`,
...(postGuard ? [`codeAgentPostAttempts=${postGuard.attempts.length}`] : [])
],
observations: {
...gate,
...(postGuard ? { codeAgentPostGuard: postGuard.observation() } : {})
}
};
} catch (error) {
return {
status: "blocked",
summary: `Browser Gate route failed: ${error.message}`,
evidence: ["browser navigation or route inspection failed"]
};
} finally {
if (browser) await browser.close();
}
}
async function installNoCodeAgentPostGuard(page) {
const attempts = [];
await page.route("**/v1/agent/chat", async (route) => {
const request = route.request();
const attempt = {
method: request.method(),
urlPath: new URL(request.url()).pathname,
status: "aborted_by_dom_only_guard"
};
attempts.push(attempt);
await route.abort("blockedbyclient");
});
return {
attempts,
observation: () => ({
installed: true,
blockedPath: "/v1/agent/chat",
attemptedCount: attempts.length,
attempts
})
};
}
async function inspectLiveCodeAgentE2e(url) {
let chromium;
try {
@@ -3364,12 +3654,13 @@ function escapeRegExp(value) {
export function printSmokeHelp() {
return {
status: "usage",
command: "node scripts/dev-cloud-workbench-smoke.mjs --source | --mobile | --local-agent-fixture | --local-agent-timeout-fixture | --live --confirm-dev-live --url http://74.48.78.17:16666/ [--report reports/dev-gate/dev-cloud-workbench-live.json]",
command: "node scripts/dev-cloud-workbench-smoke.mjs --source | --mobile | --local-agent-fixture | --local-agent-timeout-fixture | --dom-only --url http://74.48.78.17:16666/ [--report reports/dev-gate/dev-cloud-workbench-dom-only.json] | --live --confirm-dev-live --url http://74.48.78.17:16666/ [--report reports/dev-gate/dev-cloud-workbench-live.json]",
notes: [
"Default/source mode reads repository files and emits SOURCE-level evidence only; it never calls the live provider and must not claim DEV-LIVE.",
"--mobile runs a local static 390x844 browser hit-test and still emits SOURCE-level evidence only.",
"--local-agent-fixture runs two local browser send/reply paths with a sanitized SOURCE fixture delayed beyond 4500ms and does not claim DEV-LIVE.",
"--local-agent-timeout-fixture runs a local slow SOURCE fixture to verify bounded timeout UI classification and retry preservation without live acceptance.",
"--dom-only preserves runtime/web-asset identity preflight, inspects read-only deployed DOM/help wiring, and never posts /v1/agent/chat.",
"DEV-LIVE mode requires --live --confirm-dev-live, opens the deployed workbench in a browser, sends the simple Chinese prompt and 列出你能使用的所有skill, and retains only non-sensitive response fields."
]
};