test: add workbench dom-only live validation
This commit is contained in:
Code Queue Review
@@ -127,7 +127,15 @@ const requiredDevCloudWorkbenchValidationCommands = [
|
||||
"node --check scripts/dev-cloud-workbench-smoke.mjs",
|
||||
"node --check scripts/src/dev-cloud-workbench-smoke-lib.mjs",
|
||||
"node scripts/dev-cloud-workbench-smoke.mjs --source",
|
||||
"node scripts/dev-cloud-workbench-smoke.mjs --mobile",
|
||||
"node scripts/dev-cloud-workbench-smoke.mjs --local-agent-fixture",
|
||||
"node scripts/dev-cloud-workbench-smoke.mjs --local-agent-timeout-fixture"
|
||||
];
|
||||
const requiredDevCloudWorkbenchDomOnlyValidationCommands = [
|
||||
"node scripts/dev-cloud-workbench-smoke.mjs --dom-only --url http://74.48.78.17:16666/ --report reports/dev-gate/dev-cloud-workbench-dom-only.json",
|
||||
"node scripts/validate-dev-gate-report.mjs reports/dev-gate/dev-cloud-workbench-dom-only.json"
|
||||
];
|
||||
const requiredDevCloudWorkbenchLiveValidationCommands = [
|
||||
"node scripts/dev-cloud-workbench-smoke.mjs --live --confirm-dev-live --url http://74.48.78.17:16666/ --report reports/dev-gate/dev-cloud-workbench-live.json",
|
||||
"node scripts/validate-dev-gate-report.mjs reports/dev-gate/dev-cloud-workbench-live.json"
|
||||
];
|
||||
@@ -243,7 +251,7 @@ const reportFamilyTemplates = new Map([
|
||||
]
|
||||
]);
|
||||
|
||||
const statusValues = new Set(["pass", "blocked", "degraded", "not_run", "not_applicable", "failed"]);
|
||||
const statusValues = new Set(["pass", "blocked", "degraded", "not_run", "not_applicable", "not_sent", "failed"]);
|
||||
const blockerTypes = new Set([
|
||||
"contract_blocker",
|
||||
"environment_blocker",
|
||||
@@ -1359,16 +1367,21 @@ async function validateDevCloudWorkbenchLiveReport(report, label, raw) {
|
||||
`${label}.status must match blockers`
|
||||
);
|
||||
assertTimestamp(report.generatedAt, `${label}.generatedAt`);
|
||||
assert.equal(report.mode, "live", `${label}.mode`);
|
||||
assert.ok(["live", "dom-only"].includes(report.mode), `${label}.mode`);
|
||||
assert.equal(report.url, "http://74.48.78.17:16666/", `${label}.url`);
|
||||
assert.equal(
|
||||
report.evidenceLevel,
|
||||
report.status === "pass"
|
||||
? "DEV-LIVE-BROWSER"
|
||||
: report.status === "degraded" ? "DEV-LIVE-BROWSER-DEGRADED" : "BLOCKED",
|
||||
`${label}.evidenceLevel`
|
||||
);
|
||||
assert.equal(report.devLive, report.status === "pass", `${label}.devLive`);
|
||||
if (report.mode === "dom-only") {
|
||||
assert.equal(report.evidenceLevel, report.status === "pass" ? "DEV-LIVE-DOM-READONLY" : "BLOCKED", `${label}.evidenceLevel`);
|
||||
assert.equal(report.devLive, false, `${label}.devLive`);
|
||||
} else {
|
||||
assert.equal(
|
||||
report.evidenceLevel,
|
||||
report.status === "pass"
|
||||
? "DEV-LIVE-BROWSER"
|
||||
: report.status === "degraded" ? "DEV-LIVE-BROWSER-DEGRADED" : "BLOCKED",
|
||||
`${label}.evidenceLevel`
|
||||
);
|
||||
assert.equal(report.devLive, report.status === "pass", `${label}.devLive`);
|
||||
}
|
||||
assertDevCloudWorkbenchIdentity(report, label);
|
||||
const apiRuntimeReadiness = assertDevCloudWorkbenchApiRuntimeReadiness(report, label);
|
||||
assertDevCloudWorkbenchEndpoints(report, label);
|
||||
@@ -1385,8 +1398,18 @@ async function validateDevCloudWorkbenchLiveReport(report, label, raw) {
|
||||
assertStringArray(report.validationCommands, `${label}.validationCommands`, {
|
||||
minLength: requiredDevCloudWorkbenchValidationCommands.length
|
||||
});
|
||||
const requiredModeCommands = report.mode === "dom-only"
|
||||
? requiredDevCloudWorkbenchDomOnlyValidationCommands
|
||||
: requiredDevCloudWorkbenchLiveValidationCommands;
|
||||
const requiredCommands = [
|
||||
...requiredDevCloudWorkbenchValidationCommands,
|
||||
...requiredModeCommands
|
||||
];
|
||||
assertStringArray(report.validationCommands, `${label}.validationCommands`, {
|
||||
minLength: requiredCommands.length
|
||||
});
|
||||
assertUnique(report.validationCommands, `${label}.validationCommands`);
|
||||
for (const requiredCommand of requiredDevCloudWorkbenchValidationCommands) {
|
||||
for (const requiredCommand of requiredCommands) {
|
||||
assert.ok(
|
||||
report.validationCommands.includes(requiredCommand),
|
||||
`${label}.validationCommands missing ${requiredCommand}`
|
||||
@@ -1434,6 +1457,11 @@ async function validateDevCloudWorkbenchLiveReport(report, label, raw) {
|
||||
return;
|
||||
}
|
||||
|
||||
if (report.mode === "dom-only") {
|
||||
assertDevCloudWorkbenchDomOnlyReport(report, label, checks);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const requiredCheck of [
|
||||
"live-http-html",
|
||||
"live-browser-dom",
|
||||
@@ -1907,6 +1935,171 @@ function assertDevCloudWorkbenchDeploymentPreflightJourney(report, label, journe
|
||||
);
|
||||
}
|
||||
|
||||
function assertDevCloudWorkbenchDomOnlyReport(report, label, checks) {
|
||||
assert.equal(report.mode, "dom-only", `${label}.mode`);
|
||||
for (const requiredCheck of [
|
||||
"live-runtime-current-main",
|
||||
"live-api-runtime-readiness",
|
||||
"live-http-html",
|
||||
"live-web-assets-current-main",
|
||||
"live-browser-dom",
|
||||
"live-gate-route",
|
||||
"live-help-markdown-route",
|
||||
"live-code-agent-browser-journey"
|
||||
]) {
|
||||
assert.ok(checks.has(requiredCheck), `${label}.checks missing ${requiredCheck}`);
|
||||
}
|
||||
|
||||
const runtimeCheck = checks.get("live-runtime-current-main");
|
||||
const apiRuntimeReadinessCheck = checks.get("live-api-runtime-readiness");
|
||||
const webAssetCheck = checks.get("live-web-assets-current-main");
|
||||
const apiRuntimeReadiness = assertDevCloudWorkbenchApiRuntimeReadiness(report, label);
|
||||
assert.ok(["pass", "blocked"].includes(runtimeCheck.status), `${label}.live-runtime-current-main.status`);
|
||||
assert.equal(
|
||||
apiRuntimeReadinessCheck.status,
|
||||
apiRuntimeReadiness.status,
|
||||
`${label}.live-api-runtime-readiness.status`
|
||||
);
|
||||
assert.deepEqual(
|
||||
apiRuntimeReadinessCheck.observations,
|
||||
apiRuntimeReadiness,
|
||||
`${label}.live-api-runtime-readiness.observations`
|
||||
);
|
||||
assert.ok(["pass", "blocked"].includes(webAssetCheck.status), `${label}.live-web-assets-current-main.status`);
|
||||
assertObject(runtimeCheck.observations, `${label}.live-runtime-current-main.observations`);
|
||||
assertObject(webAssetCheck.observations, `${label}.live-web-assets-current-main.observations`);
|
||||
assertObject(report.expectedRuntimeIdentity, `${label}.expectedRuntimeIdentity`);
|
||||
assertObject(report.deploymentIdentity, `${label}.deploymentIdentity`);
|
||||
assert.equal(report.deploymentIdentity.status, runtimeCheck.status, `${label}.deploymentIdentity.status`);
|
||||
assertObject(report.webAssetIdentity, `${label}.webAssetIdentity`);
|
||||
assert.equal(report.webAssetIdentity.status, webAssetCheck.status, `${label}.webAssetIdentity.status`);
|
||||
assertArray(report.webAssetIdentity.assets, `${label}.webAssetIdentity.assets`);
|
||||
assertString(report.webAssetIdentity.summary, `${label}.webAssetIdentity.summary`);
|
||||
|
||||
const domCheck = checks.get("live-browser-dom");
|
||||
if (domCheck.status === "pass") {
|
||||
assertDevCloudWorkbenchDomObservations(domCheck.observations, `${label}.live-browser-dom`);
|
||||
} else {
|
||||
assert.ok(["blocked", "skip"].includes(domCheck.status), `${label}.live-browser-dom.status`);
|
||||
}
|
||||
|
||||
const helpCheck = checks.get("live-help-markdown-route");
|
||||
assert.ok(["pass", "blocked", "skip"].includes(helpCheck.status), `${label}.live-help-markdown-route.status`);
|
||||
|
||||
const gateCheck = checks.get("live-gate-route");
|
||||
if (gateCheck.status === "pass") {
|
||||
assertDevCloudWorkbenchGateRoute(gateCheck.observations, `${label}.live-gate-route`);
|
||||
} else {
|
||||
assert.ok(["blocked", "skip"].includes(gateCheck.status), `${label}.live-gate-route.status`);
|
||||
}
|
||||
|
||||
assertDevCloudWorkbenchDomOnlyJourney(report, label, checks.get("live-code-agent-browser-journey"));
|
||||
assertBlockers(report.blockers, `${label}.blockers`);
|
||||
|
||||
assertObject(report.safety, `${label}.safety`);
|
||||
for (const field of ["prodTouched", "servicesRestarted", "heavyE2E", "hardwareWriteApis", "sourceIsDevLive"]) {
|
||||
assert.equal(report.safety[field], false, `${label}.safety.${field}`);
|
||||
}
|
||||
assert.equal(report.safety.liveMode, "dom-only-readonly-no-code-agent-post", `${label}.safety.liveMode`);
|
||||
assert.equal(report.safety.codeAgentBrowserJourneySkipped, true, `${label}.safety.codeAgentBrowserJourneySkipped`);
|
||||
assert.equal(report.safety.codeAgentPostSent, false, `${label}.safety.codeAgentPostSent`);
|
||||
assertStringArray(report.safety.retainedApiFields, `${label}.safety.retainedApiFields`, { minLength: 4 });
|
||||
assertString(report.safety.statement, `${label}.safety.statement`);
|
||||
assert.equal(/Secret|M3\/M4\/M5|\/v1\/agent\/chat/u.test(report.safety.statement), true, `${label}.safety.statement`);
|
||||
if (apiRuntimeReadiness.status === "degraded") {
|
||||
assert.equal(report.status, "blocked", `${label}.status`);
|
||||
assert.match(
|
||||
report.safety.statement,
|
||||
/must not be used to infer M3\/M4\/M5 acceptance|claims M3\/M4\/M5 acceptance/u,
|
||||
`${label}.safety.statement`
|
||||
);
|
||||
}
|
||||
|
||||
const raw = JSON.stringify(report);
|
||||
assert.equal(/"method"\s*:\s*"POST"[\s\S]{0,200}"status"\s*:\s*(200|201|202)|"completed"/u.test(raw), false, `${label} must not include a sent or completed Code Agent POST`);
|
||||
}
|
||||
|
||||
function assertDevCloudWorkbenchDomObservations(dom, label) {
|
||||
assertObject(dom, `${label}.observations`);
|
||||
assert.equal(dom.title, "HWLAB 云工作台", `${label}.title`);
|
||||
assert.equal(dom.workspaceHidden, false, `${label}.workspaceHidden`);
|
||||
assert.equal(dom.gateHidden, true, `${label}.gateHidden`);
|
||||
assert.equal(dom.helpHidden, true, `${label}.helpHidden`);
|
||||
assert.equal(dom.navLabelsPresent, true, `${label}.navLabelsPresent`);
|
||||
assert.equal(dom.defaultBackstageHidden, true, `${label}.defaultBackstageHidden`);
|
||||
assert.equal(dom.gateRouteAvailable, true, `${label}.gateRouteAvailable`);
|
||||
if (Object.hasOwn(dom, "codeAgentPostGuard")) {
|
||||
assertObject(dom.codeAgentPostGuard, `${label}.codeAgentPostGuard`);
|
||||
assert.equal(dom.codeAgentPostGuard.installed, true, `${label}.codeAgentPostGuard.installed`);
|
||||
assert.equal(dom.codeAgentPostGuard.blockedPath, "/v1/agent/chat", `${label}.codeAgentPostGuard.blockedPath`);
|
||||
assert.equal(dom.codeAgentPostGuard.attemptedCount, 0, `${label}.codeAgentPostGuard.attemptedCount`);
|
||||
}
|
||||
assertStringArray(dom.wiringHeaders, `${label}.wiringHeaders`, { minLength: 8 });
|
||||
for (const column of ["源设备", "源端口", "接线盘", "目标设备", "目标端口", "状态", "证据来源", "trace/evidence"]) {
|
||||
assert.ok(dom.wiringHeaders.includes(column), `${label}.wiringHeaders missing ${column}`);
|
||||
}
|
||||
assertArray(dom.m3WiringRows, `${label}.m3WiringRows`);
|
||||
assert.ok(
|
||||
dom.m3WiringRows.some((row) =>
|
||||
Array.isArray(row) &&
|
||||
row.includes("res_boxsimu_1") &&
|
||||
row.includes("DO1") &&
|
||||
row.includes("hwlab-patch-panel") &&
|
||||
row.includes("res_boxsimu_2") &&
|
||||
row.includes("DI1")
|
||||
),
|
||||
`${label}.m3WiringRows missing required M3 route`
|
||||
);
|
||||
assertObject(dom.coreControlsVisible, `${label}.coreControlsVisible`);
|
||||
for (const control of ["commandInput", "commandSend", "agentChatStatus", "conversationList", "hardwareList", "wiringTab", "recordsTab"]) {
|
||||
assert.equal(dom.coreControlsVisible[control], true, `${label}.coreControlsVisible.${control}`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertDevCloudWorkbenchGateRoute(gate, label) {
|
||||
assertObject(gate, `${label}.observations`);
|
||||
assert.equal(gate.pathname, "/gate", `${label}.pathname`);
|
||||
assert.equal(gate.title, "HWLAB 云工作台", `${label}.title`);
|
||||
assert.equal(gate.workspaceHidden, true, `${label}.workspaceHidden`);
|
||||
assert.equal(gate.gateHidden, false, `${label}.gateHidden`);
|
||||
assert.equal(gate.helpHidden, true, `${label}.helpHidden`);
|
||||
assert.equal(gate.diagnosticsTermsPresent, true, `${label}.diagnosticsTermsPresent`);
|
||||
if (Object.hasOwn(gate, "codeAgentPostGuard")) {
|
||||
assertObject(gate.codeAgentPostGuard, `${label}.codeAgentPostGuard`);
|
||||
assert.equal(gate.codeAgentPostGuard.installed, true, `${label}.codeAgentPostGuard.installed`);
|
||||
assert.equal(gate.codeAgentPostGuard.blockedPath, "/v1/agent/chat", `${label}.codeAgentPostGuard.blockedPath`);
|
||||
assert.equal(gate.codeAgentPostGuard.attemptedCount, 0, `${label}.codeAgentPostGuard.attemptedCount`);
|
||||
}
|
||||
}
|
||||
|
||||
function assertDevCloudWorkbenchDomOnlyJourney(report, label, journey) {
|
||||
assert.equal(journey.status, "not_applicable", `${label}.live-code-agent-browser-journey.status`);
|
||||
assertStringArray(journey.evidence, `${label}.live-code-agent-browser-journey.evidence`, { minLength: 3 });
|
||||
assert.ok(
|
||||
journey.evidence.includes("POST /v1/agent/chat not sent"),
|
||||
`${label}.live-code-agent-browser-journey.evidence missing not-sent proof`
|
||||
);
|
||||
assertObject(journey.observations, `${label}.live-code-agent-browser-journey.observations`);
|
||||
assertObject(journey.observations.request, `${label}.live-code-agent-browser-journey.request`);
|
||||
assert.equal(journey.observations.request.method, "POST", `${label}.live-code-agent-browser-journey.request.method`);
|
||||
assert.equal(journey.observations.request.status, "not_applicable", `${label}.live-code-agent-browser-journey.request.status`);
|
||||
assert.equal(journey.observations.request.urlPath, "/v1/agent/chat", `${label}.live-code-agent-browser-journey.request.urlPath`);
|
||||
assertObject(journey.observations.response, `${label}.live-code-agent-browser-journey.response`);
|
||||
assert.equal(journey.observations.response.status, "not_applicable", `${label}.live-code-agent-browser-journey.response.status`);
|
||||
assert.equal(journey.observations.response.hasReply, false, `${label}.live-code-agent-browser-journey.response.hasReply`);
|
||||
assert.equal(journey.observations.response.error, null, `${label}.live-code-agent-browser-journey.response.error`);
|
||||
for (const field of ["provider", "model", "backend", "traceId"]) {
|
||||
assert.equal(journey.observations.response[field], "not_observed", `${label}.live-code-agent-browser-journey.response.${field}`);
|
||||
}
|
||||
assertObject(journey.observations.classification, `${label}.live-code-agent-browser-journey.classification`);
|
||||
assert.equal(journey.observations.classification.status, "not_applicable", `${label}.live-code-agent-browser-journey.classification.status`);
|
||||
assert.equal(journey.observations.classification.reason, "dom_only_no_code_agent_post", `${label}.live-code-agent-browser-journey.classification.reason`);
|
||||
assert.equal(journey.observations.classification.codeAgentBrowserJourneySkipped, true, `${label}.live-code-agent-browser-journey.classification.codeAgentBrowserJourneySkipped`);
|
||||
assertArray(journey.observations.networkEvents ?? [], `${label}.live-code-agent-browser-journey.networkEvents`);
|
||||
assert.equal((journey.observations.networkEvents ?? []).length, 0, `${label}.live-code-agent-browser-journey.networkEvents`);
|
||||
assert.equal(report.devLive, false, `${label}.devLive`);
|
||||
}
|
||||
|
||||
function assertDevCloudWorkbenchIdentity(report, label) {
|
||||
assertObject(report.sourceIdentity, `${label}.sourceIdentity`);
|
||||
for (const field of ["status", "source", "commitId", "shortCommitId", "ref", "worktreeState", "reportCommitId", "summary"]) {
|
||||
|
||||
Reference in New Issue
Block a user