Files
pikasTech-HWLAB/scripts/dev-cloud-workbench-smoke.test.mjs
T
2026-06-04 14:51:54 +08:00

523 lines
20 KiB
JavaScript

import assert from "node:assert/strict";
import { readFileSync } from "node:fs";
import test from "node:test";
import {
classifyCodeAgentBrowserJourney,
classifyLiveDeploymentIdentity,
classifyLiveWebAssetIdentity,
parseSmokeArgs,
runDevCloudWorkbenchStaticSmoke,
sanitizeAgentChatBody
} from "./src/dev-cloud-workbench-smoke-lib.mjs";
import {
browserLaunchRemediation,
classifyBrowserLaunchFailure,
chromiumLaunchPlan
} from "./src/browser-launcher.mjs";
import {
classifyCodeAgentBrowserFailure,
classifyCodeAgentChatReadiness
} from "./src/code-agent-response-contract.mjs";
import {
decideSmokeReportWrite,
smokeCliExitCode
} from "./dev-cloud-workbench-smoke.mjs";
import { checkProfiles } from "./src/check-plan.mjs";
const sourceIdentity = Object.freeze({
status: "observed",
commitId: "ff4b3f928f76a0b46cd7998a2a865f3e1caf02c8",
shortCommitId: "ff4b3f928f76",
reportCommitId: "ff4b3f928f76",
worktreeState: "clean",
dirty: false
});
const expectedRuntimeIdentity = Object.freeze({
status: "observed",
serviceId: "hwlab-cloud-api",
source: "deploy/artifact-catalog.dev.json+deploy/deploy.json:runtime-config",
commitId: "7de6edd",
imageTag: "7de6edd",
image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:7de6edd"
});
const rootPackage = JSON.parse(readFileSync(new URL("../package.json", import.meta.url), "utf8"));
const cloudWebPackage = JSON.parse(readFileSync(new URL("../web/hwlab-cloud-web/package.json", import.meta.url), "utf8"));
const cloudWebCheckSource = readFileSync(new URL("../web/hwlab-cloud-web/scripts/check.ts", import.meta.url), "utf8");
const cloudWebTscCheckSource = readFileSync(new URL("../web/hwlab-cloud-web/scripts/tsc-check.ts", import.meta.url), "utf8");
test("workbench smoke defaults to SOURCE mode and requires live confirmation before DEV-LIVE provider calls", () => {
const defaultArgs = parseSmokeArgs([]);
assert.equal(defaultArgs.mode, "source");
assert.equal(defaultArgs.url, "http://74.48.78.17:16666/");
assert.equal(defaultArgs.confirmDevLive, false);
const sourceArgs = parseSmokeArgs(["--source"]);
assert.equal(sourceArgs.mode, "source");
const staticAliasArgs = parseSmokeArgs(["--static"]);
assert.equal(staticAliasArgs.mode, "source");
const liveArgs = parseSmokeArgs(["--live", "--confirm-dev-live"]);
assert.equal(liveArgs.mode, "live");
assert.equal(liveArgs.confirmDevLive, true);
assert.equal(liveArgs.url, "http://74.48.78.17:16666/");
assert.throws(
() => parseSmokeArgs(["--live"]),
/requires explicit --live --confirm-dev-live/u
);
});
test("source/default workbench report cannot claim DEV-LIVE and documents the confirmed live command", () => {
const report = runDevCloudWorkbenchStaticSmoke();
assert.equal(report.mode, "source");
assert.equal(report.evidenceLevel, "SOURCE");
assert.equal(report.devLive, false);
assert.equal(report.safety.devLive, false);
assert.equal(report.validationCommands.includes("node scripts/dev-cloud-workbench-smoke.mjs --source"), true);
assert.equal(
report.validationCommands.includes("node scripts/dev-cloud-workbench-smoke.mjs --live --confirm-dev-live --url http://74.48.78.17:16666/ --report /tmp/hwlab-dev-gate/dev-cloud-workbench-live.json"),
true
);
assert.equal(
report.validationCommands.includes("node scripts/dev-cloud-workbench-smoke.mjs --live --url http://74.48.78.17:16666/ --report /tmp/hwlab-dev-gate/dev-cloud-workbench-live.json"),
false
);
assert.equal(report.checks.find((check) => check.id === "react-strict-typescript-entry")?.status, "pass");
assert.equal(report.checks.find((check) => check.id === "vite-build-and-dist-contract")?.status, "pass");
assert.equal(report.checks.find((check) => check.id === "legacy-runtime-paths-removed")?.status, "pass");
});
test("Code Agent browser failure classifier emits Chinese timeout/provider/browser categories with traceId", () => {
const timeout = classifyCodeAgentBrowserFailure(new Error("request timed out after 180000ms"), { traceId: "trc_timeout" });
assert.equal(timeout.category, "timeout");
assert.match(timeout.chineseSummary, /超时/u);
assert.match(timeout.chineseSummary, /trc_timeout/u);
const provider = classifyCodeAgentBrowserFailure(new Error("OpenAI provider returned HTTP 502"), { traceId: "trc_provider" });
assert.equal(provider.category, "provider");
assert.match(provider.chineseSummary, /Provider 故障/u);
assert.match(provider.chineseSummary, /trc_provider/u);
const runnerBusy = classifyCodeAgentBrowserFailure(new Error("session_busy: runner already busy"), { traceId: "trc_busy" });
assert.equal(runnerBusy.category, "runner_busy");
assert.equal(runnerBusy.blocker, "runner-busy");
assert.match(runnerBusy.chineseSummary, /Runner 忙碌/u);
assert.match(runnerBusy.chineseSummary, /trc_busy/u);
const sessionBlocked = classifyCodeAgentBrowserFailure(new Error("session_expired: session blocked"), { traceId: "trc_session" });
assert.equal(sessionBlocked.category, "session_blocked");
assert.equal(sessionBlocked.blocker, "session-blocked");
assert.match(sessionBlocked.chineseSummary, /Session 受阻/u);
assert.match(sessionBlocked.chineseSummary, /trc_session/u);
const apiError = classifyCodeAgentBrowserFailure(new Error("API HTTP 500 request_failed"), { traceId: "trc_api" });
assert.equal(apiError.category, "api_error");
assert.equal(apiError.blocker, "api-error");
assert.match(apiError.chineseSummary, /API 错误/u);
assert.match(apiError.chineseSummary, /trc_api/u);
const browser = classifyCodeAgentBrowserFailure(new Error("locator #command-input not found"), { traceId: "trc_browser" });
assert.equal(browser.category, "browser");
assert.match(browser.chineseSummary, /浏览器故障/u);
assert.match(browser.chineseSummary, /trc_browser/u);
});
test("workbench smoke CLI treats skipped browser evidence as blocked and refuses to write reports", () => {
const report = {
status: "skip",
mode: "static-mobile-browser",
summary: "Mobile browser smoke skipped because Playwright is unavailable."
};
const decision = decideSmokeReportWrite(
{ reportPath: "/tmp/hwlab-dev-gate/dev-cloud-workbench-mobile.json" },
report,
{ repoRoot: "/repo", cwd: "/repo" }
);
assert.equal(decision.status, "blocked");
assert.equal(decision.write, false);
assert.match(decision.summary, /skipped and does not contain required browser evidence/u);
assert.equal(smokeCliExitCode(report, decision), 2);
});
test("repo-owned browser launcher classifies cache misses and exposes system Chromium fallback metadata", () => {
assert.equal(classifyBrowserLaunchFailure(new Error("Executable doesn't exist at /root/.cache/ms-playwright/chromium_headless_shell-1217")), "browser-cache-missing");
assert.equal(classifyBrowserLaunchFailure(new Error("spawn /missing/chromium ENOENT")), "browser-executable-unavailable");
const plan = chromiumLaunchPlan({ HWLAB_PLAYWRIGHT_CHROMIUM: "/definitely/missing/chromium" });
assert.ok(["system-chromium", "playwright-cache"].includes(plan.browserSource));
assert.equal(Array.isArray(plan.candidatePaths), true);
assert.equal(browserLaunchRemediation({ executablePath: "/snap/bin/chromium" })[0], "using /snap/bin/chromium");
});
test("workbench smoke CLI refuses to overwrite live report with non-live fixture evidence", () => {
const report = {
status: "pass",
mode: "local-agent-fixture-browser"
};
const decision = decideSmokeReportWrite(
{ reportPath: "/tmp/hwlab-dev-gate/dev-cloud-workbench-live.json" },
report,
{ repoRoot: "/repo", cwd: "/repo" }
);
assert.equal(decision.status, "blocked");
assert.equal(decision.write, false);
assert.match(decision.summary, /Refusing to overwrite \/tmp\/hwlab-dev-gate\/dev-cloud-workbench-live\.json/u);
assert.equal(smokeCliExitCode(report, decision), 2);
});
test("workbench smoke CLI allows live evidence to write the live report", () => {
const report = {
status: "pass",
mode: "live"
};
const decision = decideSmokeReportWrite(
{ reportPath: "/tmp/hwlab-dev-gate/dev-cloud-workbench-live.json" },
report,
{ repoRoot: "/repo", cwd: "/repo" }
);
assert.equal(decision.status, "pass");
assert.equal(decision.write, true);
assert.equal(decision.reportPath, "/tmp/hwlab-dev-gate/dev-cloud-workbench-live.json");
assert.equal(smokeCliExitCode(report, decision), 0);
});
test("smoke args include a DOM-only live read-only mode", () => {
const args = parseSmokeArgs(["--dom-only", "--url", "http://74.48.78.17:16666/"]);
assert.equal(args.confirmDevLive, false);
assert.equal(args.mode, "dom-only");
assert.equal(args.url, "http://74.48.78.17:16666/");
assert.equal(args.urlExplicit, true);
});
test("source/default smoke covers React shell-only and module boundary contracts", () => {
const report = runDevCloudWorkbenchStaticSmoke();
assert.equal(report.checks.find((item) => item.id === "react-shell-only-index")?.status, "pass");
assert.equal(report.checks.find((item) => item.id === "react-module-boundaries")?.status, "pass");
assert.equal(report.checks.find((item) => item.id === "react-api-state-separation")?.status, "pass");
});
test("source/default smoke covers React workbench selectors and layout", () => {
const report = runDevCloudWorkbenchStaticSmoke();
const selectors = report.checks.find((item) => item.id === "react-core-workbench-selectors");
const layout = report.checks.find((item) => item.id === "react-layout-contract");
assert.equal(selectors?.status, "pass");
assert.equal(layout?.status, "pass");
assert.equal(selectors.evidence.includes("#device-pod-sidebar"), true);
assert.equal(layout.evidence.includes("@media (max-width: 720px)"), true);
});
test("source/default smoke removes legacy vanilla runtime paths", () => {
const report = runDevCloudWorkbenchStaticSmoke();
const check = report.checks.find((item) => item.id === "legacy-runtime-paths-removed");
assert.equal(check?.status, "pass");
assert.equal(check.evidence.includes("removed:app.ts"), true);
assert.equal(check.evidence.includes("removed:styles.css"), true);
assert.equal(report.checks.some((item) => item.id === "feedback-288-gate-single-table"), false);
});
test("live workbench identity passes only when runtime commit or image tag matches current source", () => {
assert.equal(
classifyLiveDeploymentIdentity(sourceIdentity, {
status: "observed",
commitId: "7de6edd2c41f",
imageTag: "c7de474"
}, expectedRuntimeIdentity).status,
"pass"
);
assert.equal(
classifyLiveDeploymentIdentity(sourceIdentity, {
status: "observed",
commitId: "unknown",
imageTag: "7de6edd"
}, expectedRuntimeIdentity).status,
"pass"
);
const stale = classifyLiveDeploymentIdentity(sourceIdentity, {
status: "observed",
commitId: "c7de474",
imageTag: "c7de474"
}, expectedRuntimeIdentity);
assert.equal(stale.status, "blocked");
assert.match(stale.summary, /Deployment drift/u);
assert.match(stale.reason, /does not match current source desired runtime identity/u);
});
test("live workbench identity blocks when source identity is dirty or runtime identity is absent", () => {
const dirty = classifyLiveDeploymentIdentity(
{ ...sourceIdentity, worktreeState: "dirty", dirty: true, reportCommitId: "unknown" },
{ status: "observed", commitId: "7de6edd", imageTag: "7de6edd" },
expectedRuntimeIdentity
);
assert.equal(dirty.status, "blocked");
assert.match(dirty.reason, /not cleanly attributable/u);
const missingRuntime = classifyLiveDeploymentIdentity(sourceIdentity, {
status: "not_observed",
commitId: "not_observed",
imageTag: "not_observed"
}, expectedRuntimeIdentity);
assert.equal(missingRuntime.status, "blocked");
assert.match(missingRuntime.reason, /not observed/u);
});
test("live workbench identity uses deploy desired runtime identity instead of git HEAD", () => {
const deployed = classifyLiveDeploymentIdentity(sourceIdentity, {
status: "observed",
commitId: "7de6edd",
imageTag: "7de6edd"
}, expectedRuntimeIdentity);
assert.equal(deployed.status, "pass");
assert.equal(deployed.expectedCommit, "7de6edd");
assert.equal(deployed.expectedReportCommitId, "ff4b3f928f76");
assert.equal(deployed.expectedSource, "deploy/artifact-catalog.dev.json+deploy/deploy.json:runtime-config");
});
test("live web asset identity blocks stale deployed primary assets", () => {
const pass = classifyLiveWebAssetIdentity([
{ path: "index.html", status: "match" },
{ path: "assets/index.css", status: "match" },
{ path: "app.js", status: "match" }
]);
assert.equal(pass.status, "pass");
const stale = classifyLiveWebAssetIdentity([
{ path: "index.html", status: "match" },
{ path: "assets/index.css", status: "mismatch" },
{ path: "app.js", status: "mismatch" }
]);
assert.equal(stale.status, "blocked");
assert.deepEqual(stale.mismatches, ["assets/index.css", "app.js"]);
assert.match(stale.summary, /Deployment drift/u);
});
test("Code Agent browser classifier blocks provider 502 response shapes", () => {
const summary = sanitizeAgentChatBody({
status: "completed",
provider: "openai-responses",
model: "gpt-5.5",
backend: "hwlab-cloud-api/openai-responses",
traceId: "trc_provider_502",
reply: {
content: "must not count"
},
error: {
code: "provider_unavailable",
message: "OpenAI Responses returned HTTP 502: request rejected",
providerStatus: 502,
missingEnv: []
}
}, { httpStatus: 502 });
assert.equal(summary.status, "failed");
assert.equal(summary.hasReply, false);
assert.equal(summary.error.code, "provider_unavailable");
assert.equal(summary.error.providerStatus, 502);
const classification = classifyCodeAgentBrowserJourney({
responseSummary: summary,
httpOk: false,
httpStatus: 502,
ui: {
agentChatStatus: "DEV-LIVE 回复",
completedMessageVisible: true,
failedMessageVisible: false
}
});
assert.equal(classification.status, "blocked");
assert.equal(classification.blocker, "provider-upstream");
});
test("Code Agent browser classifier treats sanitized 服务受阻 UI as blocked provider evidence", () => {
const summary = sanitizeAgentChatBody({
status: "failed",
provider: "openai-responses",
model: "gpt-5.5",
backend: "hwlab-cloud-api/openai-responses",
traceId: "trc_provider_502",
error: {
code: "provider_unavailable",
message: "OpenAI Responses returned HTTP 502: request rejected",
providerStatus: 502,
missingEnv: []
}
}, { httpStatus: 200 });
const classification = classifyCodeAgentBrowserJourney({
responseSummary: summary,
httpOk: true,
httpStatus: 200,
ui: {
agentChatStatus: "服务受阻",
completedMessageVisible: false,
failedMessageVisible: true
}
});
assert.equal(classification.status, "blocked");
assert.equal(classification.blocker, "provider-upstream");
});
test("Code Agent browser classifier distinguishes runner busy, session blocked, and API error payloads", () => {
const base = {
status: "failed",
provider: "codex-readonly-runner",
model: "read-only-tools",
backend: "hwlab-cloud-api/codex-readonly-runner",
traceId: "trc_runtime_block"
};
const runnerBusy = classifyCodeAgentBrowserJourney({
responseSummary: sanitizeAgentChatBody({
...base,
error: {
code: "session_busy",
message: "Code Agent runner session is already busy"
}
}, { httpStatus: 200 }),
httpOk: true,
httpStatus: 200,
ui: {
agentChatStatus: "服务受阻",
completedMessageVisible: false,
failedMessageVisible: true
}
});
assert.equal(runnerBusy.blocker, "runner-busy");
assert.equal(runnerBusy.category, "runner_busy");
const sessionBlocked = classifyCodeAgentBrowserJourney({
responseSummary: sanitizeAgentChatBody({
...base,
error: {
code: "session_expired",
message: "Code Agent runner session expired"
}
}, { httpStatus: 200 }),
httpOk: true,
httpStatus: 200,
ui: {
agentChatStatus: "服务受阻",
completedMessageVisible: false,
failedMessageVisible: true
}
});
assert.equal(sessionBlocked.blocker, "session-blocked");
assert.equal(sessionBlocked.category, "session_blocked");
const codexRuntimeBlocked = classifyCodeAgentBrowserJourney({
responseSummary: sanitizeAgentChatBody({
...base,
error: {
code: "codex_stdio_failed",
message: "Codex app-server turn finished with status=failed"
}
}, { httpStatus: 200 }),
httpOk: true,
httpStatus: 200,
ui: {
agentChatStatus: "Runner 受阻",
completedMessageVisible: false,
failedMessageVisible: true
}
});
assert.equal(codexRuntimeBlocked.blocker, "runner-blocked");
assert.equal(codexRuntimeBlocked.category, "runner_blocked");
const apiError = classifyCodeAgentBrowserJourney({
responseSummary: sanitizeAgentChatBody({
status: "failed",
traceId: "trc_proxy_timeout",
error: {
code: "cloud_api_proxy_failed",
message: "cloud web proxy failed"
}
}, { httpStatus: 502 }),
httpOk: false,
httpStatus: 502,
ui: {
agentChatStatus: "后端失败",
completedMessageVisible: false,
failedMessageVisible: true
}
});
assert.equal(apiError.blocker, "api-error");
assert.equal(apiError.category, "api_error");
});
test("Code Agent readiness classifier blocks completed payloads over any non-2xx HTTP status", () => {
const completedPayload = {
conversationId: "cnv_completed_non_2xx",
sessionId: "cnv_completed_non_2xx",
messageId: "msg_completed_non_2xx",
status: "completed",
provider: "openai-responses",
model: "gpt-5.5",
backend: "hwlab-cloud-api/openai-responses",
traceId: "trc_completed_non_2xx",
reply: {
content: "must only count over HTTP 2xx"
}
};
const accepted = classifyCodeAgentChatReadiness(completedPayload, { realDevLive: true, httpStatus: 200 });
assert.equal(accepted.status, "blocked");
assert.equal(accepted.devLiveReplyPass, false);
assert.equal(accepted.blocker, "untrusted-completion");
for (const httpStatus of [400, 401, 403, 429, 500, 502, 503, 504]) {
const blocked = classifyCodeAgentChatReadiness(completedPayload, { realDevLive: true, httpStatus });
assert.equal(blocked.status, "blocked", `HTTP ${httpStatus} must block`);
assert.equal(blocked.blocker, "provider-upstream", `HTTP ${httpStatus} blocker`);
assert.equal(blocked.devLiveReplyPass, false, `HTTP ${httpStatus} must not pass DEV-LIVE`);
assert.equal(blocked.providerStatus, httpStatus, `HTTP ${httpStatus} providerStatus`);
assert.match(blocked.reason, /HTTP non-2xx|upstream response/u, `HTTP ${httpStatus} reason`);
}
});
test("Code Agent browser classifier blocks completed payloads without backend evidence", () => {
const summary = sanitizeAgentChatBody({
status: "completed",
provider: "openai-responses",
model: "gpt-5.5",
traceId: "trc_missing_backend",
reply: {
content: "must not count"
}
}, { httpStatus: 200 });
const classification = classifyCodeAgentBrowserJourney({
responseSummary: summary,
httpOk: true,
httpStatus: 200,
ui: {
agentChatStatus: "DEV-LIVE 回复",
completedMessageVisible: true,
failedMessageVisible: false
}
});
assert.equal(classification.status, "blocked");
assert.equal(classification.blocker, "untrusted-completion");
});
test("repo-owned web checks keep cloud-web semantic checks out of old browser gates", () => {
const checkCommands = checkProfiles.check.map((task) => task.command.join(" "));
assert.match(rootPackage.scripts["web:check"], /cd web\/hwlab-cloud-web && bun run check/u);
assert.match(cloudWebPackage.scripts.check, /bun run scripts\/check\.ts && bun run scripts\/tsc-check\.ts && bun test/u);
assert.ok(checkProfiles.check.some((task) => task.cwd === "web/hwlab-cloud-web" && task.command.join(" ") === "bun run check"));
assert.doesNotMatch(checkCommands.join("\n"), /dev-cloud-workbench-layout-smoke|gate-summary\.mjs|export-web-gate-summary/u);
assert.match(cloudWebCheckSource, /React module assets present/u);
assert.match(cloudWebTscCheckSource, /explicit any/u);
assert.match(cloudWebTscCheckSource, /--strict/u);
assert.match(cloudWebCheckSource, /legacy runtime path/u);
assert.match(cloudWebCheckSource, /dist bundle built by Vite/u);
assert.doesNotMatch(cloudWebCheckSource, /m3-readonly-contract|workbench-hardware-panel-contract|gate-summary\.mjs/u);
});