From bba8c952696e79832ccdb538f360a214df50789e Mon Sep 17 00:00:00 2001 From: Code Queue Review Date: Sat, 23 May 2026 04:45:18 +0000 Subject: [PATCH] fix: harden workbench mobile smoke evidence --- package-lock.json | 47 ++++++++++++++- package.json | 3 +- scripts/dev-cloud-workbench-smoke.mjs | 59 +++++++++++++++++-- scripts/dev-cloud-workbench-smoke.test.mjs | 56 ++++++++++++++++++ scripts/src/dev-cloud-workbench-smoke-lib.mjs | 6 +- web/hwlab-cloud-web/app.mjs | 1 + web/hwlab-cloud-web/scripts/check.mjs | 21 ++++++- 7 files changed, 181 insertions(+), 12 deletions(-) diff --git a/package-lock.json b/package-lock.json index 58ea6d1c..33926997 100644 --- a/package-lock.json +++ b/package-lock.json @@ -8,7 +8,22 @@ "name": "hwlab", "version": "0.0.0-l0", "dependencies": { - "pg": "^8.21.0" + "pg": "^8.21.0", + "playwright": "1.59.1" + } + }, + "node_modules/fsevents": { + "version": "2.3.2", + "resolved": "https://registry.npmjs.org/fsevents/-/fsevents-2.3.2.tgz", + "integrity": "sha512-xiqMQR4xAeHTuB9uWm+fFRcIOgKBMiOBP+eXiyT7jsgVCq1bkVygt00oASowB7EdtpOHaaPgKt812P9ab+DDKA==", + "hasInstallScript": true, + "license": "MIT", + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, "node_modules/pg": { @@ -100,6 +115,36 @@ "split2": "^4.1.0" } }, + "node_modules/playwright": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright/-/playwright-1.59.1.tgz", + "integrity": "sha512-C8oWjPR3F81yljW9o5OxcWzfh6avkVwDD2VYdwIGqTkl+OGFISgypqzfu7dOe4QNLL2aqcWBmI3PMtLIK233lw==", + "license": "Apache-2.0", + "dependencies": { + "playwright-core": "1.59.1" + }, + "bin": { + "playwright": "cli.js" + }, + "engines": { + "node": ">=18" + }, + "optionalDependencies": { + "fsevents": "2.3.2" + } + }, + "node_modules/playwright-core": { + "version": "1.59.1", + "resolved": "https://registry.npmjs.org/playwright-core/-/playwright-core-1.59.1.tgz", + "integrity": "sha512-HBV/RJg81z5BiiZ9yPzIiClYV/QMsDCKUyogwH9p3MCP6IYjUFu/MActgYAvK0oWyV9NlwM3GLBjADyWgydVyg==", + "license": "Apache-2.0", + "bin": { + "playwright-core": "cli.js" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/postgres-array": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", diff --git a/package.json b/package.json index 8ce95052..2bc614ae 100644 --- a/package.json +++ b/package.json @@ -36,6 +36,7 @@ "dev-runtime:migration": "node scripts/dev-runtime-migration.mjs" }, "dependencies": { - "pg": "^8.21.0" + "pg": "^8.21.0", + "playwright": "1.59.1" } } diff --git a/scripts/dev-cloud-workbench-smoke.mjs b/scripts/dev-cloud-workbench-smoke.mjs index 3f9e5fdd..87de5237 100644 --- a/scripts/dev-cloud-workbench-smoke.mjs +++ b/scripts/dev-cloud-workbench-smoke.mjs @@ -10,6 +10,41 @@ import { runDevCloudWorkbenchSmoke } from "./src/dev-cloud-workbench-smoke-lib.mjs"; +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); + +export function decideSmokeReportWrite(args, report, options = {}) { + if (!args.reportPath || report.status === "usage") { + return { status: "not_requested", write: false }; + } + const root = options.repoRoot ?? repoRoot; + const cwd = options.cwd ?? process.cwd(); + const reportPath = path.resolve(cwd, args.reportPath); + const liveReportPath = path.resolve(root, "reports/dev-gate/dev-cloud-workbench-live.json"); + const displayPath = displayReportPath(reportPath, root); + if (report.status === "skip") { + return { + status: "blocked", + write: false, + reportPath, + summary: `Refusing to write ${displayPath}: ${report.mode ?? "smoke"} skipped and does not contain required browser evidence.` + }; + } + if (reportPath === liveReportPath && report.mode !== "live") { + return { + status: "blocked", + write: false, + reportPath, + summary: `Refusing to overwrite ${displayPath} with ${report.mode ?? "non-live"} evidence; run --live for the live report or choose a mode-specific report path.` + }; + } + return { status: "pass", write: true, reportPath }; +} + +export function smokeCliExitCode(report, reportWriteDecision = { status: "not_requested" }) { + if (reportWriteDecision.status === "blocked") return 2; + return report.status === "pass" || report.status === "usage" ? 0 : 2; +} + if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { try { const args = parseSmokeArgs(process.argv.slice(2)); @@ -18,16 +53,28 @@ if (process.argv[1] && fileURLToPath(import.meta.url) === process.argv[1]) { : args.mobile ? await runDevCloudWorkbenchMobileSmoke() : await runDevCloudWorkbenchSmoke(process.argv.slice(2)); - if (args.reportPath && report.status !== "usage") { - const reportPath = path.resolve(process.cwd(), args.reportPath); - await mkdir(path.dirname(reportPath), { recursive: true }); - report.artifacts = { ...(report.artifacts ?? {}), reportPath }; - await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + const reportWrite = decideSmokeReportWrite(args, report); + if (reportWrite.write) { + await mkdir(path.dirname(reportWrite.reportPath), { recursive: true }); + report.artifacts = { ...(report.artifacts ?? {}), reportPath: reportWrite.reportPath }; + await writeFile(reportWrite.reportPath, `${JSON.stringify(report, null, 2)}\n`, "utf8"); + } else if (reportWrite.status === "blocked") { + report.reportWrite = { + status: "blocked", + reportPath: reportWrite.reportPath, + summary: reportWrite.summary + }; + process.stderr.write(`[dev-cloud-workbench-smoke] ${reportWrite.summary}\n`); } process.stdout.write(`${JSON.stringify(report, null, 2)}\n`); - process.exitCode = report.status === "pass" || report.status === "usage" || report.status === "skip" ? 0 : 2; + process.exitCode = smokeCliExitCode(report, reportWrite); } catch (error) { process.stderr.write(`[dev-cloud-workbench-smoke] ${error instanceof Error ? error.stack : String(error)}\n`); process.exitCode = 1; } } + +function displayReportPath(reportPath, root) { + const relative = path.relative(root, reportPath); + return relative && !relative.startsWith("..") && !path.isAbsolute(relative) ? relative : reportPath; +} diff --git a/scripts/dev-cloud-workbench-smoke.test.mjs b/scripts/dev-cloud-workbench-smoke.test.mjs index 038458f9..39d71c76 100644 --- a/scripts/dev-cloud-workbench-smoke.test.mjs +++ b/scripts/dev-cloud-workbench-smoke.test.mjs @@ -10,6 +10,10 @@ import { import { classifyCodeAgentChatReadiness } from "./src/code-agent-response-contract.mjs"; +import { + decideSmokeReportWrite, + smokeCliExitCode +} from "./dev-cloud-workbench-smoke.mjs"; const sourceIdentity = Object.freeze({ status: "observed", @@ -29,6 +33,58 @@ const expectedRuntimeIdentity = Object.freeze({ image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:7de6edd" }); +test("workbench smoke CLI treats skipped browser evidence as blocked and refuses to write reports", () => { + const report = { + status: "skip", + mode: "static-mobile-browser", + summary: "Mobile browser smoke skipped because Playwright is unavailable." + }; + const decision = decideSmokeReportWrite( + { reportPath: "reports/dev-gate/dev-cloud-workbench-mobile.json" }, + report, + { repoRoot: "/repo", cwd: "/repo" } + ); + + assert.equal(decision.status, "blocked"); + assert.equal(decision.write, false); + assert.match(decision.summary, /skipped and does not contain required browser evidence/u); + assert.equal(smokeCliExitCode(report, decision), 2); +}); + +test("workbench smoke CLI refuses to overwrite live report with non-live fixture evidence", () => { + const report = { + status: "pass", + mode: "local-agent-fixture-browser" + }; + const decision = decideSmokeReportWrite( + { reportPath: "reports/dev-gate/dev-cloud-workbench-live.json" }, + report, + { repoRoot: "/repo", cwd: "/repo" } + ); + + assert.equal(decision.status, "blocked"); + assert.equal(decision.write, false); + assert.match(decision.summary, /Refusing to overwrite reports\/dev-gate\/dev-cloud-workbench-live\.json/u); + assert.equal(smokeCliExitCode(report, decision), 2); +}); + +test("workbench smoke CLI allows live evidence to write the live report", () => { + const report = { + status: "pass", + mode: "live" + }; + const decision = decideSmokeReportWrite( + { reportPath: "reports/dev-gate/dev-cloud-workbench-live.json" }, + report, + { repoRoot: "/repo", cwd: "/repo" } + ); + + assert.equal(decision.status, "pass"); + assert.equal(decision.write, true); + assert.equal(decision.reportPath, "/repo/reports/dev-gate/dev-cloud-workbench-live.json"); + assert.equal(smokeCliExitCode(report, decision), 0); +}); + test("live workbench identity passes only when runtime commit or image tag matches current source", () => { assert.equal( classifyLiveDeploymentIdentity(sourceIdentity, { diff --git a/scripts/src/dev-cloud-workbench-smoke-lib.mjs b/scripts/src/dev-cloud-workbench-smoke-lib.mjs index 29a00671..77d0fead 100644 --- a/scripts/src/dev-cloud-workbench-smoke-lib.mjs +++ b/scripts/src/dev-cloud-workbench-smoke-lib.mjs @@ -640,6 +640,7 @@ function baseReport({ "node --check scripts/dev-cloud-workbench-smoke.mjs", "node --check scripts/src/dev-cloud-workbench-smoke-lib.mjs", "node scripts/dev-cloud-workbench-smoke.mjs --static", + "node scripts/dev-cloud-workbench-smoke.mjs --mobile", "node scripts/dev-cloud-workbench-smoke.mjs --local-agent-fixture", "node scripts/dev-cloud-workbench-smoke.mjs --live --url http://74.48.78.17:16666/ --report reports/dev-gate/dev-cloud-workbench-live.json", "node scripts/validate-dev-gate-report.mjs reports/dev-gate/dev-cloud-workbench-live.json", @@ -649,10 +650,11 @@ function baseReport({ status: mode === "static" ? status : "not_run", commands: [ "node scripts/dev-cloud-workbench-smoke.mjs --static", + "node scripts/dev-cloud-workbench-smoke.mjs --mobile", "node scripts/dev-cloud-workbench-smoke.mjs --local-agent-fixture" ], - evidence: ["Static mode verifies the source contract and same-origin Code Agent wiring only.", "Local fixture browser mode verifies send/reply UX at SOURCE level."], - summary: "Static and local fixture workbench smokes are SOURCE-level evidence and do not prove the deployed browser journey." + evidence: ["Static mode verifies the source contract and same-origin Code Agent wiring only.", "Mobile browser mode verifies the 390x844 outer-scroll lock and help route at SOURCE level.", "Local fixture browser mode verifies send/reply UX at SOURCE level."], + summary: "Static, mobile, and local fixture workbench smokes are SOURCE-level evidence and do not prove the deployed browser journey." }, dryRun: { status: "not_applicable", diff --git a/web/hwlab-cloud-web/app.mjs b/web/hwlab-cloud-web/app.mjs index aff22325..788b841f 100644 --- a/web/hwlab-cloud-web/app.mjs +++ b/web/hwlab-cloud-web/app.mjs @@ -1406,6 +1406,7 @@ function renderGateDiagnostics() { "node --check web/hwlab-cloud-web/app.mjs", "node --check web/hwlab-cloud-web/scripts/check.mjs", "node web/hwlab-cloud-web/scripts/check.mjs", + "node scripts/dev-cloud-workbench-smoke.mjs --mobile", "node scripts/dev-cloud-workbench-smoke.mjs --local-agent-fixture", "node scripts/l6-cli-web-smoke.mjs", "git diff --check" diff --git a/web/hwlab-cloud-web/scripts/check.mjs b/web/hwlab-cloud-web/scripts/check.mjs index b01c14d1..0de50844 100644 --- a/web/hwlab-cloud-web/scripts/check.mjs +++ b/web/hwlab-cloud-web/scripts/check.mjs @@ -68,6 +68,15 @@ const mobileOuterScrollSmokeCheck = mobileWorkbenchSmoke.checks.find((check) => const mobileHelpSmokeCheck = mobileWorkbenchSmoke.checks.find((check) => check.id === "mobile-help-markdown-route"); const localAgentFixtureSmoke = await runDevCloudWorkbenchLocalAgentFixtureSmoke(); +assert.notEqual( + mobileWorkbenchSmoke.status, + "skip", + [ + mobileWorkbenchSmoke.summary, + "Install repository dependencies and run `node scripts/dev-cloud-workbench-smoke.mjs --mobile` to generate the required mobile outer-scroll evidence." + ].filter(Boolean).join(" ") +); + assert.equal(workbenchSmoke.status, "pass", JSON.stringify(workbenchSmoke.blockers, null, 2)); assert.equal(workbenchSmoke.evidenceLevel, "SOURCE"); assert.equal(workbenchSmoke.devLive, false); @@ -79,8 +88,16 @@ assert.equal(workbenchSmoke.help.status, "pass", "smoke report must not leave #1 assert.notEqual(mobileWorkbenchSmoke.status, "blocked", JSON.stringify(mobileWorkbenchSmoke.blockers, null, 2)); assert.equal(mobileWorkbenchSmoke.evidenceLevel, "SOURCE"); assert.equal(mobileWorkbenchSmoke.devLive, false); -assert.equal(mobileOuterScrollSmokeCheck?.status, "pass", "mobile smoke must prove outer page scroll remains locked"); -assert.equal(mobileHelpSmokeCheck?.status, "pass", "mobile help route must remain non-default Markdown content"); +assert.ok( + mobileOuterScrollSmokeCheck, + "mobile smoke must include the mobile-outer-scroll-lock check; run `node scripts/dev-cloud-workbench-smoke.mjs --mobile` with Playwright installed." +); +assert.equal(mobileOuterScrollSmokeCheck.status, "pass", "mobile smoke must prove outer page scroll remains locked"); +assert.ok( + mobileHelpSmokeCheck, + "mobile smoke must include the mobile-help-markdown-route check; run `node scripts/dev-cloud-workbench-smoke.mjs --mobile` with Playwright installed." +); +assert.equal(mobileHelpSmokeCheck.status, "pass", "mobile help route must remain non-default Markdown content"); assert.equal(localAgentFixtureSmoke.status, "pass", JSON.stringify(localAgentFixtureSmoke.blockers, null, 2)); assert.equal(localAgentFixtureSmoke.evidenceLevel, "SOURCE"); assert.equal(localAgentFixtureSmoke.devLive, false);