From e8c8db14981798276f053b0120db1290e30382c3 Mon Sep 17 00:00:00 2001 From: Code Queue Review Date: Sat, 23 May 2026 02:25:04 +0000 Subject: [PATCH] test: record blocked workbench journey preflight --- scripts/src/dev-cloud-workbench-smoke-lib.mjs | 43 ++++++ .../validate-dev-gate-report-guard.test.mjs | 136 ++++++++++++++++++ scripts/validate-dev-gate-report.mjs | 54 ++++++- 3 files changed, 232 insertions(+), 1 deletion(-) diff --git a/scripts/src/dev-cloud-workbench-smoke-lib.mjs b/scripts/src/dev-cloud-workbench-smoke-lib.mjs index cff7ae0e..39504765 100644 --- a/scripts/src/dev-cloud-workbench-smoke-lib.mjs +++ b/scripts/src/dev-cloud-workbench-smoke-lib.mjs @@ -408,6 +408,34 @@ async function runLiveSmoke(args) { const identityBlocked = deploymentIdentity.status !== "pass" || webAssetIdentity.status !== "pass"; if (identityBlocked) { + addCheck( + checks, + blockers, + "live-code-agent-browser-journey", + "blocked", + "Browser Code Agent journey was not posted because deployed runtime/assets do not match current source identity.", + { + blocker: "observability_blocker", + evidence: [ + "POST /v1/agent/chat not sent", + `runtimeIdentity=${deploymentIdentity.status}`, + `webAssetIdentity=${webAssetIdentity.status}` + ], + observations: { + request: { + method: "POST", + status: "not_sent", + urlPath: "/v1/agent/chat" + }, + response: blockedAgentChatResponse("deployment_identity_preflight"), + classification: { + status: "blocked", + reason: "deployment_identity_preflight", + codeAgentBrowserJourneySkipped: true + } + } + } + ); return baseReport({ mode: "live", url: args.url, @@ -1724,6 +1752,21 @@ function sanitizeAgentChatBody(body) { }; } +function blockedAgentChatResponse(code) { + return { + status: "failed", + provider: "not_observed", + model: "not_observed", + backend: "not_observed", + traceId: "not_observed", + hasReply: false, + error: { + code, + missingEnv: [] + } + }; +} + export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() { let chromium; try { diff --git a/scripts/validate-dev-gate-report-guard.test.mjs b/scripts/validate-dev-gate-report-guard.test.mjs index 68464e36..4c78b5b0 100644 --- a/scripts/validate-dev-gate-report-guard.test.mjs +++ b/scripts/validate-dev-gate-report-guard.test.mjs @@ -21,6 +21,127 @@ function cloneBaseAggregatorReport() { return JSON.parse(JSON.stringify(baseAggregatorReport)); } +function preflightBlockedReport() { + const report = cloneBaseReport(); + report.status = "blocked"; + report.evidenceLevel = "BLOCKED"; + report.devLive = false; + report.generatedAt = report.runtimeIdentity.observedAt; + report.expectedRuntimeIdentity = { + status: "observed", + serviceId: "hwlab-cloud-api", + source: "deploy/artifact-catalog.dev.json+deploy/deploy.json", + commitId: "c7de474", + imageTag: "c7de474", + image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:c7de474", + summary: "Expected DEV runtime identity was derived from current source deploy desired state, not from live health." + }; + report.deploymentIdentity = { + status: "pass", + expectedCommit: "c7de474", + expectedImageTag: "c7de474", + expectedSource: "deploy/artifact-catalog.dev.json+deploy/deploy.json", + observedRuntimeCommit: "c7de474", + observedImageTag: "c7de474", + summary: "Live API runtime identity matches the current source desired runtime identity.", + evidence: [ + "expected=c7de474", + "expectedImageTag=c7de474", + "runtimeCommit=c7de474", + "imageTag=c7de474" + ] + }; + report.webAssetIdentity = { + status: "blocked", + assets: [ + { + path: "index.html", + status: "mismatch", + httpStatus: 200, + contentType: "text/html; charset=utf-8", + sourceHash: "435edb7da5c539ff", + liveHash: "924f9d57e0bb61be", + sourceLength: 13844, + liveLength: 11375 + } + ], + mismatches: ["index.html"], + summary: "Deployment drift: live 16666 primary web assets differ from current source (index.html)." + }; + report.checks = [ + { + id: "live-runtime-current-main", + status: report.deploymentIdentity.status, + summary: report.deploymentIdentity.summary, + evidence: report.deploymentIdentity.evidence, + observations: report.deploymentIdentity + }, + report.checks.find((check) => check.id === "live-http-html"), + { + id: "live-web-assets-current-main", + status: report.webAssetIdentity.status, + summary: report.webAssetIdentity.summary, + evidence: ["index.html: mismatch"], + observations: report.webAssetIdentity + }, + { + id: "live-code-agent-browser-journey", + status: "blocked", + summary: "Browser Code Agent journey was not posted because deployed runtime/assets do not match current source identity.", + evidence: [ + "POST /v1/agent/chat not sent", + "runtimeIdentity=pass", + "webAssetIdentity=blocked" + ], + observations: { + request: { + method: "POST", + status: "not_sent", + urlPath: "/v1/agent/chat" + }, + response: { + status: "failed", + provider: "not_observed", + model: "not_observed", + backend: "not_observed", + traceId: "not_observed", + hasReply: false, + error: { + code: "deployment_identity_preflight", + missingEnv: [] + } + }, + classification: { + status: "blocked", + reason: "deployment_identity_preflight", + codeAgentBrowserJourneySkipped: true + } + } + } + ]; + report.blockers = [ + { + type: "observability_blocker", + scope: "live-web-assets-current-main", + status: "open", + summary: report.webAssetIdentity.summary + }, + { + type: "observability_blocker", + scope: "live-code-agent-browser-journey", + status: "open", + summary: "Browser Code Agent journey was not posted because deployed runtime/assets do not match current source identity." + } + ]; + report.devPreconditions.status = "blocked"; + report.devPreconditions.summary = "Deployed browser journey is blocked; see checks and blockers."; + report.safety.liveMode = "deployment-identity-preflight"; + report.safety.codeAgentBrowserJourneySkipped = true; + report.safety.retainedApiFields = ["runtime.commitId", "runtime.imageTag", "web asset hash/status"]; + report.safety.statement = "Live smoke blocked before the browser chat journey because deployed runtime/assets do not match the current source identity; it did not post Code Agent chat, call hardware write APIs, restart services, or read secret values."; + return report; +} + async function withReport(report, fn) { const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-")); const reportPath = path.join(directory, "report.json"); @@ -56,6 +177,21 @@ test("accepts the sanitized deployed workbench evidence contract", async () => { }); }); +test("accepts preflight-blocked workbench evidence when Code Agent POST is marked not_sent", async () => { + await withReport(preflightBlockedReport(), async (reportPath) => { + const result = runValidator(reportPath); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /validated 1 dev-gate report JSON file/); + }); +}); + +test("rejects preflight-blocked workbench evidence without not_sent Code Agent journey proof", async () => { + const report = preflightBlockedReport(); + report.checks = report.checks.filter((check) => check.id !== "live-code-agent-browser-journey"); + report.blockers = report.blockers.filter((blocker) => blocker.scope !== "live-code-agent-browser-journey"); + await assertRejected(report, /checks missing live-code-agent-browser-journey/); +}); + test("rejects accepted workbench evidence when runtime identity is not observed", async () => { const report = cloneBaseReport(); Object.assign(report.runtimeIdentity, { diff --git a/scripts/validate-dev-gate-report.mjs b/scripts/validate-dev-gate-report.mjs index 342b9f3d..5734e0b7 100644 --- a/scripts/validate-dev-gate-report.mjs +++ b/scripts/validate-dev-gate-report.mjs @@ -41,6 +41,7 @@ const requiredMilestoneIds = ["M0", "M1", "M2", "M3", "M4", "M5"]; const evidenceLevels = new Set(["SOURCE", "LOCAL", "DRY-RUN", "DEV-LIVE", "BLOCKED"]); const devCloudWorkbenchRuntimeIdentityMaxSkewMs = 15 * 60 * 1000; const syntheticAcceptedCodeAgentModels = new Set(["gpt-5"]); +const deploymentPreflightCodeAgentErrorCode = "deployment_identity_preflight"; const requiredValidationCommands = [ "node --check scripts/validate-dev-gate-report.mjs", "node scripts/validate-dev-gate-report.mjs" @@ -1536,15 +1537,18 @@ function assertDevCloudWorkbenchDeploymentPreflight(report, label, checks) { for (const requiredCheck of [ "live-runtime-current-main", "live-http-html", - "live-web-assets-current-main" + "live-web-assets-current-main", + "live-code-agent-browser-journey" ]) { assert.ok(checks.has(requiredCheck), `${label}.checks missing ${requiredCheck}`); } const runtimeCheck = checks.get("live-runtime-current-main"); const webAssetCheck = checks.get("live-web-assets-current-main"); + const journey = checks.get("live-code-agent-browser-journey"); assert.ok(["pass", "blocked"].includes(runtimeCheck.status), `${label}.live-runtime-current-main.status`); assert.ok(["pass", "blocked"].includes(webAssetCheck.status), `${label}.live-web-assets-current-main.status`); + assert.equal(journey.status, "blocked", `${label}.live-code-agent-browser-journey.status`); assert.ok( runtimeCheck.status === "blocked" || webAssetCheck.status === "blocked", `${label}.deployment preflight requires runtime or web asset drift` @@ -1596,6 +1600,7 @@ function assertDevCloudWorkbenchDeploymentPreflight(report, label, checks) { report.webAssetIdentity, `${label}.live-web-assets-current-main.observations` ); + assertDevCloudWorkbenchDeploymentPreflightJourney(report, label, journey); assertBlockers(report.blockers, `${label}.blockers`); if (runtimeCheck.status === "blocked") { @@ -1624,6 +1629,53 @@ function assertDevCloudWorkbenchDeploymentPreflight(report, label, checks) { assertString(report.safety.statement, `${label}.safety.statement`); } +function assertDevCloudWorkbenchDeploymentPreflightJourney(report, label, journey) { + assertStringArray(journey.evidence, `${label}.live-code-agent-browser-journey.evidence`, { minLength: 3 }); + assert.ok( + journey.evidence.includes("POST /v1/agent/chat not sent"), + `${label}.live-code-agent-browser-journey.evidence missing not_sent proof` + ); + assertObject(journey.observations, `${label}.live-code-agent-browser-journey.observations`); + assertObject(journey.observations.request, `${label}.live-code-agent-browser-journey.request`); + assert.equal(journey.observations.request.method, "POST", `${label}.live-code-agent-browser-journey.request.method`); + assert.equal(journey.observations.request.status, "not_sent", `${label}.live-code-agent-browser-journey.request.status`); + assert.equal(journey.observations.request.urlPath, "/v1/agent/chat", `${label}.live-code-agent-browser-journey.request.urlPath`); + assertObject(journey.observations.response, `${label}.live-code-agent-browser-journey.response`); + assert.equal(journey.observations.response.status, "failed", `${label}.live-code-agent-browser-journey.response.status`); + assert.equal(journey.observations.response.hasReply, false, `${label}.live-code-agent-browser-journey.response.hasReply`); + assertObject(journey.observations.response.error, `${label}.live-code-agent-browser-journey.response.error`); + assert.equal( + journey.observations.response.error.code, + deploymentPreflightCodeAgentErrorCode, + `${label}.live-code-agent-browser-journey.response.error.code` + ); + for (const field of ["provider", "model", "backend", "traceId"]) { + assert.equal(journey.observations.response[field], "not_observed", `${label}.live-code-agent-browser-journey.response.${field}`); + } + assertObject(journey.observations.classification, `${label}.live-code-agent-browser-journey.classification`); + assert.equal(journey.observations.classification.status, "blocked", `${label}.live-code-agent-browser-journey.classification.status`); + assert.equal( + journey.observations.classification.reason, + deploymentPreflightCodeAgentErrorCode, + `${label}.live-code-agent-browser-journey.classification.reason` + ); + assert.equal( + journey.observations.classification.codeAgentBrowserJourneySkipped, + true, + `${label}.live-code-agent-browser-journey.classification.codeAgentBrowserJourneySkipped` + ); + const journeyText = JSON.stringify(journey.observations); + assert.equal( + /"status"\s*:\s*200|completed|provider_unavailable|OPENAI_API_KEY/u.test(journeyText), + false, + `${label}.live-code-agent-browser-journey must not retain sent, completed, or secret-dependent evidence during deployment preflight` + ); + assert.ok( + report.blockers.some((blocker) => blocker.scope === "live-code-agent-browser-journey"), + `${label}.blockers must include live-code-agent-browser-journey` + ); +} + function assertDevCloudWorkbenchIdentity(report, label) { assertObject(report.sourceIdentity, `${label}.sourceIdentity`); for (const field of ["status", "source", "commitId", "shortCommitId", "ref", "worktreeState", "reportCommitId", "summary"]) {