From dffc3dd6b83cfa533b601a5eeffac0774e29632e Mon Sep 17 00:00:00 2001 From: Code Queue Review Date: Fri, 22 May 2026 22:28:27 +0000 Subject: [PATCH] test: guard deployed workbench evidence --- .../validate-dev-gate-report-guard.test.mjs | 110 ++++++++++++++++++ scripts/validate-dev-gate-report.mjs | 101 +++++++++++++++- 2 files changed, 209 insertions(+), 2 deletions(-) create mode 100644 scripts/validate-dev-gate-report-guard.test.mjs diff --git a/scripts/validate-dev-gate-report-guard.test.mjs b/scripts/validate-dev-gate-report-guard.test.mjs new file mode 100644 index 00000000..c637923e --- /dev/null +++ b/scripts/validate-dev-gate-report-guard.test.mjs @@ -0,0 +1,110 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import path from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); +const validator = path.join(repoRoot, "scripts/validate-dev-gate-report.mjs"); +const baseReportPath = path.join(repoRoot, "reports/dev-gate/dev-cloud-workbench-live.json"); +const baseReport = JSON.parse(await readFile(baseReportPath, "utf8")); + +function cloneBaseReport() { + return JSON.parse(JSON.stringify(baseReport)); +} + +async function withReport(report, fn) { + const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-")); + const reportPath = path.join(directory, "report.json"); + try { + await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); + return await fn(reportPath); + } finally { + await rm(directory, { recursive: true, force: true }); + } +} + +function runValidator(reportPath) { + return spawnSync(process.execPath, [validator, reportPath], { + cwd: repoRoot, + encoding: "utf8" + }); +} + +async function assertRejected(report, expectedMessage) { + await withReport(report, async (reportPath) => { + const result = runValidator(reportPath); + assert.notEqual(result.status, 0, "validator should reject stale deployed evidence"); + assert.match(`${result.stdout}\n${result.stderr}`, expectedMessage); + assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /sk-[A-Za-z0-9._-]{8,}/u); + }); +} + +test("accepts the sanitized deployed workbench evidence contract", async () => { + await withReport(cloneBaseReport(), async (reportPath) => { + const result = runValidator(reportPath); + assert.equal(result.status, 0, result.stderr); + assert.match(result.stdout, /validated 1 dev-gate report JSON file/); + }); +}); + +test("rejects accepted workbench evidence when runtime identity is not observed", async () => { + const report = cloneBaseReport(); + Object.assign(report.runtimeIdentity, { + status: "not_observed", + serviceId: "not_observed", + environment: "not_observed", + healthStatus: "not_observed", + commitId: "not_observed", + commitSource: "not_observed", + imageTag: "not_observed", + reason: "test fixture" + }); + await assertRejected(report, /runtimeIdentity\.status/); +}); + +test("rejects accepted workbench evidence when runtime identity is stale", async () => { + const report = cloneBaseReport(); + report.runtimeIdentity.observedAt = "2026-05-22T12:00:00.000Z"; + await assertRejected(report, /runtimeIdentity\.observedAt must be fresh/); +}); + +test("rejects accepted workbench evidence that carries an allowed legacy port annotation", async () => { + const report = cloneBaseReport(); + report.deprecatedEndpoints = [ + { + endpoint: "http://74.48.78.17:6666", + status: "deprecated", + activeGreenEligible: false + } + ]; + report.checks[0].evidence.push("legacy probe http://74.48.78.17:6666/"); + await assertRejected(report, /deprecated public endpoint evidence/); +}); + +test("rejects accepted workbench evidence with a hard-coded runtime commit source", async () => { + const report = cloneBaseReport(); + report.runtimeIdentity.commitSource = "hard-coded"; + await assertRejected(report, /runtimeIdentity\.commitSource/); +}); + +test("rejects accepted workbench evidence that mixes provider_unavailable blockers", async () => { + const report = cloneBaseReport(); + const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey"); + journey.observations.networkEvents[0].body.error = { + code: "provider_unavailable", + missingEnv: ["OPENAI_API_KEY"] + }; + await assertRejected(report, /provider_unavailable or OPENAI_API_KEY blockers/); +}); + +test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => { + const report = cloneBaseReport(); + const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey"); + journey.evidence = journey.evidence.map((item) => item === "model=gpt-5.5" ? "model=gpt-5" : item); + journey.observations.response.model = "gpt-5"; + journey.observations.networkEvents[0].body.model = "gpt-5"; + await assertRejected(report, /synthetic placeholder model/); +}); diff --git a/scripts/validate-dev-gate-report.mjs b/scripts/validate-dev-gate-report.mjs index 1aed10e7..576c3ca3 100644 --- a/scripts/validate-dev-gate-report.mjs +++ b/scripts/validate-dev-gate-report.mjs @@ -39,6 +39,8 @@ const allowedIssues = new Set([ const requiredPreflightIssue = issueFamily.DEV_GATE_PREFLIGHT; const requiredMilestoneIds = ["M0", "M1", "M2", "M3", "M4", "M5"]; const evidenceLevels = new Set(["SOURCE", "LOCAL", "DRY-RUN", "DEV-LIVE", "BLOCKED"]); +const devCloudWorkbenchRuntimeIdentityMaxSkewMs = 15 * 60 * 1000; +const syntheticAcceptedCodeAgentModels = new Set(["gpt-5"]); const requiredValidationCommands = [ "node --check scripts/validate-dev-gate-report.mjs", "node scripts/validate-dev-gate-report.mjs" @@ -476,7 +478,7 @@ async function validateReport(relativePath) { report.issue === issueFamily.DEV_CLOUD_WORKBENCH_LIVE || report.taskId === "dev-cloud-workbench-live" ) { - await validateDevCloudWorkbenchLiveReport(report, label); + await validateDevCloudWorkbenchLiveReport(report, label, raw); return; } @@ -1297,7 +1299,7 @@ async function validateDevM2Report(report, label) { ); } -async function validateDevCloudWorkbenchLiveReport(report, label) { +async function validateDevCloudWorkbenchLiveReport(report, label, raw) { for (const field of [ "$schema", "$id", @@ -1345,6 +1347,7 @@ async function validateDevCloudWorkbenchLiveReport(report, label) { assert.equal(report.evidenceLevel, report.status === "pass" ? "DEV-LIVE-BROWSER" : "BLOCKED", `${label}.evidenceLevel`); assert.equal(report.devLive, report.status === "pass", `${label}.devLive`); assertDevCloudWorkbenchIdentity(report, label); + assertDevCloudWorkbenchEndpoints(report, label); assertObject(report.sourceContract, `${label}.sourceContract`); assertStatus(report.sourceContract.status, `${label}.sourceContract.status`); @@ -1459,6 +1462,7 @@ async function validateDevCloudWorkbenchLiveReport(report, label) { `${label}.safety.retainedApiFields` ); assertString(report.safety.statement, `${label}.safety.statement`); + assertFreshDevCloudWorkbenchLiveEvidence(report, label, raw, journey); } function assertDevCloudWorkbenchIdentity(report, label) { @@ -1516,6 +1520,99 @@ function assertDevCloudWorkbenchIdentity(report, label) { ); } +function assertDevCloudWorkbenchEndpoints(report, label) { + assertObject(report.endpoints, `${label}.endpoints`); + assert.equal(report.endpoints.frontend, ACTIVE_DEV_BROWSER_ENDPOINT, `${label}.endpoints.frontend`); + assert.equal(report.endpoints.api, ACTIVE_DEV_PUBLIC_ENDPOINT, `${label}.endpoints.api`); + assert.equal(report.endpoints.edge, ACTIVE_DEV_PUBLIC_ENDPOINT, `${label}.endpoints.edge`); +} + +function assertFreshDevCloudWorkbenchLiveEvidence(report, label, raw, journey) { + const deprecatedEndpoints = findDeprecatedPublicEndpoints(raw); + assert.deepEqual( + deprecatedEndpoints, + [], + `${label} deployed workbench report must not contain deprecated public endpoint evidence` + ); + + if (report.status !== "pass") { + return; + } + + assert.equal(report.devLive, true, `${label}.devLive`); + assert.equal(report.evidenceLevel, "DEV-LIVE-BROWSER", `${label}.evidenceLevel`); + assert.equal(report.safety.sourceIsDevLive, false, `${label}.safety.sourceIsDevLive`); + assert.equal(report.runtimeIdentity.status, "observed", `${label}.runtimeIdentity.status`); + assert.equal(report.runtimeIdentity.source, "health-live", `${label}.runtimeIdentity.source`); + assert.equal(report.runtimeIdentity.endpoint, `${ACTIVE_DEV_PUBLIC_ENDPOINT}/health/live`, `${label}.runtimeIdentity.endpoint`); + assert.equal(report.runtimeIdentity.serviceId, "hwlab-cloud-api", `${label}.runtimeIdentity.serviceId`); + assert.equal(report.runtimeIdentity.environment, "dev", `${label}.runtimeIdentity.environment`); + assert.match(report.runtimeIdentity.commitId, /^[a-f0-9]{7,40}$/, `${label}.runtimeIdentity.commitId`); + assert.notEqual(report.runtimeIdentity.commitId, "unknown", `${label}.runtimeIdentity.commitId`); + assert.notEqual(report.runtimeIdentity.commitSource, "source-git-head", `${label}.runtimeIdentity.commitSource`); + assert.notEqual(report.runtimeIdentity.commitSource, "hard-coded", `${label}.runtimeIdentity.commitSource`); + assert.notEqual(report.runtimeIdentity.commitSource, "literal", `${label}.runtimeIdentity.commitSource`); + + const generatedAtMs = Date.parse(report.generatedAt); + const runtimeObservedAtMs = Date.parse(report.runtimeIdentity.observedAt); + assert.ok( + Math.abs(generatedAtMs - runtimeObservedAtMs) <= devCloudWorkbenchRuntimeIdentityMaxSkewMs, + `${label}.runtimeIdentity.observedAt must be fresh relative to generatedAt` + ); + + assert.equal( + report.sourceIdentity.reportCommitId === "unknown" || report.sourceIdentity.worktreeState === "clean", + true, + `${label}.sourceIdentity.reportCommitId must be unknown unless the source worktree is clean` + ); + + const journeyText = JSON.stringify(journey.observations); + assert.equal( + /provider_unavailable|OPENAI_API_KEY/u.test(journeyText), + false, + `${label}.live-code-agent-browser-journey must not mix provider_unavailable or OPENAI_API_KEY blockers into accepted evidence` + ); + + assertAcceptedCodeAgentPayload( + journey.observations.response, + `${label}.live-code-agent-browser-journey.response` + ); + const networkEvents = journey.observations.networkEvents ?? []; + assertArray(networkEvents, `${label}.live-code-agent-browser-journey.networkEvents`); + assert.ok(networkEvents.length >= 1, `${label}.live-code-agent-browser-journey.networkEvents`); + for (const [index, event] of networkEvents.entries()) { + const eventLabel = `${label}.live-code-agent-browser-journey.networkEvents[${index}]`; + assertObject(event, eventLabel); + assert.equal(event.method, "POST", `${eventLabel}.method`); + assert.equal(event.status, 200, `${eventLabel}.status`); + assert.equal(event.urlPath, "/v1/agent/chat", `${eventLabel}.urlPath`); + assertAcceptedCodeAgentPayload(event.body, `${eventLabel}.body`); + } +} + +function assertAcceptedCodeAgentPayload(payload, label) { + assertObject(payload, label); + assert.equal(payload.status, "completed", `${label}.status`); + assertString(payload.provider, `${label}.provider`); + assertString(payload.model, `${label}.model`); + assertString(payload.backend, `${label}.backend`); + assertString(payload.traceId, `${label}.traceId`); + assert.equal(payload.hasReply, true, `${label}.hasReply`); + assert.equal(payload.error, null, `${label}.error`); + assert.equal( + syntheticAcceptedCodeAgentModels.has(payload.model), + false, + `${label}.model must not use the known synthetic placeholder model` + ); + for (const forbiddenField of ["conversationId", "sessionId", "messageId", "reply", "content", "text", "output"]) { + assert.equal( + Object.hasOwn(payload, forbiddenField), + false, + `${label} must not retain ${forbiddenField}` + ); + } +} + async function validateDevM3Report(report, label) { for (const field of [ "$schema",