test: guard deployed workbench evidence

2026-05-22 22:28:27 +00:00
parent dcb496bf95
commit dffc3dd6b8
2 changed files with 209 additions and 2 deletions
@@ -0,0 +1,110 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { test } from "node:test";
+import { fileURLToPath } from "node:url";
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const validator = path.join(repoRoot, "scripts/validate-dev-gate-report.mjs");
+const baseReportPath = path.join(repoRoot, "reports/dev-gate/dev-cloud-workbench-live.json");
+const baseReport = JSON.parse(await readFile(baseReportPath, "utf8"));
+
+function cloneBaseReport() {
+  return JSON.parse(JSON.stringify(baseReport));
+}
+
+async function withReport(report, fn) {
+  const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-"));
+  const reportPath = path.join(directory, "report.json");
+  try {
+    await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+    return await fn(reportPath);
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+}
+
+function runValidator(reportPath) {
+  return spawnSync(process.execPath, [validator, reportPath], {
+    cwd: repoRoot,
+    encoding: "utf8"
+  });
+}
+
+async function assertRejected(report, expectedMessage) {
+  await withReport(report, async (reportPath) => {
+    const result = runValidator(reportPath);
+    assert.notEqual(result.status, 0, "validator should reject stale deployed evidence");
+    assert.match(`${result.stdout}\n${result.stderr}`, expectedMessage);
+    assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /sk-[A-Za-z0-9._-]{8,}/u);
+  });
+}
+
+test("accepts the sanitized deployed workbench evidence contract", async () => {
+  await withReport(cloneBaseReport(), async (reportPath) => {
+    const result = runValidator(reportPath);
+    assert.equal(result.status, 0, result.stderr);
+    assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
+  });
+});
+
+test("rejects accepted workbench evidence when runtime identity is not observed", async () => {
+  const report = cloneBaseReport();
+  Object.assign(report.runtimeIdentity, {
+    status: "not_observed",
+    serviceId: "not_observed",
+    environment: "not_observed",
+    healthStatus: "not_observed",
+    commitId: "not_observed",
+    commitSource: "not_observed",
+    imageTag: "not_observed",
+    reason: "test fixture"
+  });
+  await assertRejected(report, /runtimeIdentity\.status/);
+});
+
+test("rejects accepted workbench evidence when runtime identity is stale", async () => {
+  const report = cloneBaseReport();
+  report.runtimeIdentity.observedAt = "2026-05-22T12:00:00.000Z";
+  await assertRejected(report, /runtimeIdentity\.observedAt must be fresh/);
+});
+
+test("rejects accepted workbench evidence that carries an allowed legacy port annotation", async () => {
+  const report = cloneBaseReport();
+  report.deprecatedEndpoints = [
+    {
+      endpoint: "http://74.48.78.17:6666",
+      status: "deprecated",
+      activeGreenEligible: false
+    }
+  ];
+  report.checks[0].evidence.push("legacy probe http://74.48.78.17:6666/");
+  await assertRejected(report, /deprecated public endpoint evidence/);
+});
+
+test("rejects accepted workbench evidence with a hard-coded runtime commit source", async () => {
+  const report = cloneBaseReport();
+  report.runtimeIdentity.commitSource = "hard-coded";
+  await assertRejected(report, /runtimeIdentity\.commitSource/);
+});
+
+test("rejects accepted workbench evidence that mixes provider_unavailable blockers", async () => {
+  const report = cloneBaseReport();
+  const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
+  journey.observations.networkEvents[0].body.error = {
+    code: "provider_unavailable",
+    missingEnv: ["OPENAI_API_KEY"]
+  };
+  await assertRejected(report, /provider_unavailable or OPENAI_API_KEY blockers/);
+});
+
+test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => {
+  const report = cloneBaseReport();
+  const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
+  journey.evidence = journey.evidence.map((item) => item === "model=gpt-5.5" ? "model=gpt-5" : item);
+  journey.observations.response.model = "gpt-5";
+  journey.observations.networkEvents[0].body.model = "gpt-5";
+  await assertRejected(report, /synthetic placeholder model/);
+});