test: guard deployed workbench evidence

2026-05-22 22:28:27 +00:00
parent dcb496bf95
commit dffc3dd6b8
2 changed files with 209 additions and 2 deletions
@@ -0,0 +1,110 @@
+import assert from "node:assert/strict";
+import { spawnSync } from "node:child_process";
+import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { test } from "node:test";
+import { fileURLToPath } from "node:url";
+
+const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
+const validator = path.join(repoRoot, "scripts/validate-dev-gate-report.mjs");
+const baseReportPath = path.join(repoRoot, "reports/dev-gate/dev-cloud-workbench-live.json");
+const baseReport = JSON.parse(await readFile(baseReportPath, "utf8"));
+
+function cloneBaseReport() {
+  return JSON.parse(JSON.stringify(baseReport));
+}
+
+async function withReport(report, fn) {
+  const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-"));
+  const reportPath = path.join(directory, "report.json");
+  try {
+    await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`);
+    return await fn(reportPath);
+  } finally {
+    await rm(directory, { recursive: true, force: true });
+  }
+}
+
+function runValidator(reportPath) {
+  return spawnSync(process.execPath, [validator, reportPath], {
+    cwd: repoRoot,
+    encoding: "utf8"
+  });
+}
+
+async function assertRejected(report, expectedMessage) {
+  await withReport(report, async (reportPath) => {
+    const result = runValidator(reportPath);
+    assert.notEqual(result.status, 0, "validator should reject stale deployed evidence");
+    assert.match(`${result.stdout}\n${result.stderr}`, expectedMessage);
+    assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /sk-[A-Za-z0-9._-]{8,}/u);
+  });
+}
+
+test("accepts the sanitized deployed workbench evidence contract", async () => {
+  await withReport(cloneBaseReport(), async (reportPath) => {
+    const result = runValidator(reportPath);
+    assert.equal(result.status, 0, result.stderr);
+    assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
+  });
+});
+
+test("rejects accepted workbench evidence when runtime identity is not observed", async () => {
+  const report = cloneBaseReport();
+  Object.assign(report.runtimeIdentity, {
+    status: "not_observed",
+    serviceId: "not_observed",
+    environment: "not_observed",
+    healthStatus: "not_observed",
+    commitId: "not_observed",
+    commitSource: "not_observed",
+    imageTag: "not_observed",
+    reason: "test fixture"
+  });
+  await assertRejected(report, /runtimeIdentity\.status/);
+});
+
+test("rejects accepted workbench evidence when runtime identity is stale", async () => {
+  const report = cloneBaseReport();
+  report.runtimeIdentity.observedAt = "2026-05-22T12:00:00.000Z";
+  await assertRejected(report, /runtimeIdentity\.observedAt must be fresh/);
+});
+
+test("rejects accepted workbench evidence that carries an allowed legacy port annotation", async () => {
+  const report = cloneBaseReport();
+  report.deprecatedEndpoints = [
+    {
+      endpoint: "http://74.48.78.17:6666",
+      status: "deprecated",
+      activeGreenEligible: false
+    }
+  ];
+  report.checks[0].evidence.push("legacy probe http://74.48.78.17:6666/");
+  await assertRejected(report, /deprecated public endpoint evidence/);
+});
+
+test("rejects accepted workbench evidence with a hard-coded runtime commit source", async () => {
+  const report = cloneBaseReport();
+  report.runtimeIdentity.commitSource = "hard-coded";
+  await assertRejected(report, /runtimeIdentity\.commitSource/);
+});
+
+test("rejects accepted workbench evidence that mixes provider_unavailable blockers", async () => {
+  const report = cloneBaseReport();
+  const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
+  journey.observations.networkEvents[0].body.error = {
+    code: "provider_unavailable",
+    missingEnv: ["OPENAI_API_KEY"]
+  };
+  await assertRejected(report, /provider_unavailable or OPENAI_API_KEY blockers/);
+});
+
+test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => {
+  const report = cloneBaseReport();
+  const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
+  journey.evidence = journey.evidence.map((item) => item === "model=gpt-5.5" ? "model=gpt-5" : item);
+  journey.observations.response.model = "gpt-5";
+  journey.observations.networkEvents[0].body.model = "gpt-5";
+  await assertRejected(report, /synthetic placeholder model/);
+});
@@ -39,6 +39,8 @@ const allowedIssues = new Set([
 const requiredPreflightIssue = issueFamily.DEV_GATE_PREFLIGHT;
 const requiredMilestoneIds = ["M0", "M1", "M2", "M3", "M4", "M5"];
 const evidenceLevels = new Set(["SOURCE", "LOCAL", "DRY-RUN", "DEV-LIVE", "BLOCKED"]);
+const devCloudWorkbenchRuntimeIdentityMaxSkewMs = 15 * 60 * 1000;
+const syntheticAcceptedCodeAgentModels = new Set(["gpt-5"]);
 const requiredValidationCommands = [
  "node --check scripts/validate-dev-gate-report.mjs",
  "node scripts/validate-dev-gate-report.mjs"
@@ -476,7 +478,7 @@ async function validateReport(relativePath) {
    report.issue === issueFamily.DEV_CLOUD_WORKBENCH_LIVE ||
    report.taskId === "dev-cloud-workbench-live"
  ) {
-    await validateDevCloudWorkbenchLiveReport(report, label);
+    await validateDevCloudWorkbenchLiveReport(report, label, raw);
    return;
  }

@@ -1297,7 +1299,7 @@ async function validateDevM2Report(report, label) {
  );
 }

-async function validateDevCloudWorkbenchLiveReport(report, label) {
+async function validateDevCloudWorkbenchLiveReport(report, label, raw) {
  for (const field of [
    "$schema",
    "$id",
@@ -1345,6 +1347,7 @@ async function validateDevCloudWorkbenchLiveReport(report, label) {
  assert.equal(report.evidenceLevel, report.status === "pass" ? "DEV-LIVE-BROWSER" : "BLOCKED", `${label}.evidenceLevel`);
  assert.equal(report.devLive, report.status === "pass", `${label}.devLive`);
  assertDevCloudWorkbenchIdentity(report, label);
+  assertDevCloudWorkbenchEndpoints(report, label);

  assertObject(report.sourceContract, `${label}.sourceContract`);
  assertStatus(report.sourceContract.status, `${label}.sourceContract.status`);
@@ -1459,6 +1462,7 @@ async function validateDevCloudWorkbenchLiveReport(report, label) {
    `${label}.safety.retainedApiFields`
  );
  assertString(report.safety.statement, `${label}.safety.statement`);
+  assertFreshDevCloudWorkbenchLiveEvidence(report, label, raw, journey);
 }

 function assertDevCloudWorkbenchIdentity(report, label) {
@@ -1516,6 +1520,99 @@ function assertDevCloudWorkbenchIdentity(report, label) {
  );
 }

+function assertDevCloudWorkbenchEndpoints(report, label) {
+  assertObject(report.endpoints, `${label}.endpoints`);
+  assert.equal(report.endpoints.frontend, ACTIVE_DEV_BROWSER_ENDPOINT, `${label}.endpoints.frontend`);
+  assert.equal(report.endpoints.api, ACTIVE_DEV_PUBLIC_ENDPOINT, `${label}.endpoints.api`);
+  assert.equal(report.endpoints.edge, ACTIVE_DEV_PUBLIC_ENDPOINT, `${label}.endpoints.edge`);
+}
+
+function assertFreshDevCloudWorkbenchLiveEvidence(report, label, raw, journey) {
+  const deprecatedEndpoints = findDeprecatedPublicEndpoints(raw);
+  assert.deepEqual(
+    deprecatedEndpoints,
+    [],
+    `${label} deployed workbench report must not contain deprecated public endpoint evidence`
+  );
+
+  if (report.status !== "pass") {
+    return;
+  }
+
+  assert.equal(report.devLive, true, `${label}.devLive`);
+  assert.equal(report.evidenceLevel, "DEV-LIVE-BROWSER", `${label}.evidenceLevel`);
+  assert.equal(report.safety.sourceIsDevLive, false, `${label}.safety.sourceIsDevLive`);
+  assert.equal(report.runtimeIdentity.status, "observed", `${label}.runtimeIdentity.status`);
+  assert.equal(report.runtimeIdentity.source, "health-live", `${label}.runtimeIdentity.source`);
+  assert.equal(report.runtimeIdentity.endpoint, `${ACTIVE_DEV_PUBLIC_ENDPOINT}/health/live`, `${label}.runtimeIdentity.endpoint`);
+  assert.equal(report.runtimeIdentity.serviceId, "hwlab-cloud-api", `${label}.runtimeIdentity.serviceId`);
+  assert.equal(report.runtimeIdentity.environment, "dev", `${label}.runtimeIdentity.environment`);
+  assert.match(report.runtimeIdentity.commitId, /^[a-f0-9]{7,40}$/, `${label}.runtimeIdentity.commitId`);
+  assert.notEqual(report.runtimeIdentity.commitId, "unknown", `${label}.runtimeIdentity.commitId`);
+  assert.notEqual(report.runtimeIdentity.commitSource, "source-git-head", `${label}.runtimeIdentity.commitSource`);
+  assert.notEqual(report.runtimeIdentity.commitSource, "hard-coded", `${label}.runtimeIdentity.commitSource`);
+  assert.notEqual(report.runtimeIdentity.commitSource, "literal", `${label}.runtimeIdentity.commitSource`);
+
+  const generatedAtMs = Date.parse(report.generatedAt);
+  const runtimeObservedAtMs = Date.parse(report.runtimeIdentity.observedAt);
+  assert.ok(
+    Math.abs(generatedAtMs - runtimeObservedAtMs) <= devCloudWorkbenchRuntimeIdentityMaxSkewMs,
+    `${label}.runtimeIdentity.observedAt must be fresh relative to generatedAt`
+  );
+
+  assert.equal(
+    report.sourceIdentity.reportCommitId === "unknown" || report.sourceIdentity.worktreeState === "clean",
+    true,
+    `${label}.sourceIdentity.reportCommitId must be unknown unless the source worktree is clean`
+  );
+
+  const journeyText = JSON.stringify(journey.observations);
+  assert.equal(
+    /provider_unavailable|OPENAI_API_KEY/u.test(journeyText),
+    false,
+    `${label}.live-code-agent-browser-journey must not mix provider_unavailable or OPENAI_API_KEY blockers into accepted evidence`
+  );
+
+  assertAcceptedCodeAgentPayload(
+    journey.observations.response,
+    `${label}.live-code-agent-browser-journey.response`
+  );
+  const networkEvents = journey.observations.networkEvents ?? [];
+  assertArray(networkEvents, `${label}.live-code-agent-browser-journey.networkEvents`);
+  assert.ok(networkEvents.length >= 1, `${label}.live-code-agent-browser-journey.networkEvents`);
+  for (const [index, event] of networkEvents.entries()) {
+    const eventLabel = `${label}.live-code-agent-browser-journey.networkEvents[${index}]`;
+    assertObject(event, eventLabel);
+    assert.equal(event.method, "POST", `${eventLabel}.method`);
+    assert.equal(event.status, 200, `${eventLabel}.status`);
+    assert.equal(event.urlPath, "/v1/agent/chat", `${eventLabel}.urlPath`);
+    assertAcceptedCodeAgentPayload(event.body, `${eventLabel}.body`);
+  }
+}
+
+function assertAcceptedCodeAgentPayload(payload, label) {
+  assertObject(payload, label);
+  assert.equal(payload.status, "completed", `${label}.status`);
+  assertString(payload.provider, `${label}.provider`);
+  assertString(payload.model, `${label}.model`);
+  assertString(payload.backend, `${label}.backend`);
+  assertString(payload.traceId, `${label}.traceId`);
+  assert.equal(payload.hasReply, true, `${label}.hasReply`);
+  assert.equal(payload.error, null, `${label}.error`);
+  assert.equal(
+    syntheticAcceptedCodeAgentModels.has(payload.model),
+    false,
+    `${label}.model must not use the known synthetic placeholder model`
+  );
+  for (const forbiddenField of ["conversationId", "sessionId", "messageId", "reply", "content", "text", "output"]) {
+    assert.equal(
+      Object.hasOwn(payload, forbiddenField),
+      false,
+      `${label} must not retain ${forbiddenField}`
+    );
+  }
+}
+
 async function validateDevM3Report(report, label) {
  for (const field of [
    "$schema",