pikasTech-HWLAB/scripts/validate-dev-gate-report-guard.test.mjs

import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { test } from "node:test";
import { fileURLToPath } from "node:url";

const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const validator = path.join(repoRoot, "scripts/validate-dev-gate-report.mjs");
const baseReportPath = path.join(repoRoot, "reports/dev-gate/dev-cloud-workbench-live.json");
const baseAggregatorReportPath = path.join(repoRoot, "reports/dev-gate/dev-m5-gate-aggregator-v2.json");
const baseReport = JSON.parse(await readFile(baseReportPath, "utf8"));
const baseAggregatorReport = JSON.parse(await readFile(baseAggregatorReportPath, "utf8"));

function cloneBaseReport() {
  return JSON.parse(JSON.stringify(baseReport));
}

function cloneBaseAggregatorReport() {
  return JSON.parse(JSON.stringify(baseAggregatorReport));
}

async function withReport(report, fn) {
  const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-"));
  const reportPath = path.join(directory, "report.json");
  try {
    await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`);
    return await fn(reportPath);
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
}

function runValidator(reportPath) {
  return spawnSync(process.execPath, [validator, reportPath], {
    cwd: repoRoot,
    encoding: "utf8"
  });
}

async function assertRejected(report, expectedMessage) {
  await withReport(report, async (reportPath) => {
    const result = runValidator(reportPath);
    assert.notEqual(result.status, 0, "validator should reject stale deployed evidence");
    assert.match(`${result.stdout}\n${result.stderr}`, expectedMessage);
    assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /sk-[A-Za-z0-9._-]{8,}/u);
  });
}

test("accepts the sanitized deployed workbench evidence contract", async () => {
  await withReport(cloneBaseReport(), async (reportPath) => {
    const result = runValidator(reportPath);
    assert.equal(result.status, 0, result.stderr);
    assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
  });
});

test("rejects accepted workbench evidence when runtime identity is not observed", async () => {
  const report = cloneBaseReport();
  Object.assign(report.runtimeIdentity, {
    status: "not_observed",
    serviceId: "not_observed",
    environment: "not_observed",
    healthStatus: "not_observed",
    commitId: "not_observed",
    commitSource: "not_observed",
    imageTag: "not_observed",
    reason: "test fixture"
  });
  await assertRejected(report, /runtimeIdentity\.status/);
});

test("rejects accepted workbench evidence when runtime identity is stale", async () => {
  const report = cloneBaseReport();
  report.runtimeIdentity.observedAt = "2026-05-22T12:00:00.000Z";
  await assertRejected(report, /runtimeIdentity\.observedAt must be fresh/);
});

test("rejects accepted workbench evidence that carries an allowed legacy port annotation", async () => {
  const report = cloneBaseReport();
  report.deprecatedEndpoints = [
    {
      endpoint: "http://74.48.78.17:6666",
      status: "deprecated",
      activeGreenEligible: false
    }
  ];
  report.checks[0].evidence.push("legacy probe http://74.48.78.17:6666/");
  await assertRejected(report, /deprecated public endpoint evidence/);
});

test("rejects accepted workbench evidence with a hard-coded runtime commit source", async () => {
  const report = cloneBaseReport();
  report.runtimeIdentity.commitSource = "hard-coded";
  await assertRejected(report, /runtimeIdentity\.commitSource/);
});

test("rejects accepted workbench evidence that mixes provider_unavailable blockers", async () => {
  const report = cloneBaseReport();
  const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
  journey.observations.networkEvents[0].body.error = {
    code: "provider_unavailable",
    missingEnv: ["OPENAI_API_KEY"]
  };
  await assertRejected(report, /provider_unavailable or OPENAI_API_KEY blockers/);
});

test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => {
  const report = cloneBaseReport();
  const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
  journey.evidence = journey.evidence.map((item) => item === "model=gpt-5.5" ? "model=gpt-5" : item);
  journey.observations.response.model = "gpt-5";
  journey.observations.networkEvents[0].body.model = "gpt-5";
  await assertRejected(report, /synthetic placeholder model/);
});

test("accepts the current blocked M5 aggregator non-promotion contract", async () => {
  await withReport(cloneBaseAggregatorReport(), async (reportPath) => {
    const result = runValidator(reportPath);
    assert.equal(result.status, 0, result.stderr);
    assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
  });
});

test("rejects aggregator DB acceptance promoted from frontend route evidence", async () => {
  const report = cloneBaseAggregatorReport();
  const check = report.dod.checks.find((item) => item.id === "cloud-api-db-ready");
  Object.assign(check, {
    status: "pass",
    evidenceLevel: "DEV-LIVE",
    summary: "Frontend route HTTP 200 proves DB ready."
  });
  Object.assign(report.currentDevLayering.dbLive, {
    status: "pass",
    evidenceLevel: "DEV-LIVE",
    summary: "Promoted from active frontend/API route evidence."
  });
  await assertRejected(report, /DB DEV-LIVE acceptance source reports/);
});

test("rejects aggregator M3 acceptance promoted from source evidence", async () => {
  const report = cloneBaseAggregatorReport();
  const sourceEvidence = report.evidence.find((item) =>
    item.milestone === "M3" && item.category === "hardware-loop-cardinality"
  );
  sourceEvidence.level = "DEV-LIVE";
  report.levels["DEV-LIVE"].push({
    milestone: "M3",
    issue: sourceEvidence.issue,
    taskId: sourceEvidence.taskId,
    lifecycleState: "active",
    status: sourceEvidence.status,
    category: sourceEvidence.category,
    reportPath: sourceEvidence.reportPath,
    summary: sourceEvidence.summary
  });
  await assertRejected(report, /M3 DEV-LIVE evidence must come from hardware-loop-live/);
});

test("rejects aggregator M4 acceptance promoted from dry-run evidence", async () => {
  const report = cloneBaseAggregatorReport();
  const dryRunEvidence = report.evidence.find((item) =>
    item.milestone === "M4" && item.category === "agent-loop-dry-run"
  );
  dryRunEvidence.level = "DEV-LIVE";
  report.levels["DEV-LIVE"].push({
    milestone: "M4",
    issue: dryRunEvidence.issue,
    taskId: dryRunEvidence.taskId,
    lifecycleState: "active",
    status: dryRunEvidence.status,
    category: dryRunEvidence.category,
    reportPath: dryRunEvidence.reportPath,
    summary: dryRunEvidence.summary
  });
  await assertRejected(report, /M4 DEV-LIVE evidence must come from agent-loop-live-preflight/);
});

test("rejects aggregator M5 acceptance promoted from fixture evidence", async () => {
  const report = cloneBaseAggregatorReport();
  const fixtureEvidence = {
    milestone: "M5",
    issue: "pikasTech/HWLAB#39",
    taskId: "dev-mvp-gate-report",
    reportPath: "reports/dev-gate/dev-mvp-gate-report.json",
    commitId: "fixture",
    lifecycleState: "active",
    level: "DEV-LIVE",
    status: "pass",
    category: "fixture-mvp-e2e",
    evidence: ["fixture transcript marked green"],
    commands: ["node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run"],
    summary: "Fixture evidence was incorrectly promoted to M5 DEV-LIVE."
  };
  report.evidence.push(fixtureEvidence);
  report.levels["DEV-LIVE"].push({
    milestone: fixtureEvidence.milestone,
    issue: fixtureEvidence.issue,
    taskId: fixtureEvidence.taskId,
    lifecycleState: fixtureEvidence.lifecycleState,
    status: fixtureEvidence.status,
    category: fixtureEvidence.category,
    reportPath: fixtureEvidence.reportPath,
    summary: fixtureEvidence.summary
  });
  await assertRejected(report, /M5 DEV-LIVE evidence must come from mvp-e2e-live/);
});

test("rejects aggregator frontend fact that claims promotion authority", async () => {
  const report = cloneBaseAggregatorReport();
  report.latestFrontendDevFact.promotesM3M4M5 = true;
  await assertRejected(report, /latestFrontendDevFact\.promotesM3M4M5 must be false/);
});