pikasTech-HWLAB/scripts/validate-dev-gate-report-guard.test.mjs

import assert from "node:assert/strict";
import { spawnSync } from "node:child_process";
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { test } from "node:test";
import { fileURLToPath } from "node:url";

const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
const validator = path.join(repoRoot, "scripts/validate-dev-gate-report.mjs");
const baseReportPath = path.join(repoRoot, "reports/dev-gate/dev-cloud-workbench-live.json");
const baseAggregatorReportPath = path.join(repoRoot, "reports/dev-gate/dev-m5-gate-aggregator-v2.json");
const baseReport = JSON.parse(await readFile(baseReportPath, "utf8"));
const baseAggregatorReport = JSON.parse(await readFile(baseAggregatorReportPath, "utf8"));

function cloneBaseReport() {
  return JSON.parse(JSON.stringify(baseReport));
}

function cloneBaseAggregatorReport() {
  return JSON.parse(JSON.stringify(baseAggregatorReport));
}

async function withReport(report, fn) {
  const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-"));
  const reportPath = path.join(directory, "report.json");
  try {
    await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`);
    return await fn(reportPath);
  } finally {
    await rm(directory, { recursive: true, force: true });
  }
}

function runValidator(reportPath) {
  return spawnSync(process.execPath, [validator, reportPath], {
    cwd: repoRoot,
    encoding: "utf8"
  });
}

async function assertRejected(report, expectedMessage) {
  await withReport(report, async (reportPath) => {
    const result = runValidator(reportPath);
    assert.notEqual(result.status, 0, "validator should reject stale deployed evidence");
    assert.match(`${result.stdout}\n${result.stderr}`, expectedMessage);
    assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /sk-[A-Za-z0-9._-]{8,}/u);
  });
}

test("accepts the sanitized deployed workbench evidence contract", async () => {
  await withReport(cloneBaseReport(), async (reportPath) => {
    const result = runValidator(reportPath);
    assert.equal(result.status, 0, result.stderr);
    assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
  });
});

test("rejects accepted workbench evidence when runtime identity is not observed", async () => {
  const report = cloneBaseReport();
  Object.assign(report.runtimeIdentity, {
    status: "not_observed",
    serviceId: "not_observed",
    environment: "not_observed",
    healthStatus: "not_observed",
    commitId: "not_observed",
    commitSource: "not_observed",
    imageTag: "not_observed",
    reason: "test fixture"
  });
  await assertRejected(report, /runtimeIdentity\.status/);
});

test("rejects accepted workbench evidence when runtime identity is stale", async () => {
  const report = cloneBaseReport();
  report.runtimeIdentity.observedAt = "2026-05-22T12:00:00.000Z";
  await assertRejected(report, /runtimeIdentity\.observedAt must be fresh/);
});

test("rejects accepted workbench evidence that carries an allowed legacy port annotation", async () => {
  const report = cloneBaseReport();
  report.deprecatedEndpoints = [
    {
      endpoint: "http://74.48.78.17:6666",
      status: "deprecated",
      activeGreenEligible: false
    }
  ];
  report.checks[0].evidence.push("legacy probe http://74.48.78.17:6666/");
  await assertRejected(report, /deprecated public endpoint evidence/);
});

test("rejects accepted workbench evidence with a hard-coded runtime commit source", async () => {
  const report = cloneBaseReport();
  report.runtimeIdentity.commitSource = "hard-coded";
  await assertRejected(report, /runtimeIdentity\.commitSource/);
});

test("rejects accepted workbench evidence that mixes provider_unavailable blockers", async () => {
  const report = cloneBaseReport();
  const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
  journey.observations.networkEvents[0].body.error = {
    code: "provider_unavailable",
    missingEnv: ["OPENAI_API_KEY"]
  };
  await assertRejected(report, /provider_unavailable, OPENAI_API_KEY, or providerStatus blockers/);
});

test("accepts blocked workbench evidence for provider HTTP 502 without promoting completion", async () => {
  const report = cloneBaseReport();
  const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
  report.status = "blocked";
  report.evidenceLevel = "BLOCKED";
  report.devLive = false;
  report.devPreconditions.status = "blocked";
  report.devPreconditions.summary = "Deployed browser journey is blocked; see checks and blockers.";
  report.blockers = [
    {
      type: "runtime_blocker",
      scope: "live-code-agent-browser-journey",
      status: "open",
      summary: "Code Agent provider returned provider_unavailable with provider HTTP 502."
    }
  ];
  report.safety.retainedApiFields = [
    "status",
    "provider",
    "model",
    "backend",
    "traceId",
    "hasReply",
    "error.code",
    "error.missingEnv",
    "error.providerStatus"
  ];
  journey.status = "blocked";
  journey.summary = "Deployed browser journey reached Code Agent, but the provider returned a blocked/unavailable state instead of a completed reply.";
  journey.evidence = [
    "POST /v1/agent/chat HTTP 200",
    "status=failed",
    "provider=openai-responses",
    "model=gpt-5.5",
    "traceId=trc_provider_502",
    "uiStatus=发送失败",
    "blocker=provider-upstream"
  ];
  journey.observations.request.status = 200;
  journey.observations.response = {
    status: "failed",
    provider: "openai-responses",
    model: "gpt-5.5",
    backend: "hwlab-cloud-api/openai-responses",
    traceId: "trc_provider_502",
    hasReply: false,
    error: {
      code: "provider_unavailable",
      missingEnv: [],
      providerStatus: 502
    }
  };
  journey.observations.classification = {
    status: "blocked",
    blocker: "provider-upstream",
    providerStatus: 502,
    reason: "Code Agent HTTP non-2xx status 502 means the upstream response is blocked, not completed"
  };
  journey.observations.ui.agentChatStatus = "发送失败";
  journey.observations.ui.inputCleared = false;
  journey.observations.ui.completedMessageVisible = false;
  journey.observations.ui.failedMessageVisible = true;
  journey.observations.networkEvents = [
    {
      method: "POST",
      status: 200,
      urlPath: "/v1/agent/chat",
      body: journey.observations.response
    }
  ];

  await withReport(report, async (reportPath) => {
    const result = runValidator(reportPath);
    assert.equal(result.status, 0, result.stderr);
    assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
  });
});

test("rejects accepted workbench evidence that hides providerStatus 502 in network events", async () => {
  const report = cloneBaseReport();
  const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
  journey.observations.networkEvents[0].body.error = {
    code: null,
    missingEnv: [],
    providerStatus: 502
  };
  await assertRejected(report, /providerStatus blockers/);
});

test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => {
  const report = cloneBaseReport();
  const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
  journey.evidence = journey.evidence.map((item) => item === "model=gpt-5.5" ? "model=gpt-5" : item);
  journey.observations.response.model = "gpt-5";
  journey.observations.networkEvents[0].body.model = "gpt-5";
  await assertRejected(report, /synthetic placeholder model/);
});

test("accepts the current blocked M5 aggregator non-promotion contract", async () => {
  await withReport(cloneBaseAggregatorReport(), async (reportPath) => {
    const result = runValidator(reportPath);
    assert.equal(result.status, 0, result.stderr);
    assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
  });
});

test("rejects aggregator DB acceptance promoted from frontend route evidence", async () => {
  const report = cloneBaseAggregatorReport();
  const check = report.dod.checks.find((item) => item.id === "cloud-api-db-ready");
  Object.assign(check, {
    status: "pass",
    evidenceLevel: "DEV-LIVE",
    summary: "Frontend route HTTP 200 proves DB ready."
  });
  Object.assign(report.currentDevLayering.dbLive, {
    status: "pass",
    evidenceLevel: "DEV-LIVE",
    summary: "Promoted from active frontend/API route evidence."
  });
  await assertRejected(report, /DB DEV-LIVE acceptance source reports/);
});

test("rejects aggregator M3 acceptance promoted from source evidence", async () => {
  const report = cloneBaseAggregatorReport();
  const sourceEvidence = report.evidence.find((item) =>
    item.milestone === "M3" && item.category === "hardware-loop-cardinality"
  );
  sourceEvidence.level = "DEV-LIVE";
  report.levels["DEV-LIVE"].push({
    milestone: "M3",
    issue: sourceEvidence.issue,
    taskId: sourceEvidence.taskId,
    lifecycleState: "active",
    status: sourceEvidence.status,
    category: sourceEvidence.category,
    reportPath: sourceEvidence.reportPath,
    summary: sourceEvidence.summary
  });
  await assertRejected(report, /M3 DEV-LIVE evidence must come from hardware-loop-live/);
});

test("rejects aggregator M4 acceptance promoted from dry-run evidence", async () => {
  const report = cloneBaseAggregatorReport();
  const dryRunEvidence = report.evidence.find((item) =>
    item.milestone === "M4" && item.category === "agent-loop-dry-run"
  );
  dryRunEvidence.level = "DEV-LIVE";
  report.levels["DEV-LIVE"].push({
    milestone: "M4",
    issue: dryRunEvidence.issue,
    taskId: dryRunEvidence.taskId,
    lifecycleState: "active",
    status: dryRunEvidence.status,
    category: dryRunEvidence.category,
    reportPath: dryRunEvidence.reportPath,
    summary: dryRunEvidence.summary
  });
  await assertRejected(report, /M4 DEV-LIVE evidence must come from agent-loop-live-preflight/);
});

test("rejects aggregator M5 acceptance promoted from fixture evidence", async () => {
  const report = cloneBaseAggregatorReport();
  const fixtureEvidence = {
    milestone: "M5",
    issue: "pikasTech/HWLAB#39",
    taskId: "dev-mvp-gate-report",
    reportPath: "reports/dev-gate/dev-mvp-gate-report.json",
    commitId: "fixture",
    lifecycleState: "active",
    level: "DEV-LIVE",
    status: "pass",
    category: "fixture-mvp-e2e",
    evidence: ["fixture transcript marked green"],
    commands: ["node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run"],
    summary: "Fixture evidence was incorrectly promoted to M5 DEV-LIVE."
  };
  report.evidence.push(fixtureEvidence);
  report.levels["DEV-LIVE"].push({
    milestone: fixtureEvidence.milestone,
    issue: fixtureEvidence.issue,
    taskId: fixtureEvidence.taskId,
    lifecycleState: fixtureEvidence.lifecycleState,
    status: fixtureEvidence.status,
    category: fixtureEvidence.category,
    reportPath: fixtureEvidence.reportPath,
    summary: fixtureEvidence.summary
  });
  await assertRejected(report, /M5 DEV-LIVE evidence must come from mvp-e2e-live/);
});

test("rejects aggregator frontend fact that claims promotion authority", async () => {
  const report = cloneBaseAggregatorReport();
  report.latestFrontendDevFact.promotesM3M4M5 = true;
  await assertRejected(report, /latestFrontendDevFact\.promotesM3M4M5 must be false/);
});