Code Queue Review
111 lines
4.3 KiB
JavaScript
111 lines
4.3 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { spawnSync } from "node:child_process";
|
|
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { test } from "node:test";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const validator = path.join(repoRoot, "scripts/validate-dev-gate-report.mjs");
|
|
const baseReportPath = path.join(repoRoot, "reports/dev-gate/dev-cloud-workbench-live.json");
|
|
const baseReport = JSON.parse(await readFile(baseReportPath, "utf8"));
|
|
|
|
function cloneBaseReport() {
|
|
return JSON.parse(JSON.stringify(baseReport));
|
|
}
|
|
|
|
async function withReport(report, fn) {
|
|
const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-"));
|
|
const reportPath = path.join(directory, "report.json");
|
|
try {
|
|
await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
return await fn(reportPath);
|
|
} finally {
|
|
await rm(directory, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function runValidator(reportPath) {
|
|
return spawnSync(process.execPath, [validator, reportPath], {
|
|
cwd: repoRoot,
|
|
encoding: "utf8"
|
|
});
|
|
}
|
|
|
|
async function assertRejected(report, expectedMessage) {
|
|
await withReport(report, async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.notEqual(result.status, 0, "validator should reject stale deployed evidence");
|
|
assert.match(`${result.stdout}\n${result.stderr}`, expectedMessage);
|
|
assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /sk-[A-Za-z0-9._-]{8,}/u);
|
|
});
|
|
}
|
|
|
|
test("accepts the sanitized deployed workbench evidence contract", async () => {
|
|
await withReport(cloneBaseReport(), async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
|
});
|
|
});
|
|
|
|
test("rejects accepted workbench evidence when runtime identity is not observed", async () => {
|
|
const report = cloneBaseReport();
|
|
Object.assign(report.runtimeIdentity, {
|
|
status: "not_observed",
|
|
serviceId: "not_observed",
|
|
environment: "not_observed",
|
|
healthStatus: "not_observed",
|
|
commitId: "not_observed",
|
|
commitSource: "not_observed",
|
|
imageTag: "not_observed",
|
|
reason: "test fixture"
|
|
});
|
|
await assertRejected(report, /runtimeIdentity\.status/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence when runtime identity is stale", async () => {
|
|
const report = cloneBaseReport();
|
|
report.runtimeIdentity.observedAt = "2026-05-22T12:00:00.000Z";
|
|
await assertRejected(report, /runtimeIdentity\.observedAt must be fresh/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence that carries an allowed legacy port annotation", async () => {
|
|
const report = cloneBaseReport();
|
|
report.deprecatedEndpoints = [
|
|
{
|
|
endpoint: "http://74.48.78.17:6666",
|
|
status: "deprecated",
|
|
activeGreenEligible: false
|
|
}
|
|
];
|
|
report.checks[0].evidence.push("legacy probe http://74.48.78.17:6666/");
|
|
await assertRejected(report, /deprecated public endpoint evidence/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence with a hard-coded runtime commit source", async () => {
|
|
const report = cloneBaseReport();
|
|
report.runtimeIdentity.commitSource = "hard-coded";
|
|
await assertRejected(report, /runtimeIdentity\.commitSource/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence that mixes provider_unavailable blockers", async () => {
|
|
const report = cloneBaseReport();
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
journey.observations.networkEvents[0].body.error = {
|
|
code: "provider_unavailable",
|
|
missingEnv: ["OPENAI_API_KEY"]
|
|
};
|
|
await assertRejected(report, /provider_unavailable or OPENAI_API_KEY blockers/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => {
|
|
const report = cloneBaseReport();
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
journey.evidence = journey.evidence.map((item) => item === "model=gpt-5.5" ? "model=gpt-5" : item);
|
|
journey.observations.response.model = "gpt-5";
|
|
journey.observations.networkEvents[0].body.model = "gpt-5";
|
|
await assertRejected(report, /synthetic placeholder model/);
|
|
});
|