import assert from "node:assert/strict"; import { spawnSync } from "node:child_process"; import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import path from "node:path"; import { test } from "node:test"; import { fileURLToPath } from "node:url"; const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), ".."); const validator = path.join(repoRoot, "scripts/validate-dev-gate-report.mjs"); const baseReportPath = path.join(repoRoot, "reports/dev-gate/dev-cloud-workbench-live.json"); const baseAggregatorReportPath = path.join(repoRoot, "reports/dev-gate/dev-m5-gate-aggregator-v2.json"); const baseReport = JSON.parse(await readFile(baseReportPath, "utf8")); const baseAggregatorReport = JSON.parse(await readFile(baseAggregatorReportPath, "utf8")); function cloneBaseReport() { return JSON.parse(JSON.stringify(baseReport)); } function cloneBaseAggregatorReport() { return JSON.parse(JSON.stringify(baseAggregatorReport)); } async function withReport(report, fn) { const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-")); const reportPath = path.join(directory, "report.json"); try { await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`); return await fn(reportPath); } finally { await rm(directory, { recursive: true, force: true }); } } function runValidator(reportPath) { return spawnSync(process.execPath, [validator, reportPath], { cwd: repoRoot, encoding: "utf8" }); } async function assertRejected(report, expectedMessage) { await withReport(report, async (reportPath) => { const result = runValidator(reportPath); assert.notEqual(result.status, 0, "validator should reject stale deployed evidence"); assert.match(`${result.stdout}\n${result.stderr}`, expectedMessage); assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /sk-[A-Za-z0-9._-]{8,}/u); }); } test("accepts the sanitized deployed workbench evidence contract", async () => { await withReport(cloneBaseReport(), async (reportPath) => { const result = runValidator(reportPath); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /validated 1 dev-gate report JSON file/); }); }); test("rejects accepted workbench evidence when runtime identity is not observed", async () => { const report = cloneBaseReport(); Object.assign(report.runtimeIdentity, { status: "not_observed", serviceId: "not_observed", environment: "not_observed", healthStatus: "not_observed", commitId: "not_observed", commitSource: "not_observed", imageTag: "not_observed", reason: "test fixture" }); await assertRejected(report, /runtimeIdentity\.status/); }); test("rejects accepted workbench evidence when runtime identity is stale", async () => { const report = cloneBaseReport(); report.runtimeIdentity.observedAt = "2026-05-22T12:00:00.000Z"; await assertRejected(report, /runtimeIdentity\.observedAt must be fresh/); }); test("rejects accepted workbench evidence that carries an allowed legacy port annotation", async () => { const report = cloneBaseReport(); report.deprecatedEndpoints = [ { endpoint: "http://74.48.78.17:6666", status: "deprecated", activeGreenEligible: false } ]; report.checks[0].evidence.push("legacy probe http://74.48.78.17:6666/"); await assertRejected(report, /deprecated public endpoint evidence/); }); test("rejects accepted workbench evidence with a hard-coded runtime commit source", async () => { const report = cloneBaseReport(); report.runtimeIdentity.commitSource = "hard-coded"; await assertRejected(report, /runtimeIdentity\.commitSource/); }); test("rejects accepted workbench evidence that mixes provider_unavailable blockers", async () => { const report = cloneBaseReport(); const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey"); journey.observations.networkEvents[0].body.error = { code: "provider_unavailable", missingEnv: ["OPENAI_API_KEY"] }; await assertRejected(report, /provider_unavailable, OPENAI_API_KEY, or providerStatus blockers/); }); test("accepts blocked workbench evidence for provider HTTP 502 without promoting completion", async () => { const report = cloneBaseReport(); const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey"); report.status = "blocked"; report.evidenceLevel = "BLOCKED"; report.devLive = false; report.devPreconditions.status = "blocked"; report.devPreconditions.summary = "Deployed browser journey is blocked; see checks and blockers."; report.blockers = [ { type: "runtime_blocker", scope: "live-code-agent-browser-journey", status: "open", summary: "Code Agent provider returned provider_unavailable with provider HTTP 502." } ]; report.safety.retainedApiFields = [ "status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv", "error.providerStatus" ]; journey.status = "blocked"; journey.summary = "Deployed browser journey reached Code Agent, but the provider returned a blocked/unavailable state instead of a completed reply."; journey.evidence = [ "POST /v1/agent/chat HTTP 200", "status=failed", "provider=openai-responses", "model=gpt-5.5", "traceId=trc_provider_502", "uiStatus=发送失败", "blocker=provider-upstream" ]; journey.observations.request.status = 200; journey.observations.response = { status: "failed", provider: "openai-responses", model: "gpt-5.5", backend: "hwlab-cloud-api/openai-responses", traceId: "trc_provider_502", hasReply: false, error: { code: "provider_unavailable", missingEnv: [], providerStatus: 502 } }; journey.observations.classification = { status: "blocked", blocker: "provider-upstream", providerStatus: 502, reason: "Code Agent HTTP non-2xx status 502 means the upstream response is blocked, not completed" }; journey.observations.ui.agentChatStatus = "发送失败"; journey.observations.ui.inputCleared = false; journey.observations.ui.completedMessageVisible = false; journey.observations.ui.failedMessageVisible = true; journey.observations.networkEvents = [ { method: "POST", status: 200, urlPath: "/v1/agent/chat", body: journey.observations.response } ]; await withReport(report, async (reportPath) => { const result = runValidator(reportPath); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /validated 1 dev-gate report JSON file/); }); }); test("rejects accepted workbench evidence that hides providerStatus 502 in network events", async () => { const report = cloneBaseReport(); const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey"); journey.observations.networkEvents[0].body.error = { code: null, missingEnv: [], providerStatus: 502 }; await assertRejected(report, /providerStatus blockers/); }); test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => { const report = cloneBaseReport(); const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey"); journey.evidence = journey.evidence.map((item) => item === "model=gpt-5.5" ? "model=gpt-5" : item); journey.observations.response.model = "gpt-5"; journey.observations.networkEvents[0].body.model = "gpt-5"; await assertRejected(report, /synthetic placeholder model/); }); test("accepts the current blocked M5 aggregator non-promotion contract", async () => { await withReport(cloneBaseAggregatorReport(), async (reportPath) => { const result = runValidator(reportPath); assert.equal(result.status, 0, result.stderr); assert.match(result.stdout, /validated 1 dev-gate report JSON file/); }); }); test("rejects aggregator DB acceptance promoted from frontend route evidence", async () => { const report = cloneBaseAggregatorReport(); const check = report.dod.checks.find((item) => item.id === "cloud-api-db-ready"); Object.assign(check, { status: "pass", evidenceLevel: "DEV-LIVE", summary: "Frontend route HTTP 200 proves DB ready." }); Object.assign(report.currentDevLayering.dbLive, { status: "pass", evidenceLevel: "DEV-LIVE", summary: "Promoted from active frontend/API route evidence." }); await assertRejected(report, /DB DEV-LIVE acceptance source reports/); }); test("rejects aggregator M3 acceptance promoted from source evidence", async () => { const report = cloneBaseAggregatorReport(); const sourceEvidence = report.evidence.find((item) => item.milestone === "M3" && item.category === "hardware-loop-cardinality" ); sourceEvidence.level = "DEV-LIVE"; report.levels["DEV-LIVE"].push({ milestone: "M3", issue: sourceEvidence.issue, taskId: sourceEvidence.taskId, lifecycleState: "active", status: sourceEvidence.status, category: sourceEvidence.category, reportPath: sourceEvidence.reportPath, summary: sourceEvidence.summary }); await assertRejected(report, /M3 DEV-LIVE evidence must come from hardware-loop-live/); }); test("rejects aggregator M4 acceptance promoted from dry-run evidence", async () => { const report = cloneBaseAggregatorReport(); const dryRunEvidence = report.evidence.find((item) => item.milestone === "M4" && item.category === "agent-loop-dry-run" ); dryRunEvidence.level = "DEV-LIVE"; report.levels["DEV-LIVE"].push({ milestone: "M4", issue: dryRunEvidence.issue, taskId: dryRunEvidence.taskId, lifecycleState: "active", status: dryRunEvidence.status, category: dryRunEvidence.category, reportPath: dryRunEvidence.reportPath, summary: dryRunEvidence.summary }); await assertRejected(report, /M4 DEV-LIVE evidence must come from agent-loop-live-preflight/); }); test("rejects aggregator M5 acceptance promoted from fixture evidence", async () => { const report = cloneBaseAggregatorReport(); const fixtureEvidence = { milestone: "M5", issue: "pikasTech/HWLAB#39", taskId: "dev-mvp-gate-report", reportPath: "reports/dev-gate/dev-mvp-gate-report.json", commitId: "fixture", lifecycleState: "active", level: "DEV-LIVE", status: "pass", category: "fixture-mvp-e2e", evidence: ["fixture transcript marked green"], commands: ["node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run"], summary: "Fixture evidence was incorrectly promoted to M5 DEV-LIVE." }; report.evidence.push(fixtureEvidence); report.levels["DEV-LIVE"].push({ milestone: fixtureEvidence.milestone, issue: fixtureEvidence.issue, taskId: fixtureEvidence.taskId, lifecycleState: fixtureEvidence.lifecycleState, status: fixtureEvidence.status, category: fixtureEvidence.category, reportPath: fixtureEvidence.reportPath, summary: fixtureEvidence.summary }); await assertRejected(report, /M5 DEV-LIVE evidence must come from mvp-e2e-live/); }); test("rejects aggregator frontend fact that claims promotion authority", async () => { const report = cloneBaseAggregatorReport(); report.latestFrontendDevFact.promotesM3M4M5 = true; await assertRejected(report, /latestFrontendDevFact\.promotesM3M4M5 must be false/); });