908 lines
32 KiB
JavaScript
908 lines
32 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { spawnSync } from "node:child_process";
|
|
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { test } from "node:test";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const validator = path.join(repoRoot, "scripts/validate-dev-gate-report.mjs");
|
|
const baseReportPath = path.join(repoRoot, "reports/dev-gate/dev-cloud-workbench-live.json");
|
|
const baseAggregatorReportPath = path.join(repoRoot, "reports/dev-gate/dev-m5-gate-aggregator-v2.json");
|
|
const baseReport = JSON.parse(await readFile(baseReportPath, "utf8"));
|
|
const baseAggregatorReport = JSON.parse(await readFile(baseAggregatorReportPath, "utf8"));
|
|
|
|
function cloneBaseReport() {
|
|
return JSON.parse(JSON.stringify(baseReport));
|
|
}
|
|
|
|
function cloneBaseAggregatorReport() {
|
|
return JSON.parse(JSON.stringify(baseAggregatorReport));
|
|
}
|
|
|
|
function currentIdentityFields() {
|
|
return {
|
|
generatedAt: "2026-05-23T03:21:02.207Z",
|
|
sourceIdentity: {
|
|
status: "observed",
|
|
source: "git-head",
|
|
commitId: "7de6edd2c41f54e4265e822d2593fa67057fa41a",
|
|
shortCommitId: "7de6edd2c41f",
|
|
ref: "test/d601-dom-only-feedback-validation",
|
|
worktreeState: "clean",
|
|
dirty: false,
|
|
reportCommitId: "7de6edd2c41f",
|
|
summary: "Source identity was derived from the current worktree or source environment and is separate from the live runtime identity."
|
|
},
|
|
runtimeIdentity: {
|
|
status: "observed",
|
|
source: "health-live",
|
|
endpoint: "http://74.48.78.17:16667/health/live",
|
|
serviceId: "hwlab-cloud-api",
|
|
environment: "dev",
|
|
healthStatus: "ok",
|
|
ready: true,
|
|
commitId: "7de6edd",
|
|
commitSource: "runtime-env",
|
|
imageTag: "7de6edd",
|
|
observedAt: "2026-05-23T03:21:02.207Z",
|
|
runtime: {
|
|
durable: true,
|
|
ready: true,
|
|
status: "ok",
|
|
blocker: "none",
|
|
connection: {
|
|
queryAttempted: true,
|
|
queryResult: "ok"
|
|
}
|
|
},
|
|
readiness: {
|
|
ready: true,
|
|
status: "ok",
|
|
durability: {
|
|
ready: true,
|
|
status: "ready",
|
|
blocker: "none",
|
|
blockedLayer: "none",
|
|
queryResult: "ok"
|
|
}
|
|
},
|
|
blockerCodes: [],
|
|
summary: "Live runtime identity was observed through the existing read-only health endpoint and is not inferred from source git HEAD."
|
|
},
|
|
expectedRuntimeIdentity: {
|
|
status: "observed",
|
|
serviceId: "hwlab-cloud-api",
|
|
source: "deploy/artifact-catalog.dev.json+deploy/deploy.json",
|
|
commitId: "7de6edd",
|
|
imageTag: "7de6edd",
|
|
image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:7de6edd",
|
|
summary: "Expected DEV runtime identity was derived from current source deploy desired state, not from live health."
|
|
},
|
|
deploymentIdentity: {
|
|
status: "pass",
|
|
expectedCommit: "7de6edd",
|
|
expectedReportCommitId: "7de6edd2c41f",
|
|
expectedImageTag: "7de6edd",
|
|
expectedSource: "deploy/artifact-catalog.dev.json+deploy/deploy.json",
|
|
observedRuntimeCommit: "7de6edd",
|
|
observedImageTag: "7de6edd",
|
|
evidence: [
|
|
"expected=7de6edd",
|
|
"expectedImageTag=7de6edd",
|
|
"runtimeCommit=7de6edd",
|
|
"imageTag=7de6edd"
|
|
],
|
|
summary: "Live API runtime identity matches the current source desired runtime identity."
|
|
},
|
|
webAssetIdentity: {
|
|
status: "pass",
|
|
assets: [
|
|
{
|
|
path: "index.html",
|
|
status: "match",
|
|
httpStatus: 200,
|
|
contentType: "text/html; charset=utf-8",
|
|
sourceHash: "435edb7da5c539ff",
|
|
liveHash: "435edb7da5c539ff",
|
|
sourceLength: 13844,
|
|
liveLength: 13844
|
|
},
|
|
{
|
|
path: "styles.css",
|
|
status: "match",
|
|
httpStatus: 200,
|
|
contentType: "text/css; charset=utf-8",
|
|
sourceHash: "95574eda6090e34b",
|
|
liveHash: "95574eda6090e34b",
|
|
sourceLength: 21184,
|
|
liveLength: 21184
|
|
},
|
|
{
|
|
path: "app.mjs",
|
|
status: "match",
|
|
httpStatus: 200,
|
|
contentType: "text/javascript; charset=utf-8",
|
|
sourceHash: "3dd09b19fec790a4",
|
|
liveHash: "3dd09b19fec790a4",
|
|
sourceLength: 67912,
|
|
liveLength: 67912
|
|
}
|
|
],
|
|
mismatches: [],
|
|
summary: "Live 16666 primary web assets match the current clean source files."
|
|
}
|
|
};
|
|
}
|
|
|
|
function applyCurrentIdentity(report) {
|
|
const identity = currentIdentityFields();
|
|
const apiRuntimeReadiness = {
|
|
status: "pass",
|
|
healthStatus: "ok",
|
|
ready: true,
|
|
runtimeDurable: true,
|
|
runtimeReady: true,
|
|
durabilityReady: true,
|
|
durableBlocked: false,
|
|
blockerCodes: [],
|
|
evidence: [
|
|
"api.status=ok",
|
|
"ready=true",
|
|
"runtime.durable=true",
|
|
"runtime.ready=true",
|
|
"durability.ready=true"
|
|
],
|
|
summary: "Live API health and runtime durability are ready for browser journey evidence."
|
|
};
|
|
|
|
Object.assign(report, {
|
|
commitId: identity.sourceIdentity.reportCommitId,
|
|
generatedAt: identity.generatedAt,
|
|
sourceIdentity: identity.sourceIdentity,
|
|
runtimeIdentity: identity.runtimeIdentity,
|
|
expectedRuntimeIdentity: identity.expectedRuntimeIdentity,
|
|
deploymentIdentity: identity.deploymentIdentity,
|
|
webAssetIdentity: identity.webAssetIdentity
|
|
});
|
|
|
|
const runtimeCheck = report.checks.find((check) => check.id === "live-runtime-current-main");
|
|
if (runtimeCheck) {
|
|
Object.assign(runtimeCheck, {
|
|
status: identity.deploymentIdentity.status,
|
|
summary: identity.deploymentIdentity.summary,
|
|
evidence: identity.deploymentIdentity.evidence,
|
|
observations: identity.deploymentIdentity
|
|
});
|
|
}
|
|
|
|
const apiRuntimeReadinessCheck = report.checks.find((check) => check.id === "live-api-runtime-readiness");
|
|
if (apiRuntimeReadinessCheck) {
|
|
Object.assign(apiRuntimeReadinessCheck, {
|
|
status: apiRuntimeReadiness.status,
|
|
summary: apiRuntimeReadiness.summary,
|
|
evidence: apiRuntimeReadiness.evidence,
|
|
observations: apiRuntimeReadiness
|
|
});
|
|
}
|
|
|
|
const webAssetCheck = report.checks.find((check) => check.id === "live-web-assets-current-main");
|
|
if (webAssetCheck) {
|
|
Object.assign(webAssetCheck, {
|
|
status: identity.webAssetIdentity.status,
|
|
summary: identity.webAssetIdentity.summary,
|
|
evidence: identity.webAssetIdentity.assets.map((asset) => `${asset.path}: ${asset.status}`),
|
|
observations: identity.webAssetIdentity
|
|
});
|
|
}
|
|
}
|
|
|
|
function acceptedDomOnlyReport() {
|
|
const report = cloneBaseReport();
|
|
applyCurrentIdentity(report);
|
|
report.status = "pass";
|
|
report.mode = "dom-only";
|
|
report.evidenceLevel = "DEV-LIVE-DOM-READONLY";
|
|
report.devLive = false;
|
|
report.blockers = [];
|
|
report.devPreconditions.status = "pass";
|
|
report.devPreconditions.summary = "Deployed browser DOM was inspected read-only; Code Agent chat was not posted.";
|
|
report.safety.liveMode = "dom-only-readonly-no-code-agent-post";
|
|
report.safety.codeAgentBrowserJourneySkipped = true;
|
|
report.safety.codeAgentPostSent = false;
|
|
return report;
|
|
}
|
|
|
|
function acceptedCodeAgentResponse() {
|
|
return {
|
|
status: "completed",
|
|
provider: "openai-responses",
|
|
model: "gpt-5.5",
|
|
backend: "hwlab-cloud-api/openai-responses",
|
|
traceId: "trc_completed",
|
|
hasReply: true,
|
|
error: null
|
|
};
|
|
}
|
|
|
|
function acceptedLiveReport() {
|
|
const report = cloneBaseReport();
|
|
applyCurrentIdentity(report);
|
|
report.status = "pass";
|
|
report.mode = "live";
|
|
report.evidenceLevel = "DEV-LIVE-BROWSER";
|
|
report.devLive = true;
|
|
report.blockers = [];
|
|
report.devPreconditions.status = "pass";
|
|
report.devPreconditions.summary = "Deployed 16666 browser journey and same-origin Code Agent chat completed.";
|
|
report.safety.liveMode = "browser-user-journey-with-code-agent-post";
|
|
delete report.safety.codeAgentBrowserJourneySkipped;
|
|
delete report.safety.codeAgentPostSent;
|
|
report.safety.retainedApiFields = [
|
|
"status",
|
|
"provider",
|
|
"model",
|
|
"backend",
|
|
"traceId",
|
|
"hasReply",
|
|
"error.code",
|
|
"error.missingEnv",
|
|
"error.providerStatus"
|
|
];
|
|
report.safety.statement = "Live smoke opens the deployed workbench and sends one controlled Code Agent message; it does not call hardware write APIs and must not be used to infer M3 DEV-LIVE hardware-loop acceptance.";
|
|
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
const response = acceptedCodeAgentResponse();
|
|
Object.assign(journey, {
|
|
status: "pass",
|
|
summary: "Deployed browser journey opened the workbench, sent a Code Agent message, and observed a completed UI response.",
|
|
evidence: [
|
|
"POST /v1/agent/chat HTTP 200",
|
|
"status=completed",
|
|
"provider=openai-responses",
|
|
"model=gpt-5.5",
|
|
"traceId=trc_completed",
|
|
"uiStatus=DEV-LIVE 回复",
|
|
"blocker=none"
|
|
],
|
|
observations: {
|
|
request: {
|
|
method: "POST",
|
|
status: 200,
|
|
urlPath: "/v1/agent/chat"
|
|
},
|
|
response,
|
|
classification: {
|
|
status: "pass"
|
|
},
|
|
ui: {
|
|
agentChatStatus: "DEV-LIVE 回复",
|
|
inputCleared: true,
|
|
completedMessageVisible: true,
|
|
failedMessageVisible: false
|
|
},
|
|
networkEvents: [
|
|
{
|
|
method: "POST",
|
|
status: 200,
|
|
urlPath: "/v1/agent/chat",
|
|
body: JSON.parse(JSON.stringify(response))
|
|
}
|
|
]
|
|
}
|
|
});
|
|
|
|
return report;
|
|
}
|
|
|
|
function fullAcceptanceReport() {
|
|
return acceptedLiveReport();
|
|
}
|
|
|
|
function preflightBlockedReport() {
|
|
const report = cloneBaseReport();
|
|
report.status = "blocked";
|
|
report.evidenceLevel = "BLOCKED";
|
|
report.devLive = false;
|
|
report.generatedAt = report.runtimeIdentity.observedAt;
|
|
report.expectedRuntimeIdentity = {
|
|
status: "observed",
|
|
serviceId: "hwlab-cloud-api",
|
|
source: "deploy/artifact-catalog.dev.json+deploy/deploy.json",
|
|
commitId: "c7de474",
|
|
imageTag: "c7de474",
|
|
image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:c7de474",
|
|
summary: "Expected DEV runtime identity was derived from current source deploy desired state, not from live health."
|
|
};
|
|
report.deploymentIdentity = {
|
|
status: "pass",
|
|
expectedCommit: "c7de474",
|
|
expectedImageTag: "c7de474",
|
|
expectedSource: "deploy/artifact-catalog.dev.json+deploy/deploy.json",
|
|
observedRuntimeCommit: "c7de474",
|
|
observedImageTag: "c7de474",
|
|
summary: "Live API runtime identity matches the current source desired runtime identity.",
|
|
evidence: [
|
|
"expected=c7de474",
|
|
"expectedImageTag=c7de474",
|
|
"runtimeCommit=c7de474",
|
|
"imageTag=c7de474"
|
|
]
|
|
};
|
|
report.webAssetIdentity = {
|
|
status: "blocked",
|
|
assets: [
|
|
{
|
|
path: "index.html",
|
|
status: "mismatch",
|
|
httpStatus: 200,
|
|
contentType: "text/html; charset=utf-8",
|
|
sourceHash: "435edb7da5c539ff",
|
|
liveHash: "924f9d57e0bb61be",
|
|
sourceLength: 13844,
|
|
liveLength: 11375
|
|
}
|
|
],
|
|
mismatches: ["index.html"],
|
|
summary: "Deployment drift: live 16666 primary web assets differ from current source (index.html)."
|
|
};
|
|
report.checks = [
|
|
{
|
|
id: "live-runtime-current-main",
|
|
status: report.deploymentIdentity.status,
|
|
summary: report.deploymentIdentity.summary,
|
|
evidence: report.deploymentIdentity.evidence,
|
|
observations: report.deploymentIdentity
|
|
},
|
|
report.checks.find((check) => check.id === "live-http-html"),
|
|
{
|
|
id: "live-web-assets-current-main",
|
|
status: report.webAssetIdentity.status,
|
|
summary: report.webAssetIdentity.summary,
|
|
evidence: ["index.html: mismatch"],
|
|
observations: report.webAssetIdentity
|
|
},
|
|
{
|
|
id: "live-code-agent-browser-journey",
|
|
status: "blocked",
|
|
summary: "Browser Code Agent journey was not posted because deployed runtime/assets do not match current source identity.",
|
|
evidence: [
|
|
"POST /v1/agent/chat not sent",
|
|
"runtimeIdentity=pass",
|
|
"webAssetIdentity=blocked"
|
|
],
|
|
observations: {
|
|
request: {
|
|
method: "POST",
|
|
status: "not_sent",
|
|
urlPath: "/v1/agent/chat"
|
|
},
|
|
response: {
|
|
status: "failed",
|
|
provider: "not_observed",
|
|
model: "not_observed",
|
|
backend: "not_observed",
|
|
traceId: "not_observed",
|
|
hasReply: false,
|
|
error: {
|
|
code: "deployment_identity_preflight",
|
|
missingEnv: []
|
|
}
|
|
},
|
|
classification: {
|
|
status: "blocked",
|
|
reason: "deployment_identity_preflight",
|
|
codeAgentBrowserJourneySkipped: true
|
|
}
|
|
}
|
|
}
|
|
];
|
|
report.blockers = [
|
|
{
|
|
type: "observability_blocker",
|
|
scope: "live-web-assets-current-main",
|
|
status: "open",
|
|
summary: report.webAssetIdentity.summary
|
|
},
|
|
{
|
|
type: "observability_blocker",
|
|
scope: "live-code-agent-browser-journey",
|
|
status: "open",
|
|
summary: "Browser Code Agent journey was not posted because deployed runtime/assets do not match current source identity."
|
|
}
|
|
];
|
|
report.devPreconditions.status = "blocked";
|
|
report.devPreconditions.summary = "Deployed browser journey is blocked; see checks and blockers.";
|
|
report.safety.liveMode = "deployment-identity-preflight";
|
|
report.safety.codeAgentBrowserJourneySkipped = true;
|
|
report.safety.retainedApiFields = ["runtime.commitId", "runtime.imageTag", "web asset hash/status"];
|
|
report.safety.statement = "Live smoke blocked before the browser chat journey because deployed runtime/assets do not match the current source identity; it did not post Code Agent chat, call hardware write APIs, restart services, or read secret values.";
|
|
return report;
|
|
}
|
|
|
|
async function withReport(report, fn) {
|
|
const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-"));
|
|
const reportPath = path.join(directory, "report.json");
|
|
try {
|
|
await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
return await fn(reportPath);
|
|
} finally {
|
|
await rm(directory, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function runValidator(reportPath) {
|
|
return spawnSync(process.execPath, [validator, reportPath], {
|
|
cwd: repoRoot,
|
|
encoding: "utf8"
|
|
});
|
|
}
|
|
|
|
async function assertRejected(report, expectedMessage) {
|
|
await withReport(report, async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.notEqual(result.status, 0, "validator should reject stale deployed evidence");
|
|
assert.match(`${result.stdout}\n${result.stderr}`, expectedMessage);
|
|
assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /sk-[A-Za-z0-9._-]{8,}/u);
|
|
});
|
|
}
|
|
|
|
test("accepts the sanitized deployed workbench evidence contract", async () => {
|
|
await withReport(cloneBaseReport(), async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
|
});
|
|
});
|
|
|
|
test("accepts full workbench acceptance only when API health and runtime durability are ready", async () => {
|
|
await withReport(fullAcceptanceReport(), async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
|
});
|
|
});
|
|
|
|
test("rejects full workbench acceptance when API health is degraded", async () => {
|
|
const report = fullAcceptanceReport();
|
|
report.runtimeIdentity.healthStatus = "degraded";
|
|
report.checks.find((check) => check.id === "live-api-runtime-readiness").observations.healthStatus = "degraded";
|
|
await assertRejected(report, /degraded API or runtime durability blocker cannot be summarized as full DEV-LIVE accepted/);
|
|
});
|
|
|
|
test("rejects full workbench acceptance when runtime durability query is blocked", async () => {
|
|
const report = fullAcceptanceReport();
|
|
Object.assign(report.runtimeIdentity.runtime, {
|
|
durable: false,
|
|
ready: false,
|
|
status: "degraded",
|
|
blocker: "runtime_durable_adapter_query_blocked",
|
|
connection: {
|
|
queryAttempted: true,
|
|
queryResult: "query_blocked"
|
|
}
|
|
});
|
|
Object.assign(report.runtimeIdentity.readiness.durability, {
|
|
ready: false,
|
|
status: "blocked",
|
|
blocker: "runtime_durable_adapter_query_blocked",
|
|
blockedLayer: "durability_query",
|
|
queryResult: "query_blocked"
|
|
});
|
|
report.runtimeIdentity.blockerCodes = ["runtime_durable_adapter_query_blocked"];
|
|
await assertRejected(report, /degraded API or runtime durability blocker cannot be summarized as full DEV-LIVE accepted/);
|
|
});
|
|
|
|
test("rejects degraded workbench evidence with full-acceptance wording", async () => {
|
|
const report = acceptedLiveReport();
|
|
report.status = "degraded";
|
|
report.evidenceLevel = "DEV-LIVE-BROWSER-DEGRADED";
|
|
report.devLive = false;
|
|
report.devPreconditions.status = "degraded";
|
|
Object.assign(report.runtimeIdentity, {
|
|
healthStatus: "degraded",
|
|
ready: false
|
|
});
|
|
Object.assign(report.runtimeIdentity.runtime, {
|
|
durable: false,
|
|
ready: false,
|
|
status: "degraded",
|
|
blocker: "runtime_durable_adapter_query_blocked",
|
|
connection: {
|
|
queryAttempted: true,
|
|
queryResult: "query_blocked"
|
|
}
|
|
});
|
|
Object.assign(report.runtimeIdentity.readiness.durability, {
|
|
ready: false,
|
|
status: "blocked",
|
|
blocker: "runtime_durable_adapter_query_blocked",
|
|
blockedLayer: "durability_query",
|
|
queryResult: "query_blocked"
|
|
});
|
|
report.runtimeIdentity.blockerCodes = ["runtime_durable_adapter_query_blocked"];
|
|
const readiness = report.checks.find((check) => check.id === "live-api-runtime-readiness");
|
|
Object.assign(readiness, {
|
|
status: "degraded",
|
|
summary: "Live API is reachable but degraded/read-only; runtime durability is not ready, so this cannot be full DEV-LIVE acceptance.",
|
|
evidence: [
|
|
"api.status=degraded",
|
|
"ready=false",
|
|
"runtime.durable=false",
|
|
"runtime.ready=false",
|
|
"durability.ready=false",
|
|
"runtime_durable_adapter_query_blocked",
|
|
"blocker=runtime_durable_adapter_query_blocked"
|
|
],
|
|
observations: {
|
|
status: "degraded",
|
|
healthStatus: "degraded",
|
|
ready: false,
|
|
runtimeDurable: false,
|
|
runtimeReady: false,
|
|
durabilityReady: false,
|
|
durableBlocked: true,
|
|
blockerCodes: ["runtime_durable_adapter_query_blocked"],
|
|
evidence: [
|
|
"api.status=degraded",
|
|
"ready=false",
|
|
"runtime.durable=false",
|
|
"runtime.ready=false",
|
|
"durability.ready=false",
|
|
"runtime_durable_adapter_query_blocked",
|
|
"blocker=runtime_durable_adapter_query_blocked"
|
|
],
|
|
summary: "Live API is reachable but degraded/read-only; runtime durability is not ready, so this cannot be full DEV-LIVE acceptance."
|
|
}
|
|
});
|
|
report.reportLifecycle.summary = "Current deployed browser user journey report for the 16666 Cloud Workbench; deployed UI usable in degraded/read-only mode and does not substitute for M3/M4/M5 DEV-LIVE acceptance.";
|
|
report.devPreconditions.summary = "Deployed UI usable in degraded/read-only mode; M3/M4/M5 DEV-LIVE accepted.";
|
|
report.safety.statement = "Degraded/read-only mode only; Full DEV-LIVE accepted.";
|
|
await assertRejected(report, /degraded API\/runtime report must not claim full M3\/M4\/M5 DEV-LIVE acceptance/);
|
|
});
|
|
|
|
test("accepts preflight-blocked workbench evidence when Code Agent POST is marked not_sent", async () => {
|
|
await withReport(preflightBlockedReport(), async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
|
});
|
|
});
|
|
|
|
test("rejects preflight-blocked workbench evidence without not_sent Code Agent journey proof", async () => {
|
|
const report = preflightBlockedReport();
|
|
report.checks = report.checks.filter((check) => check.id !== "live-code-agent-browser-journey");
|
|
report.blockers = report.blockers.filter((blocker) => blocker.scope !== "live-code-agent-browser-journey");
|
|
await assertRejected(report, /checks missing live-code-agent-browser-journey/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence when runtime identity is not observed", async () => {
|
|
const report = acceptedDomOnlyReport();
|
|
Object.assign(report.runtimeIdentity, {
|
|
status: "not_observed",
|
|
serviceId: "not_observed",
|
|
environment: "not_observed",
|
|
healthStatus: "not_observed",
|
|
commitId: "not_observed",
|
|
commitSource: "not_observed",
|
|
imageTag: "not_observed",
|
|
reason: "test fixture"
|
|
});
|
|
await assertRejected(report, /runtimeIdentity\.status/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence when runtime identity is stale", async () => {
|
|
const report = acceptedDomOnlyReport();
|
|
report.runtimeIdentity.observedAt = "2026-05-22T12:00:00.000Z";
|
|
await assertRejected(report, /runtimeIdentity\.observedAt must be fresh/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence that carries an allowed legacy port annotation", async () => {
|
|
const report = acceptedDomOnlyReport();
|
|
report.deprecatedEndpoints = [
|
|
{
|
|
endpoint: "http://74.48.78.17:6666",
|
|
status: "deprecated",
|
|
activeGreenEligible: false
|
|
}
|
|
];
|
|
report.checks.find((check) => check.id === "live-http-html").evidence.push("legacy probe http://74.48.78.17:6666/");
|
|
await assertRejected(report, /deprecated public endpoint evidence/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence with a hard-coded runtime commit source", async () => {
|
|
const report = acceptedDomOnlyReport();
|
|
report.runtimeIdentity.commitSource = "hard-coded";
|
|
await assertRejected(report, /runtimeIdentity\.commitSource/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence that mixes provider_unavailable blockers", async () => {
|
|
const report = acceptedLiveReport();
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
journey.observations.networkEvents[0].body.error = {
|
|
code: "provider_unavailable",
|
|
missingEnv: ["OPENAI_API_KEY"]
|
|
};
|
|
await assertRejected(report, /provider_unavailable, OPENAI_API_KEY, or providerStatus blockers/);
|
|
});
|
|
|
|
function applyProviderBlockedJourney(report, { uiLabel = "发送失败" } = {}) {
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
report.status = "blocked";
|
|
report.evidenceLevel = "BLOCKED";
|
|
report.devLive = false;
|
|
report.devPreconditions.status = "blocked";
|
|
report.devPreconditions.summary = "Deployed browser journey is blocked; see checks and blockers.";
|
|
report.blockers = [
|
|
{
|
|
type: "runtime_blocker",
|
|
scope: "live-code-agent-browser-journey",
|
|
status: "open",
|
|
summary: "Code Agent provider returned provider_unavailable with provider HTTP 502."
|
|
}
|
|
];
|
|
report.safety.retainedApiFields = [
|
|
"status",
|
|
"provider",
|
|
"model",
|
|
"backend",
|
|
"traceId",
|
|
"hasReply",
|
|
"error.code",
|
|
"error.missingEnv",
|
|
"error.providerStatus"
|
|
];
|
|
journey.status = "blocked";
|
|
journey.summary = "Deployed browser journey reached Code Agent, but the provider returned a blocked/unavailable state instead of a completed reply.";
|
|
journey.evidence = [
|
|
"POST /v1/agent/chat HTTP 200",
|
|
"status=failed",
|
|
"provider=openai-responses",
|
|
"model=gpt-5.5",
|
|
"traceId=trc_provider_502",
|
|
`uiStatus=${uiLabel}`,
|
|
"blocker=provider-upstream"
|
|
];
|
|
journey.observations.request.status = 200;
|
|
journey.observations.response = {
|
|
status: "failed",
|
|
provider: "openai-responses",
|
|
model: "gpt-5.5",
|
|
backend: "hwlab-cloud-api/openai-responses",
|
|
traceId: "trc_provider_502",
|
|
hasReply: false,
|
|
error: {
|
|
code: "provider_unavailable",
|
|
missingEnv: [],
|
|
providerStatus: 502
|
|
}
|
|
};
|
|
journey.observations.classification = {
|
|
status: "blocked",
|
|
blocker: "provider-upstream",
|
|
providerStatus: 502,
|
|
reason: "Code Agent HTTP non-2xx status 502 means the upstream response is blocked, not completed"
|
|
};
|
|
journey.observations.ui.agentChatStatus = uiLabel;
|
|
journey.observations.ui.inputCleared = false;
|
|
journey.observations.ui.completedMessageVisible = false;
|
|
journey.observations.ui.failedMessageVisible = true;
|
|
journey.observations.networkEvents = [
|
|
{
|
|
method: "POST",
|
|
status: 200,
|
|
urlPath: "/v1/agent/chat",
|
|
body: journey.observations.response
|
|
}
|
|
];
|
|
return report;
|
|
}
|
|
|
|
test("accepts blocked workbench evidence for provider HTTP 502 without promoting completion", async () => {
|
|
const report = applyProviderBlockedJourney(acceptedLiveReport());
|
|
|
|
await withReport(report, async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
|
});
|
|
});
|
|
|
|
test("accepts sanitized 服务受阻 provider blocker wording only as blocked evidence", async () => {
|
|
const report = applyProviderBlockedJourney(acceptedLiveReport(), { uiLabel: "服务受阻" });
|
|
|
|
await withReport(report, async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
|
});
|
|
});
|
|
|
|
test("rejects sanitized 服务受阻 provider blocker evidence promoted to full workbench acceptance", async () => {
|
|
const report = fullAcceptanceReport();
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
journey.status = "pass";
|
|
journey.summary = "Incorrectly promoted provider blocker evidence.";
|
|
journey.evidence = [
|
|
"POST /v1/agent/chat HTTP 200",
|
|
"status=failed",
|
|
"provider=openai-responses",
|
|
"model=gpt-5.5",
|
|
"traceId=trc_provider_502",
|
|
"uiStatus=服务受阻",
|
|
"blocker=provider-upstream"
|
|
];
|
|
journey.observations.response = {
|
|
status: "failed",
|
|
provider: "openai-responses",
|
|
model: "gpt-5.5",
|
|
backend: "hwlab-cloud-api/openai-responses",
|
|
traceId: "trc_provider_502",
|
|
hasReply: false,
|
|
error: {
|
|
code: "provider_unavailable",
|
|
missingEnv: [],
|
|
providerStatus: 502
|
|
}
|
|
};
|
|
journey.observations.classification = {
|
|
status: "blocked",
|
|
blocker: "provider-upstream",
|
|
providerStatus: 502,
|
|
reason: "Code Agent HTTP non-2xx status 502 means the upstream response is blocked, not completed"
|
|
};
|
|
journey.observations.ui.agentChatStatus = "服务受阻";
|
|
journey.observations.ui.inputCleared = false;
|
|
journey.observations.ui.completedMessageVisible = false;
|
|
journey.observations.ui.failedMessageVisible = true;
|
|
journey.observations.networkEvents = [
|
|
{
|
|
method: "POST",
|
|
status: 200,
|
|
urlPath: "/v1/agent/chat",
|
|
body: journey.observations.response
|
|
}
|
|
];
|
|
|
|
await assertRejected(report, /response\.status|provider_unavailable, OPENAI_API_KEY, or providerStatus blockers/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence that hides providerStatus 502 in network events", async () => {
|
|
const report = acceptedLiveReport();
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
journey.observations.networkEvents[0].body.error = {
|
|
code: null,
|
|
missingEnv: [],
|
|
providerStatus: 502
|
|
};
|
|
await assertRejected(report, /providerStatus blockers/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => {
|
|
const report = acceptedLiveReport();
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
journey.evidence = journey.evidence.map((item) => item === "model=gpt-5.5" ? "model=gpt-5" : item);
|
|
journey.observations.response.model = "gpt-5";
|
|
journey.observations.networkEvents[0].body.model = "gpt-5";
|
|
await assertRejected(report, /synthetic placeholder model/);
|
|
});
|
|
|
|
test("accepts the current blocked M5 aggregator non-promotion contract", async () => {
|
|
await withReport(cloneBaseAggregatorReport(), async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
|
});
|
|
});
|
|
|
|
test("rejects aggregator DB acceptance promoted from frontend route evidence", async () => {
|
|
const report = cloneBaseAggregatorReport();
|
|
const m4Source = report.sourceReports.devM4Agent;
|
|
m4Source.path = "reports/dev-gate/dev-edge-health.json";
|
|
const check = report.dod.checks.find((item) => item.id === "cloud-api-db-ready");
|
|
Object.assign(check, {
|
|
status: "pass",
|
|
evidenceLevel: "DEV-LIVE",
|
|
summary: "Frontend route HTTP 200 proves DB ready."
|
|
});
|
|
Object.assign(report.currentDevLayering.dbLive, {
|
|
status: "pass",
|
|
evidenceLevel: "DEV-LIVE",
|
|
summary: "Promoted from active frontend/API route evidence."
|
|
});
|
|
await assertRejected(report, /DB DEV-LIVE acceptance source reports/);
|
|
});
|
|
|
|
test("rejects aggregator stale DB runtime-env blocker when DB live is pass", async () => {
|
|
const report = cloneBaseAggregatorReport();
|
|
report.blockers.push({
|
|
id: "dev-gate-preflight:cloud-api-db-health-gate",
|
|
priority: "P1",
|
|
type: "runtime_blocker",
|
|
scope: "cloud-api-db-health-gate",
|
|
status: "open",
|
|
source: "reports/dev-gate/dev-preflight-report.json",
|
|
sourceIssue: "pikasTech/HWLAB#34",
|
|
summary: "cloud-api DB runtime env is not ready; missing HWLAB_CLOUD_DB_URL, HWLAB_CLOUD_DB_SSL_MODE.",
|
|
unblockOrder: 5,
|
|
unblocks: ["pikasTech/HWLAB#34", "pikasTech/HWLAB#33", "pikasTech/HWLAB#39"],
|
|
rationale: "Stale DB runtime-env blocker fixture."
|
|
});
|
|
await assertRejected(report, /stale DB live\/runtime-env blocker/);
|
|
});
|
|
|
|
test("rejects aggregator M3 acceptance promoted from source evidence", async () => {
|
|
const report = cloneBaseAggregatorReport();
|
|
const sourceEvidence = report.evidence.find((item) =>
|
|
item.milestone === "M3" && item.category === "hardware-loop-cardinality"
|
|
);
|
|
sourceEvidence.level = "DEV-LIVE";
|
|
report.levels["DEV-LIVE"].push({
|
|
milestone: "M3",
|
|
issue: sourceEvidence.issue,
|
|
taskId: sourceEvidence.taskId,
|
|
lifecycleState: "active",
|
|
status: sourceEvidence.status,
|
|
category: sourceEvidence.category,
|
|
reportPath: sourceEvidence.reportPath,
|
|
summary: sourceEvidence.summary
|
|
});
|
|
await assertRejected(report, /M3 DEV-LIVE evidence must come from hardware-loop-live/);
|
|
});
|
|
|
|
test("rejects aggregator M4 acceptance promoted from dry-run evidence", async () => {
|
|
const report = cloneBaseAggregatorReport();
|
|
const dryRunEvidence = report.evidence.find((item) =>
|
|
item.milestone === "M4" && item.category === "agent-loop-dry-run"
|
|
);
|
|
dryRunEvidence.level = "DEV-LIVE";
|
|
report.levels["DEV-LIVE"].push({
|
|
milestone: "M4",
|
|
issue: dryRunEvidence.issue,
|
|
taskId: dryRunEvidence.taskId,
|
|
lifecycleState: "active",
|
|
status: dryRunEvidence.status,
|
|
category: dryRunEvidence.category,
|
|
reportPath: dryRunEvidence.reportPath,
|
|
summary: dryRunEvidence.summary
|
|
});
|
|
await assertRejected(report, /M4 DEV-LIVE evidence must come from agent-loop-live-preflight/);
|
|
});
|
|
|
|
test("rejects aggregator M5 acceptance promoted from fixture evidence", async () => {
|
|
const report = cloneBaseAggregatorReport();
|
|
const fixtureEvidence = {
|
|
milestone: "M5",
|
|
issue: "pikasTech/HWLAB#39",
|
|
taskId: "dev-mvp-gate-report",
|
|
reportPath: "reports/dev-gate/dev-mvp-gate-report.json",
|
|
commitId: "fixture",
|
|
lifecycleState: "active",
|
|
level: "DEV-LIVE",
|
|
status: "pass",
|
|
category: "fixture-mvp-e2e",
|
|
evidence: ["fixture transcript marked green"],
|
|
commands: ["node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run"],
|
|
summary: "Fixture evidence was incorrectly promoted to M5 DEV-LIVE."
|
|
};
|
|
report.evidence.push(fixtureEvidence);
|
|
report.levels["DEV-LIVE"].push({
|
|
milestone: fixtureEvidence.milestone,
|
|
issue: fixtureEvidence.issue,
|
|
taskId: fixtureEvidence.taskId,
|
|
lifecycleState: fixtureEvidence.lifecycleState,
|
|
status: fixtureEvidence.status,
|
|
category: fixtureEvidence.category,
|
|
reportPath: fixtureEvidence.reportPath,
|
|
summary: fixtureEvidence.summary
|
|
});
|
|
await assertRejected(report, /M5 DEV-LIVE evidence must come from mvp-e2e-live/);
|
|
});
|
|
|
|
test("rejects aggregator frontend fact that claims promotion authority", async () => {
|
|
const report = cloneBaseAggregatorReport();
|
|
report.latestFrontendDevFact.promotesM3M4M5 = true;
|
|
await assertRejected(report, /latestFrontendDevFact\.promotesM3M4M5 must be false/);
|
|
});
|