Lyon
84211e072f
Host commander merge after personal review. PR #225 is clean against current main and preserves the current-main-safe part of stale PR #213: when deployment identity preflight blocks, the Code Agent browser journey is recorded as blocked/not_sent instead of omitted or misread as a successful POST. This is validation/smoke hardening only; no live mutation, no Code Agent POST, no Secret access, and no M3/M4/M5 acceptance claim.
440 lines
16 KiB
JavaScript
440 lines
16 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import { spawnSync } from "node:child_process";
|
|
import { mkdtemp, readFile, rm, writeFile } from "node:fs/promises";
|
|
import { tmpdir } from "node:os";
|
|
import path from "node:path";
|
|
import { test } from "node:test";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const validator = path.join(repoRoot, "scripts/validate-dev-gate-report.mjs");
|
|
const baseReportPath = path.join(repoRoot, "reports/dev-gate/dev-cloud-workbench-live.json");
|
|
const baseAggregatorReportPath = path.join(repoRoot, "reports/dev-gate/dev-m5-gate-aggregator-v2.json");
|
|
const baseReport = JSON.parse(await readFile(baseReportPath, "utf8"));
|
|
const baseAggregatorReport = JSON.parse(await readFile(baseAggregatorReportPath, "utf8"));
|
|
|
|
function cloneBaseReport() {
|
|
return JSON.parse(JSON.stringify(baseReport));
|
|
}
|
|
|
|
function cloneBaseAggregatorReport() {
|
|
return JSON.parse(JSON.stringify(baseAggregatorReport));
|
|
}
|
|
|
|
function preflightBlockedReport() {
|
|
const report = cloneBaseReport();
|
|
report.status = "blocked";
|
|
report.evidenceLevel = "BLOCKED";
|
|
report.devLive = false;
|
|
report.generatedAt = report.runtimeIdentity.observedAt;
|
|
report.expectedRuntimeIdentity = {
|
|
status: "observed",
|
|
serviceId: "hwlab-cloud-api",
|
|
source: "deploy/artifact-catalog.dev.json+deploy/deploy.json",
|
|
commitId: "c7de474",
|
|
imageTag: "c7de474",
|
|
image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:c7de474",
|
|
summary: "Expected DEV runtime identity was derived from current source deploy desired state, not from live health."
|
|
};
|
|
report.deploymentIdentity = {
|
|
status: "pass",
|
|
expectedCommit: "c7de474",
|
|
expectedImageTag: "c7de474",
|
|
expectedSource: "deploy/artifact-catalog.dev.json+deploy/deploy.json",
|
|
observedRuntimeCommit: "c7de474",
|
|
observedImageTag: "c7de474",
|
|
summary: "Live API runtime identity matches the current source desired runtime identity.",
|
|
evidence: [
|
|
"expected=c7de474",
|
|
"expectedImageTag=c7de474",
|
|
"runtimeCommit=c7de474",
|
|
"imageTag=c7de474"
|
|
]
|
|
};
|
|
report.webAssetIdentity = {
|
|
status: "blocked",
|
|
assets: [
|
|
{
|
|
path: "index.html",
|
|
status: "mismatch",
|
|
httpStatus: 200,
|
|
contentType: "text/html; charset=utf-8",
|
|
sourceHash: "435edb7da5c539ff",
|
|
liveHash: "924f9d57e0bb61be",
|
|
sourceLength: 13844,
|
|
liveLength: 11375
|
|
}
|
|
],
|
|
mismatches: ["index.html"],
|
|
summary: "Deployment drift: live 16666 primary web assets differ from current source (index.html)."
|
|
};
|
|
report.checks = [
|
|
{
|
|
id: "live-runtime-current-main",
|
|
status: report.deploymentIdentity.status,
|
|
summary: report.deploymentIdentity.summary,
|
|
evidence: report.deploymentIdentity.evidence,
|
|
observations: report.deploymentIdentity
|
|
},
|
|
report.checks.find((check) => check.id === "live-http-html"),
|
|
{
|
|
id: "live-web-assets-current-main",
|
|
status: report.webAssetIdentity.status,
|
|
summary: report.webAssetIdentity.summary,
|
|
evidence: ["index.html: mismatch"],
|
|
observations: report.webAssetIdentity
|
|
},
|
|
{
|
|
id: "live-code-agent-browser-journey",
|
|
status: "blocked",
|
|
summary: "Browser Code Agent journey was not posted because deployed runtime/assets do not match current source identity.",
|
|
evidence: [
|
|
"POST /v1/agent/chat not sent",
|
|
"runtimeIdentity=pass",
|
|
"webAssetIdentity=blocked"
|
|
],
|
|
observations: {
|
|
request: {
|
|
method: "POST",
|
|
status: "not_sent",
|
|
urlPath: "/v1/agent/chat"
|
|
},
|
|
response: {
|
|
status: "failed",
|
|
provider: "not_observed",
|
|
model: "not_observed",
|
|
backend: "not_observed",
|
|
traceId: "not_observed",
|
|
hasReply: false,
|
|
error: {
|
|
code: "deployment_identity_preflight",
|
|
missingEnv: []
|
|
}
|
|
},
|
|
classification: {
|
|
status: "blocked",
|
|
reason: "deployment_identity_preflight",
|
|
codeAgentBrowserJourneySkipped: true
|
|
}
|
|
}
|
|
}
|
|
];
|
|
report.blockers = [
|
|
{
|
|
type: "observability_blocker",
|
|
scope: "live-web-assets-current-main",
|
|
status: "open",
|
|
summary: report.webAssetIdentity.summary
|
|
},
|
|
{
|
|
type: "observability_blocker",
|
|
scope: "live-code-agent-browser-journey",
|
|
status: "open",
|
|
summary: "Browser Code Agent journey was not posted because deployed runtime/assets do not match current source identity."
|
|
}
|
|
];
|
|
report.devPreconditions.status = "blocked";
|
|
report.devPreconditions.summary = "Deployed browser journey is blocked; see checks and blockers.";
|
|
report.safety.liveMode = "deployment-identity-preflight";
|
|
report.safety.codeAgentBrowserJourneySkipped = true;
|
|
report.safety.retainedApiFields = ["runtime.commitId", "runtime.imageTag", "web asset hash/status"];
|
|
report.safety.statement = "Live smoke blocked before the browser chat journey because deployed runtime/assets do not match the current source identity; it did not post Code Agent chat, call hardware write APIs, restart services, or read secret values.";
|
|
return report;
|
|
}
|
|
|
|
async function withReport(report, fn) {
|
|
const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-"));
|
|
const reportPath = path.join(directory, "report.json");
|
|
try {
|
|
await writeFile(reportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
return await fn(reportPath);
|
|
} finally {
|
|
await rm(directory, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function runValidator(reportPath) {
|
|
return spawnSync(process.execPath, [validator, reportPath], {
|
|
cwd: repoRoot,
|
|
encoding: "utf8"
|
|
});
|
|
}
|
|
|
|
async function assertRejected(report, expectedMessage) {
|
|
await withReport(report, async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.notEqual(result.status, 0, "validator should reject stale deployed evidence");
|
|
assert.match(`${result.stdout}\n${result.stderr}`, expectedMessage);
|
|
assert.doesNotMatch(`${result.stdout}\n${result.stderr}`, /sk-[A-Za-z0-9._-]{8,}/u);
|
|
});
|
|
}
|
|
|
|
test("accepts the sanitized deployed workbench evidence contract", async () => {
|
|
await withReport(cloneBaseReport(), async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
|
});
|
|
});
|
|
|
|
test("accepts preflight-blocked workbench evidence when Code Agent POST is marked not_sent", async () => {
|
|
await withReport(preflightBlockedReport(), async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
|
});
|
|
});
|
|
|
|
test("rejects preflight-blocked workbench evidence without not_sent Code Agent journey proof", async () => {
|
|
const report = preflightBlockedReport();
|
|
report.checks = report.checks.filter((check) => check.id !== "live-code-agent-browser-journey");
|
|
report.blockers = report.blockers.filter((blocker) => blocker.scope !== "live-code-agent-browser-journey");
|
|
await assertRejected(report, /checks missing live-code-agent-browser-journey/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence when runtime identity is not observed", async () => {
|
|
const report = cloneBaseReport();
|
|
Object.assign(report.runtimeIdentity, {
|
|
status: "not_observed",
|
|
serviceId: "not_observed",
|
|
environment: "not_observed",
|
|
healthStatus: "not_observed",
|
|
commitId: "not_observed",
|
|
commitSource: "not_observed",
|
|
imageTag: "not_observed",
|
|
reason: "test fixture"
|
|
});
|
|
await assertRejected(report, /runtimeIdentity\.status/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence when runtime identity is stale", async () => {
|
|
const report = cloneBaseReport();
|
|
report.runtimeIdentity.observedAt = "2026-05-22T12:00:00.000Z";
|
|
await assertRejected(report, /runtimeIdentity\.observedAt must be fresh/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence that carries an allowed legacy port annotation", async () => {
|
|
const report = cloneBaseReport();
|
|
report.deprecatedEndpoints = [
|
|
{
|
|
endpoint: "http://74.48.78.17:6666",
|
|
status: "deprecated",
|
|
activeGreenEligible: false
|
|
}
|
|
];
|
|
report.checks[0].evidence.push("legacy probe http://74.48.78.17:6666/");
|
|
await assertRejected(report, /deprecated public endpoint evidence/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence with a hard-coded runtime commit source", async () => {
|
|
const report = cloneBaseReport();
|
|
report.runtimeIdentity.commitSource = "hard-coded";
|
|
await assertRejected(report, /runtimeIdentity\.commitSource/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence that mixes provider_unavailable blockers", async () => {
|
|
const report = cloneBaseReport();
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
journey.observations.networkEvents[0].body.error = {
|
|
code: "provider_unavailable",
|
|
missingEnv: ["OPENAI_API_KEY"]
|
|
};
|
|
await assertRejected(report, /provider_unavailable, OPENAI_API_KEY, or providerStatus blockers/);
|
|
});
|
|
|
|
test("accepts blocked workbench evidence for provider HTTP 502 without promoting completion", async () => {
|
|
const report = cloneBaseReport();
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
report.status = "blocked";
|
|
report.evidenceLevel = "BLOCKED";
|
|
report.devLive = false;
|
|
report.devPreconditions.status = "blocked";
|
|
report.devPreconditions.summary = "Deployed browser journey is blocked; see checks and blockers.";
|
|
report.blockers = [
|
|
{
|
|
type: "runtime_blocker",
|
|
scope: "live-code-agent-browser-journey",
|
|
status: "open",
|
|
summary: "Code Agent provider returned provider_unavailable with provider HTTP 502."
|
|
}
|
|
];
|
|
report.safety.retainedApiFields = [
|
|
"status",
|
|
"provider",
|
|
"model",
|
|
"backend",
|
|
"traceId",
|
|
"hasReply",
|
|
"error.code",
|
|
"error.missingEnv",
|
|
"error.providerStatus"
|
|
];
|
|
journey.status = "blocked";
|
|
journey.summary = "Deployed browser journey reached Code Agent, but the provider returned a blocked/unavailable state instead of a completed reply.";
|
|
journey.evidence = [
|
|
"POST /v1/agent/chat HTTP 200",
|
|
"status=failed",
|
|
"provider=openai-responses",
|
|
"model=gpt-5.5",
|
|
"traceId=trc_provider_502",
|
|
"uiStatus=发送失败",
|
|
"blocker=provider-upstream"
|
|
];
|
|
journey.observations.request.status = 200;
|
|
journey.observations.response = {
|
|
status: "failed",
|
|
provider: "openai-responses",
|
|
model: "gpt-5.5",
|
|
backend: "hwlab-cloud-api/openai-responses",
|
|
traceId: "trc_provider_502",
|
|
hasReply: false,
|
|
error: {
|
|
code: "provider_unavailable",
|
|
missingEnv: [],
|
|
providerStatus: 502
|
|
}
|
|
};
|
|
journey.observations.classification = {
|
|
status: "blocked",
|
|
blocker: "provider-upstream",
|
|
providerStatus: 502,
|
|
reason: "Code Agent HTTP non-2xx status 502 means the upstream response is blocked, not completed"
|
|
};
|
|
journey.observations.ui.agentChatStatus = "发送失败";
|
|
journey.observations.ui.inputCleared = false;
|
|
journey.observations.ui.completedMessageVisible = false;
|
|
journey.observations.ui.failedMessageVisible = true;
|
|
journey.observations.networkEvents = [
|
|
{
|
|
method: "POST",
|
|
status: 200,
|
|
urlPath: "/v1/agent/chat",
|
|
body: journey.observations.response
|
|
}
|
|
];
|
|
|
|
await withReport(report, async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
|
});
|
|
});
|
|
|
|
test("rejects accepted workbench evidence that hides providerStatus 502 in network events", async () => {
|
|
const report = cloneBaseReport();
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
journey.observations.networkEvents[0].body.error = {
|
|
code: null,
|
|
missingEnv: [],
|
|
providerStatus: 502
|
|
};
|
|
await assertRejected(report, /providerStatus blockers/);
|
|
});
|
|
|
|
test("rejects accepted workbench evidence that uses the synthetic gpt-5 placeholder model", async () => {
|
|
const report = cloneBaseReport();
|
|
const journey = report.checks.find((check) => check.id === "live-code-agent-browser-journey");
|
|
journey.evidence = journey.evidence.map((item) => item === "model=gpt-5.5" ? "model=gpt-5" : item);
|
|
journey.observations.response.model = "gpt-5";
|
|
journey.observations.networkEvents[0].body.model = "gpt-5";
|
|
await assertRejected(report, /synthetic placeholder model/);
|
|
});
|
|
|
|
test("accepts the current blocked M5 aggregator non-promotion contract", async () => {
|
|
await withReport(cloneBaseAggregatorReport(), async (reportPath) => {
|
|
const result = runValidator(reportPath);
|
|
assert.equal(result.status, 0, result.stderr);
|
|
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
|
});
|
|
});
|
|
|
|
test("rejects aggregator DB acceptance promoted from frontend route evidence", async () => {
|
|
const report = cloneBaseAggregatorReport();
|
|
const check = report.dod.checks.find((item) => item.id === "cloud-api-db-ready");
|
|
Object.assign(check, {
|
|
status: "pass",
|
|
evidenceLevel: "DEV-LIVE",
|
|
summary: "Frontend route HTTP 200 proves DB ready."
|
|
});
|
|
Object.assign(report.currentDevLayering.dbLive, {
|
|
status: "pass",
|
|
evidenceLevel: "DEV-LIVE",
|
|
summary: "Promoted from active frontend/API route evidence."
|
|
});
|
|
await assertRejected(report, /DB DEV-LIVE acceptance source reports/);
|
|
});
|
|
|
|
test("rejects aggregator M3 acceptance promoted from source evidence", async () => {
|
|
const report = cloneBaseAggregatorReport();
|
|
const sourceEvidence = report.evidence.find((item) =>
|
|
item.milestone === "M3" && item.category === "hardware-loop-cardinality"
|
|
);
|
|
sourceEvidence.level = "DEV-LIVE";
|
|
report.levels["DEV-LIVE"].push({
|
|
milestone: "M3",
|
|
issue: sourceEvidence.issue,
|
|
taskId: sourceEvidence.taskId,
|
|
lifecycleState: "active",
|
|
status: sourceEvidence.status,
|
|
category: sourceEvidence.category,
|
|
reportPath: sourceEvidence.reportPath,
|
|
summary: sourceEvidence.summary
|
|
});
|
|
await assertRejected(report, /M3 DEV-LIVE evidence must come from hardware-loop-live/);
|
|
});
|
|
|
|
test("rejects aggregator M4 acceptance promoted from dry-run evidence", async () => {
|
|
const report = cloneBaseAggregatorReport();
|
|
const dryRunEvidence = report.evidence.find((item) =>
|
|
item.milestone === "M4" && item.category === "agent-loop-dry-run"
|
|
);
|
|
dryRunEvidence.level = "DEV-LIVE";
|
|
report.levels["DEV-LIVE"].push({
|
|
milestone: "M4",
|
|
issue: dryRunEvidence.issue,
|
|
taskId: dryRunEvidence.taskId,
|
|
lifecycleState: "active",
|
|
status: dryRunEvidence.status,
|
|
category: dryRunEvidence.category,
|
|
reportPath: dryRunEvidence.reportPath,
|
|
summary: dryRunEvidence.summary
|
|
});
|
|
await assertRejected(report, /M4 DEV-LIVE evidence must come from agent-loop-live-preflight/);
|
|
});
|
|
|
|
test("rejects aggregator M5 acceptance promoted from fixture evidence", async () => {
|
|
const report = cloneBaseAggregatorReport();
|
|
const fixtureEvidence = {
|
|
milestone: "M5",
|
|
issue: "pikasTech/HWLAB#39",
|
|
taskId: "dev-mvp-gate-report",
|
|
reportPath: "reports/dev-gate/dev-mvp-gate-report.json",
|
|
commitId: "fixture",
|
|
lifecycleState: "active",
|
|
level: "DEV-LIVE",
|
|
status: "pass",
|
|
category: "fixture-mvp-e2e",
|
|
evidence: ["fixture transcript marked green"],
|
|
commands: ["node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run"],
|
|
summary: "Fixture evidence was incorrectly promoted to M5 DEV-LIVE."
|
|
};
|
|
report.evidence.push(fixtureEvidence);
|
|
report.levels["DEV-LIVE"].push({
|
|
milestone: fixtureEvidence.milestone,
|
|
issue: fixtureEvidence.issue,
|
|
taskId: fixtureEvidence.taskId,
|
|
lifecycleState: fixtureEvidence.lifecycleState,
|
|
status: fixtureEvidence.status,
|
|
category: fixtureEvidence.category,
|
|
reportPath: fixtureEvidence.reportPath,
|
|
summary: fixtureEvidence.summary
|
|
});
|
|
await assertRejected(report, /M5 DEV-LIVE evidence must come from mvp-e2e-live/);
|
|
});
|
|
|
|
test("rejects aggregator frontend fact that claims promotion authority", async () => {
|
|
const report = cloneBaseAggregatorReport();
|
|
report.latestFrontendDevFact.promotesM3M4M5 = true;
|
|
await assertRejected(report, /latestFrontendDevFact\.promotesM3M4M5 must be false/);
|
|
});
|