test: record blocked workbench journey preflight
This commit is contained in:
Code Queue Review
@@ -408,6 +408,34 @@ async function runLiveSmoke(args) {
|
||||
|
||||
const identityBlocked = deploymentIdentity.status !== "pass" || webAssetIdentity.status !== "pass";
|
||||
if (identityBlocked) {
|
||||
addCheck(
|
||||
checks,
|
||||
blockers,
|
||||
"live-code-agent-browser-journey",
|
||||
"blocked",
|
||||
"Browser Code Agent journey was not posted because deployed runtime/assets do not match current source identity.",
|
||||
{
|
||||
blocker: "observability_blocker",
|
||||
evidence: [
|
||||
"POST /v1/agent/chat not sent",
|
||||
`runtimeIdentity=${deploymentIdentity.status}`,
|
||||
`webAssetIdentity=${webAssetIdentity.status}`
|
||||
],
|
||||
observations: {
|
||||
request: {
|
||||
method: "POST",
|
||||
status: "not_sent",
|
||||
urlPath: "/v1/agent/chat"
|
||||
},
|
||||
response: blockedAgentChatResponse("deployment_identity_preflight"),
|
||||
classification: {
|
||||
status: "blocked",
|
||||
reason: "deployment_identity_preflight",
|
||||
codeAgentBrowserJourneySkipped: true
|
||||
}
|
||||
}
|
||||
}
|
||||
);
|
||||
return baseReport({
|
||||
mode: "live",
|
||||
url: args.url,
|
||||
@@ -1724,6 +1752,21 @@ function sanitizeAgentChatBody(body) {
|
||||
};
|
||||
}
|
||||
|
||||
function blockedAgentChatResponse(code) {
|
||||
return {
|
||||
status: "failed",
|
||||
provider: "not_observed",
|
||||
model: "not_observed",
|
||||
backend: "not_observed",
|
||||
traceId: "not_observed",
|
||||
hasReply: false,
|
||||
error: {
|
||||
code,
|
||||
missingEnv: []
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
export async function runDevCloudWorkbenchLocalAgentFixtureSmoke() {
|
||||
let chromium;
|
||||
try {
|
||||
|
||||
@@ -21,6 +21,127 @@ function cloneBaseAggregatorReport() {
|
||||
return JSON.parse(JSON.stringify(baseAggregatorReport));
|
||||
}
|
||||
|
||||
function preflightBlockedReport() {
|
||||
const report = cloneBaseReport();
|
||||
report.status = "blocked";
|
||||
report.evidenceLevel = "BLOCKED";
|
||||
report.devLive = false;
|
||||
report.generatedAt = report.runtimeIdentity.observedAt;
|
||||
report.expectedRuntimeIdentity = {
|
||||
status: "observed",
|
||||
serviceId: "hwlab-cloud-api",
|
||||
source: "deploy/artifact-catalog.dev.json+deploy/deploy.json",
|
||||
commitId: "c7de474",
|
||||
imageTag: "c7de474",
|
||||
image: "127.0.0.1:5000/hwlab/hwlab-cloud-api:c7de474",
|
||||
summary: "Expected DEV runtime identity was derived from current source deploy desired state, not from live health."
|
||||
};
|
||||
report.deploymentIdentity = {
|
||||
status: "pass",
|
||||
expectedCommit: "c7de474",
|
||||
expectedImageTag: "c7de474",
|
||||
expectedSource: "deploy/artifact-catalog.dev.json+deploy/deploy.json",
|
||||
observedRuntimeCommit: "c7de474",
|
||||
observedImageTag: "c7de474",
|
||||
summary: "Live API runtime identity matches the current source desired runtime identity.",
|
||||
evidence: [
|
||||
"expected=c7de474",
|
||||
"expectedImageTag=c7de474",
|
||||
"runtimeCommit=c7de474",
|
||||
"imageTag=c7de474"
|
||||
]
|
||||
};
|
||||
report.webAssetIdentity = {
|
||||
status: "blocked",
|
||||
assets: [
|
||||
{
|
||||
path: "index.html",
|
||||
status: "mismatch",
|
||||
httpStatus: 200,
|
||||
contentType: "text/html; charset=utf-8",
|
||||
sourceHash: "435edb7da5c539ff",
|
||||
liveHash: "924f9d57e0bb61be",
|
||||
sourceLength: 13844,
|
||||
liveLength: 11375
|
||||
}
|
||||
],
|
||||
mismatches: ["index.html"],
|
||||
summary: "Deployment drift: live 16666 primary web assets differ from current source (index.html)."
|
||||
};
|
||||
report.checks = [
|
||||
{
|
||||
id: "live-runtime-current-main",
|
||||
status: report.deploymentIdentity.status,
|
||||
summary: report.deploymentIdentity.summary,
|
||||
evidence: report.deploymentIdentity.evidence,
|
||||
observations: report.deploymentIdentity
|
||||
},
|
||||
report.checks.find((check) => check.id === "live-http-html"),
|
||||
{
|
||||
id: "live-web-assets-current-main",
|
||||
status: report.webAssetIdentity.status,
|
||||
summary: report.webAssetIdentity.summary,
|
||||
evidence: ["index.html: mismatch"],
|
||||
observations: report.webAssetIdentity
|
||||
},
|
||||
{
|
||||
id: "live-code-agent-browser-journey",
|
||||
status: "blocked",
|
||||
summary: "Browser Code Agent journey was not posted because deployed runtime/assets do not match current source identity.",
|
||||
evidence: [
|
||||
"POST /v1/agent/chat not sent",
|
||||
"runtimeIdentity=pass",
|
||||
"webAssetIdentity=blocked"
|
||||
],
|
||||
observations: {
|
||||
request: {
|
||||
method: "POST",
|
||||
status: "not_sent",
|
||||
urlPath: "/v1/agent/chat"
|
||||
},
|
||||
response: {
|
||||
status: "failed",
|
||||
provider: "not_observed",
|
||||
model: "not_observed",
|
||||
backend: "not_observed",
|
||||
traceId: "not_observed",
|
||||
hasReply: false,
|
||||
error: {
|
||||
code: "deployment_identity_preflight",
|
||||
missingEnv: []
|
||||
}
|
||||
},
|
||||
classification: {
|
||||
status: "blocked",
|
||||
reason: "deployment_identity_preflight",
|
||||
codeAgentBrowserJourneySkipped: true
|
||||
}
|
||||
}
|
||||
}
|
||||
];
|
||||
report.blockers = [
|
||||
{
|
||||
type: "observability_blocker",
|
||||
scope: "live-web-assets-current-main",
|
||||
status: "open",
|
||||
summary: report.webAssetIdentity.summary
|
||||
},
|
||||
{
|
||||
type: "observability_blocker",
|
||||
scope: "live-code-agent-browser-journey",
|
||||
status: "open",
|
||||
summary: "Browser Code Agent journey was not posted because deployed runtime/assets do not match current source identity."
|
||||
}
|
||||
];
|
||||
report.devPreconditions.status = "blocked";
|
||||
report.devPreconditions.summary = "Deployed browser journey is blocked; see checks and blockers.";
|
||||
report.safety.liveMode = "deployment-identity-preflight";
|
||||
report.safety.codeAgentBrowserJourneySkipped = true;
|
||||
report.safety.retainedApiFields = ["runtime.commitId", "runtime.imageTag", "web asset hash/status"];
|
||||
report.safety.statement = "Live smoke blocked before the browser chat journey because deployed runtime/assets do not match the current source identity; it did not post Code Agent chat, call hardware write APIs, restart services, or read secret values.";
|
||||
return report;
|
||||
}
|
||||
|
||||
async function withReport(report, fn) {
|
||||
const directory = await mkdtemp(path.join(tmpdir(), "hwlab-dev-report-guard-"));
|
||||
const reportPath = path.join(directory, "report.json");
|
||||
@@ -56,6 +177,21 @@ test("accepts the sanitized deployed workbench evidence contract", async () => {
|
||||
});
|
||||
});
|
||||
|
||||
test("accepts preflight-blocked workbench evidence when Code Agent POST is marked not_sent", async () => {
|
||||
await withReport(preflightBlockedReport(), async (reportPath) => {
|
||||
const result = runValidator(reportPath);
|
||||
assert.equal(result.status, 0, result.stderr);
|
||||
assert.match(result.stdout, /validated 1 dev-gate report JSON file/);
|
||||
});
|
||||
});
|
||||
|
||||
test("rejects preflight-blocked workbench evidence without not_sent Code Agent journey proof", async () => {
|
||||
const report = preflightBlockedReport();
|
||||
report.checks = report.checks.filter((check) => check.id !== "live-code-agent-browser-journey");
|
||||
report.blockers = report.blockers.filter((blocker) => blocker.scope !== "live-code-agent-browser-journey");
|
||||
await assertRejected(report, /checks missing live-code-agent-browser-journey/);
|
||||
});
|
||||
|
||||
test("rejects accepted workbench evidence when runtime identity is not observed", async () => {
|
||||
const report = cloneBaseReport();
|
||||
Object.assign(report.runtimeIdentity, {
|
||||
|
||||
@@ -41,6 +41,7 @@ const requiredMilestoneIds = ["M0", "M1", "M2", "M3", "M4", "M5"];
|
||||
const evidenceLevels = new Set(["SOURCE", "LOCAL", "DRY-RUN", "DEV-LIVE", "BLOCKED"]);
|
||||
const devCloudWorkbenchRuntimeIdentityMaxSkewMs = 15 * 60 * 1000;
|
||||
const syntheticAcceptedCodeAgentModels = new Set(["gpt-5"]);
|
||||
const deploymentPreflightCodeAgentErrorCode = "deployment_identity_preflight";
|
||||
const requiredValidationCommands = [
|
||||
"node --check scripts/validate-dev-gate-report.mjs",
|
||||
"node scripts/validate-dev-gate-report.mjs"
|
||||
@@ -1536,15 +1537,18 @@ function assertDevCloudWorkbenchDeploymentPreflight(report, label, checks) {
|
||||
for (const requiredCheck of [
|
||||
"live-runtime-current-main",
|
||||
"live-http-html",
|
||||
"live-web-assets-current-main"
|
||||
"live-web-assets-current-main",
|
||||
"live-code-agent-browser-journey"
|
||||
]) {
|
||||
assert.ok(checks.has(requiredCheck), `${label}.checks missing ${requiredCheck}`);
|
||||
}
|
||||
|
||||
const runtimeCheck = checks.get("live-runtime-current-main");
|
||||
const webAssetCheck = checks.get("live-web-assets-current-main");
|
||||
const journey = checks.get("live-code-agent-browser-journey");
|
||||
assert.ok(["pass", "blocked"].includes(runtimeCheck.status), `${label}.live-runtime-current-main.status`);
|
||||
assert.ok(["pass", "blocked"].includes(webAssetCheck.status), `${label}.live-web-assets-current-main.status`);
|
||||
assert.equal(journey.status, "blocked", `${label}.live-code-agent-browser-journey.status`);
|
||||
assert.ok(
|
||||
runtimeCheck.status === "blocked" || webAssetCheck.status === "blocked",
|
||||
`${label}.deployment preflight requires runtime or web asset drift`
|
||||
@@ -1596,6 +1600,7 @@ function assertDevCloudWorkbenchDeploymentPreflight(report, label, checks) {
|
||||
report.webAssetIdentity,
|
||||
`${label}.live-web-assets-current-main.observations`
|
||||
);
|
||||
assertDevCloudWorkbenchDeploymentPreflightJourney(report, label, journey);
|
||||
|
||||
assertBlockers(report.blockers, `${label}.blockers`);
|
||||
if (runtimeCheck.status === "blocked") {
|
||||
@@ -1624,6 +1629,53 @@ function assertDevCloudWorkbenchDeploymentPreflight(report, label, checks) {
|
||||
assertString(report.safety.statement, `${label}.safety.statement`);
|
||||
}
|
||||
|
||||
function assertDevCloudWorkbenchDeploymentPreflightJourney(report, label, journey) {
|
||||
assertStringArray(journey.evidence, `${label}.live-code-agent-browser-journey.evidence`, { minLength: 3 });
|
||||
assert.ok(
|
||||
journey.evidence.includes("POST /v1/agent/chat not sent"),
|
||||
`${label}.live-code-agent-browser-journey.evidence missing not_sent proof`
|
||||
);
|
||||
assertObject(journey.observations, `${label}.live-code-agent-browser-journey.observations`);
|
||||
assertObject(journey.observations.request, `${label}.live-code-agent-browser-journey.request`);
|
||||
assert.equal(journey.observations.request.method, "POST", `${label}.live-code-agent-browser-journey.request.method`);
|
||||
assert.equal(journey.observations.request.status, "not_sent", `${label}.live-code-agent-browser-journey.request.status`);
|
||||
assert.equal(journey.observations.request.urlPath, "/v1/agent/chat", `${label}.live-code-agent-browser-journey.request.urlPath`);
|
||||
assertObject(journey.observations.response, `${label}.live-code-agent-browser-journey.response`);
|
||||
assert.equal(journey.observations.response.status, "failed", `${label}.live-code-agent-browser-journey.response.status`);
|
||||
assert.equal(journey.observations.response.hasReply, false, `${label}.live-code-agent-browser-journey.response.hasReply`);
|
||||
assertObject(journey.observations.response.error, `${label}.live-code-agent-browser-journey.response.error`);
|
||||
assert.equal(
|
||||
journey.observations.response.error.code,
|
||||
deploymentPreflightCodeAgentErrorCode,
|
||||
`${label}.live-code-agent-browser-journey.response.error.code`
|
||||
);
|
||||
for (const field of ["provider", "model", "backend", "traceId"]) {
|
||||
assert.equal(journey.observations.response[field], "not_observed", `${label}.live-code-agent-browser-journey.response.${field}`);
|
||||
}
|
||||
assertObject(journey.observations.classification, `${label}.live-code-agent-browser-journey.classification`);
|
||||
assert.equal(journey.observations.classification.status, "blocked", `${label}.live-code-agent-browser-journey.classification.status`);
|
||||
assert.equal(
|
||||
journey.observations.classification.reason,
|
||||
deploymentPreflightCodeAgentErrorCode,
|
||||
`${label}.live-code-agent-browser-journey.classification.reason`
|
||||
);
|
||||
assert.equal(
|
||||
journey.observations.classification.codeAgentBrowserJourneySkipped,
|
||||
true,
|
||||
`${label}.live-code-agent-browser-journey.classification.codeAgentBrowserJourneySkipped`
|
||||
);
|
||||
const journeyText = JSON.stringify(journey.observations);
|
||||
assert.equal(
|
||||
/"status"\s*:\s*200|completed|provider_unavailable|OPENAI_API_KEY/u.test(journeyText),
|
||||
false,
|
||||
`${label}.live-code-agent-browser-journey must not retain sent, completed, or secret-dependent evidence during deployment preflight`
|
||||
);
|
||||
assert.ok(
|
||||
report.blockers.some((blocker) => blocker.scope === "live-code-agent-browser-journey"),
|
||||
`${label}.blockers must include live-code-agent-browser-journey`
|
||||
);
|
||||
}
|
||||
|
||||
function assertDevCloudWorkbenchIdentity(report, label) {
|
||||
assertObject(report.sourceIdentity, `${label}.sourceIdentity`);
|
||||
for (const field of ["status", "source", "commitId", "shortCommitId", "ref", "worktreeState", "reportCommitId", "summary"]) {
|
||||
|
||||
Reference in New Issue
Block a user