test: capture deployed workbench user journey evidence
This commit is contained in:
Code Queue Review
@@ -29,7 +29,8 @@ const issueFamily = Object.freeze({
|
||||
DEV_M5_GATE_REPORT: "pikasTech/HWLAB#39",
|
||||
DEV_BASE_IMAGE_PREFLIGHT: "pikasTech/HWLAB#40",
|
||||
DEV_EVIDENCE_BLOCKER_AGGREGATOR: "pikasTech/HWLAB#41",
|
||||
DEV_M5_GATE_AGGREGATOR_V2: "pikasTech/HWLAB#58"
|
||||
DEV_M5_GATE_AGGREGATOR_V2: "pikasTech/HWLAB#58",
|
||||
DEV_CLOUD_WORKBENCH_LIVE: "pikasTech/HWLAB#7"
|
||||
});
|
||||
const allowedIssues = new Set([
|
||||
contractIssue,
|
||||
@@ -115,6 +116,18 @@ const requiredDevM2Docs = [
|
||||
"docs/dev-acceptance-matrix.md",
|
||||
"fixtures/dev-deploy-smoke/dev-deploy-smoke.json"
|
||||
];
|
||||
const requiredDevCloudWorkbenchValidationCommands = [
|
||||
"node --check scripts/dev-cloud-workbench-smoke.mjs",
|
||||
"node --check scripts/src/dev-cloud-workbench-smoke-lib.mjs",
|
||||
"node scripts/dev-cloud-workbench-smoke.mjs --static",
|
||||
"node scripts/dev-cloud-workbench-smoke.mjs --live --url http://74.48.78.17:16666/ --report reports/dev-gate/dev-cloud-workbench-live.json",
|
||||
"node scripts/validate-dev-gate-report.mjs reports/dev-gate/dev-cloud-workbench-live.json"
|
||||
];
|
||||
const requiredDevCloudWorkbenchDocs = [
|
||||
"docs/cloud-web-workbench.md",
|
||||
"docs/reference/cloud-workbench.md",
|
||||
"docs/reference/code-agent-chat-readiness.md"
|
||||
];
|
||||
const reportFamilyTemplates = new Map([
|
||||
[
|
||||
"dev-gate-report-contract",
|
||||
@@ -456,6 +469,13 @@ async function validateReport(relativePath) {
|
||||
await validateDevM2Report(report, label);
|
||||
return;
|
||||
}
|
||||
if (
|
||||
report.issue === issueFamily.DEV_CLOUD_WORKBENCH_LIVE ||
|
||||
report.taskId === "dev-cloud-workbench-live"
|
||||
) {
|
||||
await validateDevCloudWorkbenchLiveReport(report, label);
|
||||
return;
|
||||
}
|
||||
|
||||
const template = reportFamilyTemplates.get(report.taskId);
|
||||
assert.ok(
|
||||
@@ -1274,6 +1294,167 @@ async function validateDevM2Report(report, label) {
|
||||
);
|
||||
}
|
||||
|
||||
async function validateDevCloudWorkbenchLiveReport(report, label) {
|
||||
for (const field of [
|
||||
"$schema",
|
||||
"$id",
|
||||
"reportVersion",
|
||||
"issue",
|
||||
"taskId",
|
||||
"commitId",
|
||||
"acceptanceLevel",
|
||||
"devOnly",
|
||||
"prodDisabled",
|
||||
"status",
|
||||
"generatedAt",
|
||||
"mode",
|
||||
"url",
|
||||
"evidenceLevel",
|
||||
"devLive",
|
||||
"sourceContract",
|
||||
"validationCommands",
|
||||
"localSmoke",
|
||||
"dryRun",
|
||||
"devPreconditions",
|
||||
"checks",
|
||||
"blockers",
|
||||
"safety"
|
||||
]) {
|
||||
assert.ok(Object.hasOwn(report, field), `${label} missing ${field}`);
|
||||
}
|
||||
|
||||
assertString(report.$schema, `${label}.$schema`);
|
||||
assertString(report.$id, `${label}.$id`);
|
||||
assert.equal(report.reportVersion, "v1", `${label}.reportVersion`);
|
||||
assert.equal(report.issue, issueFamily.DEV_CLOUD_WORKBENCH_LIVE, `${label}.issue`);
|
||||
assert.equal(report.taskId, "dev-cloud-workbench-live", `${label}.taskId`);
|
||||
assert.match(report.commitId, /^([a-f0-9]{7,40}|unknown)$/, `${label}.commitId`);
|
||||
assert.equal(report.acceptanceLevel, "dev_cloud_workbench_live", `${label}.acceptanceLevel`);
|
||||
assert.equal(report.devOnly, true, `${label}.devOnly`);
|
||||
assert.equal(report.prodDisabled, true, `${label}.prodDisabled`);
|
||||
assertStatus(report.status, `${label}.status`);
|
||||
assert.equal(report.status, report.blockers.length === 0 ? "pass" : "blocked", `${label}.status must match blockers`);
|
||||
assertTimestamp(report.generatedAt, `${label}.generatedAt`);
|
||||
assert.equal(report.mode, "live", `${label}.mode`);
|
||||
assert.equal(report.url, "http://74.48.78.17:16666/", `${label}.url`);
|
||||
assert.equal(report.evidenceLevel, report.status === "pass" ? "DEV-LIVE-BROWSER" : "BLOCKED", `${label}.evidenceLevel`);
|
||||
assert.equal(report.devLive, report.status === "pass", `${label}.devLive`);
|
||||
|
||||
assertObject(report.sourceContract, `${label}.sourceContract`);
|
||||
assertStatus(report.sourceContract.status, `${label}.sourceContract.status`);
|
||||
await assertDocumentSet(
|
||||
report.sourceContract.documents,
|
||||
`${label}.sourceContract.documents`,
|
||||
requiredDevCloudWorkbenchDocs
|
||||
);
|
||||
assertString(report.sourceContract.summary, `${label}.sourceContract.summary`);
|
||||
|
||||
assertStringArray(report.validationCommands, `${label}.validationCommands`, {
|
||||
minLength: requiredDevCloudWorkbenchValidationCommands.length
|
||||
});
|
||||
assertUnique(report.validationCommands, `${label}.validationCommands`);
|
||||
for (const requiredCommand of requiredDevCloudWorkbenchValidationCommands) {
|
||||
assert.ok(
|
||||
report.validationCommands.includes(requiredCommand),
|
||||
`${label}.validationCommands missing ${requiredCommand}`
|
||||
);
|
||||
}
|
||||
|
||||
for (const section of ["localSmoke", "dryRun", "devPreconditions"]) {
|
||||
assertObject(report[section], `${label}.${section}`);
|
||||
assertStatus(report[section].status, `${label}.${section}.status`);
|
||||
assertStringArray(report[section].commands, `${label}.${section}.commands`, { minLength: 1 });
|
||||
assertStringArray(report[section].evidence ?? [], `${label}.${section}.evidence`, { minLength: section === "devPreconditions" ? 0 : 1 });
|
||||
assertString(report[section].summary, `${label}.${section}.summary`);
|
||||
}
|
||||
assertStringArray(report.devPreconditions.requirements, `${label}.devPreconditions.requirements`, {
|
||||
minLength: 4
|
||||
});
|
||||
|
||||
assertArray(report.checks, `${label}.checks`);
|
||||
assert.ok(report.checks.length >= 3, `${label}.checks must include HTTP, DOM, and Code Agent journey checks`);
|
||||
assertUnique(report.checks.map((check) => check.id), `${label}.checks ids`);
|
||||
const checks = new Map(report.checks.map((check, index) => {
|
||||
const checkLabel = `${label}.checks[${index}]`;
|
||||
assertObject(check, checkLabel);
|
||||
for (const field of ["id", "status", "summary"]) {
|
||||
assert.ok(Object.hasOwn(check, field), `${checkLabel} missing ${field}`);
|
||||
}
|
||||
assertString(check.id, `${checkLabel}.id`);
|
||||
assertStatus(check.status, `${checkLabel}.status`);
|
||||
assertString(check.summary, `${checkLabel}.summary`);
|
||||
if (Object.hasOwn(check, "evidence")) {
|
||||
assertStringArray(check.evidence, `${checkLabel}.evidence`, { minLength: 1 });
|
||||
}
|
||||
if (Object.hasOwn(check, "observations")) {
|
||||
assertObject(check.observations, `${checkLabel}.observations`);
|
||||
}
|
||||
return [check.id, check];
|
||||
}));
|
||||
for (const requiredCheck of [
|
||||
"live-http-html",
|
||||
"live-browser-dom",
|
||||
"live-code-agent-browser-journey"
|
||||
]) {
|
||||
assert.ok(checks.has(requiredCheck), `${label}.checks missing ${requiredCheck}`);
|
||||
}
|
||||
|
||||
const httpCheck = checks.get("live-http-html");
|
||||
assert.equal(httpCheck.status, "pass", `${label}.live-http-html.status`);
|
||||
assert.ok(httpCheck.evidence.includes("HTTP 200"), `${label}.live-http-html.evidence missing HTTP 200`);
|
||||
assert.ok(httpCheck.evidence.includes("text/html; charset=utf-8"), `${label}.live-http-html.evidence missing content-type`);
|
||||
|
||||
const dom = checks.get("live-browser-dom").observations;
|
||||
assertObject(dom, `${label}.live-browser-dom.observations`);
|
||||
assert.equal(dom.title, "HWLAB 云工作台", `${label}.live-browser-dom.title`);
|
||||
assert.equal(dom.workspaceHidden, false, `${label}.live-browser-dom.workspaceHidden`);
|
||||
assert.equal(dom.gateHidden, true, `${label}.live-browser-dom.gateHidden`);
|
||||
assert.equal(dom.helpHidden, true, `${label}.live-browser-dom.helpHidden`);
|
||||
assertObject(dom.coreControlsVisible, `${label}.live-browser-dom.coreControlsVisible`);
|
||||
for (const control of ["commandInput", "commandSend", "agentChatStatus", "conversationList", "hardwareList"]) {
|
||||
assert.equal(dom.coreControlsVisible[control], true, `${label}.live-browser-dom.coreControlsVisible.${control}`);
|
||||
}
|
||||
|
||||
const journey = checks.get("live-code-agent-browser-journey");
|
||||
assert.equal(journey.status, report.status, `${label}.live-code-agent-browser-journey.status`);
|
||||
assertObject(journey.observations, `${label}.live-code-agent-browser-journey.observations`);
|
||||
assertObject(journey.observations.request, `${label}.live-code-agent-browser-journey.request`);
|
||||
assert.equal(journey.observations.request.method, "POST", `${label}.live-code-agent-browser-journey.request.method`);
|
||||
assert.equal(journey.observations.request.urlPath, "/v1/agent/chat", `${label}.live-code-agent-browser-journey.request.urlPath`);
|
||||
assertObject(journey.observations.response, `${label}.live-code-agent-browser-journey.response`);
|
||||
assert.equal(journey.observations.response.status, "completed", `${label}.live-code-agent-browser-journey.response.status`);
|
||||
for (const field of ["provider", "model", "backend", "traceId"]) {
|
||||
assertString(journey.observations.response[field], `${label}.live-code-agent-browser-journey.response.${field}`);
|
||||
}
|
||||
assert.equal(journey.observations.response.hasReply, true, `${label}.live-code-agent-browser-journey.response.hasReply`);
|
||||
assert.equal(journey.observations.response.error, null, `${label}.live-code-agent-browser-journey.response.error`);
|
||||
for (const forbiddenField of ["conversationId", "sessionId", "messageId", "reply"]) {
|
||||
assert.equal(
|
||||
Object.hasOwn(journey.observations.response, forbiddenField),
|
||||
false,
|
||||
`${label}.live-code-agent-browser-journey.response must not retain ${forbiddenField}`
|
||||
);
|
||||
}
|
||||
assertObject(journey.observations.ui, `${label}.live-code-agent-browser-journey.ui`);
|
||||
assert.equal(journey.observations.ui.agentChatStatus, "已回复", `${label}.live-code-agent-browser-journey.ui.agentChatStatus`);
|
||||
assert.equal(journey.observations.ui.completedMessageVisible, true, `${label}.live-code-agent-browser-journey.ui.completedMessageVisible`);
|
||||
assert.equal(journey.observations.ui.failedMessageVisible, false, `${label}.live-code-agent-browser-journey.ui.failedMessageVisible`);
|
||||
|
||||
assertBlockers(report.blockers, `${label}.blockers`);
|
||||
|
||||
assertObject(report.safety, `${label}.safety`);
|
||||
for (const field of ["prodTouched", "servicesRestarted", "heavyE2E", "hardwareWriteApis", "sourceIsDevLive"]) {
|
||||
assert.equal(report.safety[field], false, `${label}.safety.${field}`);
|
||||
}
|
||||
assert.equal(report.safety.liveMode, "browser-user-journey-with-code-agent-post", `${label}.safety.liveMode`);
|
||||
assert.deepEqual(
|
||||
report.safety.retainedApiFields,
|
||||
["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv"],
|
||||
`${label}.safety.retainedApiFields`
|
||||
);
|
||||
assertString(report.safety.statement, `${label}.safety.statement`);
|
||||
}
|
||||
|
||||
async function validateDevM3Report(report, label) {
|
||||
for (const field of [
|
||||
"$schema",
|
||||
|
||||
Reference in New Issue
Block a user