Code Queue Review
2936 lines
134 KiB
JavaScript
2936 lines
134 KiB
JavaScript
#!/usr/bin/env node
|
|
import assert from "node:assert/strict";
|
|
import { access, readdir, readFile } from "node:fs/promises";
|
|
import { constants as fsConstants } from "node:fs";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import {
|
|
ACTIVE_DEV_BROWSER_ENDPOINT,
|
|
ACTIVE_DEV_PUBLIC_ENDPOINT,
|
|
REPORT_LIFECYCLE_VERSION,
|
|
deprecatedPublicEndpoints,
|
|
findDeprecatedPublicEndpoints,
|
|
findForbiddenActiveDeprecatedEndpoints,
|
|
reportIsHistorical
|
|
} from "../internal/dev-report-lifecycle.mjs";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..");
|
|
const reportsDir = path.join(repoRoot, "reports/dev-gate");
|
|
|
|
const contractIssue = "pikasTech/HWLAB#31";
|
|
const issueFamily = Object.freeze({
|
|
DEV_DEPLOY_APPLY: "pikasTech/HWLAB#33",
|
|
DEV_GATE_PREFLIGHT: "pikasTech/HWLAB#34",
|
|
DEV_ARTIFACT_PUBLISH: "pikasTech/HWLAB#35",
|
|
DEV_EDGE_HEALTH: "pikasTech/HWLAB#36",
|
|
DEV_M4_AGENT_LOOP: "pikasTech/HWLAB#37",
|
|
DEV_M3_HARDWARE_LOOP: "pikasTech/HWLAB#38",
|
|
DEV_M5_GATE_REPORT: "pikasTech/HWLAB#39",
|
|
DEV_BASE_IMAGE_PREFLIGHT: "pikasTech/HWLAB#40",
|
|
DEV_EVIDENCE_BLOCKER_AGGREGATOR: "pikasTech/HWLAB#41",
|
|
DEV_M5_GATE_AGGREGATOR_V2: "pikasTech/HWLAB#58",
|
|
DEV_CLOUD_WORKBENCH_LIVE: "pikasTech/HWLAB#7"
|
|
});
|
|
const allowedIssues = new Set([
|
|
contractIssue,
|
|
...Object.values(issueFamily)
|
|
]);
|
|
const requiredPreflightIssue = issueFamily.DEV_GATE_PREFLIGHT;
|
|
const requiredMilestoneIds = ["M0", "M1", "M2", "M3", "M4", "M5"];
|
|
const evidenceLevels = new Set(["SOURCE", "LOCAL", "DRY-RUN", "DEV-LIVE", "BLOCKED"]);
|
|
const devCloudWorkbenchRuntimeIdentityMaxSkewMs = 15 * 60 * 1000;
|
|
const syntheticAcceptedCodeAgentModels = new Set(["gpt-5"]);
|
|
const deploymentPreflightCodeAgentErrorCode = "deployment_identity_preflight";
|
|
const blockedCodeAgentUiLabels = new Set(["发送失败", "服务受阻", "BLOCKED 凭证缺口"]);
|
|
const requiredValidationCommands = [
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs"
|
|
];
|
|
const requiredAggregatorV2Commands = [
|
|
"node --check scripts/dev-evidence-blocker-aggregator.mjs",
|
|
"node --check scripts/src/dev-evidence-blocker-aggregator.mjs",
|
|
"node scripts/dev-evidence-blocker-aggregator.mjs --check",
|
|
"node scripts/dev-evidence-blocker-aggregator.mjs --markdown",
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs"
|
|
];
|
|
const requiredDevDeployApplySupports = [
|
|
"pikasTech/HWLAB#63",
|
|
"pikasTech/HWLAB#50",
|
|
"pikasTech/HWLAB#7",
|
|
"pikasTech/HWLAB#33",
|
|
"pikasTech/HWLAB#17",
|
|
"pikasTech/HWLAB#32",
|
|
"pikasTech/HWLAB#30"
|
|
];
|
|
const requiredDevDeployApplyValidationCommands = [
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs",
|
|
"node --check scripts/dev-deploy-apply.mjs",
|
|
"node --check scripts/src/dev-deploy-apply.mjs",
|
|
"node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked"
|
|
];
|
|
const requiredPreflightValidationCommands = [
|
|
"node --check scripts/dev-gate-preflight.mjs",
|
|
"node --check scripts/src/dev-gate-preflight.mjs",
|
|
"node --check scripts/src/registry-capabilities.mjs",
|
|
"node --check scripts/refresh-artifact-catalog.mjs",
|
|
"node scripts/dev-gate-preflight.mjs"
|
|
];
|
|
const requiredSmokeCommand = "node scripts/m1-contract-smoke.mjs";
|
|
const requiredDryRunCommand =
|
|
"node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run";
|
|
const requiredDocs = [
|
|
"docs/dev-acceptance-matrix.md",
|
|
"docs/m0-contract-audit.md"
|
|
];
|
|
const requiredDevM3Issue = issueFamily.DEV_M3_HARDWARE_LOOP;
|
|
const requiredDevM3ValidationCommands = [
|
|
"node --check scripts/dev-m3-hardware-loop-smoke.mjs",
|
|
"node --check scripts/validate-dev-m3-cardinality.mjs",
|
|
"node scripts/validate-dev-m3-cardinality.mjs",
|
|
"node scripts/dev-m3-hardware-loop-smoke.mjs --dry-run",
|
|
"node scripts/dev-m3-hardware-loop-smoke.mjs --live --confirm-dev --expect-non-prod"
|
|
];
|
|
const legacyDevM3LiveCommand =
|
|
"node scripts/dev-m3-hardware-loop-smoke.mjs --live --confirm-dev --confirmed-non-production";
|
|
const requiredDevM3Docs = [
|
|
"docs/dev-acceptance-matrix.md",
|
|
"docs/m3-hardware-loop.md"
|
|
];
|
|
const requiredDevEdgeIssue = issueFamily.DEV_EDGE_HEALTH;
|
|
const requiredDevEdgeValidationCommands = [
|
|
"node --check scripts/dev-edge-health-smoke.mjs",
|
|
"node --check scripts/src/dev-edge-health-smoke-lib.mjs",
|
|
"node scripts/dev-edge-health-smoke.mjs --live --write-report"
|
|
];
|
|
const requiredDevEdgeDocs = [
|
|
"docs/dev-acceptance-matrix.md",
|
|
"docs/m0-contract-audit.md",
|
|
"docs/dev-edge-health.md"
|
|
];
|
|
const requiredDevM2Issue = "pikasTech/HWLAB#23";
|
|
const requiredDevM2ValidationCommands = [
|
|
"node --check scripts/m2-dev-deploy-smoke.mjs",
|
|
"node scripts/m2-dev-deploy-smoke.mjs --dry-run",
|
|
"node scripts/m2-dev-deploy-smoke.mjs --live --confirm-dev --confirmed-non-production --write-report",
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs"
|
|
];
|
|
const requiredDevM2Docs = [
|
|
"docs/m2-dev-deploy-smoke.md",
|
|
"docs/dev-acceptance-matrix.md",
|
|
"fixtures/dev-deploy-smoke/dev-deploy-smoke.json"
|
|
];
|
|
const requiredDevCloudWorkbenchValidationCommands = [
|
|
"node --check scripts/dev-cloud-workbench-smoke.mjs",
|
|
"node --check scripts/src/dev-cloud-workbench-smoke-lib.mjs",
|
|
"node scripts/dev-cloud-workbench-smoke.mjs --static",
|
|
"node scripts/dev-cloud-workbench-smoke.mjs --live --url http://74.48.78.17:16666/ --report reports/dev-gate/dev-cloud-workbench-live.json",
|
|
"node scripts/validate-dev-gate-report.mjs reports/dev-gate/dev-cloud-workbench-live.json"
|
|
];
|
|
const requiredDevCloudWorkbenchDocs = [
|
|
"docs/cloud-web-workbench.md",
|
|
"docs/reference/cloud-workbench.md",
|
|
"docs/reference/code-agent-chat-readiness.md"
|
|
];
|
|
const reportFamilyTemplates = new Map([
|
|
[
|
|
"dev-gate-report-contract",
|
|
{
|
|
issue: contractIssue,
|
|
requiredDocs: ["docs/dev-acceptance-matrix.md", "docs/m0-contract-audit.md"],
|
|
requiredValidationCommands,
|
|
requiredSmokeCommand,
|
|
requiredDryRunCommand
|
|
}
|
|
],
|
|
[
|
|
"dev-mvp-gate-report",
|
|
{
|
|
issue: issueFamily.DEV_M5_GATE_REPORT,
|
|
requiredDocs: [
|
|
"docs/dev-gate-report.md",
|
|
"docs/dev-acceptance-matrix.md",
|
|
"docs/m0-contract-audit.md",
|
|
"docs/m1-local-smoke.md",
|
|
"docs/m2-dev-deploy-smoke.md",
|
|
"docs/m3-hardware-loop.md",
|
|
"docs/m4-agent-loop.md",
|
|
"docs/m5-mvp-e2e.md",
|
|
"docs/runtime-boundary-guard.md",
|
|
"docs/topology-constraints.md",
|
|
"docs/artifact-catalog.md",
|
|
"docs/schema-drift-map.md",
|
|
"protocol/README.md",
|
|
"protocol/audit.md",
|
|
"protocol/evidence-chain.md",
|
|
"protocol/mvp-e2e-contract.md"
|
|
],
|
|
requiredValidationCommands: [
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs"
|
|
],
|
|
requiredSmokeCommand: "node scripts/m1-contract-smoke.mjs",
|
|
requiredDryRunCommand:
|
|
"node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run"
|
|
}
|
|
],
|
|
[
|
|
"dev-deploy-apply",
|
|
{
|
|
issue: issueFamily.DEV_DEPLOY_APPLY,
|
|
requiredDocs: [
|
|
"docs/dev-acceptance-matrix.md",
|
|
"docs/m0-contract-audit.md",
|
|
"docs/dev-deploy-apply.md",
|
|
"docs/artifact-catalog.md",
|
|
"deploy/README.md"
|
|
],
|
|
requiredValidationCommands: [
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs",
|
|
"node --check scripts/dev-deploy-apply.mjs",
|
|
"node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked"
|
|
],
|
|
requiredSmokeCommand,
|
|
requiredDryRunCommand
|
|
}
|
|
],
|
|
[
|
|
"dev-artifact-publish",
|
|
{
|
|
issue: issueFamily.DEV_ARTIFACT_PUBLISH,
|
|
requiredDocs: [
|
|
"docs/dev-acceptance-matrix.md",
|
|
"docs/m0-contract-audit.md",
|
|
"docs/artifact-catalog.md",
|
|
"docs/dev-artifact-publish.md",
|
|
"docs/dev-base-image-preflight.md"
|
|
],
|
|
requiredValidationCommands: [
|
|
"node --check scripts/dev-artifact-publish.mjs",
|
|
"node --check scripts/src/dev-artifact-services.mjs",
|
|
"node --check scripts/src/registry-capabilities.mjs",
|
|
"node --check scripts/preflight-dev-base-image.mjs",
|
|
"node --check scripts/src/dev-base-image-preflight.mjs",
|
|
"node scripts/preflight-dev-base-image.mjs",
|
|
"node scripts/dev-artifact-publish.mjs --preflight --no-report",
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs"
|
|
],
|
|
requiredSmokeCommand,
|
|
requiredDryRunCommand
|
|
}
|
|
],
|
|
[
|
|
"dev-m4-agent-loop",
|
|
{
|
|
issue: issueFamily.DEV_M4_AGENT_LOOP,
|
|
requiredDocs: ["docs/dev-acceptance-matrix.md", "docs/dev-m4-agent-loop.md"],
|
|
requiredValidationCommands: [
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs",
|
|
"node --check scripts/dev-m4-agent-loop-smoke.mjs",
|
|
"node scripts/dev-m4-agent-loop-smoke.mjs --dry-run",
|
|
"node scripts/dev-m4-agent-loop-smoke.mjs --live --confirm-dev --confirmed-non-production"
|
|
],
|
|
requiredSmokeCommand: "node scripts/m4-agent-loop-smoke.mjs",
|
|
requiredDryRunCommand: "node scripts/dev-m4-agent-loop-smoke.mjs --dry-run"
|
|
}
|
|
]
|
|
]);
|
|
|
|
const statusValues = new Set(["pass", "blocked", "degraded", "not_run", "not_applicable", "failed"]);
|
|
const blockerTypes = new Set([
|
|
"contract_blocker",
|
|
"environment_blocker",
|
|
"network_blocker",
|
|
"runtime_blocker",
|
|
"agent_blocker",
|
|
"observability_blocker",
|
|
"safety_blocker"
|
|
]);
|
|
const blockerStates = new Set(["open", "acknowledged", "closed"]);
|
|
const artifactSourceStates = new Set(["source-present", "intentionally-disabled"]);
|
|
const preflightConclusions = new Set(["ready", "blocked"]);
|
|
const registryCapabilityDimensionIds = new Set([
|
|
"process-http-access",
|
|
"docker-daemon-push-access",
|
|
"k3s-pull-access"
|
|
]);
|
|
const registryCapabilityDimensionKeys = new Set([
|
|
"processHttpAccess",
|
|
"dockerDaemonPushAccess",
|
|
"k3sPullAccess"
|
|
]);
|
|
const requiredPreflightSupports = [
|
|
"pikasTech/HWLAB#7",
|
|
"pikasTech/HWLAB#12",
|
|
"pikasTech/HWLAB#22",
|
|
"pikasTech/HWLAB#23",
|
|
"pikasTech/HWLAB#29",
|
|
"pikasTech/HWLAB#30",
|
|
"pikasTech/HWLAB#31",
|
|
"pikasTech/HWLAB#66"
|
|
];
|
|
const protectedAggregatorLiveCategories = Object.freeze({
|
|
M3: "hardware-loop-live",
|
|
M4: "agent-loop-live-preflight",
|
|
M5: "mvp-e2e-live"
|
|
});
|
|
const protectedAggregatorMilestones = new Set(Object.keys(protectedAggregatorLiveCategories));
|
|
|
|
function assertObject(value, label) {
|
|
assert.ok(value && typeof value === "object" && !Array.isArray(value), `${label} must be an object`);
|
|
}
|
|
|
|
function assertString(value, label) {
|
|
assert.equal(typeof value, "string", `${label} must be a string`);
|
|
assert.ok(value.trim().length > 0, `${label} must not be empty`);
|
|
assert.ok(!value.includes("\n"), `${label} must be a single line`);
|
|
}
|
|
|
|
function assertArray(value, label) {
|
|
assert.ok(Array.isArray(value), `${label} must be an array`);
|
|
}
|
|
|
|
function assertStatus(value, label) {
|
|
assertString(value, label);
|
|
assert.ok(statusValues.has(value), `${label} must be one of ${Array.from(statusValues).join(", ")}`);
|
|
}
|
|
|
|
function assertTimestamp(value, label) {
|
|
assertString(value, label);
|
|
assert.ok(!Number.isNaN(Date.parse(value)), `${label} must be an RFC 3339 timestamp`);
|
|
}
|
|
|
|
function assertDbLayerBooleans(value, label) {
|
|
assertObject(value, label);
|
|
for (const field of ["configReady", "liveConnected", "connectionAttempted"]) {
|
|
assertBoolean(value[field], `${label}.${field}`);
|
|
}
|
|
assertString(value.connectionResult, `${label}.connectionResult`);
|
|
assertNoDbSecretMaterial(value, label);
|
|
}
|
|
|
|
function assertDbRuntimeReadiness(value, label) {
|
|
assertDbLayerBooleans(value, label);
|
|
for (const field of ["envInjected", "valuesRedacted", "secretMaterialRead", "liveDbEvidence"]) {
|
|
assertBoolean(value[field], `${label}.${field}`);
|
|
}
|
|
assert.equal(value.valuesRedacted, true, `${label}.valuesRedacted`);
|
|
assert.equal(value.secretMaterialRead, false, `${label}.secretMaterialRead`);
|
|
assertObject(value.secret, `${label}.secret`);
|
|
assertString(value.secret.secretName, `${label}.secret.secretName`);
|
|
assertString(value.secret.secretKey, `${label}.secret.secretKey`);
|
|
assert.equal(value.secret.redacted, true, `${label}.secret.redacted`);
|
|
assert.equal(value.secret.secretValueRead, false, `${label}.secret.secretValueRead`);
|
|
assertArray(value.evidence, `${label}.evidence`);
|
|
if (value.status === "blocked") {
|
|
assertString(value.blocker, `${label}.blocker`);
|
|
}
|
|
}
|
|
|
|
function assertNoDbSecretMaterial(value, label) {
|
|
const serialized = JSON.stringify(value);
|
|
for (const forbidden of ["postgres://", "postgresql://", "password=", "://user:"]) {
|
|
assert.equal(serialized.includes(forbidden), false, `${label} must not include ${forbidden}`);
|
|
}
|
|
}
|
|
|
|
function assertRepoRelativePath(value, label) {
|
|
assertString(value, label);
|
|
assert.ok(!path.isAbsolute(value), `${label} must be repo-relative`);
|
|
assert.ok(!value.startsWith(".."), `${label} must stay inside the repository`);
|
|
|
|
const absolutePath = path.resolve(repoRoot, value);
|
|
assert.ok(
|
|
absolutePath.startsWith(`${repoRoot}${path.sep}`),
|
|
`${label} must stay inside the repository`
|
|
);
|
|
|
|
return absolutePath;
|
|
}
|
|
|
|
function assertStringArray(value, label, { minLength = 0 } = {}) {
|
|
assertArray(value, label);
|
|
assert.ok(value.length >= minLength, `${label} must contain at least ${minLength} item(s)`);
|
|
for (const [index, item] of value.entries()) {
|
|
assertString(item, `${label}[${index}]`);
|
|
}
|
|
}
|
|
|
|
function assertUnique(values, label) {
|
|
assert.equal(new Set(values).size, values.length, `${label} must be unique`);
|
|
}
|
|
|
|
function assertReportLifecycle(report, raw, label) {
|
|
assertObject(report.reportLifecycle, `${label}.reportLifecycle`);
|
|
const lifecycle = report.reportLifecycle;
|
|
assert.equal(lifecycle.version, REPORT_LIFECYCLE_VERSION, `${label}.reportLifecycle.version`);
|
|
assert.ok(["active", "historical"].includes(lifecycle.state), `${label}.reportLifecycle.state`);
|
|
assert.equal(lifecycle.activeEndpoint, ACTIVE_DEV_PUBLIC_ENDPOINT, `${label}.reportLifecycle.activeEndpoint`);
|
|
assert.equal(lifecycle.activeBrowserEndpoint, ACTIVE_DEV_BROWSER_ENDPOINT, `${label}.reportLifecycle.activeBrowserEndpoint`);
|
|
assertString(lifecycle.summary, `${label}.reportLifecycle.summary`);
|
|
|
|
if (lifecycle.state === "active") {
|
|
assert.equal(lifecycle.deprecatedEndpoint, null, `${label}.reportLifecycle.deprecatedEndpoint`);
|
|
assert.deepEqual(
|
|
findForbiddenActiveDeprecatedEndpoints(report, raw),
|
|
[],
|
|
`${label} active report must not use deprecated public endpoints as active evidence`
|
|
);
|
|
return;
|
|
}
|
|
|
|
assert.ok(
|
|
deprecatedPublicEndpoints.includes(lifecycle.deprecatedEndpoint),
|
|
`${label}.reportLifecycle.deprecatedEndpoint must name a deprecated public endpoint`
|
|
);
|
|
assert.ok(
|
|
lifecycle.summary.toLowerCase().includes("historical") ||
|
|
lifecycle.summary.toLowerCase().includes("deprecated"),
|
|
`${label}.reportLifecycle.summary must explain the historical/deprecated state`
|
|
);
|
|
const deprecatedMatches = findDeprecatedPublicEndpoints(raw);
|
|
assert.ok(
|
|
deprecatedMatches.length === 0 || deprecatedMatches.includes(lifecycle.deprecatedEndpoint),
|
|
`${label}.reportLifecycle.deprecatedEndpoint must cover deprecated public endpoint evidence`
|
|
);
|
|
}
|
|
|
|
function primaryReportStatus(report) {
|
|
return report.status ??
|
|
report.gateStatus ??
|
|
report.conclusion ??
|
|
report.artifactPublish?.status ??
|
|
report.summary?.status ??
|
|
(Array.isArray(report.blockers) && report.blockers.length > 0 ? "blocked" : "not_run");
|
|
}
|
|
|
|
function statusIsPass(value) {
|
|
return value === "pass";
|
|
}
|
|
|
|
function validateHistoricalReport(report, label) {
|
|
assert.equal(report.devOnly, true, `${label}.devOnly`);
|
|
assert.equal(report.prodDisabled, true, `${label}.prodDisabled`);
|
|
assertString(report.issue, `${label}.issue`);
|
|
assert.ok(allowedIssues.has(report.issue), `${label}.issue must be a known HWLAB DEV gate issue`);
|
|
assertString(report.taskId ?? report.reportKind, `${label}.taskId`);
|
|
assert.ok(
|
|
["blocked", "not_run", "not_applicable", "failed", "degraded"].includes(primaryReportStatus(report)),
|
|
`${label} historical report must not present active green status`
|
|
);
|
|
assert.ok(
|
|
Array.isArray(report.blockers) && report.blockers.length >= 1,
|
|
`${label}.blockers must explain why the report is historical`
|
|
);
|
|
assertBlockers(report.blockers, `${label}.blockers`);
|
|
}
|
|
|
|
async function readJsonReport(relativePath) {
|
|
const raw = await readFile(path.join(repoRoot, relativePath), "utf8");
|
|
return { report: JSON.parse(raw), raw };
|
|
}
|
|
|
|
async function collectReportFiles() {
|
|
if (process.argv.length > 2) {
|
|
return process.argv.slice(2).map((entry) => path.relative(repoRoot, path.resolve(process.cwd(), entry)));
|
|
}
|
|
return collectReportJsonFiles(reportsDir, "reports/dev-gate");
|
|
}
|
|
|
|
async function collectReportJsonFiles(directory, relativeDirectory) {
|
|
const entries = await readdir(directory, { withFileTypes: true });
|
|
const nested = await Promise.all(entries.map(async (entry) => {
|
|
const relativePath = path.join(relativeDirectory, entry.name);
|
|
if (entry.isDirectory()) {
|
|
return collectReportJsonFiles(path.join(directory, entry.name), relativePath);
|
|
}
|
|
return entry.isFile() && entry.name.endsWith(".json") ? [relativePath] : [];
|
|
}));
|
|
return nested.flat().sort();
|
|
}
|
|
|
|
async function validateReport(relativePath) {
|
|
const { report, raw } = await readJsonReport(relativePath);
|
|
const label = relativePath;
|
|
assertObject(report, label);
|
|
assertReportLifecycle(report, raw, label);
|
|
if (reportIsHistorical(report)) {
|
|
validateHistoricalReport(report, label);
|
|
return;
|
|
}
|
|
|
|
if (report.reportKind === "dev-gate-preflight") {
|
|
await validatePreflightReport(relativePath, report);
|
|
return;
|
|
}
|
|
if (report.reportKind === "dev-m5-gate-aggregator") {
|
|
await validateAggregatorV2Report(relativePath, report);
|
|
return;
|
|
}
|
|
|
|
if (report.issue === requiredDevM3Issue || report.taskId === "dev-m3-hardware-loop") {
|
|
await validateDevM3Report(report, label);
|
|
return;
|
|
}
|
|
if (report.issue === requiredDevEdgeIssue || report.taskId === "dev-edge-health") {
|
|
await validateDevEdgeReport(report, label);
|
|
return;
|
|
}
|
|
if (report.issue === requiredDevM2Issue || report.taskId === "m2-dev-deploy-smoke") {
|
|
await validateDevM2Report(report, label);
|
|
return;
|
|
}
|
|
if (
|
|
report.issue === issueFamily.DEV_CLOUD_WORKBENCH_LIVE ||
|
|
report.taskId === "dev-cloud-workbench-live"
|
|
) {
|
|
await validateDevCloudWorkbenchLiveReport(report, label, raw);
|
|
return;
|
|
}
|
|
|
|
const template = reportFamilyTemplates.get(report.taskId);
|
|
assert.ok(
|
|
template,
|
|
`${label}.taskId must be one of ${Array.from(reportFamilyTemplates.keys()).join(", ")}`
|
|
);
|
|
|
|
for (const field of [
|
|
"$schema",
|
|
"$id",
|
|
"reportVersion",
|
|
"issue",
|
|
"taskId",
|
|
"commitId",
|
|
"acceptanceLevel",
|
|
"devOnly",
|
|
"prodDisabled",
|
|
"sourceContract",
|
|
"validationCommands",
|
|
"localSmoke",
|
|
"dryRun",
|
|
"devPreconditions",
|
|
"blockers"
|
|
]) {
|
|
assert.ok(Object.hasOwn(report, field), `${label} missing ${field}`);
|
|
}
|
|
|
|
assertString(report.$schema, `${label}.$schema`);
|
|
assertString(report.$id, `${label}.$id`);
|
|
assert.equal(report.reportVersion, "v1", `${label}.reportVersion`);
|
|
assert.equal(report.issue, template.issue, `${label}.issue`);
|
|
assert.match(report.taskId, /^[a-z][a-z0-9-]*$/, `${label}.taskId`);
|
|
assert.match(report.commitId, /^[a-f0-9]{7,40}$/, `${label}.commitId`);
|
|
assert.match(report.acceptanceLevel, /^[a-z][a-z0-9_-]*$/, `${label}.acceptanceLevel`);
|
|
assert.equal(report.devOnly, true, `${label}.devOnly`);
|
|
assert.equal(report.prodDisabled, true, `${label}.prodDisabled`);
|
|
|
|
assertObject(report.sourceContract, `${label}.sourceContract`);
|
|
assertStatus(report.sourceContract.status, `${label}.sourceContract.status`);
|
|
assertString(report.sourceContract.summary, `${label}.sourceContract.summary`);
|
|
assertStringArray(report.sourceContract.documents, `${label}.sourceContract.documents`, {
|
|
minLength: template.requiredDocs.length
|
|
});
|
|
|
|
const documents = new Set(report.sourceContract.documents);
|
|
assertUnique(report.sourceContract.documents, `${label}.sourceContract.documents`);
|
|
for (const requiredDoc of template.requiredDocs) {
|
|
assert.ok(documents.has(requiredDoc), `${label}.sourceContract.documents missing ${requiredDoc}`);
|
|
}
|
|
for (const [index, documentPath] of report.sourceContract.documents.entries()) {
|
|
const absolutePath = assertRepoRelativePath(
|
|
documentPath,
|
|
`${label}.sourceContract.documents[${index}]`
|
|
);
|
|
await access(absolutePath, fsConstants.R_OK);
|
|
}
|
|
|
|
assertStringArray(report.validationCommands, `${label}.validationCommands`, { minLength: 2 });
|
|
assertUnique(report.validationCommands, `${label}.validationCommands`);
|
|
for (const requiredCommand of template.requiredValidationCommands) {
|
|
assert.ok(
|
|
report.validationCommands.includes(requiredCommand),
|
|
`${label}.validationCommands missing ${requiredCommand}`
|
|
);
|
|
}
|
|
|
|
assertObject(report.localSmoke, `${label}.localSmoke`);
|
|
assertStatus(report.localSmoke.status, `${label}.localSmoke.status`);
|
|
assertStringArray(report.localSmoke.commands, `${label}.localSmoke.commands`, { minLength: 1 });
|
|
assertUnique(report.localSmoke.commands, `${label}.localSmoke.commands`);
|
|
assert.ok(
|
|
report.localSmoke.commands.includes(template.requiredSmokeCommand),
|
|
`${label}.localSmoke.commands missing ${template.requiredSmokeCommand}`
|
|
);
|
|
assertStringArray(report.localSmoke.evidence, `${label}.localSmoke.evidence`, { minLength: 1 });
|
|
assertString(report.localSmoke.summary, `${label}.localSmoke.summary`);
|
|
|
|
assertObject(report.dryRun, `${label}.dryRun`);
|
|
assertStatus(report.dryRun.status, `${label}.dryRun.status`);
|
|
assertStringArray(report.dryRun.commands, `${label}.dryRun.commands`, { minLength: 1 });
|
|
assertUnique(report.dryRun.commands, `${label}.dryRun.commands`);
|
|
assert.ok(
|
|
report.dryRun.commands.includes(template.requiredDryRunCommand),
|
|
`${label}.dryRun.commands missing ${template.requiredDryRunCommand}`
|
|
);
|
|
assertStringArray(report.dryRun.evidence, `${label}.dryRun.evidence`, { minLength: 1 });
|
|
assertString(report.dryRun.summary, `${label}.dryRun.summary`);
|
|
|
|
assertObject(report.devPreconditions, `${label}.devPreconditions`);
|
|
assertStatus(report.devPreconditions.status, `${label}.devPreconditions.status`);
|
|
assertStringArray(report.devPreconditions.requirements, `${label}.devPreconditions.requirements`, {
|
|
minLength: 1
|
|
});
|
|
assertString(report.devPreconditions.summary, `${label}.devPreconditions.summary`);
|
|
|
|
assertArray(report.blockers, `${label}.blockers`);
|
|
assertUnique(
|
|
report.blockers.map((blocker) => `${blocker.type}::${blocker.scope}`),
|
|
`${label}.blockers`
|
|
);
|
|
for (const [index, blocker] of report.blockers.entries()) {
|
|
const blockerLabel = `${label}.blockers[${index}]`;
|
|
assertObject(blocker, blockerLabel);
|
|
for (const field of ["type", "scope", "status", "summary"]) {
|
|
assert.ok(Object.hasOwn(blocker, field), `${blockerLabel} missing ${field}`);
|
|
}
|
|
assert.ok(blockerTypes.has(blocker.type), `${blockerLabel}.type must be a known blocker type`);
|
|
assertString(blocker.scope, `${blockerLabel}.scope`);
|
|
assert.ok(blockerStates.has(blocker.status), `${blockerLabel}.status must be open, acknowledged, or closed`);
|
|
assertString(blocker.summary, `${blockerLabel}.summary`);
|
|
}
|
|
|
|
if (Object.hasOwn(report, "notes")) {
|
|
assertString(report.notes, `${label}.notes`);
|
|
}
|
|
|
|
if (Object.hasOwn(report, "gateStatus")) {
|
|
assertStatus(report.gateStatus, `${label}.gateStatus`);
|
|
}
|
|
|
|
if (Object.hasOwn(report, "artifactPublish")) {
|
|
assertObject(report.artifactPublish, `${label}.artifactPublish`);
|
|
assertRegistryCapabilities(report.artifactPublish.registryCapabilities, `${label}.artifactPublish.registryCapabilities`);
|
|
assertRegistryCapabilities(report.artifactPublish.publishPlan?.registryCapabilities, `${label}.artifactPublish.publishPlan.registryCapabilities`);
|
|
assertArray(report.artifactPublish.services ?? [], `${label}.artifactPublish.services`);
|
|
for (const [index, service] of report.artifactPublish.services.entries()) {
|
|
const serviceLabel = `${label}.artifactPublish.services[${index}]`;
|
|
assertObject(service, serviceLabel);
|
|
assertString(service.serviceId, `${serviceLabel}.serviceId`);
|
|
if (Object.hasOwn(service, "sourceCommitId")) {
|
|
assertString(service.sourceCommitId, `${serviceLabel}.sourceCommitId`);
|
|
}
|
|
assertString(service.status, `${serviceLabel}.status`);
|
|
assertString(service.image, `${serviceLabel}.image`);
|
|
assertString(service.imageTag, `${serviceLabel}.imageTag`);
|
|
assertShaOrNotPublished(service.digest, `${serviceLabel}.digest`);
|
|
assertString(service.implementationState, `${serviceLabel}.implementationState`);
|
|
assertString(service.sourceState, `${serviceLabel}.sourceState`);
|
|
assert.ok(
|
|
artifactSourceStates.has(service.sourceState),
|
|
`${serviceLabel}.sourceState must be source-present or intentionally-disabled`
|
|
);
|
|
if (Object.hasOwn(service, "distFreshness")) {
|
|
assertObject(service.distFreshness, `${serviceLabel}.distFreshness`);
|
|
assert.equal(service.distFreshness.status, "pass", `${serviceLabel}.distFreshness.status`);
|
|
assert.equal(service.distFreshness.distPath, "web/hwlab-cloud-web/dist", `${serviceLabel}.distFreshness.distPath`);
|
|
assert.equal(
|
|
service.distFreshness.buildCommand,
|
|
"node web/hwlab-cloud-web/scripts/build.mjs",
|
|
`${serviceLabel}.distFreshness.buildCommand`
|
|
);
|
|
assert.equal(
|
|
service.distFreshness.freshnessCommand,
|
|
"node web/hwlab-cloud-web/scripts/build.mjs",
|
|
`${serviceLabel}.distFreshness.freshnessCommand`
|
|
);
|
|
assertArray(service.distFreshness.files, `${serviceLabel}.distFreshness.files`);
|
|
assertArray(service.distFreshness.mismatches, `${serviceLabel}.distFreshness.mismatches`);
|
|
assert.equal(service.distFreshness.mismatches.length, 0, `${serviceLabel}.distFreshness.mismatches`);
|
|
}
|
|
if (service.sourceState === "intentionally-disabled") {
|
|
assert.equal(service.entrypoint, null, `${serviceLabel}.entrypoint must be null when intentionally disabled`);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (Object.hasOwn(report, "milestones")) {
|
|
assertArray(report.milestones, `${label}.milestones`);
|
|
assert.deepEqual(
|
|
report.milestones.map((milestone) => milestone.id),
|
|
requiredMilestoneIds,
|
|
`${label}.milestones must cover M0-M5 in order`
|
|
);
|
|
assertUnique(
|
|
report.milestones.map((milestone) => milestone.id),
|
|
`${label}.milestones`
|
|
);
|
|
|
|
for (const [index, milestone] of report.milestones.entries()) {
|
|
const milestoneLabel = `${label}.milestones[${index}]`;
|
|
assertObject(milestone, milestoneLabel);
|
|
for (const field of ["id", "status", "commands", "evidence", "summary"]) {
|
|
assert.ok(Object.hasOwn(milestone, field), `${milestoneLabel} missing ${field}`);
|
|
}
|
|
assert.match(milestone.id, /^M[0-5]$/, `${milestoneLabel}.id`);
|
|
assertStatus(milestone.status, `${milestoneLabel}.status`);
|
|
assertStringArray(milestone.commands, `${milestoneLabel}.commands`, { minLength: 1 });
|
|
assertUnique(milestone.commands, `${milestoneLabel}.commands`);
|
|
assertStringArray(milestone.evidence, `${milestoneLabel}.evidence`, { minLength: 1 });
|
|
assertUnique(milestone.evidence, `${milestoneLabel}.evidence`);
|
|
assertString(milestone.summary, `${milestoneLabel}.summary`);
|
|
if (Object.hasOwn(milestone, "blocker")) {
|
|
assertString(milestone.blocker, `${milestoneLabel}.blocker`);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (Object.hasOwn(report.devPreconditions, "commands")) {
|
|
assertStringArray(report.devPreconditions.commands, `${label}.devPreconditions.commands`, {
|
|
minLength: 1
|
|
});
|
|
assertUnique(report.devPreconditions.commands, `${label}.devPreconditions.commands`);
|
|
}
|
|
|
|
if (Object.hasOwn(report.devPreconditions, "evidence")) {
|
|
assertStringArray(report.devPreconditions.evidence, `${label}.devPreconditions.evidence`, {
|
|
minLength: 1
|
|
});
|
|
assertUnique(report.devPreconditions.evidence, `${label}.devPreconditions.evidence`);
|
|
}
|
|
|
|
if (report.taskId === "dev-deploy-apply") {
|
|
assertDevDeployApplyReport(report, label);
|
|
}
|
|
}
|
|
|
|
async function assertDocumentSet(documentsValue, label, requiredDocs) {
|
|
assertStringArray(documentsValue, label, { minLength: requiredDocs.length });
|
|
const documents = new Set(documentsValue);
|
|
assertUnique(documentsValue, label);
|
|
for (const requiredDoc of requiredDocs) {
|
|
assert.ok(documents.has(requiredDoc), `${label} missing ${requiredDoc}`);
|
|
}
|
|
for (const [index, documentPath] of documentsValue.entries()) {
|
|
const absolutePath = assertRepoRelativePath(documentPath, `${label}[${index}]`);
|
|
await access(absolutePath, fsConstants.R_OK);
|
|
}
|
|
}
|
|
|
|
function assertBlockers(blockers, label) {
|
|
assertArray(blockers, label);
|
|
assertUnique(
|
|
blockers.map((blocker) => `${blocker.type}::${blocker.scope}`),
|
|
label
|
|
);
|
|
for (const [index, blocker] of blockers.entries()) {
|
|
const blockerLabel = `${label}[${index}]`;
|
|
assertObject(blocker, blockerLabel);
|
|
for (const field of ["type", "scope", "status", "summary"]) {
|
|
assert.ok(Object.hasOwn(blocker, field), `${blockerLabel} missing ${field}`);
|
|
}
|
|
assert.ok(blockerTypes.has(blocker.type), `${blockerLabel}.type must be a known blocker type`);
|
|
assertString(blocker.scope, `${blockerLabel}.scope`);
|
|
assert.ok(blockerStates.has(blocker.status), `${blockerLabel}.status must be open, acknowledged, or closed`);
|
|
assertString(blocker.summary, `${blockerLabel}.summary`);
|
|
}
|
|
}
|
|
|
|
function assertCommandList(value, label, { minLength = 1 } = {}) {
|
|
assertStringArray(value, label, { minLength });
|
|
assertUnique(value, label);
|
|
for (const command of value) {
|
|
assert.ok(!command.includes("--prod"), `${label} must not include PROD command`);
|
|
assert.ok(!command.includes("--production"), `${label} must not include production command`);
|
|
}
|
|
}
|
|
|
|
function assertDevDeployApplyReport(report, label) {
|
|
assert.equal(report.issue, issueFamily.DEV_DEPLOY_APPLY, `${label}.issue`);
|
|
|
|
assertStringArray(report.supports, `${label}.supports`, {
|
|
minLength: requiredDevDeployApplySupports.length
|
|
});
|
|
for (const supportedIssue of requiredDevDeployApplySupports) {
|
|
assert.ok(report.supports.includes(supportedIssue), `${label}.supports missing ${supportedIssue}`);
|
|
}
|
|
|
|
for (const requiredCommand of requiredDevDeployApplyValidationCommands) {
|
|
assert.ok(
|
|
report.validationCommands.includes(requiredCommand),
|
|
`${label}.validationCommands missing ${requiredCommand}`
|
|
);
|
|
}
|
|
|
|
assertObject(report.devDeployApply, `${label}.devDeployApply`);
|
|
const plan = report.devDeployApply;
|
|
for (const field of [
|
|
"conclusion",
|
|
"applyBoundary",
|
|
"target",
|
|
"artifactPlan",
|
|
"workloadPlan",
|
|
"servicePlan",
|
|
"templateJobReplacementPolicy",
|
|
"templateJobReplacements",
|
|
"manualCommands",
|
|
"rollbackHint",
|
|
"remainingBlockers"
|
|
]) {
|
|
assert.ok(Object.hasOwn(plan, field), `${label}.devDeployApply missing ${field}`);
|
|
}
|
|
|
|
assertObject(plan.conclusion, `${label}.devDeployApply.conclusion`);
|
|
assert.ok(["blocked", "ready"].includes(plan.conclusion.status), `${label}.devDeployApply.conclusion.status`);
|
|
assertString(plan.conclusion.reason, `${label}.devDeployApply.conclusion.reason`);
|
|
assert.equal(plan.conclusion.blockerCount, report.blockers.length, `${label}.devDeployApply.conclusion.blockerCount`);
|
|
|
|
assertObject(plan.applyBoundary, `${label}.devDeployApply.applyBoundary`);
|
|
assert.equal(plan.applyBoundary.defaultNoWrite, plan.mode !== "apply", `${label}.devDeployApply.applyBoundary.defaultNoWrite`);
|
|
assert.equal(plan.applyBoundary.mutationAttempted, report.devDeployApply.mutationAttempted, `${label}.devDeployApply.applyBoundary.mutationAttempted`);
|
|
assertStringArray(plan.applyBoundary.applyRequiresFlags, `${label}.devDeployApply.applyBoundary.applyRequiresFlags`, {
|
|
minLength: 3
|
|
});
|
|
for (const flag of ["--apply", "--confirm-dev", "--confirmed-non-production"]) {
|
|
assert.ok(plan.applyBoundary.applyRequiresFlags.includes(flag), `${label}.devDeployApply.applyBoundary.applyRequiresFlags missing ${flag}`);
|
|
}
|
|
assertStringArray(plan.applyBoundary.forbiddenActions, `${label}.devDeployApply.applyBoundary.forbiddenActions`, {
|
|
minLength: 6
|
|
});
|
|
for (const forbiddenAction of ["prod-deploy", "secret-read", "runtime-substitute", "heavy-master-e2e"]) {
|
|
assert.ok(plan.applyBoundary.forbiddenActions.includes(forbiddenAction), `${label}.devDeployApply.applyBoundary.forbiddenActions missing ${forbiddenAction}`);
|
|
}
|
|
assertString(plan.applyBoundary.writeScope, `${label}.devDeployApply.applyBoundary.writeScope`);
|
|
assertString(plan.applyBoundary.noWriteScope, `${label}.devDeployApply.applyBoundary.noWriteScope`);
|
|
|
|
assertObject(plan.target, `${label}.devDeployApply.target`);
|
|
assert.equal(plan.target.environment, "dev", `${label}.devDeployApply.target.environment`);
|
|
assert.equal(plan.target.namespace, "hwlab-dev", `${label}.devDeployApply.target.namespace`);
|
|
assert.equal(plan.target.endpoint, "http://74.48.78.17:16667", `${label}.devDeployApply.target.endpoint`);
|
|
assert.equal(plan.target.prodDisabled, true, `${label}.devDeployApply.target.prodDisabled`);
|
|
|
|
assertObject(plan.artifactPlan, `${label}.devDeployApply.artifactPlan`);
|
|
for (const field of ["expectedArtifactCommit", "deployCommitId", "catalogCommitId", "sourceCommitId"]) {
|
|
assertString(plan.artifactPlan[field], `${label}.devDeployApply.artifactPlan.${field}`);
|
|
}
|
|
assertStringArray(plan.artifactPlan.serviceCommitIds, `${label}.devDeployApply.artifactPlan.serviceCommitIds`, {
|
|
minLength: 1
|
|
});
|
|
assert.equal(typeof plan.artifactPlan.matchesSourceCommit, "boolean", `${label}.devDeployApply.artifactPlan.matchesSourceCommit`);
|
|
assert.equal(typeof plan.artifactPlan.published, "boolean", `${label}.devDeployApply.artifactPlan.published`);
|
|
assert.equal(typeof plan.artifactPlan.registryVerified, "boolean", `${label}.devDeployApply.artifactPlan.registryVerified`);
|
|
assert.ok(Number.isInteger(plan.artifactPlan.imageCount), `${label}.devDeployApply.artifactPlan.imageCount`);
|
|
assertStringArray(plan.artifactPlan.unpublishedServices, `${label}.devDeployApply.artifactPlan.unpublishedServices`);
|
|
|
|
assertArray(plan.workloadPlan, `${label}.devDeployApply.workloadPlan`);
|
|
assert.ok(plan.workloadPlan.length >= 1, `${label}.devDeployApply.workloadPlan must not be empty`);
|
|
assertArray(plan.servicePlan, `${label}.devDeployApply.servicePlan`);
|
|
assert.ok(plan.servicePlan.length >= 1, `${label}.devDeployApply.servicePlan must not be empty`);
|
|
for (const [index, workload] of plan.workloadPlan.entries()) {
|
|
const workloadLabel = `${label}.devDeployApply.workloadPlan[${index}]`;
|
|
assertObject(workload, workloadLabel);
|
|
for (const field of ["kind", "name", "namespace", "serviceIds", "containers"]) {
|
|
assert.ok(Object.hasOwn(workload, field), `${workloadLabel} missing ${field}`);
|
|
}
|
|
assert.equal(workload.namespace, "hwlab-dev", `${workloadLabel}.namespace`);
|
|
assertStringArray(workload.serviceIds, `${workloadLabel}.serviceIds`, { minLength: 1 });
|
|
assertArray(workload.containers, `${workloadLabel}.containers`);
|
|
}
|
|
|
|
assertObject(plan.templateJobReplacementPolicy, `${label}.devDeployApply.templateJobReplacementPolicy`);
|
|
assert.equal(plan.templateJobReplacementPolicy.namespace, "hwlab-dev", `${label}.devDeployApply.templateJobReplacementPolicy.namespace`);
|
|
assert.equal(plan.templateJobReplacementPolicy.status, "active", `${label}.devDeployApply.templateJobReplacementPolicy.status`);
|
|
assertArray(plan.templateJobReplacementPolicy.allowedJobs, `${label}.devDeployApply.templateJobReplacementPolicy.allowedJobs`);
|
|
const allowedTemplateJobNames = plan.templateJobReplacementPolicy.allowedJobs.map((job) => job.name);
|
|
assert.deepEqual(
|
|
allowedTemplateJobNames,
|
|
["hwlab-agent-worker-template", "hwlab-cli-template"],
|
|
`${label}.devDeployApply.templateJobReplacementPolicy.allowedJobs`
|
|
);
|
|
|
|
assertArray(plan.templateJobReplacements, `${label}.devDeployApply.templateJobReplacements`);
|
|
for (const [index, replacement] of plan.templateJobReplacements.entries()) {
|
|
const replacementLabel = `${label}.devDeployApply.templateJobReplacements[${index}]`;
|
|
assertObject(replacement, replacementLabel);
|
|
for (const field of ["namespace", "jobName", "allowed", "desiredSuspended", "oldImages", "newImages", "replace", "result", "reason"]) {
|
|
assert.ok(Object.hasOwn(replacement, field), `${replacementLabel} missing ${field}`);
|
|
}
|
|
assert.equal(replacement.namespace, "hwlab-dev", `${replacementLabel}.namespace`);
|
|
assert.ok(allowedTemplateJobNames.includes(replacement.jobName), `${replacementLabel}.jobName must be allowlisted`);
|
|
assert.equal(replacement.allowed, true, `${replacementLabel}.allowed`);
|
|
assert.equal(replacement.desiredSuspended, true, `${replacementLabel}.desiredSuspended`);
|
|
assertArray(replacement.oldImages, `${replacementLabel}.oldImages`);
|
|
assertArray(replacement.newImages, `${replacementLabel}.newImages`);
|
|
assert.equal(typeof replacement.replace, "boolean", `${replacementLabel}.replace`);
|
|
assert.ok(
|
|
["not_needed", "planned", "replaced", "not_found", "not_evaluated", "blocked", "delete_failed", "delete_succeeded_apply_failed"].includes(replacement.result),
|
|
`${replacementLabel}.result`
|
|
);
|
|
assertString(replacement.reason, `${replacementLabel}.reason`);
|
|
}
|
|
|
|
if (Object.hasOwn(plan, "cloudWebRollout")) {
|
|
assertObject(plan.cloudWebRollout, `${label}.devDeployApply.cloudWebRollout`);
|
|
for (const field of [
|
|
"serviceId",
|
|
"namespace",
|
|
"status",
|
|
"sourceCommitId",
|
|
"image",
|
|
"imageTag",
|
|
"digest",
|
|
"publishState",
|
|
"rolloutRevision",
|
|
"liveImage",
|
|
"imageMatchesDesired",
|
|
"verificationCommand",
|
|
"readCommand"
|
|
]) {
|
|
assert.ok(Object.hasOwn(plan.cloudWebRollout, field), `${label}.devDeployApply.cloudWebRollout missing ${field}`);
|
|
}
|
|
assert.equal(plan.cloudWebRollout.serviceId, "hwlab-cloud-web", `${label}.devDeployApply.cloudWebRollout.serviceId`);
|
|
assert.equal(plan.cloudWebRollout.namespace, "hwlab-dev", `${label}.devDeployApply.cloudWebRollout.namespace`);
|
|
assert.ok(["observed", "blocked", "not_evaluated"].includes(plan.cloudWebRollout.status), `${label}.devDeployApply.cloudWebRollout.status`);
|
|
assertString(plan.cloudWebRollout.sourceCommitId, `${label}.devDeployApply.cloudWebRollout.sourceCommitId`);
|
|
assertString(plan.cloudWebRollout.image, `${label}.devDeployApply.cloudWebRollout.image`);
|
|
assertString(plan.cloudWebRollout.imageTag, `${label}.devDeployApply.cloudWebRollout.imageTag`);
|
|
assertShaOrNotPublished(plan.cloudWebRollout.digest, `${label}.devDeployApply.cloudWebRollout.digest`);
|
|
if (plan.cloudWebRollout.rolloutRevision !== null) {
|
|
assertString(plan.cloudWebRollout.rolloutRevision, `${label}.devDeployApply.cloudWebRollout.rolloutRevision`);
|
|
}
|
|
if (plan.cloudWebRollout.liveImage !== null) {
|
|
assertString(plan.cloudWebRollout.liveImage, `${label}.devDeployApply.cloudWebRollout.liveImage`);
|
|
}
|
|
assert.equal(typeof plan.cloudWebRollout.imageMatchesDesired, "boolean", `${label}.devDeployApply.cloudWebRollout.imageMatchesDesired`);
|
|
assert.match(plan.cloudWebRollout.verificationCommand, /rollout status deployment\/hwlab-cloud-web/u, `${label}.devDeployApply.cloudWebRollout.verificationCommand`);
|
|
assert.match(plan.cloudWebRollout.verificationCommand, /^KUBECONFIG=(?:[A-Za-z0-9_@%+=:,./-]+|'[^']+') kubectl /u, `${label}.devDeployApply.cloudWebRollout.verificationCommand`);
|
|
assert.match(plan.cloudWebRollout.readCommand, /get deployment hwlab-cloud-web/u, `${label}.devDeployApply.cloudWebRollout.readCommand`);
|
|
assert.match(plan.cloudWebRollout.readCommand, /^KUBECONFIG=(?:[A-Za-z0-9_@%+=:,./-]+|'[^']+') kubectl /u, `${label}.devDeployApply.cloudWebRollout.readCommand`);
|
|
if (plan.cloudWebRollout.status === "blocked") {
|
|
assertString(plan.cloudWebRollout.blocker, `${label}.devDeployApply.cloudWebRollout.blocker`);
|
|
}
|
|
}
|
|
|
|
assertObject(plan.manualCommands, `${label}.devDeployApply.manualCommands`);
|
|
assert.ok(["blocked", "ready"].includes(plan.manualCommands.status), `${label}.devDeployApply.manualCommands.status`);
|
|
assertCommandList(plan.manualCommands.beforeHumanApproval, `${label}.devDeployApply.manualCommands.beforeHumanApproval`);
|
|
assertCommandList(plan.manualCommands.afterHumanApproval, `${label}.devDeployApply.manualCommands.afterHumanApproval`, {
|
|
minLength: 0
|
|
});
|
|
assertString(plan.manualCommands.summary, `${label}.devDeployApply.manualCommands.summary`);
|
|
if (plan.conclusion.status === "blocked") {
|
|
assert.equal(plan.manualCommands.status, "blocked", `${label}.devDeployApply.manualCommands.status`);
|
|
assert.equal(plan.manualCommands.afterHumanApproval.length, 0, `${label}.devDeployApply.manualCommands.afterHumanApproval`);
|
|
}
|
|
if (plan.conclusion.status === "ready") {
|
|
assert.equal(plan.manualCommands.status, "ready", `${label}.devDeployApply.manualCommands.status`);
|
|
assert.ok(
|
|
plan.manualCommands.afterHumanApproval.includes("node scripts/dev-deploy-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report"),
|
|
`${label}.devDeployApply.manualCommands.afterHumanApproval missing apply command`
|
|
);
|
|
}
|
|
|
|
assertObject(plan.rollbackHint, `${label}.devDeployApply.rollbackHint`);
|
|
assert.equal(plan.rollbackHint.namespace, "hwlab-dev", `${label}.devDeployApply.rollbackHint.namespace`);
|
|
assertString(plan.rollbackHint.strategy, `${label}.devDeployApply.rollbackHint.strategy`);
|
|
assertCommandList(plan.rollbackHint.captureBeforeApply, `${label}.devDeployApply.rollbackHint.captureBeforeApply`);
|
|
assertCommandList(plan.rollbackHint.deploymentRollbackCommands, `${label}.devDeployApply.rollbackHint.deploymentRollbackCommands`);
|
|
assertCommandList(plan.rollbackHint.jobCleanupCommands, `${label}.devDeployApply.rollbackHint.jobCleanupCommands`);
|
|
assertCommandList(plan.rollbackHint.postRollbackChecks, `${label}.devDeployApply.rollbackHint.postRollbackChecks`);
|
|
|
|
assertArray(plan.remainingBlockers, `${label}.devDeployApply.remainingBlockers`);
|
|
assert.equal(plan.remainingBlockers.length, report.blockers.length, `${label}.devDeployApply.remainingBlockers.length`);
|
|
for (const [index, blocker] of plan.remainingBlockers.entries()) {
|
|
const blockerLabel = `${label}.devDeployApply.remainingBlockers[${index}]`;
|
|
assertObject(blocker, blockerLabel);
|
|
for (const field of ["type", "scope", "status", "summary", "unblockHint"]) {
|
|
assert.ok(Object.hasOwn(blocker, field), `${blockerLabel} missing ${field}`);
|
|
}
|
|
assert.ok(blockerTypes.has(blocker.type), `${blockerLabel}.type must be known`);
|
|
assertString(blocker.unblockHint, `${blockerLabel}.unblockHint`);
|
|
}
|
|
}
|
|
|
|
|
|
function assertShaOrNotPublished(value, label) {
|
|
assertString(value, label);
|
|
assert.ok(
|
|
value === "not_published" || /^sha256:[a-f0-9]{64}$/.test(value),
|
|
`${label} must be not_published or a sha256 digest`
|
|
);
|
|
}
|
|
|
|
function assertRegistryCapabilityDimension(dimension, label, expectedId) {
|
|
assertObject(dimension, label);
|
|
for (const field of [
|
|
"id",
|
|
"status",
|
|
"requiredForPublish",
|
|
"requiredForDeploy",
|
|
"endpoint",
|
|
"probeKind",
|
|
"summary",
|
|
"evidence"
|
|
]) {
|
|
assert.ok(Object.hasOwn(dimension, field), `${label} missing ${field}`);
|
|
}
|
|
assert.equal(dimension.id, expectedId, `${label}.id`);
|
|
assert.ok(registryCapabilityDimensionIds.has(dimension.id), `${label}.id must be a known registry capability dimension`);
|
|
assertStatus(dimension.status, `${label}.status`);
|
|
assert.equal(typeof dimension.requiredForPublish, "boolean", `${label}.requiredForPublish`);
|
|
assert.equal(typeof dimension.requiredForDeploy, "boolean", `${label}.requiredForDeploy`);
|
|
assertString(dimension.endpoint, `${label}.endpoint`);
|
|
assertString(dimension.probeKind, `${label}.probeKind`);
|
|
assertString(dimension.summary, `${label}.summary`);
|
|
assertObject(dimension.evidence, `${label}.evidence`);
|
|
}
|
|
|
|
function assertRegistryCapabilities(capabilities, label) {
|
|
assertObject(capabilities, label);
|
|
for (const field of [
|
|
"version",
|
|
"registryPrefix",
|
|
"generatedAt",
|
|
"classification",
|
|
"dimensions",
|
|
"interpretation"
|
|
]) {
|
|
assert.ok(Object.hasOwn(capabilities, field), `${label} missing ${field}`);
|
|
}
|
|
assert.equal(capabilities.version, "v1", `${label}.version`);
|
|
assertString(capabilities.registryPrefix, `${label}.registryPrefix`);
|
|
assertTimestamp(capabilities.generatedAt, `${label}.generatedAt`);
|
|
assertStatus(capabilities.classification, `${label}.classification`);
|
|
assertObject(capabilities.dimensions, `${label}.dimensions`);
|
|
assert.deepEqual(
|
|
new Set(Object.keys(capabilities.dimensions)),
|
|
registryCapabilityDimensionKeys,
|
|
`${label}.dimensions must contain processHttpAccess, dockerDaemonPushAccess, and k3sPullAccess`
|
|
);
|
|
assertRegistryCapabilityDimension(capabilities.dimensions.processHttpAccess, `${label}.dimensions.processHttpAccess`, "process-http-access");
|
|
assertRegistryCapabilityDimension(capabilities.dimensions.dockerDaemonPushAccess, `${label}.dimensions.dockerDaemonPushAccess`, "docker-daemon-push-access");
|
|
assertRegistryCapabilityDimension(capabilities.dimensions.k3sPullAccess, `${label}.dimensions.k3sPullAccess`, "k3s-pull-access");
|
|
assert.equal(capabilities.dimensions.processHttpAccess.requiredForPublish, false, `${label}.dimensions.processHttpAccess.requiredForPublish`);
|
|
assert.equal(capabilities.dimensions.dockerDaemonPushAccess.requiredForPublish, true, `${label}.dimensions.dockerDaemonPushAccess.requiredForPublish`);
|
|
assert.equal(capabilities.dimensions.k3sPullAccess.requiredForDeploy, true, `${label}.dimensions.k3sPullAccess.requiredForDeploy`);
|
|
assertObject(capabilities.interpretation, `${label}.interpretation`);
|
|
for (const key of registryCapabilityDimensionKeys) {
|
|
assertString(capabilities.interpretation[key], `${label}.interpretation.${key}`);
|
|
}
|
|
}
|
|
|
|
function assertArtifactIdentity(identity, label, target) {
|
|
assertObject(identity, label);
|
|
for (const field of [
|
|
"source",
|
|
"deployManifest",
|
|
"artifactCatalog",
|
|
"services",
|
|
"serviceCommitIds",
|
|
"matchesSource",
|
|
"publishVerified",
|
|
"refreshCommands"
|
|
]) {
|
|
assert.ok(Object.hasOwn(identity, field), `${label} missing ${field}`);
|
|
}
|
|
|
|
assertObject(identity.source, `${label}.source`);
|
|
assert.equal(identity.source.ref, target.ref, `${label}.source.ref`);
|
|
assert.equal(identity.source.commitId, target.commitId, `${label}.source.commitId`);
|
|
assert.equal(identity.source.shortCommitId, target.shortCommitId, `${label}.source.shortCommitId`);
|
|
|
|
assertObject(identity.deployManifest, `${label}.deployManifest`);
|
|
assert.equal(identity.deployManifest.path, "deploy/deploy.json", `${label}.deployManifest.path`);
|
|
assert.match(identity.deployManifest.commitId, /^[a-f0-9]{7,40}$/, `${label}.deployManifest.commitId`);
|
|
assert.equal(typeof identity.deployManifest.matchesSource, "boolean", `${label}.deployManifest.matchesSource`);
|
|
|
|
assertObject(identity.artifactCatalog, `${label}.artifactCatalog`);
|
|
assert.equal(identity.artifactCatalog.path, "deploy/artifact-catalog.dev.json", `${label}.artifactCatalog.path`);
|
|
assert.match(identity.artifactCatalog.commitId, /^[a-f0-9]{7,40}$/, `${label}.artifactCatalog.commitId`);
|
|
assert.ok(["contract-skeleton", "published"].includes(identity.artifactCatalog.artifactState), `${label}.artifactCatalog.artifactState`);
|
|
assert.equal(typeof identity.artifactCatalog.ciPublished, "boolean", `${label}.artifactCatalog.ciPublished`);
|
|
assert.equal(typeof identity.artifactCatalog.registryVerified, "boolean", `${label}.artifactCatalog.registryVerified`);
|
|
assertString(identity.artifactCatalog.provenance, `${label}.artifactCatalog.provenance`);
|
|
assert.equal(typeof identity.artifactCatalog.matchesSource, "boolean", `${label}.artifactCatalog.matchesSource`);
|
|
assertObject(identity.artifactCatalog.digestCounts, `${label}.artifactCatalog.digestCounts`);
|
|
for (const field of ["sha256", "notPublished", "invalid"]) {
|
|
assert.equal(typeof identity.artifactCatalog.digestCounts[field], "number", `${label}.artifactCatalog.digestCounts.${field}`);
|
|
}
|
|
|
|
assertArray(identity.services, `${label}.services`);
|
|
assert.ok(identity.services.length >= 1, `${label}.services must not be empty`);
|
|
assertUnique(identity.services.map((service) => service.serviceId), `${label}.services`);
|
|
for (const [index, service] of identity.services.entries()) {
|
|
const serviceLabel = `${label}.services[${index}]`;
|
|
assertObject(service, serviceLabel);
|
|
for (const field of ["serviceId", "commitId", "matchesSource", "image", "imageTag", "digest", "publishState"]) {
|
|
assert.ok(Object.hasOwn(service, field), `${serviceLabel} missing ${field}`);
|
|
}
|
|
assertString(service.serviceId, `${serviceLabel}.serviceId`);
|
|
assert.match(service.commitId, /^[a-f0-9]{7,40}$/, `${serviceLabel}.commitId`);
|
|
assert.equal(typeof service.matchesSource, "boolean", `${serviceLabel}.matchesSource`);
|
|
assertString(service.image, `${serviceLabel}.image`);
|
|
assertString(service.imageTag, `${serviceLabel}.imageTag`);
|
|
assertShaOrNotPublished(service.digest, `${serviceLabel}.digest`);
|
|
assert.ok(["skeleton-only", "published"].includes(service.publishState), `${serviceLabel}.publishState`);
|
|
}
|
|
|
|
assertStringArray(identity.serviceCommitIds, `${label}.serviceCommitIds`, { minLength: 1 });
|
|
assert.equal(typeof identity.matchesSource, "boolean", `${label}.matchesSource`);
|
|
assert.equal(typeof identity.publishVerified, "boolean", `${label}.publishVerified`);
|
|
assertObject(identity.refreshCommands, `${label}.refreshCommands`);
|
|
assert.equal(
|
|
identity.refreshCommands.blocked,
|
|
`node scripts/refresh-artifact-catalog.mjs --target-ref ${target.ref} --blocked`,
|
|
`${label}.refreshCommands.blocked`
|
|
);
|
|
assert.equal(
|
|
identity.refreshCommands.published,
|
|
`node scripts/refresh-artifact-catalog.mjs --target-ref ${target.ref} --publish-report reports/dev-gate/dev-artifacts.json`,
|
|
`${label}.refreshCommands.published`
|
|
);
|
|
|
|
if (identity.publishVerified) {
|
|
assert.equal(identity.artifactCatalog.ciPublished, true, `${label}.artifactCatalog.ciPublished`);
|
|
assert.equal(identity.artifactCatalog.registryVerified, true, `${label}.artifactCatalog.registryVerified`);
|
|
assert.equal(identity.artifactCatalog.digestCounts.sha256, identity.services.length, `${label}.artifactCatalog.digestCounts.sha256`);
|
|
assert.equal(identity.artifactCatalog.digestCounts.notPublished, 0, `${label}.artifactCatalog.digestCounts.notPublished`);
|
|
assert.equal(identity.artifactCatalog.digestCounts.invalid, 0, `${label}.artifactCatalog.digestCounts.invalid`);
|
|
}
|
|
}
|
|
async function validateDevEdgeReport(report, label) {
|
|
for (const field of [
|
|
"$schema",
|
|
"$id",
|
|
"reportVersion",
|
|
"issue",
|
|
"taskId",
|
|
"commitId",
|
|
"acceptanceLevel",
|
|
"devOnly",
|
|
"prodDisabled",
|
|
"sourceContract",
|
|
"validationCommands",
|
|
"localSmoke",
|
|
"dryRun",
|
|
"devPreconditions",
|
|
"blockers",
|
|
"edgeHealth"
|
|
]) {
|
|
assert.ok(Object.hasOwn(report, field), `${label} missing ${field}`);
|
|
}
|
|
|
|
assertString(report.$schema, `${label}.$schema`);
|
|
assertString(report.$id, `${label}.$id`);
|
|
assert.equal(report.reportVersion, "v1", `${label}.reportVersion`);
|
|
assert.equal(report.issue, requiredDevEdgeIssue, `${label}.issue`);
|
|
assert.equal(report.taskId, "dev-edge-health", `${label}.taskId`);
|
|
assert.match(report.commitId, /^([a-f0-9]{7,40}|unknown)$/, `${label}.commitId`);
|
|
assert.equal(report.acceptanceLevel, "dev_edge_health", `${label}.acceptanceLevel`);
|
|
assert.equal(report.devOnly, true, `${label}.devOnly`);
|
|
assert.equal(report.prodDisabled, true, `${label}.prodDisabled`);
|
|
|
|
assertObject(report.sourceContract, `${label}.sourceContract`);
|
|
assertStatus(report.sourceContract.status, `${label}.sourceContract.status`);
|
|
await assertDocumentSet(
|
|
report.sourceContract.documents,
|
|
`${label}.sourceContract.documents`,
|
|
requiredDevEdgeDocs
|
|
);
|
|
assertString(report.sourceContract.summary, `${label}.sourceContract.summary`);
|
|
|
|
assertStringArray(report.validationCommands, `${label}.validationCommands`, { minLength: 3 });
|
|
assertUnique(report.validationCommands, `${label}.validationCommands`);
|
|
for (const requiredCommand of requiredDevEdgeValidationCommands) {
|
|
assert.ok(
|
|
report.validationCommands.includes(requiredCommand),
|
|
`${label}.validationCommands missing ${requiredCommand}`
|
|
);
|
|
}
|
|
|
|
for (const section of ["localSmoke", "dryRun", "devPreconditions"]) {
|
|
assertObject(report[section], `${label}.${section}`);
|
|
assertStatus(report[section].status, `${label}.${section}.status`);
|
|
assertString(report[section].summary, `${label}.${section}.summary`);
|
|
}
|
|
assertStringArray(report.localSmoke.commands, `${label}.localSmoke.commands`, { minLength: 1 });
|
|
assertStringArray(report.localSmoke.evidence, `${label}.localSmoke.evidence`, { minLength: 1 });
|
|
assertStringArray(report.dryRun.commands, `${label}.dryRun.commands`, { minLength: 1 });
|
|
assertStringArray(report.dryRun.evidence, `${label}.dryRun.evidence`, { minLength: 1 });
|
|
assertStringArray(report.devPreconditions.requirements, `${label}.devPreconditions.requirements`, {
|
|
minLength: 1
|
|
});
|
|
assertBlockers(report.blockers, `${label}.blockers`);
|
|
|
|
assertObject(report.edgeHealth, `${label}.edgeHealth`);
|
|
assertTimestamp(report.edgeHealth.generatedAt, `${label}.edgeHealth.generatedAt`);
|
|
assert.equal(report.edgeHealth.endpoint, "http://74.48.78.17:16667", `${label}.edgeHealth.endpoint`);
|
|
assert.ok(
|
|
["pass", "blocker", "not_run"].includes(report.edgeHealth.status),
|
|
`${label}.edgeHealth.status must be pass, blocker, or not_run`
|
|
);
|
|
assertString(report.edgeHealth.classification, `${label}.edgeHealth.classification`);
|
|
assertObject(report.edgeHealth.contracts, `${label}.edgeHealth.contracts`);
|
|
assertDbLayerBooleans(
|
|
report.edgeHealth.contracts?.deploy?.cloudApiDb,
|
|
`${label}.edgeHealth.contracts.deploy.cloudApiDb`
|
|
);
|
|
assertDbRuntimeReadiness(
|
|
report.edgeHealth.runtimeDbReadiness,
|
|
`${label}.edgeHealth.runtimeDbReadiness`
|
|
);
|
|
assertArray(report.edgeHealth.publicTcp ?? [], `${label}.edgeHealth.publicTcp`);
|
|
assertArray(report.edgeHealth.publicHttp ?? [], `${label}.edgeHealth.publicHttp`);
|
|
assertObject(report.edgeHealth.kubernetes ?? {}, `${label}.edgeHealth.kubernetes`);
|
|
}
|
|
|
|
async function validateDevM2Report(report, label) {
|
|
for (const field of [
|
|
"$schema",
|
|
"$id",
|
|
"reportVersion",
|
|
"issue",
|
|
"taskId",
|
|
"commitId",
|
|
"acceptanceLevel",
|
|
"devOnly",
|
|
"prodDisabled",
|
|
"status",
|
|
"generatedAt",
|
|
"endpoint",
|
|
"frontendEndpoint",
|
|
"sourceContract",
|
|
"validationCommands",
|
|
"localSmoke",
|
|
"dryRun",
|
|
"devPreconditions",
|
|
"blockers",
|
|
"runtimeSmoke"
|
|
]) {
|
|
assert.ok(Object.hasOwn(report, field), `${label} missing ${field}`);
|
|
}
|
|
|
|
assertString(report.$schema, `${label}.$schema`);
|
|
assertString(report.$id, `${label}.$id`);
|
|
assert.equal(report.reportVersion, "v1", `${label}.reportVersion`);
|
|
assert.equal(report.issue, requiredDevM2Issue, `${label}.issue`);
|
|
assert.equal(report.taskId, "m2-dev-deploy-smoke", `${label}.taskId`);
|
|
assert.match(report.commitId, /^([a-f0-9]{7,40}|unknown)$/, `${label}.commitId`);
|
|
assert.equal(report.acceptanceLevel, "dev_m2_deploy_smoke", `${label}.acceptanceLevel`);
|
|
assert.equal(report.devOnly, true, `${label}.devOnly`);
|
|
assert.equal(report.prodDisabled, true, `${label}.prodDisabled`);
|
|
assert.equal(report.status, "pass", `${label}.status`);
|
|
assertTimestamp(report.generatedAt, `${label}.generatedAt`);
|
|
assert.equal(report.endpoint, "http://74.48.78.17:16667", `${label}.endpoint`);
|
|
assert.equal(report.frontendEndpoint, "http://74.48.78.17:16666", `${label}.frontendEndpoint`);
|
|
|
|
assertObject(report.sourceContract, `${label}.sourceContract`);
|
|
assertStatus(report.sourceContract.status, `${label}.sourceContract.status`);
|
|
await assertDocumentSet(
|
|
report.sourceContract.documents,
|
|
`${label}.sourceContract.documents`,
|
|
requiredDevM2Docs
|
|
);
|
|
assertString(report.sourceContract.summary, `${label}.sourceContract.summary`);
|
|
|
|
assertStringArray(report.validationCommands, `${label}.validationCommands`, {
|
|
minLength: requiredDevM2ValidationCommands.length
|
|
});
|
|
assertUnique(report.validationCommands, `${label}.validationCommands`);
|
|
for (const requiredCommand of requiredDevM2ValidationCommands) {
|
|
assert.ok(
|
|
report.validationCommands.includes(requiredCommand),
|
|
`${label}.validationCommands missing ${requiredCommand}`
|
|
);
|
|
}
|
|
|
|
for (const section of ["localSmoke", "dryRun", "devPreconditions"]) {
|
|
assertObject(report[section], `${label}.${section}`);
|
|
assertStatus(report[section].status, `${label}.${section}.status`);
|
|
assertStringArray(report[section].commands, `${label}.${section}.commands`, { minLength: 1 });
|
|
assertStringArray(report[section].evidence, `${label}.${section}.evidence`, { minLength: 1 });
|
|
assertString(report[section].summary, `${label}.${section}.summary`);
|
|
}
|
|
assertStringArray(report.devPreconditions.requirements, `${label}.devPreconditions.requirements`, {
|
|
minLength: 1
|
|
});
|
|
for (const command of [
|
|
"curl -fsS --max-time 10 http://74.48.78.17:16667/health",
|
|
"curl -fsS --max-time 10 http://74.48.78.17:16667/health/live",
|
|
"curl -fsS --max-time 10 http://74.48.78.17:16666/"
|
|
]) {
|
|
assert.ok(
|
|
report.devPreconditions.commands.includes(command),
|
|
`${label}.devPreconditions.commands missing ${command}`
|
|
);
|
|
}
|
|
assertBlockers(report.blockers, `${label}.blockers`);
|
|
|
|
assertObject(report.runtimeSmoke, `${label}.runtimeSmoke`);
|
|
assert.equal(report.runtimeSmoke.mode, "live-read-only", `${label}.runtimeSmoke.mode`);
|
|
assert.equal(report.runtimeSmoke.status, "pass", `${label}.runtimeSmoke.status`);
|
|
assertTimestamp(report.runtimeSmoke.generatedAt, `${label}.runtimeSmoke.generatedAt`);
|
|
assert.equal(report.runtimeSmoke.apiEndpoint, "http://74.48.78.17:16667", `${label}.runtimeSmoke.apiEndpoint`);
|
|
assert.equal(report.runtimeSmoke.frontendEndpoint, "http://74.48.78.17:16666", `${label}.runtimeSmoke.frontendEndpoint`);
|
|
|
|
assertObject(report.runtimeSmoke.safety, `${label}.runtimeSmoke.safety`);
|
|
assert.equal(report.runtimeSmoke.safety.environment, "dev", `${label}.runtimeSmoke.safety.environment`);
|
|
for (const field of [
|
|
"prodTouched",
|
|
"secretsRead",
|
|
"restarts",
|
|
"deployAttempted",
|
|
"heavyE2E",
|
|
"unideskRuntimeSubstitute"
|
|
]) {
|
|
assert.equal(report.runtimeSmoke.safety[field], false, `${label}.runtimeSmoke.safety.${field}`);
|
|
}
|
|
|
|
assertArray(report.runtimeSmoke.probes, `${label}.runtimeSmoke.probes`);
|
|
assert.ok(report.runtimeSmoke.probes.length >= 3, `${label}.runtimeSmoke.probes must include api and frontend probes`);
|
|
assertObject(report.runtimeSmoke.legacyPublicEndpoint, `${label}.runtimeSmoke.legacyPublicEndpoint`);
|
|
assert.equal(
|
|
report.runtimeSmoke.legacyPublicEndpoint.endpoint,
|
|
"http://74.48.78.17:6667",
|
|
`${label}.runtimeSmoke.legacyPublicEndpoint.endpoint`
|
|
);
|
|
assert.equal(
|
|
report.runtimeSmoke.legacyPublicEndpoint.status,
|
|
"deprecated",
|
|
`${label}.runtimeSmoke.legacyPublicEndpoint.status`
|
|
);
|
|
assert.equal(
|
|
report.runtimeSmoke.legacyPublicEndpoint.activeGreenEligible,
|
|
false,
|
|
`${label}.runtimeSmoke.legacyPublicEndpoint.activeGreenEligible`
|
|
);
|
|
}
|
|
|
|
async function validateDevCloudWorkbenchLiveReport(report, label, raw) {
|
|
for (const field of [
|
|
"$schema",
|
|
"$id",
|
|
"reportVersion",
|
|
"issue",
|
|
"taskId",
|
|
"commitId",
|
|
"acceptanceLevel",
|
|
"devOnly",
|
|
"prodDisabled",
|
|
"status",
|
|
"generatedAt",
|
|
"mode",
|
|
"url",
|
|
"evidenceLevel",
|
|
"devLive",
|
|
"sourceIdentity",
|
|
"runtimeIdentity",
|
|
"sourceContract",
|
|
"validationCommands",
|
|
"localSmoke",
|
|
"dryRun",
|
|
"devPreconditions",
|
|
"checks",
|
|
"blockers",
|
|
"safety"
|
|
]) {
|
|
assert.ok(Object.hasOwn(report, field), `${label} missing ${field}`);
|
|
}
|
|
|
|
assertString(report.$schema, `${label}.$schema`);
|
|
assertString(report.$id, `${label}.$id`);
|
|
assert.equal(report.reportVersion, "v1", `${label}.reportVersion`);
|
|
assert.equal(report.issue, issueFamily.DEV_CLOUD_WORKBENCH_LIVE, `${label}.issue`);
|
|
assert.equal(report.taskId, "dev-cloud-workbench-live", `${label}.taskId`);
|
|
assert.match(report.commitId, /^([a-f0-9]{7,40}|unknown)$/, `${label}.commitId`);
|
|
assert.equal(report.acceptanceLevel, "dev_cloud_workbench_live", `${label}.acceptanceLevel`);
|
|
assert.equal(report.devOnly, true, `${label}.devOnly`);
|
|
assert.equal(report.prodDisabled, true, `${label}.prodDisabled`);
|
|
assertStatus(report.status, `${label}.status`);
|
|
assert.equal(
|
|
report.blockers.length === 0 ? ["pass", "degraded"].includes(report.status) : report.status === "blocked",
|
|
true,
|
|
`${label}.status must match blockers`
|
|
);
|
|
assertTimestamp(report.generatedAt, `${label}.generatedAt`);
|
|
assert.equal(report.mode, "live", `${label}.mode`);
|
|
assert.equal(report.url, "http://74.48.78.17:16666/", `${label}.url`);
|
|
assert.equal(
|
|
report.evidenceLevel,
|
|
report.status === "pass"
|
|
? "DEV-LIVE-BROWSER"
|
|
: report.status === "degraded" ? "DEV-LIVE-BROWSER-DEGRADED" : "BLOCKED",
|
|
`${label}.evidenceLevel`
|
|
);
|
|
assert.equal(report.devLive, report.status === "pass", `${label}.devLive`);
|
|
assertDevCloudWorkbenchIdentity(report, label);
|
|
const apiRuntimeReadiness = assertDevCloudWorkbenchApiRuntimeReadiness(report, label);
|
|
assertDevCloudWorkbenchEndpoints(report, label);
|
|
|
|
assertObject(report.sourceContract, `${label}.sourceContract`);
|
|
assertStatus(report.sourceContract.status, `${label}.sourceContract.status`);
|
|
await assertDocumentSet(
|
|
report.sourceContract.documents,
|
|
`${label}.sourceContract.documents`,
|
|
requiredDevCloudWorkbenchDocs
|
|
);
|
|
assertString(report.sourceContract.summary, `${label}.sourceContract.summary`);
|
|
|
|
assertStringArray(report.validationCommands, `${label}.validationCommands`, {
|
|
minLength: requiredDevCloudWorkbenchValidationCommands.length
|
|
});
|
|
assertUnique(report.validationCommands, `${label}.validationCommands`);
|
|
for (const requiredCommand of requiredDevCloudWorkbenchValidationCommands) {
|
|
assert.ok(
|
|
report.validationCommands.includes(requiredCommand),
|
|
`${label}.validationCommands missing ${requiredCommand}`
|
|
);
|
|
}
|
|
|
|
for (const section of ["localSmoke", "dryRun", "devPreconditions"]) {
|
|
assertObject(report[section], `${label}.${section}`);
|
|
assertStatus(report[section].status, `${label}.${section}.status`);
|
|
assertStringArray(report[section].commands, `${label}.${section}.commands`, { minLength: 1 });
|
|
assertStringArray(report[section].evidence ?? [], `${label}.${section}.evidence`, { minLength: section === "devPreconditions" ? 0 : 1 });
|
|
assertString(report[section].summary, `${label}.${section}.summary`);
|
|
}
|
|
assertStringArray(report.devPreconditions.requirements, `${label}.devPreconditions.requirements`, {
|
|
minLength: 4
|
|
});
|
|
|
|
assertArray(report.checks, `${label}.checks`);
|
|
assert.ok(report.checks.length >= 3, `${label}.checks must include HTTP plus either deployment identity or browser journey checks`);
|
|
assertUnique(report.checks.map((check) => check.id), `${label}.checks ids`);
|
|
const checks = new Map(report.checks.map((check, index) => {
|
|
const checkLabel = `${label}.checks[${index}]`;
|
|
assertObject(check, checkLabel);
|
|
for (const field of ["id", "status", "summary"]) {
|
|
assert.ok(Object.hasOwn(check, field), `${checkLabel} missing ${field}`);
|
|
}
|
|
assertString(check.id, `${checkLabel}.id`);
|
|
assertStatus(check.status, `${checkLabel}.status`);
|
|
assertString(check.summary, `${checkLabel}.summary`);
|
|
if (Object.hasOwn(check, "evidence")) {
|
|
assertStringArray(check.evidence, `${checkLabel}.evidence`, { minLength: 1 });
|
|
}
|
|
if (Object.hasOwn(check, "observations")) {
|
|
assertObject(check.observations, `${checkLabel}.observations`);
|
|
}
|
|
return [check.id, check];
|
|
}));
|
|
const deploymentIdentityPreflight =
|
|
report.status === "blocked" &&
|
|
report.safety?.liveMode === "deployment-identity-preflight" &&
|
|
checks.has("live-runtime-current-main") &&
|
|
checks.has("live-web-assets-current-main");
|
|
if (deploymentIdentityPreflight) {
|
|
assertDevCloudWorkbenchDeploymentPreflight(report, label, checks);
|
|
return;
|
|
}
|
|
|
|
for (const requiredCheck of [
|
|
"live-http-html",
|
|
"live-browser-dom",
|
|
"live-api-runtime-readiness",
|
|
"live-code-agent-browser-journey"
|
|
]) {
|
|
assert.ok(checks.has(requiredCheck), `${label}.checks missing ${requiredCheck}`);
|
|
}
|
|
|
|
const httpCheck = checks.get("live-http-html");
|
|
assert.equal(httpCheck.status, "pass", `${label}.live-http-html.status`);
|
|
assert.ok(httpCheck.evidence.includes("HTTP 200"), `${label}.live-http-html.evidence missing HTTP 200`);
|
|
assert.ok(httpCheck.evidence.includes("text/html; charset=utf-8"), `${label}.live-http-html.evidence missing content-type`);
|
|
|
|
const dom = checks.get("live-browser-dom").observations;
|
|
assertObject(dom, `${label}.live-browser-dom.observations`);
|
|
assert.equal(dom.title, "HWLAB 云工作台", `${label}.live-browser-dom.title`);
|
|
assert.equal(dom.workspaceHidden, false, `${label}.live-browser-dom.workspaceHidden`);
|
|
assert.equal(dom.gateHidden, true, `${label}.live-browser-dom.gateHidden`);
|
|
assert.equal(dom.helpHidden, true, `${label}.live-browser-dom.helpHidden`);
|
|
assertObject(dom.coreControlsVisible, `${label}.live-browser-dom.coreControlsVisible`);
|
|
for (const control of ["commandInput", "commandSend", "agentChatStatus", "conversationList", "hardwareList"]) {
|
|
assert.equal(dom.coreControlsVisible[control], true, `${label}.live-browser-dom.coreControlsVisible.${control}`);
|
|
}
|
|
|
|
const apiRuntimeReadinessCheck = checks.get("live-api-runtime-readiness");
|
|
assert.equal(
|
|
apiRuntimeReadinessCheck.status,
|
|
apiRuntimeReadiness.status,
|
|
`${label}.live-api-runtime-readiness.status`
|
|
);
|
|
assert.deepEqual(
|
|
apiRuntimeReadinessCheck.observations,
|
|
apiRuntimeReadiness,
|
|
`${label}.live-api-runtime-readiness.observations`
|
|
);
|
|
|
|
const journey = checks.get("live-code-agent-browser-journey");
|
|
assert.equal(
|
|
journey.status,
|
|
report.status === "degraded" ? "pass" : report.status,
|
|
`${label}.live-code-agent-browser-journey.status`
|
|
);
|
|
assertObject(journey.observations, `${label}.live-code-agent-browser-journey.observations`);
|
|
assertObject(journey.observations.request, `${label}.live-code-agent-browser-journey.request`);
|
|
assert.equal(journey.observations.request.method, "POST", `${label}.live-code-agent-browser-journey.request.method`);
|
|
assert.equal(journey.observations.request.urlPath, "/v1/agent/chat", `${label}.live-code-agent-browser-journey.request.urlPath`);
|
|
assertObject(journey.observations.response, `${label}.live-code-agent-browser-journey.response`);
|
|
if (report.status === "blocked" && journey.status === "blocked") {
|
|
assertDevCloudWorkbenchCodeAgentBlocked(report, label, journey);
|
|
return;
|
|
}
|
|
assert.equal(journey.observations.response.status, "completed", `${label}.live-code-agent-browser-journey.response.status`);
|
|
for (const field of ["provider", "model", "backend", "traceId"]) {
|
|
assertString(journey.observations.response[field], `${label}.live-code-agent-browser-journey.response.${field}`);
|
|
}
|
|
assert.equal(journey.observations.response.hasReply, true, `${label}.live-code-agent-browser-journey.response.hasReply`);
|
|
assert.equal(journey.observations.response.error, null, `${label}.live-code-agent-browser-journey.response.error`);
|
|
for (const forbiddenField of ["conversationId", "sessionId", "messageId", "reply"]) {
|
|
assert.equal(
|
|
Object.hasOwn(journey.observations.response, forbiddenField),
|
|
false,
|
|
`${label}.live-code-agent-browser-journey.response must not retain ${forbiddenField}`
|
|
);
|
|
}
|
|
assertObject(journey.observations.ui, `${label}.live-code-agent-browser-journey.ui`);
|
|
assert.ok(
|
|
["已回复", "DEV-LIVE 回复"].includes(journey.observations.ui.agentChatStatus),
|
|
`${label}.live-code-agent-browser-journey.ui.agentChatStatus`
|
|
);
|
|
assert.equal(journey.observations.ui.completedMessageVisible, true, `${label}.live-code-agent-browser-journey.ui.completedMessageVisible`);
|
|
assert.equal(journey.observations.ui.failedMessageVisible, false, `${label}.live-code-agent-browser-journey.ui.failedMessageVisible`);
|
|
|
|
assertBlockers(report.blockers, `${label}.blockers`);
|
|
|
|
assertObject(report.safety, `${label}.safety`);
|
|
for (const field of ["prodTouched", "servicesRestarted", "heavyE2E", "hardwareWriteApis", "sourceIsDevLive"]) {
|
|
assert.equal(report.safety[field], false, `${label}.safety.${field}`);
|
|
}
|
|
assert.equal(report.safety.liveMode, "browser-user-journey-with-code-agent-post", `${label}.safety.liveMode`);
|
|
assertCodeAgentRetainedApiFields(report.safety.retainedApiFields, `${label}.safety.retainedApiFields`);
|
|
assertString(report.safety.statement, `${label}.safety.statement`);
|
|
if (report.status === "degraded") {
|
|
assertDevCloudWorkbenchDegradedSummary(report, label, apiRuntimeReadiness);
|
|
}
|
|
assertFreshDevCloudWorkbenchLiveEvidence(report, label, raw, journey);
|
|
}
|
|
|
|
function assertDevCloudWorkbenchCodeAgentBlocked(report, label, journey) {
|
|
const response = journey.observations.response;
|
|
assert.ok(["failed", "blocked"].includes(response.status), `${label}.live-code-agent-browser-journey.response.status`);
|
|
assert.equal(response.hasReply, false, `${label}.live-code-agent-browser-journey.response.hasReply`);
|
|
assertObject(response.error, `${label}.live-code-agent-browser-journey.response.error`);
|
|
assert.ok(
|
|
["provider_unavailable", "provider_blocked", "untrusted_completion"].includes(response.error.code),
|
|
`${label}.live-code-agent-browser-journey.response.error.code`
|
|
);
|
|
if (response.error.providerStatus !== null && response.error.providerStatus !== undefined) {
|
|
assert.ok(
|
|
Number.isInteger(response.error.providerStatus) && response.error.providerStatus >= 100 && response.error.providerStatus <= 599,
|
|
`${label}.live-code-agent-browser-journey.response.error.providerStatus`
|
|
);
|
|
}
|
|
for (const field of ["provider", "model", "backend", "traceId"]) {
|
|
assertString(response[field], `${label}.live-code-agent-browser-journey.response.${field}`);
|
|
}
|
|
for (const forbiddenField of ["conversationId", "sessionId", "messageId", "reply"]) {
|
|
assert.equal(
|
|
Object.hasOwn(response, forbiddenField),
|
|
false,
|
|
`${label}.live-code-agent-browser-journey.response must not retain ${forbiddenField}`
|
|
);
|
|
}
|
|
|
|
assertObject(journey.observations.ui, `${label}.live-code-agent-browser-journey.ui`);
|
|
assert.ok(
|
|
blockedCodeAgentUiLabels.has(journey.observations.ui.agentChatStatus),
|
|
`${label}.live-code-agent-browser-journey.ui.agentChatStatus`
|
|
);
|
|
assert.equal(journey.observations.ui.completedMessageVisible, false, `${label}.live-code-agent-browser-journey.ui.completedMessageVisible`);
|
|
assert.equal(journey.observations.ui.failedMessageVisible, true, `${label}.live-code-agent-browser-journey.ui.failedMessageVisible`);
|
|
assertBlockers(report.blockers, `${label}.blockers`);
|
|
assert.ok(
|
|
report.blockers.some((blocker) => blocker.scope === "live-code-agent-browser-journey"),
|
|
`${label}.blockers must include live-code-agent-browser-journey`
|
|
);
|
|
|
|
assertObject(report.safety, `${label}.safety`);
|
|
for (const field of ["prodTouched", "servicesRestarted", "heavyE2E", "hardwareWriteApis", "sourceIsDevLive"]) {
|
|
assert.equal(report.safety[field], false, `${label}.safety.${field}`);
|
|
}
|
|
assert.equal(report.safety.liveMode, "browser-user-journey-with-code-agent-post", `${label}.safety.liveMode`);
|
|
assertCodeAgentRetainedApiFields(report.safety.retainedApiFields, `${label}.safety.retainedApiFields`);
|
|
assertString(report.safety.statement, `${label}.safety.statement`);
|
|
}
|
|
|
|
function assertDevCloudWorkbenchApiRuntimeReadiness(report, label) {
|
|
assertObject(report.runtimeIdentity, `${label}.runtimeIdentity`);
|
|
if (Object.hasOwn(report.runtimeIdentity, "ready")) {
|
|
assertBoolean(report.runtimeIdentity.ready, `${label}.runtimeIdentity.ready`);
|
|
}
|
|
|
|
if (Object.hasOwn(report.runtimeIdentity, "runtime")) {
|
|
assertObject(report.runtimeIdentity.runtime, `${label}.runtimeIdentity.runtime`);
|
|
for (const field of ["durable", "ready"]) {
|
|
if (Object.hasOwn(report.runtimeIdentity.runtime, field)) {
|
|
assertBoolean(report.runtimeIdentity.runtime[field], `${label}.runtimeIdentity.runtime.${field}`);
|
|
}
|
|
}
|
|
if (Object.hasOwn(report.runtimeIdentity.runtime, "status")) {
|
|
assertString(report.runtimeIdentity.runtime.status, `${label}.runtimeIdentity.runtime.status`);
|
|
}
|
|
if (Object.hasOwn(report.runtimeIdentity.runtime, "blocker")) {
|
|
assertString(report.runtimeIdentity.runtime.blocker, `${label}.runtimeIdentity.runtime.blocker`);
|
|
}
|
|
}
|
|
|
|
if (Object.hasOwn(report.runtimeIdentity, "readiness")) {
|
|
assertObject(report.runtimeIdentity.readiness, `${label}.runtimeIdentity.readiness`);
|
|
if (Object.hasOwn(report.runtimeIdentity.readiness, "ready")) {
|
|
assertBoolean(report.runtimeIdentity.readiness.ready, `${label}.runtimeIdentity.readiness.ready`);
|
|
}
|
|
if (Object.hasOwn(report.runtimeIdentity.readiness, "status")) {
|
|
assertString(report.runtimeIdentity.readiness.status, `${label}.runtimeIdentity.readiness.status`);
|
|
}
|
|
if (Object.hasOwn(report.runtimeIdentity.readiness, "durability")) {
|
|
assertObject(report.runtimeIdentity.readiness.durability, `${label}.runtimeIdentity.readiness.durability`);
|
|
if (Object.hasOwn(report.runtimeIdentity.readiness.durability, "ready")) {
|
|
assertBoolean(report.runtimeIdentity.readiness.durability.ready, `${label}.runtimeIdentity.readiness.durability.ready`);
|
|
}
|
|
for (const field of ["status", "blocker", "blockedLayer", "queryResult"]) {
|
|
if (Object.hasOwn(report.runtimeIdentity.readiness.durability, field)) {
|
|
assertString(report.runtimeIdentity.readiness.durability[field], `${label}.runtimeIdentity.readiness.durability.${field}`);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (Object.hasOwn(report.runtimeIdentity, "blockerCodes")) {
|
|
assertStringArray(report.runtimeIdentity.blockerCodes, `${label}.runtimeIdentity.blockerCodes`);
|
|
}
|
|
|
|
const runtime = report.runtimeIdentity.runtime ?? {};
|
|
const readiness = report.runtimeIdentity.readiness ?? {};
|
|
const durability = readiness.durability ?? {};
|
|
const blockerCodes = Array.isArray(report.runtimeIdentity.blockerCodes) ? report.runtimeIdentity.blockerCodes : [];
|
|
const durableBlocked = runtime.blocker === "runtime_durable_adapter_query_blocked" ||
|
|
durability.blocker === "runtime_durable_adapter_query_blocked" ||
|
|
blockerCodes.includes("runtime_durable_adapter_query_blocked") ||
|
|
durability.blockedLayer === "durability_query" ||
|
|
runtime.connection?.queryResult === "query_blocked" ||
|
|
durability.queryResult === "query_blocked";
|
|
const degraded = report.runtimeIdentity.healthStatus === "degraded" ||
|
|
report.runtimeIdentity.ready === false ||
|
|
runtime.ready === false ||
|
|
durability.ready === false ||
|
|
runtime.durable === false ||
|
|
durableBlocked;
|
|
|
|
const status = degraded ? "degraded" : "pass";
|
|
const result = {
|
|
status,
|
|
healthStatus: report.runtimeIdentity.healthStatus,
|
|
ready: report.runtimeIdentity.ready === true,
|
|
runtimeDurable: runtime.durable === true,
|
|
runtimeReady: runtime.ready === true,
|
|
durabilityReady: durability.ready === true,
|
|
durableBlocked,
|
|
blockerCodes,
|
|
evidence: [
|
|
`api.status=${report.runtimeIdentity.healthStatus}`,
|
|
`ready=${report.runtimeIdentity.ready === true}`,
|
|
`runtime.durable=${runtime.durable === true}`,
|
|
`runtime.ready=${runtime.ready === true}`,
|
|
`durability.ready=${durability.ready === true}`,
|
|
durableBlocked ? "runtime_durable_adapter_query_blocked" : null,
|
|
...blockerCodes.map((code) => `blocker=${code}`)
|
|
].filter(Boolean),
|
|
summary: degraded
|
|
? "Live API is reachable but degraded/read-only; runtime durability is not ready, so this cannot be full DEV-LIVE acceptance."
|
|
: "Live API health and runtime durability are ready for browser journey evidence."
|
|
};
|
|
|
|
if (["pass", "degraded"].includes(report.status)) {
|
|
assert.equal(report.runtimeIdentity.status, "observed", `${label}.runtimeIdentity.status`);
|
|
assert.equal(report.runtimeIdentity.source, "health-live", `${label}.runtimeIdentity.source`);
|
|
assert.equal(report.runtimeIdentity.endpoint, `${ACTIVE_DEV_PUBLIC_ENDPOINT}/health/live`, `${label}.runtimeIdentity.endpoint`);
|
|
assert.equal(report.runtimeIdentity.serviceId, "hwlab-cloud-api", `${label}.runtimeIdentity.serviceId`);
|
|
assert.equal(report.runtimeIdentity.environment, "dev", `${label}.runtimeIdentity.environment`);
|
|
assert.match(report.runtimeIdentity.commitId, /^[a-f0-9]{7,40}$/, `${label}.runtimeIdentity.commitId`);
|
|
assert.notEqual(report.runtimeIdentity.commitId, "unknown", `${label}.runtimeIdentity.commitId`);
|
|
assert.notEqual(report.runtimeIdentity.commitSource, "source-git-head", `${label}.runtimeIdentity.commitSource`);
|
|
assert.notEqual(report.runtimeIdentity.commitSource, "hard-coded", `${label}.runtimeIdentity.commitSource`);
|
|
assert.notEqual(report.runtimeIdentity.commitSource, "literal", `${label}.runtimeIdentity.commitSource`);
|
|
|
|
const generatedAtMs = Date.parse(report.generatedAt);
|
|
const runtimeObservedAtMs = Date.parse(report.runtimeIdentity.observedAt);
|
|
assert.ok(
|
|
Math.abs(generatedAtMs - runtimeObservedAtMs) <= devCloudWorkbenchRuntimeIdentityMaxSkewMs,
|
|
`${label}.runtimeIdentity.observedAt must be fresh relative to generatedAt`
|
|
);
|
|
}
|
|
|
|
if (report.status === "pass") {
|
|
assert.equal(
|
|
degraded,
|
|
false,
|
|
`${label}.runtimeIdentity degraded API or runtime durability blocker cannot be summarized as full DEV-LIVE accepted`
|
|
);
|
|
}
|
|
|
|
if (degraded) {
|
|
assert.ok(
|
|
["degraded", "blocked"].includes(report.status),
|
|
`${label}.status must be degraded or blocked when API health or runtime durability is degraded`
|
|
);
|
|
assert.equal(report.devLive, false, `${label}.devLive must remain false for degraded/read-only workbench evidence`);
|
|
assert.equal(
|
|
report.evidenceLevel,
|
|
report.status === "degraded" ? "DEV-LIVE-BROWSER-DEGRADED" : "BLOCKED",
|
|
`${label}.evidenceLevel`
|
|
);
|
|
}
|
|
|
|
return result;
|
|
}
|
|
|
|
function assertDevCloudWorkbenchDegradedSummary(report, label, apiRuntimeReadiness) {
|
|
assert.equal(apiRuntimeReadiness.status, "degraded", `${label}.live-api-runtime-readiness.status`);
|
|
assert.equal(report.devPreconditions.status, "degraded", `${label}.devPreconditions.status`);
|
|
const summaryText = [
|
|
report.reportLifecycle.summary,
|
|
report.devPreconditions.summary,
|
|
report.safety.statement
|
|
].join(" ");
|
|
assert.match(
|
|
summaryText,
|
|
/degraded\/read-only mode|degraded\/read-only|read-only mode/u,
|
|
`${label} degraded API/runtime report must be summarized as deployed UI usable in degraded/read-only mode`
|
|
);
|
|
assert.doesNotMatch(
|
|
summaryText,
|
|
/full DEV-LIVE accepted|full DEV-LIVE acceptance passed|M3\/M4\/M5 DEV-LIVE accepted|M[345]\s+(?:DEV-LIVE\s+)?(?:accepted|acceptance passed|green)/u,
|
|
`${label} degraded API/runtime report must not claim full M3/M4/M5 DEV-LIVE acceptance`
|
|
);
|
|
}
|
|
|
|
function assertCodeAgentRetainedApiFields(value, label) {
|
|
assertArray(value, label);
|
|
for (const field of ["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv"]) {
|
|
assert.ok(value.includes(field), `${label} missing ${field}`);
|
|
}
|
|
for (const field of value) {
|
|
assert.ok(
|
|
["status", "provider", "model", "backend", "traceId", "hasReply", "error.code", "error.missingEnv", "error.providerStatus"].includes(field),
|
|
`${label} contains unexpected retained field ${field}`
|
|
);
|
|
}
|
|
}
|
|
|
|
function assertDevCloudWorkbenchDeploymentPreflight(report, label, checks) {
|
|
for (const requiredCheck of [
|
|
"live-runtime-current-main",
|
|
"live-http-html",
|
|
"live-web-assets-current-main",
|
|
"live-code-agent-browser-journey"
|
|
]) {
|
|
assert.ok(checks.has(requiredCheck), `${label}.checks missing ${requiredCheck}`);
|
|
}
|
|
|
|
const runtimeCheck = checks.get("live-runtime-current-main");
|
|
const webAssetCheck = checks.get("live-web-assets-current-main");
|
|
const journey = checks.get("live-code-agent-browser-journey");
|
|
assert.ok(["pass", "blocked"].includes(runtimeCheck.status), `${label}.live-runtime-current-main.status`);
|
|
assert.ok(["pass", "blocked"].includes(webAssetCheck.status), `${label}.live-web-assets-current-main.status`);
|
|
assert.equal(journey.status, "blocked", `${label}.live-code-agent-browser-journey.status`);
|
|
assert.ok(
|
|
runtimeCheck.status === "blocked" || webAssetCheck.status === "blocked",
|
|
`${label}.deployment preflight requires runtime or web asset drift`
|
|
);
|
|
assertObject(runtimeCheck.observations, `${label}.live-runtime-current-main.observations`);
|
|
assertObject(webAssetCheck.observations, `${label}.live-web-assets-current-main.observations`);
|
|
assertObject(report.expectedRuntimeIdentity, `${label}.expectedRuntimeIdentity`);
|
|
for (const field of ["status", "serviceId", "source", "commitId", "imageTag", "image", "summary"]) {
|
|
assertString(report.expectedRuntimeIdentity[field], `${label}.expectedRuntimeIdentity.${field}`);
|
|
}
|
|
assert.ok(["observed", "not_observed"].includes(report.expectedRuntimeIdentity.status), `${label}.expectedRuntimeIdentity.status`);
|
|
assertObject(report.deploymentIdentity, `${label}.deploymentIdentity`);
|
|
assert.equal(report.deploymentIdentity.status, runtimeCheck.status, `${label}.deploymentIdentity.status`);
|
|
assertString(report.deploymentIdentity.expectedCommit, `${label}.deploymentIdentity.expectedCommit`);
|
|
assertString(report.deploymentIdentity.expectedImageTag, `${label}.deploymentIdentity.expectedImageTag`);
|
|
assertString(report.deploymentIdentity.expectedSource, `${label}.deploymentIdentity.expectedSource`);
|
|
assertString(report.deploymentIdentity.observedRuntimeCommit, `${label}.deploymentIdentity.observedRuntimeCommit`);
|
|
assertString(report.deploymentIdentity.observedImageTag, `${label}.deploymentIdentity.observedImageTag`);
|
|
if (report.deploymentIdentity.status === "blocked") {
|
|
assertString(report.deploymentIdentity.reason, `${label}.deploymentIdentity.reason`);
|
|
}
|
|
assertString(report.deploymentIdentity.summary, `${label}.deploymentIdentity.summary`);
|
|
assertStringArray(report.deploymentIdentity.evidence, `${label}.deploymentIdentity.evidence`, { minLength: 4 });
|
|
assert.deepEqual(
|
|
runtimeCheck.observations,
|
|
report.deploymentIdentity,
|
|
`${label}.live-runtime-current-main.observations`
|
|
);
|
|
|
|
assertObject(report.webAssetIdentity, `${label}.webAssetIdentity`);
|
|
assert.equal(report.webAssetIdentity.status, webAssetCheck.status, `${label}.webAssetIdentity.status`);
|
|
assertArray(report.webAssetIdentity.assets, `${label}.webAssetIdentity.assets`);
|
|
assertStringArray(report.webAssetIdentity.mismatches, `${label}.webAssetIdentity.mismatches`, {
|
|
minLength: report.webAssetIdentity.status === "blocked" ? 1 : 0
|
|
});
|
|
assertString(report.webAssetIdentity.summary, `${label}.webAssetIdentity.summary`);
|
|
for (const [index, asset] of report.webAssetIdentity.assets.entries()) {
|
|
const assetLabel = `${label}.webAssetIdentity.assets[${index}]`;
|
|
assertObject(asset, assetLabel);
|
|
for (const field of ["path", "status", "contentType", "sourceHash", "liveHash"]) {
|
|
assertString(asset[field], `${assetLabel}.${field}`);
|
|
}
|
|
assert.ok(["match", "mismatch", "fetch_failed"].includes(asset.status), `${assetLabel}.status`);
|
|
assert.ok(Number.isInteger(asset.sourceLength), `${assetLabel}.sourceLength`);
|
|
assert.ok(Number.isInteger(asset.liveLength), `${assetLabel}.liveLength`);
|
|
}
|
|
assert.deepEqual(
|
|
webAssetCheck.observations,
|
|
report.webAssetIdentity,
|
|
`${label}.live-web-assets-current-main.observations`
|
|
);
|
|
assertDevCloudWorkbenchDeploymentPreflightJourney(report, label, journey);
|
|
|
|
assertBlockers(report.blockers, `${label}.blockers`);
|
|
if (runtimeCheck.status === "blocked") {
|
|
assert.ok(
|
|
report.blockers.some((blocker) => blocker.scope === "live-runtime-current-main"),
|
|
`${label}.blockers must include runtime identity drift`
|
|
);
|
|
}
|
|
if (webAssetCheck.status === "blocked") {
|
|
assert.ok(
|
|
report.blockers.some((blocker) => blocker.scope === "live-web-assets-current-main"),
|
|
`${label}.blockers must include web asset drift`
|
|
);
|
|
}
|
|
|
|
assertObject(report.safety, `${label}.safety`);
|
|
for (const field of ["prodTouched", "servicesRestarted", "heavyE2E", "hardwareWriteApis", "sourceIsDevLive", "codeAgentBrowserJourneySkipped"]) {
|
|
assert.equal(report.safety[field], field === "codeAgentBrowserJourneySkipped", `${label}.safety.${field}`);
|
|
}
|
|
assert.equal(report.safety.liveMode, "deployment-identity-preflight", `${label}.safety.liveMode`);
|
|
assert.deepEqual(
|
|
report.safety.retainedApiFields,
|
|
["runtime.commitId", "runtime.imageTag", "web asset hash/status"],
|
|
`${label}.safety.retainedApiFields`
|
|
);
|
|
assertString(report.safety.statement, `${label}.safety.statement`);
|
|
}
|
|
|
|
function assertDevCloudWorkbenchDeploymentPreflightJourney(report, label, journey) {
|
|
assertStringArray(journey.evidence, `${label}.live-code-agent-browser-journey.evidence`, { minLength: 3 });
|
|
assert.ok(
|
|
journey.evidence.includes("POST /v1/agent/chat not sent"),
|
|
`${label}.live-code-agent-browser-journey.evidence missing not_sent proof`
|
|
);
|
|
assertObject(journey.observations, `${label}.live-code-agent-browser-journey.observations`);
|
|
assertObject(journey.observations.request, `${label}.live-code-agent-browser-journey.request`);
|
|
assert.equal(journey.observations.request.method, "POST", `${label}.live-code-agent-browser-journey.request.method`);
|
|
assert.equal(journey.observations.request.status, "not_sent", `${label}.live-code-agent-browser-journey.request.status`);
|
|
assert.equal(journey.observations.request.urlPath, "/v1/agent/chat", `${label}.live-code-agent-browser-journey.request.urlPath`);
|
|
assertObject(journey.observations.response, `${label}.live-code-agent-browser-journey.response`);
|
|
assert.equal(journey.observations.response.status, "failed", `${label}.live-code-agent-browser-journey.response.status`);
|
|
assert.equal(journey.observations.response.hasReply, false, `${label}.live-code-agent-browser-journey.response.hasReply`);
|
|
assertObject(journey.observations.response.error, `${label}.live-code-agent-browser-journey.response.error`);
|
|
assert.equal(
|
|
journey.observations.response.error.code,
|
|
deploymentPreflightCodeAgentErrorCode,
|
|
`${label}.live-code-agent-browser-journey.response.error.code`
|
|
);
|
|
for (const field of ["provider", "model", "backend", "traceId"]) {
|
|
assert.equal(journey.observations.response[field], "not_observed", `${label}.live-code-agent-browser-journey.response.${field}`);
|
|
}
|
|
assertObject(journey.observations.classification, `${label}.live-code-agent-browser-journey.classification`);
|
|
assert.equal(journey.observations.classification.status, "blocked", `${label}.live-code-agent-browser-journey.classification.status`);
|
|
assert.equal(
|
|
journey.observations.classification.reason,
|
|
deploymentPreflightCodeAgentErrorCode,
|
|
`${label}.live-code-agent-browser-journey.classification.reason`
|
|
);
|
|
assert.equal(
|
|
journey.observations.classification.codeAgentBrowserJourneySkipped,
|
|
true,
|
|
`${label}.live-code-agent-browser-journey.classification.codeAgentBrowserJourneySkipped`
|
|
);
|
|
const journeyText = JSON.stringify(journey.observations);
|
|
assert.equal(
|
|
/"status"\s*:\s*200|completed|provider_unavailable|OPENAI_API_KEY/u.test(journeyText),
|
|
false,
|
|
`${label}.live-code-agent-browser-journey must not retain sent, completed, or secret-dependent evidence during deployment preflight`
|
|
);
|
|
assert.ok(
|
|
report.blockers.some((blocker) => blocker.scope === "live-code-agent-browser-journey"),
|
|
`${label}.blockers must include live-code-agent-browser-journey`
|
|
);
|
|
}
|
|
|
|
function assertDevCloudWorkbenchIdentity(report, label) {
|
|
assertObject(report.sourceIdentity, `${label}.sourceIdentity`);
|
|
for (const field of ["status", "source", "commitId", "shortCommitId", "ref", "worktreeState", "reportCommitId", "summary"]) {
|
|
assert.ok(Object.hasOwn(report.sourceIdentity, field), `${label}.sourceIdentity missing ${field}`);
|
|
assertString(report.sourceIdentity[field], `${label}.sourceIdentity.${field}`);
|
|
}
|
|
assert.ok(["observed", "unknown"].includes(report.sourceIdentity.status), `${label}.sourceIdentity.status`);
|
|
assert.ok(["git-head", "environment", "unknown"].includes(report.sourceIdentity.source), `${label}.sourceIdentity.source`);
|
|
assert.match(report.sourceIdentity.commitId, /^([a-f0-9]{7,40}|unknown)$/, `${label}.sourceIdentity.commitId`);
|
|
assert.match(report.sourceIdentity.shortCommitId, /^([a-f0-9]{7,40}|unknown)$/, `${label}.sourceIdentity.shortCommitId`);
|
|
assert.ok(["clean", "dirty", "unknown"].includes(report.sourceIdentity.worktreeState), `${label}.sourceIdentity.worktreeState`);
|
|
if (report.sourceIdentity.source === "environment") {
|
|
assertString(report.sourceIdentity.environmentKey, `${label}.sourceIdentity.environmentKey`);
|
|
}
|
|
if (report.sourceIdentity.worktreeState === "clean" && report.sourceIdentity.commitId !== "unknown") {
|
|
assert.match(report.sourceIdentity.reportCommitId, /^[a-f0-9]{7,40}$/, `${label}.sourceIdentity.reportCommitId`);
|
|
assert.equal(report.commitId, report.sourceIdentity.reportCommitId, `${label}.commitId must match clean source reportCommitId`);
|
|
} else {
|
|
assert.equal(report.sourceIdentity.reportCommitId, "unknown", `${label}.sourceIdentity.reportCommitId`);
|
|
assert.equal(report.commitId, "unknown", `${label}.commitId must be unknown when source worktree identity is dirty or unknown`);
|
|
}
|
|
if (report.sourceIdentity.dirty !== null) {
|
|
assertBoolean(report.sourceIdentity.dirty, `${label}.sourceIdentity.dirty`);
|
|
assert.equal(report.sourceIdentity.dirty, report.sourceIdentity.worktreeState === "dirty", `${label}.sourceIdentity.dirty`);
|
|
}
|
|
|
|
assertObject(report.runtimeIdentity, `${label}.runtimeIdentity`);
|
|
for (const field of ["status", "source", "endpoint", "serviceId", "environment", "healthStatus", "commitId", "commitSource", "imageTag", "observedAt", "summary"]) {
|
|
assert.ok(Object.hasOwn(report.runtimeIdentity, field), `${label}.runtimeIdentity missing ${field}`);
|
|
assertString(report.runtimeIdentity[field], `${label}.runtimeIdentity.${field}`);
|
|
}
|
|
assert.ok(["observed", "not_observed"].includes(report.runtimeIdentity.status), `${label}.runtimeIdentity.status`);
|
|
assert.equal(report.runtimeIdentity.source, "health-live", `${label}.runtimeIdentity.source`);
|
|
assert.equal(report.runtimeIdentity.endpoint, "http://74.48.78.17:16667/health/live", `${label}.runtimeIdentity.endpoint`);
|
|
assertTimestamp(report.runtimeIdentity.observedAt, `${label}.runtimeIdentity.observedAt`);
|
|
assert.match(report.runtimeIdentity.commitId, /^([a-f0-9]{7,40}|unknown|not_observed)$/, `${label}.runtimeIdentity.commitId`);
|
|
if (report.runtimeIdentity.status === "not_observed") {
|
|
assert.equal(report.runtimeIdentity.commitId, "not_observed", `${label}.runtimeIdentity.commitId`);
|
|
assertString(report.runtimeIdentity.reason, `${label}.runtimeIdentity.reason`);
|
|
} else {
|
|
assert.ok(
|
|
["hwlab-cloud-api", "unknown"].includes(report.runtimeIdentity.serviceId),
|
|
`${label}.runtimeIdentity.serviceId`
|
|
);
|
|
assert.ok(["dev", "unknown"].includes(report.runtimeIdentity.environment), `${label}.runtimeIdentity.environment`);
|
|
}
|
|
assert.ok(
|
|
report.sourceIdentity.commitId === "unknown" ||
|
|
report.runtimeIdentity.commitId === "unknown" ||
|
|
report.runtimeIdentity.commitId === "not_observed" ||
|
|
report.runtimeIdentity.commitSource !== "source-git-head",
|
|
`${label}.runtimeIdentity must not present source git HEAD as the deployed live revision`
|
|
);
|
|
}
|
|
|
|
function assertDevCloudWorkbenchEndpoints(report, label) {
|
|
assertObject(report.endpoints, `${label}.endpoints`);
|
|
assert.equal(report.endpoints.frontend, ACTIVE_DEV_BROWSER_ENDPOINT, `${label}.endpoints.frontend`);
|
|
assert.equal(report.endpoints.api, ACTIVE_DEV_PUBLIC_ENDPOINT, `${label}.endpoints.api`);
|
|
assert.equal(report.endpoints.edge, ACTIVE_DEV_PUBLIC_ENDPOINT, `${label}.endpoints.edge`);
|
|
}
|
|
|
|
function assertFreshDevCloudWorkbenchLiveEvidence(report, label, raw, journey) {
|
|
const deprecatedEndpoints = findDeprecatedPublicEndpoints(raw);
|
|
assert.deepEqual(
|
|
deprecatedEndpoints,
|
|
[],
|
|
`${label} deployed workbench report must not contain deprecated public endpoint evidence`
|
|
);
|
|
|
|
if (!["pass", "degraded"].includes(report.status)) {
|
|
return;
|
|
}
|
|
|
|
assert.equal(report.devLive, report.status === "pass", `${label}.devLive`);
|
|
assert.equal(
|
|
report.evidenceLevel,
|
|
report.status === "pass" ? "DEV-LIVE-BROWSER" : "DEV-LIVE-BROWSER-DEGRADED",
|
|
`${label}.evidenceLevel`
|
|
);
|
|
assert.equal(report.safety.sourceIsDevLive, false, `${label}.safety.sourceIsDevLive`);
|
|
if (report.status === "pass") {
|
|
assert.notEqual(report.runtimeIdentity.healthStatus, "degraded", `${label}.runtimeIdentity.healthStatus`);
|
|
assert.equal(report.runtimeIdentity.ready, true, `${label}.runtimeIdentity.ready`);
|
|
assert.equal(report.runtimeIdentity.runtime?.durable, true, `${label}.runtimeIdentity.runtime.durable`);
|
|
assert.equal(report.runtimeIdentity.runtime?.ready, true, `${label}.runtimeIdentity.runtime.ready`);
|
|
assert.equal(report.runtimeIdentity.readiness?.durability?.ready, true, `${label}.runtimeIdentity.readiness.durability.ready`);
|
|
assert.notEqual(
|
|
report.runtimeIdentity.runtime?.blocker,
|
|
"runtime_durable_adapter_query_blocked",
|
|
`${label}.runtimeIdentity.runtime.blocker`
|
|
);
|
|
assert.notEqual(
|
|
report.runtimeIdentity.readiness?.durability?.blocker,
|
|
"runtime_durable_adapter_query_blocked",
|
|
`${label}.runtimeIdentity.readiness.durability.blocker`
|
|
);
|
|
assert.equal(
|
|
(report.runtimeIdentity.blockerCodes ?? []).includes("runtime_durable_adapter_query_blocked"),
|
|
false,
|
|
`${label}.runtimeIdentity.blockerCodes`
|
|
);
|
|
}
|
|
assert.equal(
|
|
report.sourceIdentity.reportCommitId === "unknown" || report.sourceIdentity.worktreeState === "clean",
|
|
true,
|
|
`${label}.sourceIdentity.reportCommitId must be unknown unless the source worktree is clean`
|
|
);
|
|
|
|
const journeyText = JSON.stringify(journey.observations);
|
|
assert.equal(
|
|
/provider_unavailable|OPENAI_API_KEY|providerStatus|provider HTTP|HTTP 502/u.test(journeyText),
|
|
false,
|
|
`${label}.live-code-agent-browser-journey must not mix provider_unavailable, OPENAI_API_KEY, or providerStatus blockers into accepted evidence`
|
|
);
|
|
|
|
assertAcceptedCodeAgentPayload(
|
|
journey.observations.response,
|
|
`${label}.live-code-agent-browser-journey.response`
|
|
);
|
|
const networkEvents = journey.observations.networkEvents ?? [];
|
|
assertArray(networkEvents, `${label}.live-code-agent-browser-journey.networkEvents`);
|
|
assert.ok(networkEvents.length >= 1, `${label}.live-code-agent-browser-journey.networkEvents`);
|
|
for (const [index, event] of networkEvents.entries()) {
|
|
const eventLabel = `${label}.live-code-agent-browser-journey.networkEvents[${index}]`;
|
|
assertObject(event, eventLabel);
|
|
assert.equal(event.method, "POST", `${eventLabel}.method`);
|
|
assert.equal(event.status, 200, `${eventLabel}.status`);
|
|
assert.equal(event.urlPath, "/v1/agent/chat", `${eventLabel}.urlPath`);
|
|
assertAcceptedCodeAgentPayload(event.body, `${eventLabel}.body`);
|
|
}
|
|
}
|
|
|
|
function assertAcceptedCodeAgentPayload(payload, label) {
|
|
assertObject(payload, label);
|
|
assert.equal(payload.status, "completed", `${label}.status`);
|
|
assertString(payload.provider, `${label}.provider`);
|
|
assertString(payload.model, `${label}.model`);
|
|
assertString(payload.backend, `${label}.backend`);
|
|
assertString(payload.traceId, `${label}.traceId`);
|
|
assert.equal(payload.hasReply, true, `${label}.hasReply`);
|
|
assert.equal(payload.error, null, `${label}.error`);
|
|
assert.equal(
|
|
syntheticAcceptedCodeAgentModels.has(payload.model),
|
|
false,
|
|
`${label}.model must not use the known synthetic placeholder model`
|
|
);
|
|
for (const forbiddenField of ["conversationId", "sessionId", "messageId", "reply", "content", "text", "output"]) {
|
|
assert.equal(
|
|
Object.hasOwn(payload, forbiddenField),
|
|
false,
|
|
`${label} must not retain ${forbiddenField}`
|
|
);
|
|
}
|
|
}
|
|
|
|
async function validateDevM3Report(report, label) {
|
|
for (const field of [
|
|
"$schema",
|
|
"$id",
|
|
"reportVersion",
|
|
"issue",
|
|
"taskId",
|
|
"commitId",
|
|
"acceptanceLevel",
|
|
"devOnly",
|
|
"prodDisabled",
|
|
"sourceContract",
|
|
"validationCommands",
|
|
"runtimeTarget",
|
|
"safetyGates",
|
|
"liveChecks",
|
|
"liveOperation",
|
|
"blockers",
|
|
"summary"
|
|
]) {
|
|
assert.ok(Object.hasOwn(report, field), `${label} missing ${field}`);
|
|
}
|
|
|
|
assertString(report.$schema, `${label}.$schema`);
|
|
assertString(report.$id, `${label}.$id`);
|
|
assert.equal(report.reportVersion, "v1", `${label}.reportVersion`);
|
|
assert.equal(report.issue, requiredDevM3Issue, `${label}.issue`);
|
|
assert.equal(report.taskId, "dev-m3-hardware-loop", `${label}.taskId`);
|
|
assert.match(report.commitId, /^([a-f0-9]{7,40}|unknown)$/, `${label}.commitId`);
|
|
assert.equal(report.acceptanceLevel, "dev_m3_hardware_loop", `${label}.acceptanceLevel`);
|
|
assert.equal(report.devOnly, true, `${label}.devOnly`);
|
|
assert.equal(report.prodDisabled, true, `${label}.prodDisabled`);
|
|
|
|
assertObject(report.sourceContract, `${label}.sourceContract`);
|
|
assertStatus(report.sourceContract.status, `${label}.sourceContract.status`);
|
|
assertStringArray(report.sourceContract.documents, `${label}.sourceContract.documents`, {
|
|
minLength: requiredDevM3Docs.length
|
|
});
|
|
assertString(report.sourceContract.summary, `${label}.sourceContract.summary`);
|
|
const documents = new Set(report.sourceContract.documents);
|
|
assertUnique(report.sourceContract.documents, `${label}.sourceContract.documents`);
|
|
for (const requiredDoc of requiredDevM3Docs) {
|
|
assert.ok(documents.has(requiredDoc), `${label}.sourceContract.documents missing ${requiredDoc}`);
|
|
}
|
|
for (const [index, documentPath] of report.sourceContract.documents.entries()) {
|
|
const absolutePath = assertRepoRelativePath(
|
|
documentPath,
|
|
`${label}.sourceContract.documents[${index}]`
|
|
);
|
|
await access(absolutePath, fsConstants.R_OK);
|
|
}
|
|
|
|
assertStringArray(report.validationCommands, `${label}.validationCommands`, { minLength: 2 });
|
|
assertUnique(report.validationCommands, `${label}.validationCommands`);
|
|
for (const requiredCommand of requiredDevM3ValidationCommands) {
|
|
assert.ok(
|
|
report.validationCommands.includes(requiredCommand),
|
|
`${label}.validationCommands missing ${requiredCommand}`
|
|
);
|
|
}
|
|
assert.ok(
|
|
report.validationCommands.includes(legacyDevM3LiveCommand),
|
|
`${label}.validationCommands missing legacy compatibility command ${legacyDevM3LiveCommand}`
|
|
);
|
|
|
|
assertObject(report.runtimeTarget, `${label}.runtimeTarget`);
|
|
assert.equal(report.runtimeTarget.endpoint, "http://74.48.78.17:16667", `${label}.runtimeTarget.endpoint`);
|
|
if (Object.hasOwn(report.runtimeTarget, "frontendEndpoint")) {
|
|
assert.equal(
|
|
report.runtimeTarget.frontendEndpoint,
|
|
"http://74.48.78.17:16666",
|
|
`${label}.runtimeTarget.frontendEndpoint`
|
|
);
|
|
}
|
|
assert.equal(report.runtimeTarget.namespace, "hwlab-dev", `${label}.runtimeTarget.namespace`);
|
|
assert.equal(report.runtimeTarget.environment, "dev", `${label}.runtimeTarget.environment`);
|
|
assert.equal(report.runtimeTarget.requiredBoxSimulators, 2, `${label}.runtimeTarget.requiredBoxSimulators`);
|
|
assert.equal(report.runtimeTarget.requiredGatewaySimulators, 2, `${label}.runtimeTarget.requiredGatewaySimulators`);
|
|
if (Object.hasOwn(report.runtimeTarget, "requiredPatchPanels")) {
|
|
assert.equal(report.runtimeTarget.requiredPatchPanels, 1, `${label}.runtimeTarget.requiredPatchPanels`);
|
|
}
|
|
if (Object.hasOwn(report.runtimeTarget, "requiredRoute")) {
|
|
assert.equal(
|
|
report.runtimeTarget.requiredRoute,
|
|
"res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1",
|
|
`${label}.runtimeTarget.requiredRoute`
|
|
);
|
|
}
|
|
assert.equal(report.runtimeTarget.realHardwareAllowed, false, `${label}.runtimeTarget.realHardwareAllowed`);
|
|
assert.equal(report.runtimeTarget.prodAllowed, false, `${label}.runtimeTarget.prodAllowed`);
|
|
|
|
assertObject(report.safetyGates, `${label}.safetyGates`);
|
|
for (const field of [
|
|
"liveFlagRequired",
|
|
"confirmDevRequired",
|
|
"expectNonProdRequired",
|
|
"confirmedNonProductionRequired",
|
|
"dryRunPlanSupported",
|
|
"prodForbidden",
|
|
"realHardwareForbidden",
|
|
"secretReadForbidden",
|
|
"forcePushForbidden",
|
|
"unideskRuntimeSubstitutionForbidden"
|
|
]) {
|
|
assert.equal(report.safetyGates[field], true, `${label}.safetyGates.${field}`);
|
|
}
|
|
if (Object.hasOwn(report.safetyGates, "dryRunCallsLiveEndpoints")) {
|
|
assert.equal(report.safetyGates.dryRunCallsLiveEndpoints, false, `${label}.safetyGates.dryRunCallsLiveEndpoints`);
|
|
}
|
|
if (Object.hasOwn(report.safetyGates, "defaultSourceReadOnly")) {
|
|
assert.equal(report.safetyGates.defaultSourceReadOnly, true, `${label}.safetyGates.defaultSourceReadOnly`);
|
|
}
|
|
|
|
assertArray(report.liveChecks, `${label}.liveChecks`);
|
|
assert.ok(report.liveChecks.length >= 8, `${label}.liveChecks must include DEV M3 checks`);
|
|
assertUnique(report.liveChecks.map((check) => check.id), `${label}.liveChecks`);
|
|
for (const [index, check] of report.liveChecks.entries()) {
|
|
const checkLabel = `${label}.liveChecks[${index}]`;
|
|
assertObject(check, checkLabel);
|
|
for (const field of ["id", "status", "summary", "evidence"]) {
|
|
assert.ok(Object.hasOwn(check, field), `${checkLabel} missing ${field}`);
|
|
}
|
|
assertString(check.id, `${checkLabel}.id`);
|
|
assertStatus(check.status, `${checkLabel}.status`);
|
|
assertString(check.summary, `${checkLabel}.summary`);
|
|
assertStringArray(check.evidence, `${checkLabel}.evidence`, { minLength: 1 });
|
|
if (Object.hasOwn(check, "blockerClass")) {
|
|
assert.ok(blockerTypes.has(check.blockerClass), `${checkLabel}.blockerClass must be known`);
|
|
}
|
|
}
|
|
|
|
if (Object.hasOwn(report, "localRuntimeObservations")) {
|
|
assertArray(report.localRuntimeObservations, `${label}.localRuntimeObservations`);
|
|
for (const [index, observation] of report.localRuntimeObservations.entries()) {
|
|
const observationLabel = `${label}.localRuntimeObservations[${index}]`;
|
|
assertObject(observation, observationLabel);
|
|
for (const field of ["id", "status", "command", "summary"]) {
|
|
assert.ok(Object.hasOwn(observation, field), `${observationLabel} missing ${field}`);
|
|
assertString(observation[field], `${observationLabel}.${field}`);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (Object.hasOwn(report, "d601Observability")) {
|
|
assertObject(report.d601Observability, `${label}.d601Observability`);
|
|
for (const field of [
|
|
"runnerKubeconfigReadable",
|
|
"runnerKubeconfigProbeExitCode",
|
|
"runnerKubeconfigProbeStderr",
|
|
"d601PublicEndpointsReachable",
|
|
"d601K3sUnavailable",
|
|
"classification",
|
|
"sourceIssue",
|
|
"inferenceRule"
|
|
]) {
|
|
assert.ok(Object.hasOwn(report.d601Observability, field), `${label}.d601Observability missing ${field}`);
|
|
}
|
|
assertBoolean(report.d601Observability.runnerKubeconfigReadable, `${label}.d601Observability.runnerKubeconfigReadable`);
|
|
if (Object.hasOwn(report.d601Observability, "runnerKubeconfigProbeOk")) {
|
|
assertBoolean(report.d601Observability.runnerKubeconfigProbeOk, `${label}.d601Observability.runnerKubeconfigProbeOk`);
|
|
}
|
|
assert.ok(
|
|
Number.isInteger(report.d601Observability.runnerKubeconfigProbeExitCode) ||
|
|
report.d601Observability.runnerKubeconfigProbeExitCode === null,
|
|
`${label}.d601Observability.runnerKubeconfigProbeExitCode`
|
|
);
|
|
assertString(report.d601Observability.runnerKubeconfigProbeStderr, `${label}.d601Observability.runnerKubeconfigProbeStderr`);
|
|
assertBoolean(report.d601Observability.d601PublicEndpointsReachable, `${label}.d601Observability.d601PublicEndpointsReachable`);
|
|
assertBoolean(report.d601Observability.d601K3sUnavailable, `${label}.d601Observability.d601K3sUnavailable`);
|
|
if (
|
|
report.d601Observability.runnerKubeconfigReadable === false &&
|
|
report.d601Observability.d601PublicEndpointsReachable === true
|
|
) {
|
|
assert.equal(
|
|
report.d601Observability.d601K3sUnavailable,
|
|
false,
|
|
`${label}.d601Observability runner gap must not imply D601 k3s unavailable`
|
|
);
|
|
}
|
|
assertString(report.d601Observability.classification, `${label}.d601Observability.classification`);
|
|
assertString(report.d601Observability.sourceIssue, `${label}.d601Observability.sourceIssue`);
|
|
assertString(report.d601Observability.inferenceRule, `${label}.d601Observability.inferenceRule`);
|
|
}
|
|
|
|
if (Object.hasOwn(report, "readOnlySupplementalEvidence")) {
|
|
assertArray(report.readOnlySupplementalEvidence, `${label}.readOnlySupplementalEvidence`);
|
|
for (const [index, evidence] of report.readOnlySupplementalEvidence.entries()) {
|
|
const evidenceLabel = `${label}.readOnlySupplementalEvidence[${index}]`;
|
|
assertObject(evidence, evidenceLabel);
|
|
for (const field of ["id", "status", "source", "summary", "evidence", "requiredFollowUp"]) {
|
|
assert.ok(Object.hasOwn(evidence, field), `${evidenceLabel} missing ${field}`);
|
|
}
|
|
assertString(evidence.id, `${evidenceLabel}.id`);
|
|
assert.ok(
|
|
["pass", "gap", "source-ready", "manifest-ready"].includes(evidence.status),
|
|
`${evidenceLabel}.status must be pass, gap, source-ready, or manifest-ready`
|
|
);
|
|
if (Array.isArray(evidence.source)) {
|
|
assertStringArray(evidence.source, `${evidenceLabel}.source`, { minLength: 1 });
|
|
for (const [sourceIndex, sourcePath] of evidence.source.entries()) {
|
|
assertRepoRelativePath(sourcePath, `${evidenceLabel}.source[${sourceIndex}]`);
|
|
}
|
|
} else {
|
|
assertRepoRelativePath(evidence.source, `${evidenceLabel}.source`);
|
|
}
|
|
assertString(evidence.summary, `${evidenceLabel}.summary`);
|
|
assertObject(evidence.evidence, `${evidenceLabel}.evidence`);
|
|
assertString(evidence.requiredFollowUp, `${evidenceLabel}.requiredFollowUp`);
|
|
if (Object.hasOwn(evidence, "validationCommand")) {
|
|
assertString(evidence.validationCommand, `${evidenceLabel}.validationCommand`);
|
|
}
|
|
if (Object.hasOwn(evidence, "blockerClass") && evidence.blockerClass !== null) {
|
|
assert.ok(blockerTypes.has(evidence.blockerClass), `${evidenceLabel}.blockerClass must be known`);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (Object.hasOwn(report, "dryRunPlan")) {
|
|
const plan = report.dryRunPlan;
|
|
assertObject(plan, `${label}.dryRunPlan`);
|
|
assert.equal(plan.mode, "plan-only", `${label}.dryRunPlan.mode`);
|
|
assert.equal(plan.evidenceLevel, "DRY-RUN", `${label}.dryRunPlan.evidenceLevel`);
|
|
assert.equal(plan.route, "res_boxsimu_1:DO1 -> hwlab-patch-panel -> res_boxsimu_2:DI1", `${label}.dryRunPlan.route`);
|
|
assert.equal(plan.dryRunCallsLiveEndpoints, false, `${label}.dryRunPlan.dryRunCallsLiveEndpoints`);
|
|
assert.equal(plan.liveWriteWillRun, false, `${label}.dryRunPlan.liveWriteWillRun`);
|
|
if (Object.hasOwn(plan, "failureClassifications")) {
|
|
assertStringArray(plan.failureClassifications, `${label}.dryRunPlan.failureClassifications`, { minLength: 5 });
|
|
for (const classification of [
|
|
"target_missing",
|
|
"identity_not_distinct",
|
|
"patch_panel_wiring_missing",
|
|
"operation_failed",
|
|
"evidence_missing"
|
|
]) {
|
|
assert.ok(
|
|
plan.failureClassifications.includes(classification),
|
|
`${label}.dryRunPlan.failureClassifications missing ${classification}`
|
|
);
|
|
}
|
|
}
|
|
assertStringArray(plan.liveWritePreconditions, `${label}.dryRunPlan.liveWritePreconditions`, { minLength: 4 });
|
|
assertStringArray(plan.evidenceFields, `${label}.dryRunPlan.evidenceFields`, { minLength: 4 });
|
|
assertArray(plan.endpointPlan, `${label}.dryRunPlan.endpointPlan`);
|
|
assert.ok(plan.endpointPlan.length >= 8, `${label}.dryRunPlan.endpointPlan must enumerate precondition and live endpoints`);
|
|
assert.ok(
|
|
plan.endpointPlan.some((endpoint) => endpoint.serviceId === "hwlab-patch-panel" && endpoint.path === "/signals/route" && endpoint.mutatesDevState === true),
|
|
`${label}.dryRunPlan.endpointPlan must include the patch-panel route write endpoint`
|
|
);
|
|
assert.ok(
|
|
plan.endpointPlan.some((endpoint) => endpoint.serviceId === "hwlab-box-simu" && endpoint.path === "/ports/write" && endpoint.mutatesDevState === true),
|
|
`${label}.dryRunPlan.endpointPlan must include the source box DO1 write endpoint`
|
|
);
|
|
assert.ok(
|
|
plan.endpointPlan.every((endpoint) => endpoint.calledInDryRun === false),
|
|
`${label}.dryRunPlan.endpointPlan endpoints must not be called in dry-run`
|
|
);
|
|
for (const requiredField of [
|
|
"operationId",
|
|
"traceId",
|
|
"auditId",
|
|
"evidenceId",
|
|
"routeResponse.propagatedBy=hwlab-patch-panel",
|
|
"targetState.ports.DI1.propagatedBy=hwlab-patch-panel"
|
|
]) {
|
|
assert.ok(plan.evidenceFields.includes(requiredField), `${label}.dryRunPlan.evidenceFields missing ${requiredField}`);
|
|
}
|
|
}
|
|
|
|
assertObject(report.liveOperation, `${label}.liveOperation`);
|
|
assertStatus(report.liveOperation.status, `${label}.liveOperation.status`);
|
|
for (const field of ["operationId", "traceId", "auditId", "evidenceId", "summary"]) {
|
|
assertString(report.liveOperation[field], `${label}.liveOperation.${field}`);
|
|
}
|
|
|
|
assertArray(report.blockers, `${label}.blockers`);
|
|
assertUnique(
|
|
report.blockers.map((blocker) => `${blocker.type}::${blocker.scope}`),
|
|
`${label}.blockers`
|
|
);
|
|
for (const [index, blocker] of report.blockers.entries()) {
|
|
const blockerLabel = `${label}.blockers[${index}]`;
|
|
assertObject(blocker, blockerLabel);
|
|
for (const field of ["type", "scope", "status", "summary"]) {
|
|
assert.ok(Object.hasOwn(blocker, field), `${blockerLabel} missing ${field}`);
|
|
}
|
|
assert.ok(blockerTypes.has(blocker.type), `${blockerLabel}.type must be a known blocker type`);
|
|
assertString(blocker.scope, `${blockerLabel}.scope`);
|
|
assert.ok(blockerStates.has(blocker.status), `${blockerLabel}.status must be open, acknowledged, or closed`);
|
|
assertString(blocker.summary, `${blockerLabel}.summary`);
|
|
}
|
|
|
|
assertObject(report.summary, `${label}.summary`);
|
|
assertStatus(report.summary.status, `${label}.summary.status`);
|
|
assertTimestamp(report.summary.observedAt, `${label}.summary.observedAt`);
|
|
assertString(report.summary.result, `${label}.summary.result`);
|
|
}
|
|
|
|
async function validatePreflightReport(relativePath, report) {
|
|
const label = relativePath;
|
|
|
|
assertObject(report, label);
|
|
for (const field of [
|
|
"$schema",
|
|
"$id",
|
|
"reportVersion",
|
|
"reportKind",
|
|
"issue",
|
|
"supports",
|
|
"target",
|
|
"generatedAt",
|
|
"mode",
|
|
"devOnly",
|
|
"prodDisabled",
|
|
"forbiddenActions",
|
|
"validationCommands",
|
|
"artifactIdentity",
|
|
"registryCapabilities",
|
|
"conclusion",
|
|
"checks",
|
|
"blockers"
|
|
]) {
|
|
assert.ok(Object.hasOwn(report, field), `${label} missing ${field}`);
|
|
}
|
|
|
|
assertString(report.$schema, `${label}.$schema`);
|
|
assertString(report.$id, `${label}.$id`);
|
|
assert.equal(report.reportVersion, "v1", `${label}.reportVersion`);
|
|
assert.equal(report.reportKind, "dev-gate-preflight", `${label}.reportKind`);
|
|
assert.equal(report.issue, requiredPreflightIssue, `${label}.issue`);
|
|
assert.equal(report.mode, "read-only", `${label}.mode`);
|
|
assert.equal(report.devOnly, true, `${label}.devOnly`);
|
|
assert.equal(report.prodDisabled, true, `${label}.prodDisabled`);
|
|
assert.ok(preflightConclusions.has(report.conclusion), `${label}.conclusion must be ready or blocked`);
|
|
assert.match(report.generatedAt, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{3}Z$/, `${label}.generatedAt`);
|
|
|
|
assertStringArray(report.supports, `${label}.supports`, { minLength: requiredPreflightSupports.length });
|
|
for (const supportedIssue of requiredPreflightSupports) {
|
|
assert.ok(report.supports.includes(supportedIssue), `${label}.supports missing ${supportedIssue}`);
|
|
}
|
|
|
|
assertObject(report.target, `${label}.target`);
|
|
assert.equal(report.target.ref, "origin/main", `${label}.target.ref`);
|
|
assert.match(report.target.commitId, /^[a-f0-9]{40}$/, `${label}.target.commitId`);
|
|
assert.equal(report.target.shortCommitId, report.target.commitId.slice(0, 7), `${label}.target.shortCommitId`);
|
|
|
|
assertStringArray(report.forbiddenActions, `${label}.forbiddenActions`, { minLength: 1 });
|
|
for (const forbiddenAction of ["prod-deploy", "secret-material-read", "unidesk-runtime-substitute", "force-push"]) {
|
|
assert.ok(report.forbiddenActions.includes(forbiddenAction), `${label}.forbiddenActions missing ${forbiddenAction}`);
|
|
}
|
|
|
|
assertStringArray(report.validationCommands, `${label}.validationCommands`, { minLength: 2 });
|
|
for (const requiredCommand of requiredPreflightValidationCommands) {
|
|
assert.ok(
|
|
report.validationCommands.includes(requiredCommand),
|
|
`${label}.validationCommands missing ${requiredCommand}`
|
|
);
|
|
}
|
|
|
|
assertArtifactIdentity(report.artifactIdentity, `${label}.artifactIdentity`, report.target);
|
|
assertRegistryCapabilities(report.registryCapabilities, `${label}.registryCapabilities`);
|
|
|
|
assertArray(report.checks, `${label}.checks`);
|
|
assert.ok(report.checks.length >= 1, `${label}.checks must not be empty`);
|
|
assertUnique(report.checks.map((check) => check.id), `${label}.checks`);
|
|
for (const [index, check] of report.checks.entries()) {
|
|
const checkLabel = `${label}.checks[${index}]`;
|
|
assertObject(check, checkLabel);
|
|
for (const field of ["id", "category", "status", "summary", "evidence"]) {
|
|
assert.ok(Object.hasOwn(check, field), `${checkLabel} missing ${field}`);
|
|
}
|
|
assert.match(check.id, /^[a-z][a-z0-9-]*$/, `${checkLabel}.id`);
|
|
assertString(check.category, `${checkLabel}.category`);
|
|
assertStatus(check.status, `${checkLabel}.status`);
|
|
assertString(check.summary, `${checkLabel}.summary`);
|
|
assertArray(check.evidence, `${checkLabel}.evidence`);
|
|
}
|
|
|
|
assertArray(report.blockers, `${label}.blockers`);
|
|
if (report.conclusion === "blocked") {
|
|
assert.ok(report.blockers.length >= 1, `${label}.blockers required when conclusion is blocked`);
|
|
}
|
|
assertUnique(
|
|
report.blockers.map((blocker) => `${blocker.type}::${blocker.scope}`),
|
|
`${label}.blockers`
|
|
);
|
|
for (const [index, blocker] of report.blockers.entries()) {
|
|
const blockerLabel = `${label}.blockers[${index}]`;
|
|
assertObject(blocker, blockerLabel);
|
|
for (const field of ["type", "scope", "status", "summary", "nextTask"]) {
|
|
assert.ok(Object.hasOwn(blocker, field), `${blockerLabel} missing ${field}`);
|
|
}
|
|
assert.ok(blockerTypes.has(blocker.type), `${blockerLabel}.type must be a known blocker type`);
|
|
assertString(blocker.scope, `${blockerLabel}.scope`);
|
|
assert.ok(blockerStates.has(blocker.status), `${blockerLabel}.status must be open, acknowledged, or closed`);
|
|
assertString(blocker.summary, `${blockerLabel}.summary`);
|
|
assertString(blocker.nextTask, `${blockerLabel}.nextTask`);
|
|
}
|
|
|
|
if (report.conclusion === "ready") {
|
|
assert.equal(report.blockers.length, 0, `${label}.blockers must be empty when ready`);
|
|
}
|
|
if (Object.hasOwn(report, "notes")) {
|
|
assertString(report.notes, `${label}.notes`);
|
|
}
|
|
}
|
|
|
|
function assertEvidenceLevel(value, label) {
|
|
assertString(value, label);
|
|
assert.ok(evidenceLevels.has(value), `${label} must be one of ${Array.from(evidenceLevels).join(", ")}`);
|
|
}
|
|
|
|
function assertBoolean(value, label) {
|
|
assert.equal(typeof value, "boolean", `${label} must be a boolean`);
|
|
}
|
|
|
|
async function readAggregatorSourceReports(report, label) {
|
|
const sources = {};
|
|
for (const key of ["devEdgeHealth", "devM3Hardware", "devM4Agent", "devM5Gate"]) {
|
|
const sourceReport = report.sourceReports[key];
|
|
const absolutePath = assertRepoRelativePath(
|
|
sourceReport.path,
|
|
`${label}.sourceReports.${key}.path`
|
|
);
|
|
sources[key] = JSON.parse(await readFile(absolutePath, "utf8"));
|
|
}
|
|
return sources;
|
|
}
|
|
|
|
function observedLiveIdentifier(value) {
|
|
return typeof value === "string" &&
|
|
value.trim().length > 0 &&
|
|
!["not_observed", "not_run", "unknown", "blocked", "missing"].includes(value);
|
|
}
|
|
|
|
function aggregatorLiveStateFromSources(sources) {
|
|
const cloudApiDb = sources.devEdgeHealth.edgeHealth?.contracts?.deploy?.cloudApiDb ?? {};
|
|
const dbLive = cloudApiDb.ready === true &&
|
|
(cloudApiDb.connected === true || cloudApiDb.liveConnected === true) &&
|
|
cloudApiDb.liveDbEvidence === true;
|
|
const m3Operation = sources.devM3Hardware.liveOperation ?? {};
|
|
const m3Live = statusIsPass(m3Operation.status) &&
|
|
["operationId", "traceId", "auditId", "evidenceId"].every((field) =>
|
|
observedLiveIdentifier(m3Operation[field])
|
|
);
|
|
const m4Live = statusIsPass(sources.devM4Agent.livePreflight?.status);
|
|
const m5Live = dbLive &&
|
|
m3Live &&
|
|
m4Live &&
|
|
statusIsPass(sources.devM5Gate.devPreconditions?.status);
|
|
|
|
return {
|
|
dbLive,
|
|
m3Live,
|
|
m4Live,
|
|
m5Live
|
|
};
|
|
}
|
|
|
|
function findById(items, id) {
|
|
return items.find((item) => item.id === id);
|
|
}
|
|
|
|
function findByMilestone(items, milestone) {
|
|
return items.find((item) => item.milestone === milestone || item.id === milestone);
|
|
}
|
|
|
|
function assertAggregatorBlockedDod(check, label, reason) {
|
|
assertObject(check, label);
|
|
assert.equal(check.status, "blocked", `${label}.status must stay blocked until ${reason}`);
|
|
assert.equal(check.evidenceLevel, "BLOCKED", `${label}.evidenceLevel must stay BLOCKED until ${reason}`);
|
|
}
|
|
|
|
function assertAggregatorBlockedLayer(layer, label, reason) {
|
|
assertObject(layer, label);
|
|
assert.equal(layer.status, "blocked", `${label}.status must stay blocked until ${reason}`);
|
|
assert.equal(layer.evidenceLevel, "BLOCKED", `${label}.evidenceLevel must stay BLOCKED until ${reason}`);
|
|
}
|
|
|
|
function assertAggregatorBlockedMilestone(milestone, label, reason) {
|
|
assertObject(milestone, label);
|
|
assert.equal(milestone.status, "blocked", `${label}.status must stay blocked until ${reason}`);
|
|
assert.equal(milestone.liveEvidence, "missing_or_blocked", `${label}.liveEvidence must stay missing_or_blocked until ${reason}`);
|
|
assert.notEqual(milestone.highestVisibleLevel, "DEV-LIVE", `${label}.highestVisibleLevel must not promote non-live evidence`);
|
|
}
|
|
|
|
function assertAggregatorBlockedClassification(classification, label, reason) {
|
|
assertObject(classification, label);
|
|
assert.equal(classification.status, "blocked", `${label}.status must stay blocked until ${reason}`);
|
|
assert.equal(classification.currentLevel, "BLOCKED", `${label}.currentLevel must stay BLOCKED until ${reason}`);
|
|
}
|
|
|
|
function assertProtectedAggregatorDevLiveEvidence(item, label, liveState, assumedLevel = item.level) {
|
|
if (!protectedAggregatorMilestones.has(item.milestone) || assumedLevel !== "DEV-LIVE") {
|
|
return;
|
|
}
|
|
|
|
const requiredCategory = protectedAggregatorLiveCategories[item.milestone];
|
|
assert.equal(
|
|
item.category,
|
|
requiredCategory,
|
|
`${label} ${item.milestone} DEV-LIVE evidence must come from ${requiredCategory}, not frontend, route, source, dry-run, or fixture evidence`
|
|
);
|
|
|
|
const sourceLive = {
|
|
M3: liveState.m3Live,
|
|
M4: liveState.m4Live,
|
|
M5: liveState.m5Live
|
|
}[item.milestone];
|
|
assert.equal(
|
|
sourceLive,
|
|
true,
|
|
`${label} ${item.milestone} DEV-LIVE evidence must be backed by its live source report`
|
|
);
|
|
}
|
|
|
|
async function assertAggregatorNonPromotionBoundaries(report, label) {
|
|
const sources = await readAggregatorSourceReports(report, label);
|
|
const liveState = aggregatorLiveStateFromSources(sources);
|
|
const anyProtectedLiveBlocked = !liveState.dbLive || !liveState.m3Live || !liveState.m4Live || !liveState.m5Live;
|
|
|
|
assertObject(report.latestFrontendDevFact, `${label}.latestFrontendDevFact`);
|
|
assert.equal(
|
|
report.latestFrontendDevFact.promotesM3M4M5,
|
|
false,
|
|
`${label}.latestFrontendDevFact.promotesM3M4M5 must be false`
|
|
);
|
|
|
|
if (anyProtectedLiveBlocked) {
|
|
assert.equal(report.overall.status, "blocked", `${label}.overall.status must remain blocked while DB/M3/M4/M5 live evidence is incomplete`);
|
|
assert.equal(report.overall.green, false, `${label}.overall.green must remain false while DB/M3/M4/M5 live evidence is incomplete`);
|
|
}
|
|
|
|
if (!liveState.dbLive) {
|
|
assertAggregatorBlockedDod(
|
|
findById(report.dod.checks, "cloud-api-db-ready"),
|
|
`${label}.dod.checks.cloud-api-db-ready`,
|
|
"DB DEV-LIVE acceptance source reports prove ready=true, connected=true, and liveDbEvidence=true"
|
|
);
|
|
if (report.currentDevLayering?.dbLive) {
|
|
assertAggregatorBlockedLayer(
|
|
report.currentDevLayering.dbLive,
|
|
`${label}.currentDevLayering.dbLive`,
|
|
"DB DEV-LIVE acceptance source reports prove ready=true, connected=true, and liveDbEvidence=true"
|
|
);
|
|
}
|
|
}
|
|
|
|
if (!liveState.m3Live) {
|
|
assertAggregatorBlockedDod(
|
|
findById(report.dod.checks, "m3-hardware-trusted-loop"),
|
|
`${label}.dod.checks.m3-hardware-trusted-loop`,
|
|
"M3 liveOperation proves operationId, traceId, auditId, and evidenceId"
|
|
);
|
|
assertAggregatorBlockedMilestone(
|
|
findByMilestone(report.milestones, "M3"),
|
|
`${label}.milestones.M3`,
|
|
"M3 liveOperation proves operationId, traceId, auditId, and evidenceId"
|
|
);
|
|
if (report.currentDevLayering?.m3HardwareTrustedLoop) {
|
|
assertAggregatorBlockedLayer(
|
|
report.currentDevLayering.m3HardwareTrustedLoop,
|
|
`${label}.currentDevLayering.m3HardwareTrustedLoop`,
|
|
"M3 liveOperation proves operationId, traceId, auditId, and evidenceId"
|
|
);
|
|
}
|
|
if (Array.isArray(report.milestoneLevelClassification)) {
|
|
assertAggregatorBlockedClassification(
|
|
findByMilestone(report.milestoneLevelClassification, "M3"),
|
|
`${label}.milestoneLevelClassification.M3`,
|
|
"M3 liveOperation proves operationId, traceId, auditId, and evidenceId"
|
|
);
|
|
}
|
|
if (Array.isArray(report.milestoneBlockerClassification)) {
|
|
assertAggregatorBlockedClassification(
|
|
findByMilestone(report.milestoneBlockerClassification, "M3"),
|
|
`${label}.milestoneBlockerClassification.M3`,
|
|
"M3 liveOperation proves operationId, traceId, auditId, and evidenceId"
|
|
);
|
|
}
|
|
}
|
|
|
|
if (!liveState.m4Live) {
|
|
assertAggregatorBlockedDod(
|
|
findById(report.dod.checks, "m4-agent-loop-live"),
|
|
`${label}.dod.checks.m4-agent-loop-live`,
|
|
"M4 livePreflight source report passes"
|
|
);
|
|
assertAggregatorBlockedMilestone(
|
|
findByMilestone(report.milestones, "M4"),
|
|
`${label}.milestones.M4`,
|
|
"M4 livePreflight source report passes"
|
|
);
|
|
if (report.currentDevLayering?.m4AgentLoop) {
|
|
assertAggregatorBlockedLayer(
|
|
report.currentDevLayering.m4AgentLoop,
|
|
`${label}.currentDevLayering.m4AgentLoop`,
|
|
"M4 livePreflight source report passes"
|
|
);
|
|
}
|
|
if (Array.isArray(report.milestoneLevelClassification)) {
|
|
assertAggregatorBlockedClassification(
|
|
findByMilestone(report.milestoneLevelClassification, "M4"),
|
|
`${label}.milestoneLevelClassification.M4`,
|
|
"M4 livePreflight source report passes"
|
|
);
|
|
}
|
|
if (Array.isArray(report.milestoneBlockerClassification)) {
|
|
assertAggregatorBlockedClassification(
|
|
findByMilestone(report.milestoneBlockerClassification, "M4"),
|
|
`${label}.milestoneBlockerClassification.M4`,
|
|
"M4 livePreflight source report passes"
|
|
);
|
|
}
|
|
}
|
|
|
|
if (!liveState.m5Live) {
|
|
assertAggregatorBlockedDod(
|
|
findById(report.dod.checks, "m5-mvp-dev-live"),
|
|
`${label}.dod.checks.m5-mvp-dev-live`,
|
|
"M5 source report passes with DB, M3, and M4 live evidence"
|
|
);
|
|
assertAggregatorBlockedMilestone(
|
|
findByMilestone(report.milestones, "M5"),
|
|
`${label}.milestones.M5`,
|
|
"M5 source report passes with DB, M3, and M4 live evidence"
|
|
);
|
|
if (Array.isArray(report.milestoneLevelClassification)) {
|
|
assertAggregatorBlockedClassification(
|
|
findByMilestone(report.milestoneLevelClassification, "M5"),
|
|
`${label}.milestoneLevelClassification.M5`,
|
|
"M5 source report passes with DB, M3, and M4 live evidence"
|
|
);
|
|
}
|
|
if (Array.isArray(report.milestoneBlockerClassification)) {
|
|
assertAggregatorBlockedClassification(
|
|
findByMilestone(report.milestoneBlockerClassification, "M5"),
|
|
`${label}.milestoneBlockerClassification.M5`,
|
|
"M5 source report passes with DB, M3, and M4 live evidence"
|
|
);
|
|
}
|
|
}
|
|
|
|
for (const [index, evidence] of report.evidence.entries()) {
|
|
assertProtectedAggregatorDevLiveEvidence(evidence, `${label}.evidence[${index}]`, liveState);
|
|
}
|
|
for (const [index, evidence] of (report.levels["DEV-LIVE"] ?? []).entries()) {
|
|
assertProtectedAggregatorDevLiveEvidence(evidence, `${label}.levels.DEV-LIVE[${index}]`, liveState, "DEV-LIVE");
|
|
}
|
|
}
|
|
|
|
async function validateAggregatorV2Report(relativePath, report) {
|
|
const label = relativePath;
|
|
|
|
assertObject(report, label);
|
|
for (const field of [
|
|
"$schema",
|
|
"$id",
|
|
"reportVersion",
|
|
"reportKind",
|
|
"issue",
|
|
"supports",
|
|
"generatedAt",
|
|
"generatedFromCommit",
|
|
"environment",
|
|
"endpoint",
|
|
"devOnly",
|
|
"prodDisabled",
|
|
"safety",
|
|
"sourceReports",
|
|
"overall",
|
|
"dod",
|
|
"milestones",
|
|
"evidence",
|
|
"levels",
|
|
"blockers",
|
|
"nextSteps",
|
|
"validationCommands"
|
|
]) {
|
|
assert.ok(Object.hasOwn(report, field), `${label} missing ${field}`);
|
|
}
|
|
|
|
assertString(report.$schema, `${label}.$schema`);
|
|
assertString(report.$id, `${label}.$id`);
|
|
assert.equal(report.reportVersion, "v2", `${label}.reportVersion`);
|
|
assert.equal(report.reportKind, "dev-m5-gate-aggregator", `${label}.reportKind`);
|
|
assert.equal(report.issue, issueFamily.DEV_M5_GATE_AGGREGATOR_V2, `${label}.issue`);
|
|
assertTimestamp(report.generatedAt, `${label}.generatedAt`);
|
|
assert.match(report.generatedFromCommit, /^([a-f0-9]{7,40}|unknown)$/, `${label}.generatedFromCommit`);
|
|
assert.equal(report.environment, "dev", `${label}.environment`);
|
|
assert.equal(report.endpoint, "http://74.48.78.17:16667", `${label}.endpoint`);
|
|
assert.equal(report.devOnly, true, `${label}.devOnly`);
|
|
assert.equal(report.prodDisabled, true, `${label}.prodDisabled`);
|
|
|
|
assertStringArray(report.supports, `${label}.supports`, { minLength: 10 });
|
|
for (const requiredIssue of [
|
|
"pikasTech/HWLAB#7",
|
|
"pikasTech/HWLAB#9",
|
|
"pikasTech/HWLAB#33",
|
|
"pikasTech/HWLAB#34",
|
|
"pikasTech/HWLAB#35",
|
|
"pikasTech/HWLAB#36",
|
|
"pikasTech/HWLAB#37",
|
|
"pikasTech/HWLAB#38",
|
|
"pikasTech/HWLAB#39",
|
|
"pikasTech/HWLAB#46"
|
|
]) {
|
|
assert.ok(report.supports.includes(requiredIssue), `${label}.supports missing ${requiredIssue}`);
|
|
}
|
|
|
|
assertObject(report.safety, `${label}.safety`);
|
|
for (const field of [
|
|
"reportOnly",
|
|
"noDeploy",
|
|
"noProd",
|
|
"noSecretRead",
|
|
"noRuntimeRestart",
|
|
"noLiveProbe",
|
|
"noHeavyE2E",
|
|
"noUniDeskRuntimeSubstitute"
|
|
]) {
|
|
assert.equal(report.safety[field], true, `${label}.safety.${field}`);
|
|
}
|
|
|
|
assertObject(report.sourceReports, `${label}.sourceReports`);
|
|
for (const requiredReport of [
|
|
"devPreflight",
|
|
"devDeploy",
|
|
"devArtifacts",
|
|
"devEdgeHealth",
|
|
"devM3Hardware",
|
|
"devM4Agent",
|
|
"devM5Gate",
|
|
"d601Observability"
|
|
]) {
|
|
assert.ok(Object.hasOwn(report.sourceReports, requiredReport), `${label}.sourceReports missing ${requiredReport}`);
|
|
const sourceReport = report.sourceReports[requiredReport];
|
|
assertObject(sourceReport, `${label}.sourceReports.${requiredReport}`);
|
|
for (const field of ["path", "issue", "taskId", "status", "commitId"]) {
|
|
assert.ok(Object.hasOwn(sourceReport, field), `${label}.sourceReports.${requiredReport} missing ${field}`);
|
|
assertString(sourceReport[field], `${label}.sourceReports.${requiredReport}.${field}`);
|
|
}
|
|
assertRepoRelativePath(sourceReport.path, `${label}.sourceReports.${requiredReport}.path`);
|
|
}
|
|
|
|
assertObject(report.overall, `${label}.overall`);
|
|
assert.ok(["green", "blocked"].includes(report.overall.status), `${label}.overall.status`);
|
|
assertBoolean(report.overall.green, `${label}.overall.green`);
|
|
assertString(report.overall.reason, `${label}.overall.reason`);
|
|
assert.equal(report.overall.green, report.overall.status === "green", `${label}.overall.green must match status`);
|
|
|
|
assertObject(report.dod, `${label}.dod`);
|
|
assert.ok(["green", "blocked"].includes(report.dod.status), `${label}.dod.status`);
|
|
assertBoolean(report.dod.green, `${label}.dod.green`);
|
|
assertArray(report.dod.checks, `${label}.dod.checks`);
|
|
assert.ok(report.dod.checks.length >= 7, `${label}.dod.checks must cover #9 DoD gates`);
|
|
assertUnique(report.dod.checks.map((check) => check.id), `${label}.dod.checks`);
|
|
for (const [index, check] of report.dod.checks.entries()) {
|
|
const checkLabel = `${label}.dod.checks[${index}]`;
|
|
assertObject(check, checkLabel);
|
|
for (const field of ["id", "status", "evidenceLevel", "summary"]) {
|
|
assert.ok(Object.hasOwn(check, field), `${checkLabel} missing ${field}`);
|
|
}
|
|
assert.match(check.id, /^[a-z][a-z0-9-]*$/, `${checkLabel}.id`);
|
|
assert.ok(["pass", "blocked"].includes(check.status), `${checkLabel}.status`);
|
|
assertEvidenceLevel(check.evidenceLevel, `${checkLabel}.evidenceLevel`);
|
|
assertString(check.summary, `${checkLabel}.summary`);
|
|
}
|
|
|
|
assertArray(report.milestones, `${label}.milestones`);
|
|
assert.deepEqual(
|
|
report.milestones.map((milestone) => milestone.id),
|
|
requiredMilestoneIds,
|
|
`${label}.milestones must cover M0-M5 in order`
|
|
);
|
|
for (const [index, milestone] of report.milestones.entries()) {
|
|
const milestoneLabel = `${label}.milestones[${index}]`;
|
|
assertObject(milestone, milestoneLabel);
|
|
for (const field of ["id", "status", "highestVisibleLevel", "liveEvidence", "evidenceCount", "blockerCount", "summary"]) {
|
|
assert.ok(Object.hasOwn(milestone, field), `${milestoneLabel} missing ${field}`);
|
|
}
|
|
assertStatus(milestone.status, `${milestoneLabel}.status`);
|
|
assertEvidenceLevel(milestone.highestVisibleLevel, `${milestoneLabel}.highestVisibleLevel`);
|
|
assert.ok(["pass", "missing_or_blocked"].includes(milestone.liveEvidence), `${milestoneLabel}.liveEvidence`);
|
|
assert.ok(Number.isInteger(milestone.evidenceCount), `${milestoneLabel}.evidenceCount`);
|
|
assert.ok(Number.isInteger(milestone.blockerCount), `${milestoneLabel}.blockerCount`);
|
|
assertString(milestone.summary, `${milestoneLabel}.summary`);
|
|
}
|
|
|
|
assertArray(report.evidence, `${label}.evidence`);
|
|
assert.ok(report.evidence.length >= 10, `${label}.evidence must include M0-M5 entries`);
|
|
for (const [index, evidence] of report.evidence.entries()) {
|
|
const evidenceLabel = `${label}.evidence[${index}]`;
|
|
assertObject(evidence, evidenceLabel);
|
|
for (const field of ["milestone", "issue", "level", "status", "category", "summary"]) {
|
|
assert.ok(Object.hasOwn(evidence, field), `${evidenceLabel} missing ${field}`);
|
|
}
|
|
assert.match(evidence.milestone, /^M[0-5]$/, `${evidenceLabel}.milestone`);
|
|
assertString(evidence.issue, `${evidenceLabel}.issue`);
|
|
assertEvidenceLevel(evidence.level, `${evidenceLabel}.level`);
|
|
assertString(evidence.status, `${evidenceLabel}.status`);
|
|
assertString(evidence.category, `${evidenceLabel}.category`);
|
|
assertString(evidence.summary, `${evidenceLabel}.summary`);
|
|
if (Object.hasOwn(evidence, "reportPath")) {
|
|
assertRepoRelativePath(evidence.reportPath, `${evidenceLabel}.reportPath`);
|
|
}
|
|
if (Object.hasOwn(evidence, "commands")) {
|
|
assertStringArray(evidence.commands, `${evidenceLabel}.commands`);
|
|
}
|
|
if (Object.hasOwn(evidence, "evidence")) {
|
|
assertStringArray(evidence.evidence, `${evidenceLabel}.evidence`);
|
|
}
|
|
if (Object.hasOwn(evidence, "sources")) {
|
|
assertStringArray(evidence.sources, `${evidenceLabel}.sources`);
|
|
}
|
|
}
|
|
|
|
assertObject(report.levels, `${label}.levels`);
|
|
assert.deepEqual(Object.keys(report.levels), Array.from(evidenceLevels), `${label}.levels keys`);
|
|
for (const level of evidenceLevels) {
|
|
assertArray(report.levels[level], `${label}.levels.${level}`);
|
|
}
|
|
|
|
assertArray(report.blockers, `${label}.blockers`);
|
|
assert.ok(report.blockers.length >= 1, `${label}.blockers must not be empty while blocked`);
|
|
for (const [index, blocker] of report.blockers.entries()) {
|
|
const blockerLabel = `${label}.blockers[${index}]`;
|
|
assertObject(blocker, blockerLabel);
|
|
for (const field of ["id", "priority", "type", "scope", "status", "source", "sourceIssue", "summary", "unblockOrder", "unblocks", "rationale"]) {
|
|
assert.ok(Object.hasOwn(blocker, field), `${blockerLabel} missing ${field}`);
|
|
}
|
|
assertString(blocker.id, `${blockerLabel}.id`);
|
|
assert.match(blocker.priority, /^P[0-3]$/, `${blockerLabel}.priority`);
|
|
assert.ok(blockerTypes.has(blocker.type), `${blockerLabel}.type`);
|
|
assertString(blocker.scope, `${blockerLabel}.scope`);
|
|
assert.equal(blocker.status, "open", `${blockerLabel}.status`);
|
|
assertRepoRelativePath(blocker.source, `${blockerLabel}.source`);
|
|
assertString(blocker.sourceIssue, `${blockerLabel}.sourceIssue`);
|
|
assertString(blocker.summary, `${blockerLabel}.summary`);
|
|
assert.ok(Number.isInteger(blocker.unblockOrder), `${blockerLabel}.unblockOrder`);
|
|
assertStringArray(blocker.unblocks, `${blockerLabel}.unblocks`, { minLength: 1 });
|
|
assertString(blocker.rationale, `${blockerLabel}.rationale`);
|
|
if (Object.hasOwn(blocker, "nextTask")) {
|
|
assertString(blocker.nextTask, `${blockerLabel}.nextTask`);
|
|
}
|
|
}
|
|
|
|
assertArray(report.nextSteps, `${label}.nextSteps`);
|
|
assert.ok(report.nextSteps.length >= 1, `${label}.nextSteps must contain blocker order`);
|
|
assert.deepEqual(
|
|
report.nextSteps.map((step) => step.blockerOrder).sort((a, b) => a - b),
|
|
[...new Set(report.blockers.map((blocker) => blocker.unblockOrder))].sort((a, b) => a - b),
|
|
`${label}.nextSteps must cover active blocker orders`
|
|
);
|
|
assert.deepEqual(
|
|
report.nextSteps.map((step) => step.order),
|
|
[...report.nextSteps].map((step) => step.order).sort((a, b) => a - b),
|
|
`${label}.nextSteps must be sorted by order`
|
|
);
|
|
for (const [index, step] of report.nextSteps.entries()) {
|
|
const stepLabel = `${label}.nextSteps[${index}]`;
|
|
assertObject(step, stepLabel);
|
|
for (const field of ["order", "priority", "scopes", "sourceIssues", "rationale", "action", "evidenceRequired"]) {
|
|
assert.ok(Object.hasOwn(step, field), `${stepLabel} missing ${field}`);
|
|
}
|
|
assert.ok(Number.isInteger(step.order), `${stepLabel}.order`);
|
|
if (Object.hasOwn(step, "blockerOrder")) {
|
|
assert.ok(Number.isInteger(step.blockerOrder), `${stepLabel}.blockerOrder`);
|
|
}
|
|
assert.match(step.priority, /^P[0-3]$/, `${stepLabel}.priority`);
|
|
assertStringArray(step.scopes, `${stepLabel}.scopes`, { minLength: 1 });
|
|
assertStringArray(step.sourceIssues, `${stepLabel}.sourceIssues`, { minLength: 1 });
|
|
assertString(step.rationale, `${stepLabel}.rationale`);
|
|
assertString(step.action, `${stepLabel}.action`);
|
|
assertString(step.evidenceRequired, `${stepLabel}.evidenceRequired`);
|
|
}
|
|
|
|
assertStringArray(report.validationCommands, `${label}.validationCommands`, {
|
|
minLength: requiredAggregatorV2Commands.length
|
|
});
|
|
for (const requiredCommand of requiredAggregatorV2Commands) {
|
|
assert.ok(
|
|
report.validationCommands.includes(requiredCommand),
|
|
`${label}.validationCommands missing ${requiredCommand}`
|
|
);
|
|
}
|
|
|
|
await assertAggregatorNonPromotionBoundaries(report, label);
|
|
}
|
|
|
|
async function main() {
|
|
const reportFiles = await collectReportFiles();
|
|
assert.ok(reportFiles.length >= 1, "expected at least one dev-gate report JSON file");
|
|
|
|
for (const reportFile of reportFiles) {
|
|
await validateReport(reportFile);
|
|
}
|
|
|
|
console.log(`validated ${reportFiles.length} dev-gate report JSON file(s)`);
|
|
}
|
|
|
|
await main();
|