chore: add dev deploy apply preflight
This commit is contained in:
HWLAB Code Queue
Executable
+468
@@ -0,0 +1,468 @@
|
||||
#!/usr/bin/env node
|
||||
import { constants as fsConstants } from "node:fs";
|
||||
import { access, readFile, writeFile } from "node:fs/promises";
|
||||
import { request as httpRequest } from "node:http";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath, pathToFileURL } from "node:url";
|
||||
import { spawn } from "node:child_process";
|
||||
|
||||
import { DEV_ENDPOINT, ENVIRONMENT_DEV, SERVICE_IDS } from "../../internal/protocol/index.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const reportPath = "reports/dev-gate/dev-deploy-report.json";
|
||||
const namespace = "hwlab-dev";
|
||||
const healthPath = "/health/live";
|
||||
const requiredValidationCommands = [
|
||||
"node --check scripts/validate-dev-gate-report.mjs",
|
||||
"node scripts/validate-dev-gate-report.mjs",
|
||||
"node --check scripts/dev-deploy-apply.mjs",
|
||||
"node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked"
|
||||
];
|
||||
|
||||
function parseArgs(argv) {
|
||||
const flags = new Set(argv.filter((arg) => arg.startsWith("--")));
|
||||
return {
|
||||
apply: flags.has("--apply"),
|
||||
dryRun: !flags.has("--apply"),
|
||||
expectBlocked: flags.has("--expect-blocked"),
|
||||
skipLiveProbe: flags.has("--skip-live-probe"),
|
||||
writeReport: flags.has("--write-report"),
|
||||
flags
|
||||
};
|
||||
}
|
||||
|
||||
function addBlocker(blockers, type, scope, summary) {
|
||||
const key = `${type}::${scope}`;
|
||||
if (!blockers.some((blocker) => `${blocker.type}::${blocker.scope}` === key)) {
|
||||
blockers.push({ type, scope, status: "open", summary: oneLine(summary) });
|
||||
}
|
||||
}
|
||||
|
||||
function oneLine(value) {
|
||||
return String(value).replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
async function readJson(relativePath, blockers) {
|
||||
try {
|
||||
return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8"));
|
||||
} catch (error) {
|
||||
addBlocker(blockers, "contract_blocker", relativePath, `Cannot parse ${relativePath}: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
async function readText(relativePath) {
|
||||
return readFile(path.join(repoRoot, relativePath), "utf8");
|
||||
}
|
||||
|
||||
async function commandResult(command, args, timeoutMs = 15000) {
|
||||
return new Promise((resolve) => {
|
||||
const child = spawn(command, args, { cwd: repoRoot, stdio: ["ignore", "pipe", "pipe"] });
|
||||
const timer = setTimeout(() => child.kill("SIGTERM"), timeoutMs);
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
child.stdout.on("data", (chunk) => {
|
||||
stdout += chunk;
|
||||
});
|
||||
child.stderr.on("data", (chunk) => {
|
||||
stderr += chunk;
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ ok: false, code: null, stdout, stderr: error.message });
|
||||
});
|
||||
child.on("close", (code, signal) => {
|
||||
clearTimeout(timer);
|
||||
resolve({ ok: code === 0, code, signal, stdout: stdout.trim(), stderr: stderr.trim() });
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
async function findExecutable(name) {
|
||||
for (const dir of (process.env.PATH || "").split(path.delimiter)) {
|
||||
if (!dir) continue;
|
||||
const candidate = path.join(dir, name);
|
||||
try {
|
||||
await access(candidate, fsConstants.X_OK);
|
||||
return candidate;
|
||||
} catch {}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function listItems(document) {
|
||||
if (!document) return [];
|
||||
return document.kind === "List" ? document.items ?? [] : [document];
|
||||
}
|
||||
|
||||
function containersFor(item) {
|
||||
return item?.spec?.template?.spec?.containers ?? [];
|
||||
}
|
||||
|
||||
function serviceIdFor(item, container) {
|
||||
return (
|
||||
item?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
|
||||
item?.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
|
||||
container?.name
|
||||
);
|
||||
}
|
||||
|
||||
function validateArgs(args, blockers) {
|
||||
for (const forbidden of ["--prod", "--production", "--read-secret", "--force-push", "--heavyweight-e2e"]) {
|
||||
if (args.flags.has(forbidden)) {
|
||||
addBlocker(blockers, "safety_blocker", forbidden.slice(2), `${forbidden} is forbidden for DEV deploy apply`);
|
||||
}
|
||||
}
|
||||
if (args.apply && (!args.flags.has("--confirm-dev") || !args.flags.has("--confirmed-non-production"))) {
|
||||
addBlocker(
|
||||
blockers,
|
||||
"safety_blocker",
|
||||
"apply-confirmation",
|
||||
"Live apply requires --confirm-dev and --confirmed-non-production"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function commitMatchesSource(commitId, sourceCommitId) {
|
||||
return sourceCommitId.startsWith(commitId) || commitId.startsWith(sourceCommitId);
|
||||
}
|
||||
|
||||
function validateDeployAndCatalog(deploy, catalog, sourceCommitId, blockers) {
|
||||
const artifactEvidence = [];
|
||||
if (!deploy || !catalog) return artifactEvidence;
|
||||
|
||||
if (deploy.environment !== ENVIRONMENT_DEV || deploy.namespace !== namespace) {
|
||||
addBlocker(blockers, "safety_blocker", "deploy-target", "deploy/deploy.json must target only dev hwlab-dev");
|
||||
}
|
||||
if (deploy.profiles?.prod?.enabled !== false) {
|
||||
addBlocker(blockers, "safety_blocker", "prod-profile", "PROD profile must remain disabled");
|
||||
}
|
||||
if (catalog.environment !== ENVIRONMENT_DEV || catalog.namespace !== namespace) {
|
||||
addBlocker(blockers, "safety_blocker", "catalog-target", "artifact catalog must target only dev hwlab-dev");
|
||||
}
|
||||
if (!commitMatchesSource(deploy.commitId, sourceCommitId) || !commitMatchesSource(catalog.commitId, sourceCommitId)) {
|
||||
addBlocker(
|
||||
blockers,
|
||||
"observability_blocker",
|
||||
"artifact-source-commit",
|
||||
`deploy/catalog artifact commit ${deploy.commitId}/${catalog.commitId} does not match source ${sourceCommitId}`
|
||||
);
|
||||
}
|
||||
|
||||
const deployById = new Map((deploy.services ?? []).map((service) => [service.serviceId, service]));
|
||||
const catalogById = new Map((catalog.services ?? []).map((service) => [service.serviceId, service]));
|
||||
for (const serviceId of SERVICE_IDS) {
|
||||
const deployService = deployById.get(serviceId);
|
||||
const catalogService = catalogById.get(serviceId);
|
||||
if (!deployService || !catalogService) {
|
||||
addBlocker(blockers, "contract_blocker", `service-${serviceId}`, `${serviceId} missing from deploy or catalog`);
|
||||
continue;
|
||||
}
|
||||
if (deployService.namespace !== namespace || deployService.profile !== ENVIRONMENT_DEV) {
|
||||
addBlocker(blockers, "safety_blocker", `service-target-${serviceId}`, `${serviceId} is not DEV-only`);
|
||||
}
|
||||
if (deployService.image !== catalogService.image) {
|
||||
addBlocker(blockers, "contract_blocker", `service-image-${serviceId}`, `${serviceId} image differs in deploy/catalog`);
|
||||
}
|
||||
artifactEvidence.push({
|
||||
serviceId,
|
||||
deployCommitId: deploy.commitId,
|
||||
catalogCommitId: catalogService.commitId,
|
||||
image: deployService.image,
|
||||
digest: catalogService.digest,
|
||||
publishState: catalogService.publishState
|
||||
});
|
||||
}
|
||||
|
||||
const hasPublishedCatalog =
|
||||
catalog.publish?.ciPublished === true &&
|
||||
catalog.publish?.registryVerified === true &&
|
||||
catalog.services?.every(
|
||||
(service) => service.digest?.startsWith("sha256:") && service.publishState !== "skeleton-only"
|
||||
);
|
||||
if (!hasPublishedCatalog) {
|
||||
addBlocker(
|
||||
blockers,
|
||||
"observability_blocker",
|
||||
"artifact-publish",
|
||||
"DEV artifact catalog has no CI publish, registry verification, or registry digests"
|
||||
);
|
||||
}
|
||||
return artifactEvidence;
|
||||
}
|
||||
|
||||
function validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, blockers) {
|
||||
const items = [...listItems(namespaceDoc), ...listItems(workloads), ...listItems(services), ...listItems(healthContract)];
|
||||
if (devKustomization?.namespace !== namespace) {
|
||||
addBlocker(blockers, "safety_blocker", "dev-kustomization", "DEV kustomization must pin namespace hwlab-dev");
|
||||
}
|
||||
if ((devKustomization?.resources ?? []).some((resource) => resource.includes("prod"))) {
|
||||
addBlocker(blockers, "safety_blocker", "dev-kustomization-prod", "DEV kustomization must not reference PROD resources");
|
||||
}
|
||||
for (const item of items) {
|
||||
const itemNamespace = item?.metadata?.namespace ?? (item?.kind === "Namespace" ? item?.metadata?.name : namespace);
|
||||
if (itemNamespace !== namespace) {
|
||||
addBlocker(blockers, "safety_blocker", `k8s-${item?.kind}-${item?.metadata?.name}`, "Kubernetes resource is not in hwlab-dev");
|
||||
}
|
||||
}
|
||||
|
||||
const expectedImages = new Map((deploy?.services ?? []).map((service) => [service.serviceId, service.image]));
|
||||
const workloadItems = listItems(workloads);
|
||||
const images = [];
|
||||
for (const item of workloadItems) {
|
||||
for (const container of containersFor(item)) {
|
||||
const serviceId = serviceIdFor(item, container);
|
||||
const expectedImage = expectedImages.get(serviceId);
|
||||
images.push({ serviceId, image: container.image, kind: item.kind, name: item.metadata?.name });
|
||||
if (expectedImage && container.image !== expectedImage) {
|
||||
addBlocker(blockers, "contract_blocker", `k8s-image-${serviceId}`, `${serviceId} image differs in k8s workload`);
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
resourceCounts: {
|
||||
workloads: workloadItems.length,
|
||||
services: listItems(services).length,
|
||||
configMaps: listItems(healthContract).filter((item) => item.kind === "ConfigMap").length
|
||||
},
|
||||
images
|
||||
};
|
||||
}
|
||||
|
||||
async function checkCloudApiDb(deploy, workloads, blockers) {
|
||||
const envNames = new Set(Object.keys(deploy?.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {}));
|
||||
for (const item of listItems(workloads)) {
|
||||
for (const container of containersFor(item)) {
|
||||
if (serviceIdFor(item, container) === "hwlab-cloud-api") {
|
||||
for (const env of container.env ?? []) envNames.add(env.name);
|
||||
}
|
||||
}
|
||||
}
|
||||
const dbEnvNames = [...envNames].filter((name) => /DATABASE|POSTGRES|MYSQL|SQLITE|MONGO|_DB(_|$)/u.test(name));
|
||||
if (dbEnvNames.length === 0) {
|
||||
addBlocker(blockers, "runtime_blocker", "cloud-api-db-config", "cloud-api has no DEV DB connection env configured");
|
||||
}
|
||||
|
||||
try {
|
||||
const modulePath = pathToFileURL(path.join(repoRoot, "internal/cloud/server.mjs")).href;
|
||||
const { buildHealthPayload } = await import(modulePath);
|
||||
const health = buildHealthPayload();
|
||||
if (health.db?.connected !== true) {
|
||||
addBlocker(blockers, "runtime_blocker", "cloud-api-db", "cloud-api health reports DB not connected in this artifact");
|
||||
}
|
||||
} catch (error) {
|
||||
addBlocker(blockers, "observability_blocker", "cloud-api-health", `Cannot inspect cloud-api health payload: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
async function probeDevEndpoint(blockers) {
|
||||
const url = new URL(healthPath, DEV_ENDPOINT).href;
|
||||
try {
|
||||
const response = await httpGet(url, 8000);
|
||||
const text = response.body;
|
||||
let json = null;
|
||||
try {
|
||||
json = JSON.parse(text);
|
||||
} catch {}
|
||||
if (response.statusCode < 200 || response.statusCode >= 300) {
|
||||
addBlocker(blockers, "network_blocker", "dev-health", `DEV health returned HTTP ${response.statusCode}`);
|
||||
}
|
||||
if (json?.serviceId !== "hwlab-cloud-api" || json?.environment !== ENVIRONMENT_DEV) {
|
||||
addBlocker(blockers, "runtime_blocker", "dev-health-identity", "DEV health did not identify hwlab-cloud-api in dev");
|
||||
}
|
||||
if (json?.db?.connected !== true) {
|
||||
addBlocker(blockers, "runtime_blocker", "dev-health-db", "DEV health did not confirm cloud-api DB connectivity");
|
||||
}
|
||||
return {
|
||||
status: response.statusCode >= 200 && response.statusCode < 300 ? "pass" : "blocked",
|
||||
url,
|
||||
httpStatus: response.statusCode,
|
||||
body: text.slice(0, 500)
|
||||
};
|
||||
} catch (error) {
|
||||
const reason = oneLine(error.cause?.message ?? error.message);
|
||||
addBlocker(blockers, "network_blocker", "dev-health", `Cannot reach ${url}: ${reason}`);
|
||||
return { status: "blocked", url, error: reason };
|
||||
}
|
||||
}
|
||||
|
||||
function httpGet(url, timeoutMs) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const request = httpRequest(url, { method: "GET", timeout: timeoutMs }, (response) => {
|
||||
response.setEncoding("utf8");
|
||||
let body = "";
|
||||
response.on("data", (chunk) => {
|
||||
body += chunk;
|
||||
});
|
||||
response.on("end", () => resolve({ statusCode: response.statusCode ?? 0, body }));
|
||||
});
|
||||
request.on("timeout", () => {
|
||||
request.destroy(new Error(`timeout after ${timeoutMs}ms`));
|
||||
});
|
||||
request.on("error", reject);
|
||||
request.end();
|
||||
});
|
||||
}
|
||||
|
||||
async function observeCluster(kubectlPath, blockers) {
|
||||
if (!kubectlPath) {
|
||||
addBlocker(blockers, "environment_blocker", "kubectl", "kubectl is not installed in the runner");
|
||||
return { status: "blocked", executor: "missing", resources: [] };
|
||||
}
|
||||
const version = await commandResult(kubectlPath, ["version", "--client=true", "--output=yaml"]);
|
||||
if (!version.ok) {
|
||||
addBlocker(blockers, "environment_blocker", "kubectl-version", `kubectl client failed: ${version.stderr || version.stdout}`);
|
||||
return { status: "blocked", executor: kubectlPath, resources: [] };
|
||||
}
|
||||
const get = await commandResult(kubectlPath, ["get", "pods,services,configmaps,deployments,jobs", "-n", namespace, "-o", "json"], 20000);
|
||||
if (!get.ok) {
|
||||
addBlocker(blockers, "environment_blocker", "cluster-read", `Cannot read hwlab-dev resources: ${get.stderr || get.stdout}`);
|
||||
return { status: "blocked", executor: kubectlPath, version: version.stdout, resources: [] };
|
||||
}
|
||||
const parsed = JSON.parse(get.stdout);
|
||||
return {
|
||||
status: "pass",
|
||||
executor: kubectlPath,
|
||||
version: version.stdout,
|
||||
resources: (parsed.items ?? []).map((item) => ({
|
||||
kind: item.kind,
|
||||
name: item.metadata?.name,
|
||||
phase: item.status?.phase,
|
||||
readyReplicas: item.status?.readyReplicas,
|
||||
replicas: item.status?.replicas
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
async function runApplyStep(args, kubectlPath, blockers) {
|
||||
if (blockers.length > 0) {
|
||||
return { status: "not_run", summary: "Skipped because preflight blockers are open" };
|
||||
}
|
||||
const commandArgs = args.apply ? ["apply", "-k", "deploy/k8s/dev"] : ["apply", "--dry-run=server", "-k", "deploy/k8s/dev"];
|
||||
const result = await commandResult(kubectlPath, commandArgs, 30000);
|
||||
if (!result.ok) {
|
||||
addBlocker(blockers, "environment_blocker", args.apply ? "kubectl-apply" : "kubectl-dry-run", result.stderr || result.stdout);
|
||||
}
|
||||
return { status: result.ok ? "pass" : "blocked", command: `kubectl ${commandArgs.join(" ")}`, stdout: result.stdout, stderr: result.stderr };
|
||||
}
|
||||
|
||||
async function sourceCommit() {
|
||||
const result = await commandResult("git", ["rev-parse", "--short=12", "HEAD"]);
|
||||
return result.ok ? result.stdout : "unknown";
|
||||
}
|
||||
|
||||
export async function runDevDeployApply(argv, io = {}) {
|
||||
const stdout = io.stdout ?? process.stdout;
|
||||
const args = parseArgs(argv);
|
||||
const blockers = [];
|
||||
validateArgs(args, blockers);
|
||||
const commitId = await sourceCommit();
|
||||
|
||||
const [deploy, catalog, devKustomization, namespaceDoc, workloads, services, healthContract] = await Promise.all([
|
||||
readJson("deploy/deploy.json", blockers),
|
||||
readJson("deploy/artifact-catalog.dev.json", blockers),
|
||||
readJson("deploy/k8s/dev/kustomization.yaml", blockers),
|
||||
readJson("deploy/k8s/base/namespace.yaml", blockers),
|
||||
readJson("deploy/k8s/base/workloads.yaml", blockers),
|
||||
readJson("deploy/k8s/base/services.yaml", blockers),
|
||||
readJson("deploy/k8s/dev/health-contract.yaml", blockers)
|
||||
]);
|
||||
|
||||
const artifactEvidence = validateDeployAndCatalog(deploy, catalog, commitId, blockers);
|
||||
const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, blockers);
|
||||
await checkCloudApiDb(deploy, workloads, blockers);
|
||||
if (!(await readText("internal/cloud/server.mjs")).includes('url.pathname === "/health/live"')) {
|
||||
addBlocker(blockers, "contract_blocker", "cloud-api-health-path", "cloud-api does not implement the k8s /health/live path");
|
||||
}
|
||||
|
||||
const kubectlPath = await findExecutable("kubectl");
|
||||
const clusterObservation = await observeCluster(kubectlPath, blockers);
|
||||
const liveProbe = args.skipLiveProbe ? { status: "not_run", reason: "--skip-live-probe" } : await probeDevEndpoint(blockers);
|
||||
const applyStep = await runApplyStep(args, kubectlPath, blockers);
|
||||
const status = blockers.length > 0 ? "blocked" : "pass";
|
||||
|
||||
const report = {
|
||||
$schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json",
|
||||
$id: "https://hwlab.pikastech.local/reports/dev-gate/dev-deploy-report.json",
|
||||
reportVersion: "v1",
|
||||
issue: "pikasTech/HWLAB#33",
|
||||
taskId: "dev-deploy-apply",
|
||||
commitId,
|
||||
acceptanceLevel: "dev_deploy_apply",
|
||||
devOnly: true,
|
||||
prodDisabled: true,
|
||||
status,
|
||||
generatedAt: new Date().toISOString(),
|
||||
namespace,
|
||||
endpoint: DEV_ENDPOINT,
|
||||
sourceContract: {
|
||||
status: "pass",
|
||||
documents: [
|
||||
"docs/dev-acceptance-matrix.md",
|
||||
"docs/m0-contract-audit.md",
|
||||
"docs/dev-deploy-apply.md",
|
||||
"docs/artifact-catalog.md",
|
||||
"deploy/README.md"
|
||||
],
|
||||
summary: "DEV deploy apply is pinned to hwlab-dev and the frozen HWLAB service IDs"
|
||||
},
|
||||
validationCommands: requiredValidationCommands,
|
||||
localSmoke: {
|
||||
status: "not_run",
|
||||
commands: ["node scripts/m1-contract-smoke.mjs"],
|
||||
evidence: ["This deploy apply task did not run local smoke because artifact/executor blockers stop promotion first."],
|
||||
summary: "Local smoke remains available but is not a substitute for DEV artifact publish evidence."
|
||||
},
|
||||
dryRun: {
|
||||
status,
|
||||
commands: [
|
||||
"node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run",
|
||||
"node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked"
|
||||
],
|
||||
evidence: [
|
||||
`artifact services checked: ${artifactEvidence.length}`,
|
||||
`kubectl executor: ${kubectlPath ?? "missing"}`,
|
||||
`live health: ${liveProbe.status}`
|
||||
],
|
||||
summary: status === "blocked" ? "DEV apply is blocked before mutation." : "DEV dry-run preflight passed."
|
||||
},
|
||||
devPreconditions: {
|
||||
status,
|
||||
requirements: [
|
||||
"DEV artifact catalog must contain CI publish evidence and registry digests",
|
||||
"kubectl must be available for D601 hwlab-dev and must not target PROD",
|
||||
"hwlab-cloud-api /health/live must report serviceId, dev environment, and DB connectivity",
|
||||
"pods, services, configmaps, and image commit tags must be observable in hwlab-dev"
|
||||
],
|
||||
summary: status === "blocked" ? "One or more required DEV deploy preconditions are blocked." : "DEV deploy preconditions are satisfied."
|
||||
},
|
||||
devDeployApply: {
|
||||
mode: args.apply ? "apply" : "dry-run",
|
||||
mutationAttempted: applyStep.status === "pass" && args.apply,
|
||||
applyStep,
|
||||
artifactEvidence,
|
||||
k8sManifest,
|
||||
clusterObservation,
|
||||
liveProbe
|
||||
},
|
||||
blockers,
|
||||
notes: "DEV-only result. No PROD resources, secret values, force push, heavy e2e, browser e2e, or UniDesk runtime substitute were used."
|
||||
};
|
||||
|
||||
if (args.writeReport) {
|
||||
await writeFile(path.join(repoRoot, reportPath), `${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
|
||||
stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
return status === "blocked" && !args.expectBlocked ? 2 : 0;
|
||||
}
|
||||
|
||||
export function formatFailure(error) {
|
||||
return {
|
||||
reportVersion: "v1",
|
||||
issue: "pikasTech/HWLAB#33",
|
||||
taskId: "dev-deploy-apply",
|
||||
status: "failed",
|
||||
blockers: [{ type: "observability_blocker", scope: "script", status: "open", summary: oneLine(error.message) }]
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user