Code Queue Review
1452 lines
55 KiB
JavaScript
Executable File
1452 lines
55 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
import { constants as fsConstants } from "node:fs";
|
|
import { access, readFile, writeFile } from "node:fs/promises";
|
|
import { request as httpRequest } from "node:http";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { spawn } from "node:child_process";
|
|
|
|
import {
|
|
DEV_DB_ENV_CONTRACT,
|
|
summarizeDbContract
|
|
} from "../../internal/cloud/db-contract.mjs";
|
|
import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs";
|
|
import { DEV_ENDPOINT, ENVIRONMENT_DEV, SERVICE_IDS } from "../../internal/protocol/index.mjs";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
const reportPath = "reports/dev-gate/dev-deploy-report.json";
|
|
const namespace = "hwlab-dev";
|
|
const healthPath = "/health/live";
|
|
const d601KubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
|
|
const devTemplateJobReplacementPolicy = {
|
|
status: "active",
|
|
namespace,
|
|
scope: "DEV-only suspended template Jobs",
|
|
allowedJobs: [
|
|
{
|
|
name: "hwlab-agent-worker-template",
|
|
serviceId: "hwlab-agent-worker",
|
|
reason: "suspended session worker template Job has immutable spec.template"
|
|
},
|
|
{
|
|
name: "hwlab-cli-template",
|
|
serviceId: "hwlab-cli",
|
|
reason: "suspended CLI template Job has immutable spec.template"
|
|
}
|
|
]
|
|
};
|
|
const replaceableDevTemplateJobNames = new Set(devTemplateJobReplacementPolicy.allowedJobs.map((job) => job.name));
|
|
const requiredValidationCommands = [
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs",
|
|
"node --check scripts/dev-deploy-apply.mjs",
|
|
"node --check scripts/src/dev-deploy-apply.mjs",
|
|
"node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked"
|
|
];
|
|
const applyCommand = "node scripts/dev-deploy-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report";
|
|
const dryRunPlanCommand = "node scripts/dev-deploy-apply.mjs --dry-run --write-report";
|
|
const blockedDryRunCommand = "node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked --write-report";
|
|
const forbiddenActions = [
|
|
"prod-deploy",
|
|
"secret-read",
|
|
"runtime-substitute",
|
|
"unidesk-restart",
|
|
"code-queue-restart",
|
|
"backend-core-restart",
|
|
"heavy-master-e2e",
|
|
"force-push"
|
|
];
|
|
const runtimeIdentityEnvNames = ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"];
|
|
|
|
function parseArgs(argv) {
|
|
const flags = new Set(argv.filter((arg) => arg.startsWith("--")));
|
|
return {
|
|
apply: flags.has("--apply"),
|
|
dryRun: !flags.has("--apply"),
|
|
expectBlocked: flags.has("--expect-blocked"),
|
|
skipLiveProbe: flags.has("--skip-live-probe"),
|
|
writeReport: flags.has("--write-report"),
|
|
flags
|
|
};
|
|
}
|
|
|
|
function addBlocker(blockers, type, scope, summary) {
|
|
const key = `${type}::${scope}`;
|
|
if (!blockers.some((blocker) => `${blocker.type}::${blocker.scope}` === key)) {
|
|
blockers.push({ type, scope, status: "open", summary: oneLine(summary) });
|
|
}
|
|
}
|
|
|
|
function oneLine(value) {
|
|
return String(value).replace(/\s+/g, " ").trim();
|
|
}
|
|
|
|
async function readJson(relativePath, blockers) {
|
|
try {
|
|
return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8"));
|
|
} catch (error) {
|
|
addBlocker(blockers, "contract_blocker", relativePath, `Cannot parse ${relativePath}: ${error.message}`);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function readText(relativePath) {
|
|
return readFile(path.join(repoRoot, relativePath), "utf8");
|
|
}
|
|
|
|
async function commandResult(command, args, timeoutMs = 15000, options = {}) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn(command, args, {
|
|
cwd: repoRoot,
|
|
env: options.env ?? process.env,
|
|
stdio: ["pipe", "pipe", "pipe"]
|
|
});
|
|
const timer = setTimeout(() => child.kill("SIGTERM"), timeoutMs);
|
|
let stdout = "";
|
|
let stderr = "";
|
|
child.stdin.on("error", () => {});
|
|
child.stdin.end(options.input ?? "");
|
|
child.stdout.on("data", (chunk) => {
|
|
stdout += chunk;
|
|
});
|
|
child.stderr.on("data", (chunk) => {
|
|
stderr += chunk;
|
|
});
|
|
child.on("error", (error) => {
|
|
clearTimeout(timer);
|
|
resolve({ ok: false, code: null, stdout, stderr: error.message });
|
|
});
|
|
child.on("close", (code, signal) => {
|
|
clearTimeout(timer);
|
|
resolve({ ok: code === 0, code, signal, stdout: stdout.trim(), stderr: stderr.trim() });
|
|
});
|
|
});
|
|
}
|
|
|
|
async function findExecutable(name) {
|
|
for (const dir of (process.env.PATH || "").split(path.delimiter)) {
|
|
if (!dir) continue;
|
|
const candidate = path.join(dir, name);
|
|
try {
|
|
await access(candidate, fsConstants.X_OK);
|
|
return candidate;
|
|
} catch {}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function resolveD601Kubectl() {
|
|
const kubectlPath = await findExecutable("kubectl");
|
|
const commandPrefix = `KUBECONFIG=${d601KubeconfigPath} kubectl`;
|
|
const base = {
|
|
executor: kubectlPath,
|
|
kubeconfig: d601KubeconfigPath,
|
|
commandPrefix,
|
|
env: {
|
|
...process.env,
|
|
KUBECONFIG: d601KubeconfigPath
|
|
}
|
|
};
|
|
|
|
if (!kubectlPath) {
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
reason: "kubectl is not installed in the runner"
|
|
};
|
|
}
|
|
|
|
try {
|
|
await access(d601KubeconfigPath, fsConstants.R_OK);
|
|
} catch {
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
reason: `D601 k3s kubeconfig is not readable at ${d601KubeconfigPath}`
|
|
};
|
|
}
|
|
|
|
return {
|
|
...base,
|
|
status: "ready",
|
|
reason: "D601 k3s kubeconfig is readable"
|
|
};
|
|
}
|
|
|
|
function kubectlCommand(kubectl, args) {
|
|
return `${kubectl.commandPrefix} ${args.join(" ")}`;
|
|
}
|
|
|
|
async function kubectlResult(kubectl, args, timeoutMs = 15000, options = {}) {
|
|
if (kubectl.status !== "ready") {
|
|
return {
|
|
ok: false,
|
|
code: null,
|
|
stdout: "",
|
|
stderr: kubectl.reason,
|
|
blocked: true
|
|
};
|
|
}
|
|
return commandResult(kubectl.executor, args, timeoutMs, {
|
|
...options,
|
|
env: kubectl.env
|
|
});
|
|
}
|
|
|
|
function listItems(document) {
|
|
if (!document) return [];
|
|
return document.kind === "List" ? document.items ?? [] : [document];
|
|
}
|
|
|
|
function containersFor(item) {
|
|
return item?.spec?.template?.spec?.containers ?? [];
|
|
}
|
|
|
|
function serviceIdFor(item, container) {
|
|
return (
|
|
item?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
|
|
item?.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
|
|
container?.name
|
|
);
|
|
}
|
|
|
|
function itemNamespace(item) {
|
|
return item?.metadata?.namespace ?? (item?.kind === "Namespace" ? item?.metadata?.name : namespace);
|
|
}
|
|
|
|
function uniqueStrings(values) {
|
|
return [...new Set(values.filter((value) => typeof value === "string" && value.length > 0))];
|
|
}
|
|
|
|
function imageRecordsFor(item) {
|
|
return containersFor(item).map((container) => ({
|
|
name: container.name,
|
|
serviceId: serviceIdFor(item, container),
|
|
image: container.image
|
|
}));
|
|
}
|
|
|
|
function primaryImage(images) {
|
|
if (!images.length) return null;
|
|
if (images.length === 1) return images[0].image ?? null;
|
|
return images.map((image) => `${image.name}=${image.image}`).join(", ");
|
|
}
|
|
|
|
function imagesDiffer(oldImages, newImages) {
|
|
if (oldImages.length !== newImages.length) return true;
|
|
const oldByName = new Map(oldImages.map((image) => [image.name, image.image]));
|
|
return newImages.some((image) => oldByName.get(image.name) !== image.image);
|
|
}
|
|
|
|
function isAllowedDesiredTemplateJob(item) {
|
|
return (
|
|
item?.kind === "Job" &&
|
|
itemNamespace(item) === namespace &&
|
|
item?.spec?.suspend === true &&
|
|
replaceableDevTemplateJobNames.has(item?.metadata?.name)
|
|
);
|
|
}
|
|
|
|
export function decideDevTemplateJobReplacement({
|
|
jobName,
|
|
jobNamespace = namespace,
|
|
desiredSuspended,
|
|
liveExists,
|
|
liveSuspended,
|
|
oldImages = [],
|
|
newImages = []
|
|
}) {
|
|
const base = {
|
|
namespace: jobNamespace,
|
|
jobName,
|
|
allowed: replaceableDevTemplateJobNames.has(jobName),
|
|
desiredSuspended: desiredSuspended === true,
|
|
liveExists: liveExists === true,
|
|
liveSuspended: liveExists === true ? liveSuspended === true : null,
|
|
oldImage: primaryImage(oldImages),
|
|
newImage: primaryImage(newImages),
|
|
oldImages,
|
|
newImages,
|
|
replace: false,
|
|
result: "not_needed",
|
|
reason: "Live suspended template Job already matches the desired image."
|
|
};
|
|
|
|
if (!base.allowed) {
|
|
return {
|
|
...base,
|
|
result: "ignored",
|
|
reason: "Job is not in the DEV template Job replacement allowlist."
|
|
};
|
|
}
|
|
if (jobNamespace !== namespace) {
|
|
return {
|
|
...base,
|
|
result: "blocked",
|
|
reason: `Replacement policy is DEV-only and refuses namespace ${jobNamespace}.`
|
|
};
|
|
}
|
|
if (desiredSuspended !== true) {
|
|
return {
|
|
...base,
|
|
result: "blocked",
|
|
reason: "Desired Job is not suspended, so it is not a template Job replacement candidate."
|
|
};
|
|
}
|
|
if (liveExists !== true) {
|
|
return {
|
|
...base,
|
|
result: "not_found",
|
|
reason: "Live Job is absent; kubectl apply can create it without replacement."
|
|
};
|
|
}
|
|
if (liveSuspended !== true) {
|
|
return {
|
|
...base,
|
|
result: "blocked",
|
|
reason: "Live Job is not suspended; refusing to delete a potentially active Job."
|
|
};
|
|
}
|
|
if (imagesDiffer(oldImages, newImages)) {
|
|
return {
|
|
...base,
|
|
replace: true,
|
|
result: "planned",
|
|
reason: "Live suspended template Job image differs from the desired manifest; delete/recreate is required because spec.template is immutable."
|
|
};
|
|
}
|
|
return base;
|
|
}
|
|
|
|
function replicaPlan(item) {
|
|
if (item?.kind === "Job") {
|
|
return item?.spec?.suspend === true ? 0 : 1;
|
|
}
|
|
return item?.spec?.replicas ?? null;
|
|
}
|
|
|
|
function buildWorkloadPlan(workloads) {
|
|
return listItems(workloads).map((item) => {
|
|
const containers = containersFor(item).map((container) => ({
|
|
name: container.name,
|
|
serviceId: serviceIdFor(item, container),
|
|
image: container.image
|
|
}));
|
|
return {
|
|
kind: item?.kind ?? "unknown",
|
|
name: item?.metadata?.name ?? "unknown",
|
|
namespace: item?.metadata?.namespace ?? namespace,
|
|
replicas: replicaPlan(item),
|
|
serviceIds: uniqueStrings(containers.map((container) => container.serviceId)),
|
|
containers
|
|
};
|
|
});
|
|
}
|
|
|
|
function buildServicePlan(services) {
|
|
return listItems(services).map((item) => ({
|
|
kind: item?.kind ?? "unknown",
|
|
name: item?.metadata?.name ?? "unknown",
|
|
namespace: item?.metadata?.namespace ?? namespace,
|
|
serviceId: item?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ?? item?.metadata?.name ?? "unknown",
|
|
type: item?.spec?.type ?? "unknown",
|
|
ports: (item?.spec?.ports ?? []).map((port) => ({
|
|
name: port.name,
|
|
port: port.port,
|
|
targetPort: port.targetPort
|
|
}))
|
|
}));
|
|
}
|
|
|
|
function buildArtifactPlan(deploy, catalog, sourceCommitId, artifactEvidence) {
|
|
const requiredServices = (catalog?.services ?? []).filter((service) => service.artifactRequired !== false);
|
|
const serviceCommitIds = uniqueStrings([
|
|
...(catalog?.services ?? []).map((service) => service.commitId),
|
|
...(artifactEvidence ?? []).map((service) => service.catalogCommitId)
|
|
]);
|
|
const expectedArtifactCommit = deploy?.commitId ?? catalog?.commitId ?? "unknown";
|
|
return {
|
|
expectedArtifactCommit,
|
|
deployCommitId: deploy?.commitId ?? "unknown",
|
|
catalogCommitId: catalog?.commitId ?? "unknown",
|
|
sourceCommitId,
|
|
serviceCommitIds,
|
|
matchesSourceCommit:
|
|
deploy?.commitId && catalog?.commitId
|
|
? commitMatchesSource(deploy.commitId, sourceCommitId) && commitMatchesSource(catalog.commitId, sourceCommitId)
|
|
: false,
|
|
published: catalog?.publish?.ciPublished === true,
|
|
registryVerified: catalog?.publish?.registryVerified === true,
|
|
imageCount: artifactEvidence.length,
|
|
requiredServiceCount: requiredServices.length,
|
|
disabledServiceCount: (catalog?.services ?? []).filter((service) => service.artifactRequired === false).length,
|
|
unpublishedServices: requiredServices
|
|
.filter((service) => service.publishState !== "published" || !service.digest?.startsWith("sha256:"))
|
|
.map((service) => service.serviceId)
|
|
};
|
|
}
|
|
|
|
function artifactIdentityForService(deploy, catalog, serviceId) {
|
|
const deployService = deploy?.services?.find((service) => service.serviceId === serviceId) ?? null;
|
|
const catalogService = catalog?.services?.find((service) => service.serviceId === serviceId) ?? null;
|
|
return {
|
|
serviceId,
|
|
sourceCommitId: catalogService?.commitId ?? deploy?.commitId ?? "unknown",
|
|
image: deployService?.image ?? catalogService?.image ?? null,
|
|
imageTag: catalogService?.imageTag ?? parseImageTag(deployService?.image),
|
|
digest: catalogService?.digest ?? "not_published",
|
|
repositoryDigest: catalogService?.repositoryDigest ?? null,
|
|
publishState: catalogService?.publishState ?? "unknown"
|
|
};
|
|
}
|
|
|
|
function parseImageTag(image) {
|
|
if (typeof image !== "string") return null;
|
|
const slashIndex = image.lastIndexOf("/");
|
|
const colonIndex = image.lastIndexOf(":");
|
|
if (colonIndex <= slashIndex) return null;
|
|
return image.slice(colonIndex + 1);
|
|
}
|
|
|
|
function expectedRuntimeIdentityEnv(artifact) {
|
|
return {
|
|
HWLAB_COMMIT_ID: artifact.sourceCommitId,
|
|
HWLAB_IMAGE: artifact.image,
|
|
HWLAB_IMAGE_TAG: artifact.imageTag
|
|
};
|
|
}
|
|
|
|
function envObjectFromEnvList(envList) {
|
|
const env = {};
|
|
if (!Array.isArray(envList)) return env;
|
|
for (const entry of envList) {
|
|
if (!entry?.name) continue;
|
|
env[entry.name] = Object.hasOwn(entry, "value") ? entry.value : null;
|
|
}
|
|
return env;
|
|
}
|
|
|
|
function runtimeIdentityValueMatches(name, actual, expected) {
|
|
if (typeof actual !== "string" || typeof expected !== "string") return false;
|
|
if (name === "HWLAB_COMMIT_ID") return commitMatchesSource(actual, expected);
|
|
return actual === expected;
|
|
}
|
|
|
|
export function compareRuntimeIdentityEnv(env, expected) {
|
|
const fields = runtimeIdentityEnvNames.map((name) => {
|
|
const actual = Object.hasOwn(env ?? {}, name) ? env[name] : null;
|
|
const expectedValue = expected?.[name] ?? null;
|
|
const status = runtimeIdentityValueMatches(name, actual, expectedValue) ? "match" : "drift";
|
|
return {
|
|
name,
|
|
expected: expectedValue,
|
|
actual,
|
|
status
|
|
};
|
|
});
|
|
return {
|
|
status: fields.every((field) => field.status === "match") ? "match" : "drift",
|
|
matchesDesired: fields.every((field) => field.status === "match"),
|
|
fields,
|
|
drift: fields.filter((field) => field.status === "drift")
|
|
};
|
|
}
|
|
|
|
function formatRuntimeIdentityDrift(drift) {
|
|
return drift.map((field) => `${field.name} expected ${field.expected ?? "missing"} actual ${field.actual ?? "missing"}`).join("; ");
|
|
}
|
|
|
|
function deploymentRevision(deployment) {
|
|
return deployment?.metadata?.annotations?.["deployment.kubernetes.io/revision"] ?? null;
|
|
}
|
|
|
|
function containerImageForDeployment(deployment, containerName) {
|
|
const containers = deployment?.spec?.template?.spec?.containers ?? [];
|
|
return containers.find((container) => container.name === containerName)?.image ?? null;
|
|
}
|
|
|
|
function buildRolloutVerificationCommand(serviceId) {
|
|
return `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout status deployment/${serviceId} --timeout=180s`;
|
|
}
|
|
|
|
function buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase) {
|
|
const artifact = artifactIdentityForService(deploy, catalog, serviceId);
|
|
const commandArgs = ["-n", namespace, "get", "deployment", serviceId, "-o", "json"];
|
|
return {
|
|
serviceId,
|
|
namespace,
|
|
status: "not_evaluated",
|
|
observationPhase,
|
|
sourceCommitId: artifact.sourceCommitId,
|
|
image: artifact.image ?? "unknown",
|
|
imageTag: artifact.imageTag ?? "unknown",
|
|
digest: artifact.digest,
|
|
repositoryDigest: artifact.repositoryDigest,
|
|
publishState: artifact.publishState,
|
|
rolloutRevision: null,
|
|
liveImage: null,
|
|
imageMatchesDesired: false,
|
|
expectedRuntimeEnv: expectedRuntimeIdentityEnv(artifact),
|
|
liveRuntimeEnv: null,
|
|
runtimeEnvMatchesDesired: false,
|
|
runtimeEnvDrift: [],
|
|
verificationCommand: buildRolloutVerificationCommand(serviceId),
|
|
readCommand: kubectlCommand(kubectl, commandArgs),
|
|
kubeconfig: kubectl.kubeconfig
|
|
};
|
|
}
|
|
|
|
function skippedDeploymentRolloutObservation(kubectl, deploy, catalog, serviceId, observationPhase, reason) {
|
|
return {
|
|
...buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase),
|
|
status: "not_evaluated",
|
|
blocker: reason
|
|
};
|
|
}
|
|
|
|
async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blockers, options = {}) {
|
|
const observationPhase = options.observationPhase ?? "read";
|
|
const blockRuntimeEnvDrift = options.blockRuntimeEnvDrift === true;
|
|
const commandArgs = ["-n", namespace, "get", "deployment", serviceId, "-o", "json"];
|
|
const base = buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase);
|
|
|
|
if (kubectl.status !== "ready") {
|
|
addBlocker(blockers, "environment_blocker", `rollout-read-${serviceId}`, kubectl.reason);
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
blocker: kubectl.reason
|
|
};
|
|
}
|
|
|
|
const result = await kubectlResult(kubectl, commandArgs, 15000);
|
|
if (!result.ok) {
|
|
const summary = oneLine(result.stderr || result.stdout);
|
|
addBlocker(blockers, "environment_blocker", `rollout-read-${serviceId}`, `Cannot read DEV Deployment ${serviceId} rollout revision: ${summary}`);
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
blocker: summary
|
|
};
|
|
}
|
|
|
|
try {
|
|
const deployment = JSON.parse(result.stdout);
|
|
const liveImage = containerImageForDeployment(deployment, serviceId);
|
|
const liveRuntimeEnv = compareRuntimeIdentityEnv(
|
|
envObjectFromEnvList(deployment?.spec?.template?.spec?.containers?.find((container) => container.name === serviceId)?.env),
|
|
base.expectedRuntimeEnv
|
|
);
|
|
if (blockRuntimeEnvDrift && !liveRuntimeEnv.matchesDesired) {
|
|
addBlocker(
|
|
blockers,
|
|
"runtime_blocker",
|
|
`rollout-runtime-env-${observationPhase}-${serviceId}`,
|
|
`DEV Deployment ${serviceId} runtime identity env drift after apply: ${formatRuntimeIdentityDrift(liveRuntimeEnv.drift)}`
|
|
);
|
|
}
|
|
return {
|
|
...base,
|
|
status: "observed",
|
|
rolloutRevision: deploymentRevision(deployment),
|
|
liveImage,
|
|
imageMatchesDesired: Boolean(base.image && liveImage === base.image),
|
|
liveRuntimeEnv,
|
|
runtimeEnvMatchesDesired: liveRuntimeEnv.matchesDesired,
|
|
runtimeEnvDrift: liveRuntimeEnv.drift
|
|
};
|
|
} catch (error) {
|
|
const summary = oneLine(error.message);
|
|
addBlocker(blockers, "contract_blocker", `rollout-read-${serviceId}`, `Cannot parse DEV Deployment ${serviceId} rollout revision: ${summary}`);
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
blocker: summary
|
|
};
|
|
}
|
|
}
|
|
|
|
function blockerHint(blocker) {
|
|
if (blocker.scope === "artifact-publish") {
|
|
return "Publish DEV images from the intended commit and update the catalog with registry digests and verification evidence.";
|
|
}
|
|
if (blocker.scope === "artifact-source-commit") {
|
|
return "Regenerate deploy/deploy.json and deploy/artifact-catalog.dev.json for the source commit that will be promoted.";
|
|
}
|
|
if (blocker.scope === "kubectl" || blocker.scope === "d601-kubeconfig" || blocker.scope === "kubectl-version") {
|
|
return `Provide kubectl with readable D601 k3s kubeconfig at ${d601KubeconfigPath} without exposing token or secret values.`;
|
|
}
|
|
if (blocker.scope === "cluster-read") {
|
|
return "Restore read-only visibility for pods, services, configmaps, deployments, and jobs in hwlab-dev.";
|
|
}
|
|
if (blocker.scope.startsWith("rollout-read-")) {
|
|
return "Restore read-only Deployment visibility in hwlab-dev so rollout revision, image tag, digest, and source commit can be reported after apply.";
|
|
}
|
|
if (blocker.scope.startsWith("deploy-runtime-env-") || blocker.scope.startsWith("k8s-runtime-env-")) {
|
|
return "Update deploy/deploy.json and deploy/k8s/base/workloads.yaml so the service owns HWLAB_COMMIT_ID, HWLAB_IMAGE, and HWLAB_IMAGE_TAG.";
|
|
}
|
|
if (blocker.scope.startsWith("rollout-runtime-env-")) {
|
|
return "Reconcile the DEV Deployment runtime identity env to the desired source image and rerun post-apply read-only verification.";
|
|
}
|
|
if (blocker.scope === "dev-health" || blocker.scope === "dev-health-identity") {
|
|
return "Restore the public DEV health path and confirm it identifies HWLAB dev, not a substitute runtime.";
|
|
}
|
|
if (blocker.scope === "dev-health-db" || blocker.scope === "cloud-api-db" || blocker.scope === "cloud-api-db-config") {
|
|
return "Configure and verify the DEV cloud-api DB Secret URL host readiness through health output, without reading secret values.";
|
|
}
|
|
return blocker.summary;
|
|
}
|
|
|
|
function buildRemainingBlockers(blockers) {
|
|
return blockers.map((blocker) => ({
|
|
type: blocker.type,
|
|
scope: blocker.scope,
|
|
status: blocker.status,
|
|
summary: blocker.summary,
|
|
unblockHint: oneLine(blockerHint(blocker))
|
|
}));
|
|
}
|
|
|
|
function buildRollbackHint(workloads) {
|
|
const deploymentNames = buildWorkloadPlan(workloads)
|
|
.filter((workload) => workload.kind === "Deployment")
|
|
.map((workload) => workload.name);
|
|
const jobNames = buildWorkloadPlan(workloads)
|
|
.filter((workload) => workload.kind === "Job")
|
|
.map((workload) => workload.name);
|
|
|
|
return {
|
|
namespace,
|
|
strategy: "DEV-only Kubernetes rollback using deployment revision history; do not touch PROD or UniDesk services.",
|
|
captureBeforeApply: [
|
|
`KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev get pods,services,configmaps,deployments,jobs -o wide`,
|
|
`KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev get deployments -o json`
|
|
],
|
|
deploymentRollbackCommands: deploymentNames.map((name) => `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout undo deployment/${name}`),
|
|
jobCleanupCommands: jobNames.map((name) => `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev delete job ${name} --ignore-not-found`),
|
|
postRollbackChecks: [
|
|
`KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout status deployment/hwlab-cloud-api --timeout=120s`,
|
|
`curl -fsS ${new URL(healthPath, DEV_ENDPOINT).href}`
|
|
]
|
|
};
|
|
}
|
|
|
|
function buildManualCommands(status) {
|
|
if (status === "blocked") {
|
|
return {
|
|
status: "blocked",
|
|
beforeHumanApproval: [blockedDryRunCommand, "node scripts/validate-dev-gate-report.mjs"],
|
|
afterHumanApproval: [],
|
|
summary: "No apply command is offered while blockers remain open."
|
|
};
|
|
}
|
|
|
|
return {
|
|
status: "ready",
|
|
beforeHumanApproval: [dryRunPlanCommand, "node scripts/validate-dev-gate-report.mjs"],
|
|
afterHumanApproval: [applyCommand],
|
|
summary: "Run the dry-run plan and validator immediately before requesting human approval; live apply still requires the explicit DEV confirmation flags."
|
|
};
|
|
}
|
|
|
|
function buildApplyBoundary(args, status, applyStep) {
|
|
return {
|
|
currentMode: args.apply ? "apply-requested" : "dry-run",
|
|
defaultNoWrite: !args.apply,
|
|
mutationAttempted: applyStep.mutationAttempted === true && args.apply,
|
|
mutationAllowed: status === "pass" && args.apply,
|
|
applyRequiresFlags: ["--apply", "--confirm-dev", "--confirmed-non-production"],
|
|
writeScope: `KUBECONFIG=${d601KubeconfigPath} kubectl apply -k deploy/k8s/dev, namespace hwlab-dev only; delete/recreate is allowed only for explicitly allowlisted suspended DEV template Jobs`,
|
|
noWriteScope: `source/catalog/k8s validation, optional DEV health probe, KUBECONFIG=${d601KubeconfigPath} kubectl read, replacement planning, and server-side dry-run only`,
|
|
forbiddenActions
|
|
};
|
|
}
|
|
|
|
function buildPlanConclusion(status, blockers) {
|
|
return {
|
|
status: status === "blocked" ? "blocked" : "ready",
|
|
reason:
|
|
status === "blocked"
|
|
? `${blockers.length} blocker(s) must be cleared before any live DEV apply.`
|
|
: "All DEV-only preconditions passed; human approval is still required before apply.",
|
|
blockerCount: blockers.length
|
|
};
|
|
}
|
|
|
|
function validateArgs(args, blockers) {
|
|
for (const forbidden of ["--prod", "--production", "--read-secret", "--force-push", "--heavyweight-e2e"]) {
|
|
if (args.flags.has(forbidden)) {
|
|
addBlocker(blockers, "safety_blocker", forbidden.slice(2), `${forbidden} is forbidden for DEV deploy apply`);
|
|
}
|
|
}
|
|
if (args.apply && (!args.flags.has("--confirm-dev") || !args.flags.has("--confirmed-non-production"))) {
|
|
addBlocker(
|
|
blockers,
|
|
"safety_blocker",
|
|
"apply-confirmation",
|
|
"Live apply requires --confirm-dev and --confirmed-non-production"
|
|
);
|
|
}
|
|
}
|
|
|
|
function commitMatchesSource(commitId, sourceCommitId) {
|
|
return sourceCommitId.startsWith(commitId) || commitId.startsWith(sourceCommitId);
|
|
}
|
|
|
|
function validateDeployRuntimeIdentityEnv(deploy, catalog, serviceId, blockers) {
|
|
const deployService = deploy?.services?.find((service) => service.serviceId === serviceId) ?? null;
|
|
if (!deployService) return null;
|
|
const artifact = artifactIdentityForService(deploy, catalog, serviceId);
|
|
const expected = expectedRuntimeIdentityEnv(artifact);
|
|
const comparison = compareRuntimeIdentityEnv(deployService.env ?? {}, expected);
|
|
if (!comparison.matchesDesired) {
|
|
addBlocker(
|
|
blockers,
|
|
"contract_blocker",
|
|
`deploy-runtime-env-${serviceId}`,
|
|
`${serviceId} deploy env does not own desired runtime identity: ${formatRuntimeIdentityDrift(comparison.drift)}`
|
|
);
|
|
}
|
|
return {
|
|
serviceId,
|
|
source: "deploy",
|
|
expected,
|
|
...comparison
|
|
};
|
|
}
|
|
|
|
function validateDeployAndCatalog(deploy, catalog, sourceCommitId, blockers) {
|
|
const artifactEvidence = [];
|
|
if (!deploy || !catalog) return artifactEvidence;
|
|
|
|
if (deploy.environment !== ENVIRONMENT_DEV || deploy.namespace !== namespace) {
|
|
addBlocker(blockers, "safety_blocker", "deploy-target", "deploy/deploy.json must target only dev hwlab-dev");
|
|
}
|
|
if (deploy.profiles?.prod?.enabled !== false) {
|
|
addBlocker(blockers, "safety_blocker", "prod-profile", "PROD profile must remain disabled");
|
|
}
|
|
if (catalog.environment !== ENVIRONMENT_DEV || catalog.namespace !== namespace) {
|
|
addBlocker(blockers, "safety_blocker", "catalog-target", "artifact catalog must target only dev hwlab-dev");
|
|
}
|
|
if (!commitMatchesSource(deploy.commitId, sourceCommitId) || !commitMatchesSource(catalog.commitId, sourceCommitId)) {
|
|
addBlocker(
|
|
blockers,
|
|
"observability_blocker",
|
|
"artifact-source-commit",
|
|
`deploy/catalog artifact commit ${deploy.commitId}/${catalog.commitId} does not match source ${sourceCommitId}`
|
|
);
|
|
}
|
|
|
|
const deployById = new Map((deploy.services ?? []).map((service) => [service.serviceId, service]));
|
|
const catalogById = new Map((catalog.services ?? []).map((service) => [service.serviceId, service]));
|
|
for (const serviceId of SERVICE_IDS) {
|
|
const deployService = deployById.get(serviceId);
|
|
const catalogService = catalogById.get(serviceId);
|
|
if (!deployService || !catalogService) {
|
|
addBlocker(blockers, "contract_blocker", `service-${serviceId}`, `${serviceId} missing from deploy or catalog`);
|
|
continue;
|
|
}
|
|
if (deployService.namespace !== namespace || deployService.profile !== ENVIRONMENT_DEV) {
|
|
addBlocker(blockers, "safety_blocker", `service-target-${serviceId}`, `${serviceId} is not DEV-only`);
|
|
}
|
|
if (deployService.image !== catalogService.image) {
|
|
addBlocker(blockers, "contract_blocker", `service-image-${serviceId}`, `${serviceId} image differs in deploy/catalog`);
|
|
}
|
|
artifactEvidence.push({
|
|
serviceId,
|
|
deployCommitId: deploy.commitId,
|
|
catalogCommitId: catalogService.commitId,
|
|
image: deployService.image,
|
|
digest: catalogService.digest,
|
|
publishState: catalogService.publishState
|
|
});
|
|
}
|
|
validateDeployRuntimeIdentityEnv(deploy, catalog, "hwlab-cloud-web", blockers);
|
|
|
|
const requiredServices = (catalog.services ?? []).filter((service) => service.artifactRequired !== false);
|
|
const disabledServices = (catalog.services ?? []).filter((service) => service.artifactRequired === false);
|
|
const hasPublishedCatalog =
|
|
catalog.publish?.ciPublished === true &&
|
|
catalog.publish?.registryVerified === true &&
|
|
requiredServices.every((service) => service.digest?.startsWith("sha256:") && service.publishState === "published") &&
|
|
disabledServices.every((service) => service.digest === "not_published" && service.publishState === "skeleton-only");
|
|
if (!hasPublishedCatalog) {
|
|
addBlocker(
|
|
blockers,
|
|
"observability_blocker",
|
|
"artifact-publish",
|
|
"DEV artifact catalog has no CI publish, registry verification, or required-service registry digests"
|
|
);
|
|
}
|
|
return artifactEvidence;
|
|
}
|
|
|
|
function validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers) {
|
|
const items = [...listItems(namespaceDoc), ...listItems(workloads), ...listItems(services), ...listItems(healthContract)];
|
|
if (devKustomization?.namespace !== namespace) {
|
|
addBlocker(blockers, "safety_blocker", "dev-kustomization", "DEV kustomization must pin namespace hwlab-dev");
|
|
}
|
|
if ((devKustomization?.resources ?? []).some((resource) => resource.includes("prod"))) {
|
|
addBlocker(blockers, "safety_blocker", "dev-kustomization-prod", "DEV kustomization must not reference PROD resources");
|
|
}
|
|
for (const item of items) {
|
|
const itemNamespace = item?.metadata?.namespace ?? (item?.kind === "Namespace" ? item?.metadata?.name : namespace);
|
|
if (itemNamespace !== namespace) {
|
|
addBlocker(blockers, "safety_blocker", `k8s-${item?.kind}-${item?.metadata?.name}`, "Kubernetes resource is not in hwlab-dev");
|
|
}
|
|
}
|
|
|
|
const expectedImages = new Map((deploy?.services ?? []).map((service) => [service.serviceId, service.image]));
|
|
const cloudWebExpectedRuntimeEnv = expectedRuntimeIdentityEnv(artifactIdentityForService(deploy, catalog, "hwlab-cloud-web"));
|
|
const workloadItems = listItems(workloads);
|
|
const images = [];
|
|
const runtimeIdentityEnv = [];
|
|
for (const item of workloadItems) {
|
|
for (const container of containersFor(item)) {
|
|
const serviceId = serviceIdFor(item, container);
|
|
const expectedImage = expectedImages.get(serviceId);
|
|
images.push({ serviceId, image: container.image, kind: item.kind, name: item.metadata?.name });
|
|
if (expectedImage && container.image !== expectedImage) {
|
|
addBlocker(blockers, "contract_blocker", `k8s-image-${serviceId}`, `${serviceId} image differs in k8s workload`);
|
|
}
|
|
if (serviceId === "hwlab-cloud-web") {
|
|
const comparison = compareRuntimeIdentityEnv(envObjectFromEnvList(container.env), cloudWebExpectedRuntimeEnv);
|
|
runtimeIdentityEnv.push({
|
|
serviceId,
|
|
kind: item.kind,
|
|
name: item.metadata?.name,
|
|
container: container.name,
|
|
expected: cloudWebExpectedRuntimeEnv,
|
|
...comparison
|
|
});
|
|
if (!comparison.matchesDesired) {
|
|
addBlocker(
|
|
blockers,
|
|
"contract_blocker",
|
|
`k8s-runtime-env-${serviceId}`,
|
|
`${serviceId} workload env does not own desired runtime identity: ${formatRuntimeIdentityDrift(comparison.drift)}`
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return {
|
|
resourceCounts: {
|
|
workloads: workloadItems.length,
|
|
services: listItems(services).length,
|
|
configMaps: listItems(healthContract).filter((item) => item.kind === "ConfigMap").length
|
|
},
|
|
images,
|
|
runtimeIdentityEnv
|
|
};
|
|
}
|
|
|
|
async function checkCloudApiDb(deploy, workloads, services, blockers) {
|
|
const deployEnv = deploy?.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {};
|
|
const envNames = new Set(Object.keys(deployEnv));
|
|
const workloadEnv = new Map();
|
|
for (const item of listItems(workloads)) {
|
|
for (const container of containersFor(item)) {
|
|
if (serviceIdFor(item, container) === "hwlab-cloud-api") {
|
|
for (const env of container.env ?? []) {
|
|
envNames.add(env.name);
|
|
workloadEnv.set(env.name, env);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
const requiredEnv = DEV_DB_ENV_CONTRACT.requiredEnv.map((name) => {
|
|
const secretRef = DEV_DB_ENV_CONTRACT.secretRefs.find((ref) => ref.env === name) ?? null;
|
|
const workloadEntry = workloadEnv.get(name) ?? null;
|
|
const secretRefPresent = secretRef
|
|
? deployEnv[name] === `secretRef:${secretRef.secretName}/${secretRef.secretKey}` &&
|
|
workloadEntry?.valueFrom?.secretKeyRef?.name === secretRef.secretName &&
|
|
workloadEntry?.valueFrom?.secretKeyRef?.key === secretRef.secretKey
|
|
: true;
|
|
return {
|
|
name,
|
|
manifestPresent: Object.hasOwn(deployEnv, name),
|
|
k8sPresent: workloadEnv.has(name),
|
|
redacted: Boolean(secretRef),
|
|
source: secretRef ? "k8s-secret-ref" : "runtime-env",
|
|
secretRef: secretRef
|
|
? {
|
|
secretName: secretRef.secretName,
|
|
secretKey: secretRef.secretKey,
|
|
present: secretRefPresent,
|
|
redacted: true
|
|
}
|
|
: null
|
|
};
|
|
});
|
|
const missingManifest = requiredEnv.filter((env) => !env.manifestPresent).map((env) => env.name);
|
|
const missingK8s = requiredEnv.filter((env) => !env.k8sPresent).map((env) => env.name);
|
|
const missingSecretRefs = requiredEnv
|
|
.filter((env) => env.secretRef && !env.secretRef.present)
|
|
.map((env) => `${env.secretRef.secretName}/${env.secretRef.secretKey}`);
|
|
const dnsContract = inspectCloudApiDbOptionalAliasContract(deployEnv, workloadEnv, services);
|
|
const missingDnsContract = dnsContract.missing;
|
|
if (missingManifest.length > 0 || missingK8s.length > 0 || missingSecretRefs.length > 0 || missingDnsContract.length > 0) {
|
|
addBlocker(
|
|
blockers,
|
|
"runtime_blocker",
|
|
"cloud-api-db-config",
|
|
`cloud-api DEV DB env/authority contract is incomplete; missing ${[...missingManifest, ...missingK8s, ...missingSecretRefs, ...missingDnsContract].join(", ")}`
|
|
);
|
|
}
|
|
|
|
const configReady =
|
|
missingManifest.length === 0 &&
|
|
missingK8s.length === 0 &&
|
|
missingSecretRefs.length === 0 &&
|
|
missingDnsContract.length === 0;
|
|
const health = {
|
|
status: configReady ? "degraded" : "blocked",
|
|
ready: false,
|
|
configReady,
|
|
connected: false,
|
|
connectionChecked: false,
|
|
fields: requiredEnv.map((env) => ({
|
|
name: env.name,
|
|
present: env.manifestPresent && env.k8sPresent && (!env.secretRef || env.secretRef.present),
|
|
redacted: env.redacted
|
|
})),
|
|
missingEnv: uniqueStrings([...missingManifest, ...missingK8s]),
|
|
secretRefs: requiredEnv
|
|
.filter((env) => env.secretRef)
|
|
.map((env) => ({
|
|
env: env.name,
|
|
secretName: env.secretRef.secretName,
|
|
secretKey: env.secretRef.secretKey,
|
|
present: env.secretRef.present,
|
|
redacted: true
|
|
})),
|
|
safety: {
|
|
liveDbEvidence: false
|
|
}
|
|
};
|
|
|
|
return {
|
|
status: configReady ? "contract-ready" : "contract-blocked",
|
|
requiredEnv,
|
|
missingManifest,
|
|
missingK8s,
|
|
missingSecretRefs,
|
|
missingDnsContract,
|
|
dns: dnsContract,
|
|
runtimeHealth: summarizeDbContract(health),
|
|
secretMaterialRead: false,
|
|
valuesRedacted: true,
|
|
liveDbEvidence: false,
|
|
fixtureEvidence: false,
|
|
note: "This check records DB env, Secret reference, and Secret URL host authority only. Optional cloud-api-db alias presence is diagnostic and is not live DB evidence."
|
|
};
|
|
}
|
|
|
|
function inspectCloudApiDbOptionalAliasContract(deployEnv, workloadEnv, services) {
|
|
const alias = DEV_DB_ENV_CONTRACT.dns;
|
|
const service = listItems(services).find((item) => item?.metadata?.name === alias.serviceName) ?? null;
|
|
const actual = {
|
|
deployServiceName: deployEnv.HWLAB_CLOUD_DB_SERVICE_NAME,
|
|
deployServiceNamespace: deployEnv.HWLAB_CLOUD_DB_SERVICE_NAMESPACE,
|
|
deployHost: deployEnv.HWLAB_CLOUD_DB_HOST,
|
|
deployPort: deployEnv.HWLAB_CLOUD_DB_PORT,
|
|
workloadServiceName: workloadEnv.get("HWLAB_CLOUD_DB_SERVICE_NAME")?.value,
|
|
workloadServiceNamespace: workloadEnv.get("HWLAB_CLOUD_DB_SERVICE_NAMESPACE")?.value,
|
|
workloadHost: workloadEnv.get("HWLAB_CLOUD_DB_HOST")?.value,
|
|
workloadPort: workloadEnv.get("HWLAB_CLOUD_DB_PORT")?.value,
|
|
serviceName: service?.metadata?.name
|
|
};
|
|
const missing = [];
|
|
for (const [name, value] of Object.entries(actual)) {
|
|
if (value !== undefined) {
|
|
missing.push(`${name} must be absent because ${alias.serviceName} is not the readiness authority`);
|
|
}
|
|
}
|
|
|
|
return {
|
|
ready: missing.length === 0,
|
|
source: alias.source,
|
|
serviceName: alias.serviceName,
|
|
namespace: alias.namespace,
|
|
port: alias.port,
|
|
portName: alias.portName,
|
|
requiredForReadiness: alias.requiredForReadiness,
|
|
usedForProbe: alias.usedForProbe,
|
|
servicePresent: Boolean(service),
|
|
secretMaterialRead: false,
|
|
liveDbEvidence: false,
|
|
missing,
|
|
actual
|
|
};
|
|
}
|
|
|
|
async function probeDevEndpoint(blockers) {
|
|
const url = new URL(healthPath, DEV_ENDPOINT).href;
|
|
try {
|
|
const response = await httpGet(url, 8000);
|
|
const text = response.body;
|
|
let json = null;
|
|
try {
|
|
json = JSON.parse(text);
|
|
} catch {}
|
|
if (response.statusCode < 200 || response.statusCode >= 300) {
|
|
addBlocker(blockers, "network_blocker", "dev-health", `DEV health returned HTTP ${response.statusCode}`);
|
|
}
|
|
if (json?.serviceId !== "hwlab-cloud-api" || json?.environment !== ENVIRONMENT_DEV) {
|
|
addBlocker(blockers, "runtime_blocker", "dev-health-identity", "DEV health did not identify hwlab-cloud-api in dev");
|
|
}
|
|
if (json?.db?.ready !== true && json?.db?.configReady !== true) {
|
|
addBlocker(blockers, "runtime_blocker", "dev-health-db", "DEV health did not confirm cloud-api DB config readiness");
|
|
}
|
|
return {
|
|
status: response.statusCode >= 200 && response.statusCode < 300 ? "pass" : "blocked",
|
|
url,
|
|
httpStatus: response.statusCode,
|
|
body: text.slice(0, 500)
|
|
};
|
|
} catch (error) {
|
|
const reason = oneLine(error.cause?.message ?? error.message);
|
|
addBlocker(blockers, "network_blocker", "dev-health", `Cannot reach ${url}: ${reason}`);
|
|
return { status: "blocked", url, error: reason };
|
|
}
|
|
}
|
|
|
|
function httpGet(url, timeoutMs) {
|
|
return new Promise((resolve, reject) => {
|
|
const request = httpRequest(url, { method: "GET", timeout: timeoutMs }, (response) => {
|
|
response.setEncoding("utf8");
|
|
let body = "";
|
|
response.on("data", (chunk) => {
|
|
body += chunk;
|
|
});
|
|
response.on("end", () => resolve({ statusCode: response.statusCode ?? 0, body }));
|
|
});
|
|
request.on("timeout", () => {
|
|
request.destroy(new Error(`timeout after ${timeoutMs}ms`));
|
|
});
|
|
request.on("error", reject);
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
async function observeCluster(kubectl, blockers) {
|
|
const blockedBase = {
|
|
status: "blocked",
|
|
executor: kubectl.executor ?? "missing",
|
|
kubeconfig: kubectl.kubeconfig,
|
|
commandPrefix: kubectl.commandPrefix,
|
|
resources: []
|
|
};
|
|
if (!kubectl.executor) {
|
|
addBlocker(blockers, "environment_blocker", "kubectl", kubectl.reason);
|
|
return blockedBase;
|
|
}
|
|
if (kubectl.status !== "ready") {
|
|
addBlocker(blockers, "environment_blocker", "d601-kubeconfig", kubectl.reason);
|
|
return blockedBase;
|
|
}
|
|
const version = await kubectlResult(kubectl, ["version", "--client=true", "--output=yaml"]);
|
|
if (!version.ok) {
|
|
addBlocker(blockers, "environment_blocker", "kubectl-version", `kubectl client failed: ${version.stderr || version.stdout}`);
|
|
return blockedBase;
|
|
}
|
|
const getArgs = ["get", "pods,services,configmaps,deployments,jobs", "-n", namespace, "-o", "json"];
|
|
const get = await kubectlResult(kubectl, getArgs, 20000);
|
|
if (!get.ok) {
|
|
addBlocker(blockers, "environment_blocker", "cluster-read", `Cannot read hwlab-dev resources: ${get.stderr || get.stdout}`);
|
|
return { ...blockedBase, version: version.stdout, readCommand: kubectlCommand(kubectl, getArgs) };
|
|
}
|
|
const parsed = JSON.parse(get.stdout);
|
|
return {
|
|
status: "pass",
|
|
executor: kubectl.executor,
|
|
kubeconfig: kubectl.kubeconfig,
|
|
commandPrefix: kubectl.commandPrefix,
|
|
readCommand: kubectlCommand(kubectl, getArgs),
|
|
version: version.stdout,
|
|
resources: (parsed.items ?? []).map((item) => ({
|
|
kind: item.kind,
|
|
name: item.metadata?.name,
|
|
phase: item.status?.phase,
|
|
readyReplicas: item.status?.readyReplicas,
|
|
replicas: item.status?.replicas
|
|
}))
|
|
};
|
|
}
|
|
|
|
function isNotFoundOutput(value) {
|
|
return /\bnot\s+found\b|notfound/iu.test(value);
|
|
}
|
|
|
|
async function readLiveTemplateJob(kubectl, jobName, blockers) {
|
|
if (kubectl.status !== "ready") {
|
|
return { status: "not_evaluated", reason: kubectl.reason };
|
|
}
|
|
const get = await kubectlResult(kubectl, ["-n", namespace, "get", "job", jobName, "-o", "json"], 15000);
|
|
if (!get.ok) {
|
|
const output = get.stderr || get.stdout;
|
|
if (isNotFoundOutput(output)) {
|
|
return { status: "not_found", liveJob: null };
|
|
}
|
|
addBlocker(blockers, "environment_blocker", `template-job-read-${jobName}`, `Cannot read live DEV template Job ${jobName}: ${output}`);
|
|
return { status: "blocked", reason: oneLine(output), liveJob: null };
|
|
}
|
|
try {
|
|
return { status: "found", liveJob: JSON.parse(get.stdout) };
|
|
} catch (error) {
|
|
addBlocker(blockers, "contract_blocker", `template-job-read-${jobName}`, `Cannot parse live DEV template Job ${jobName}: ${error.message}`);
|
|
return { status: "blocked", reason: oneLine(error.message), liveJob: null };
|
|
}
|
|
}
|
|
|
|
async function buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers) {
|
|
const desiredJobs = listItems(workloads).filter(isAllowedDesiredTemplateJob);
|
|
const replacements = [];
|
|
for (const desiredJob of desiredJobs) {
|
|
const jobName = desiredJob.metadata.name;
|
|
const live = await readLiveTemplateJob(kubectl, jobName, blockers);
|
|
if (live.status === "not_evaluated") {
|
|
replacements.push({
|
|
namespace,
|
|
jobName,
|
|
allowed: true,
|
|
desiredSuspended: true,
|
|
liveExists: null,
|
|
liveSuspended: null,
|
|
oldImage: null,
|
|
newImage: primaryImage(imageRecordsFor(desiredJob)),
|
|
oldImages: [],
|
|
newImages: imageRecordsFor(desiredJob),
|
|
replace: false,
|
|
result: "not_evaluated",
|
|
reason: live.reason
|
|
});
|
|
continue;
|
|
}
|
|
if (live.status === "blocked") {
|
|
replacements.push({
|
|
namespace,
|
|
jobName,
|
|
allowed: true,
|
|
desiredSuspended: true,
|
|
liveExists: null,
|
|
liveSuspended: null,
|
|
oldImage: null,
|
|
newImage: primaryImage(imageRecordsFor(desiredJob)),
|
|
oldImages: [],
|
|
newImages: imageRecordsFor(desiredJob),
|
|
replace: false,
|
|
result: "blocked",
|
|
reason: live.reason
|
|
});
|
|
continue;
|
|
}
|
|
|
|
const decision = decideDevTemplateJobReplacement({
|
|
jobName,
|
|
jobNamespace: itemNamespace(desiredJob),
|
|
desiredSuspended: desiredJob.spec?.suspend,
|
|
liveExists: live.status === "found",
|
|
liveSuspended: live.liveJob?.spec?.suspend,
|
|
oldImages: live.liveJob ? imageRecordsFor(live.liveJob) : [],
|
|
newImages: imageRecordsFor(desiredJob)
|
|
});
|
|
if (decision.result === "blocked") {
|
|
addBlocker(blockers, "safety_blocker", `template-job-replace-${jobName}`, decision.reason);
|
|
}
|
|
replacements.push(decision);
|
|
}
|
|
return replacements;
|
|
}
|
|
|
|
async function executeDevTemplateJobReplacements(kubectl, replacements, blockers) {
|
|
const planned = replacements.filter((replacement) => replacement.replace && replacement.result === "planned");
|
|
for (const replacement of planned) {
|
|
const commandArgs = [
|
|
"-n",
|
|
replacement.namespace,
|
|
"delete",
|
|
"job",
|
|
replacement.jobName,
|
|
"--ignore-not-found=true"
|
|
];
|
|
const result = await kubectlResult(kubectl, commandArgs, 20000);
|
|
replacement.deleteCommand = kubectlCommand(kubectl, commandArgs);
|
|
replacement.deleteStdout = result.stdout;
|
|
replacement.deleteStderr = result.stderr;
|
|
if (!result.ok) {
|
|
replacement.result = "delete_failed";
|
|
replacement.reason = `Failed to delete live suspended template Job before apply: ${oneLine(result.stderr || result.stdout)}`;
|
|
addBlocker(blockers, "environment_blocker", `template-job-replace-${replacement.jobName}`, replacement.reason);
|
|
continue;
|
|
}
|
|
replacement.result = "deleted_pending_recreate";
|
|
replacement.reason = "Live suspended template Job was deleted; kubectl apply must recreate it from the desired manifest.";
|
|
}
|
|
return planned.length;
|
|
}
|
|
|
|
function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) {
|
|
const plannedNames = replacementsNeeded.map((replacement) => replacement.jobName);
|
|
if (result.ok || plannedNames.length === 0) return false;
|
|
const output = `${result.stdout}\n${result.stderr}`;
|
|
const errorSections = output.split("Error from server").slice(1);
|
|
const immutableSections = errorSections.filter(
|
|
(section) => section.includes("spec.template") && section.includes("field is immutable")
|
|
);
|
|
if (immutableSections.length === 0 || immutableSections.length !== errorSections.length) return false;
|
|
for (const name of plannedNames) {
|
|
if (!immutableSections.some((section) => section.includes(name))) return false;
|
|
}
|
|
return immutableSections.every(
|
|
(section) =>
|
|
section.includes("Resource=jobs") &&
|
|
section.includes("Kind=Job") &&
|
|
plannedNames.some((name) => section.includes(name))
|
|
);
|
|
}
|
|
|
|
async function runApplyStep(args, kubectl, blockers, templateJobReplacements) {
|
|
if (blockers.length > 0) {
|
|
return { status: "not_run", summary: "Skipped because preflight blockers are open" };
|
|
}
|
|
let mutationAttempted = false;
|
|
const replacementsNeeded = templateJobReplacements.filter((replacement) => replacement.replace);
|
|
|
|
if (args.apply) {
|
|
const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers);
|
|
mutationAttempted = replaceCount > 0;
|
|
if (blockers.length > 0) {
|
|
return {
|
|
status: "blocked",
|
|
command: kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"]),
|
|
mutationAttempted,
|
|
summary: "Skipped kubectl apply because DEV template Job replacement failed"
|
|
};
|
|
}
|
|
}
|
|
|
|
const commandArgs = args.apply
|
|
? ["apply", "-k", "deploy/k8s/dev"]
|
|
: ["apply", "--dry-run=server", "-k", "deploy/k8s/dev"];
|
|
const result = await kubectlResult(kubectl, commandArgs, 30000);
|
|
const expectedImmutableDryRun = !args.apply && isExpectedTemplateJobImmutableFailure(result, replacementsNeeded);
|
|
mutationAttempted = mutationAttempted || args.apply;
|
|
if (!result.ok && !expectedImmutableDryRun) {
|
|
addBlocker(blockers, "environment_blocker", args.apply ? "kubectl-apply" : "kubectl-dry-run", result.stderr || result.stdout);
|
|
}
|
|
for (const replacement of templateJobReplacements) {
|
|
if (replacement.result === "deleted_pending_recreate") {
|
|
replacement.result = result.ok ? "replaced" : "delete_succeeded_apply_failed";
|
|
replacement.reason = result.ok
|
|
? "Live suspended template Job was deleted and recreated by kubectl apply."
|
|
: "Live suspended template Job was deleted, but kubectl apply failed before confirming recreation.";
|
|
}
|
|
}
|
|
return {
|
|
status: result.ok || expectedImmutableDryRun ? "pass" : "blocked",
|
|
command: kubectlCommand(kubectl, commandArgs),
|
|
stdout: result.stdout,
|
|
stderr: result.stderr,
|
|
mutationAttempted,
|
|
expectedImmutableTemplateJobDryRun: expectedImmutableDryRun,
|
|
expectedImmutableTemplateJobs: expectedImmutableDryRun
|
|
? replacementsNeeded.map((replacement) => replacement.jobName)
|
|
: []
|
|
};
|
|
}
|
|
|
|
async function gitHeadCommit() {
|
|
const result = await commandResult("git", ["rev-parse", "--short=12", "HEAD"]);
|
|
return result.ok ? result.stdout : "unknown";
|
|
}
|
|
|
|
export function resolveApplySourceCommit(deploy, catalog, gitHeadCommitId) {
|
|
const deployCommit = deploy?.commitId;
|
|
const catalogCommit = catalog?.commitId;
|
|
if (
|
|
typeof deployCommit === "string" &&
|
|
typeof catalogCommit === "string" &&
|
|
commitMatchesSource(deployCommit, catalogCommit)
|
|
) {
|
|
return deployCommit.length >= catalogCommit.length ? deployCommit : catalogCommit;
|
|
}
|
|
return gitHeadCommitId;
|
|
}
|
|
|
|
export async function runDevDeployApply(argv, io = {}) {
|
|
const stdout = io.stdout ?? process.stdout;
|
|
const args = parseArgs(argv);
|
|
const blockers = [];
|
|
validateArgs(args, blockers);
|
|
const gitHeadCommitIdPromise = gitHeadCommit();
|
|
|
|
const [deploy, catalog, devKustomization, namespaceDoc, workloads, services, healthContract] = await Promise.all([
|
|
readJson("deploy/deploy.json", blockers),
|
|
readJson("deploy/artifact-catalog.dev.json", blockers),
|
|
readJson("deploy/k8s/dev/kustomization.yaml", blockers),
|
|
readJson("deploy/k8s/base/namespace.yaml", blockers),
|
|
readJson("deploy/k8s/base/workloads.yaml", blockers),
|
|
readJson("deploy/k8s/base/services.yaml", blockers),
|
|
readJson("deploy/k8s/dev/health-contract.yaml", blockers)
|
|
]);
|
|
const gitHeadCommitId = await gitHeadCommitIdPromise;
|
|
const commitId = resolveApplySourceCommit(deploy, catalog, gitHeadCommitId);
|
|
|
|
const artifactEvidence = validateDeployAndCatalog(deploy, catalog, commitId, blockers);
|
|
const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers);
|
|
const cloudApiDb = await checkCloudApiDb(deploy, workloads, services, blockers);
|
|
if (!(await readText("internal/cloud/server.mjs")).includes('url.pathname === "/health/live"')) {
|
|
addBlocker(blockers, "contract_blocker", "cloud-api-health-path", "cloud-api does not implement the k8s /health/live path");
|
|
}
|
|
|
|
const kubectl = await resolveD601Kubectl();
|
|
const clusterObservation = await observeCluster(kubectl, blockers);
|
|
const cloudWebRolloutBeforeApply = await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
|
|
observationPhase: "before_apply",
|
|
blockRuntimeEnvDrift: false
|
|
});
|
|
const liveProbe = args.skipLiveProbe ? { status: "not_run", reason: "--skip-live-probe" } : await probeDevEndpoint(blockers);
|
|
const templateJobReplacements = await buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers);
|
|
const applyStep = await runApplyStep(
|
|
args,
|
|
kubectl,
|
|
blockers,
|
|
templateJobReplacements
|
|
);
|
|
const cloudWebRolloutAfterApply = args.apply && applyStep.status === "pass"
|
|
? await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
|
|
observationPhase: "after_apply",
|
|
blockRuntimeEnvDrift: true
|
|
})
|
|
: skippedDeploymentRolloutObservation(
|
|
kubectl,
|
|
deploy,
|
|
catalog,
|
|
"hwlab-cloud-web",
|
|
"after_apply",
|
|
args.apply ? "apply did not complete successfully; post-apply rollout env verification was not run" : "dry-run mode; no post-apply rollout env verification"
|
|
);
|
|
const cloudWebRollout = args.apply ? cloudWebRolloutAfterApply : cloudWebRolloutBeforeApply;
|
|
const status = blockers.length > 0 ? "blocked" : "pass";
|
|
const artifactPlan = buildArtifactPlan(deploy, catalog, commitId, artifactEvidence);
|
|
const workloadPlan = buildWorkloadPlan(workloads);
|
|
const servicePlan = buildServicePlan(services);
|
|
const planConclusion = buildPlanConclusion(status, blockers);
|
|
const manualCommands = buildManualCommands(status);
|
|
const remainingBlockers = buildRemainingBlockers(blockers);
|
|
|
|
const report = {
|
|
$schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json",
|
|
$id: "https://hwlab.pikastech.local/reports/dev-gate/dev-deploy-report.json",
|
|
reportVersion: "v1",
|
|
issue: "pikasTech/HWLAB#33",
|
|
supports: ["pikasTech/HWLAB#63", "pikasTech/HWLAB#50", "pikasTech/HWLAB#7", "pikasTech/HWLAB#33", "pikasTech/HWLAB#17", "pikasTech/HWLAB#32", "pikasTech/HWLAB#30"],
|
|
taskId: "dev-deploy-apply",
|
|
commitId,
|
|
acceptanceLevel: "dev_deploy_apply",
|
|
devOnly: true,
|
|
prodDisabled: true,
|
|
reportLifecycle: activeReportLifecycle("Current DEV deploy apply report; active public API/health endpoint is 16667 and browser endpoint is 16666."),
|
|
status,
|
|
generatedAt: new Date().toISOString(),
|
|
namespace,
|
|
endpoint: DEV_ENDPOINT,
|
|
sourceContract: {
|
|
status: "pass",
|
|
documents: [
|
|
"docs/dev-acceptance-matrix.md",
|
|
"docs/m0-contract-audit.md",
|
|
"docs/dev-deploy-apply.md",
|
|
"docs/artifact-catalog.md",
|
|
"deploy/README.md"
|
|
],
|
|
summary: "DEV deploy apply is pinned to hwlab-dev and the frozen HWLAB service IDs"
|
|
},
|
|
validationCommands: requiredValidationCommands,
|
|
localSmoke: {
|
|
status: "not_run",
|
|
commands: ["node scripts/m1-contract-smoke.mjs"],
|
|
evidence: ["This deploy apply task did not run local smoke because artifact/executor blockers stop promotion first."],
|
|
summary: "Local smoke remains available but is not a substitute for DEV artifact publish evidence."
|
|
},
|
|
dryRun: {
|
|
status,
|
|
commands: [
|
|
"node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run",
|
|
"node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked"
|
|
],
|
|
evidence: [
|
|
`artifact services checked: ${artifactEvidence.length}`,
|
|
`expected artifact commit: ${artifactPlan.expectedArtifactCommit}`,
|
|
`namespace: ${namespace}`,
|
|
`workloads planned: ${workloadPlan.length}`,
|
|
`kubectl executor: ${kubectl.executor ?? "missing"}`,
|
|
`kubectl kubeconfig: ${kubectl.kubeconfig}`,
|
|
`live health: ${liveProbe.status}`,
|
|
`template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}`
|
|
],
|
|
summary: status === "blocked" ? "DEV apply is blocked before mutation." : "DEV dry-run preflight passed."
|
|
},
|
|
devPreconditions: {
|
|
status,
|
|
requirements: [
|
|
"DEV artifact catalog must contain CI publish evidence and registry digests",
|
|
"kubectl must be available for D601 hwlab-dev and must not target PROD",
|
|
"hwlab-cloud-api /health/live must report serviceId, dev environment, and DB env readiness without exposing secret values",
|
|
"pods, services, configmaps, and image commit tags must be observable in hwlab-dev"
|
|
],
|
|
summary: status === "blocked" ? "One or more required DEV deploy preconditions are blocked." : "DEV deploy preconditions are satisfied."
|
|
},
|
|
devDeployApply: {
|
|
mode: args.apply ? "apply" : "dry-run",
|
|
conclusion: planConclusion,
|
|
applyBoundary: buildApplyBoundary(args, status, applyStep),
|
|
mutationAttempted: applyStep.mutationAttempted === true && args.apply,
|
|
target: {
|
|
environment: ENVIRONMENT_DEV,
|
|
namespace,
|
|
endpoint: DEV_ENDPOINT,
|
|
prodDisabled: true
|
|
},
|
|
artifactPlan,
|
|
workloadPlan,
|
|
servicePlan,
|
|
templateJobReplacementPolicy: devTemplateJobReplacementPolicy,
|
|
templateJobReplacements,
|
|
cloudWebRollout,
|
|
cloudWebRolloutBeforeApply,
|
|
cloudWebRolloutAfterApply,
|
|
applyStep,
|
|
manualCommands,
|
|
rollbackHint: buildRollbackHint(workloads),
|
|
remainingBlockers,
|
|
artifactEvidence,
|
|
cloudApiDb,
|
|
k8sManifest,
|
|
clusterObservation,
|
|
liveProbe
|
|
},
|
|
blockers,
|
|
notes: "DEV-only result. No PROD resources, secret values, force push, heavy e2e, browser e2e, or UniDesk runtime substitute were used."
|
|
};
|
|
|
|
if (args.writeReport) {
|
|
await writeFile(path.join(repoRoot, reportPath), `${JSON.stringify(report, null, 2)}\n`);
|
|
}
|
|
|
|
stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
return status === "blocked" && !args.expectBlocked ? 2 : 0;
|
|
}
|
|
|
|
export function formatFailure(error) {
|
|
return {
|
|
reportVersion: "v1",
|
|
issue: "pikasTech/HWLAB#33",
|
|
taskId: "dev-deploy-apply",
|
|
status: "failed",
|
|
blockers: [{ type: "observability_blocker", scope: "script", status: "open", summary: oneLine(error.message) }]
|
|
};
|
|
}
|