Codex
2796 lines
105 KiB
JavaScript
Executable File
2796 lines
105 KiB
JavaScript
Executable File
#!/usr/bin/env node
|
|
import { constants as fsConstants } from "node:fs";
|
|
import { access, mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import { request as httpRequest } from "node:http";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
import { spawn } from "node:child_process";
|
|
|
|
import {
|
|
DEV_DB_ENV_CONTRACT,
|
|
summarizeDbContract
|
|
} from "../../internal/cloud/db-contract.mjs";
|
|
import {
|
|
DEV_CODE_AGENT_PROVIDER_CONTRACT,
|
|
inspectCodeAgentProviderManifestRefs,
|
|
inspectCodeAgentProviderWorkloadEnv
|
|
} from "../../internal/cloud/code-agent-contract.mjs";
|
|
import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs";
|
|
import { DEV_ENDPOINT, ENVIRONMENT_DEV, SERVICE_IDS } from "../../internal/protocol/index.mjs";
|
|
import { requireDevCdTransactionForSideEffect } from "./dev-cd-transaction-guard.mjs";
|
|
import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs";
|
|
|
|
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
const defaultReportPath = tempReportPath("dev-deploy-report.json");
|
|
const namespace = "hwlab-dev";
|
|
const healthPath = "/health/live";
|
|
const defaultD601KubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
|
|
const devKubeconfigEnvName = "HWLAB_DEV_KUBECONFIG";
|
|
const kubeconfigEnvName = "KUBECONFIG";
|
|
const registryManifestBaseUrlEnvName = "HWLAB_DEV_REGISTRY_MANIFEST_BASE_URL";
|
|
const registryVerifyModeEnvName = "HWLAB_DEV_REGISTRY_VERIFY_MODE";
|
|
const registryHostNetworkHelperImageEnvName = "HWLAB_DEV_REGISTRY_HOSTNET_HELPER_IMAGE";
|
|
const defaultRegistryHostNetworkHelperImage = "unidesk-code-queue:d601";
|
|
const devTemplateJobReplacementPolicy = {
|
|
status: "active",
|
|
namespace,
|
|
scope: "DEV-only suspended template Jobs",
|
|
allowedJobs: [
|
|
{
|
|
name: "hwlab-agent-worker-template",
|
|
serviceId: "hwlab-agent-worker",
|
|
reason: "suspended session worker template Job has immutable spec.template"
|
|
},
|
|
{
|
|
name: "hwlab-cli-template",
|
|
serviceId: "hwlab-cli",
|
|
reason: "suspended CLI template Job has immutable spec.template"
|
|
}
|
|
]
|
|
};
|
|
const replaceableDevTemplateJobNames = new Set(devTemplateJobReplacementPolicy.allowedJobs.map((job) => job.name));
|
|
const devLegacySimulatorDeploymentCleanupPolicy = {
|
|
status: "active",
|
|
namespace,
|
|
scope: "DEV-only stale simulator Deployments replaced by indexed StatefulSets",
|
|
allowedDeployments: [
|
|
{
|
|
name: "hwlab-gateway-simu",
|
|
serviceId: "hwlab-gateway-simu",
|
|
replacementKind: "StatefulSet",
|
|
reason: "M3 DEV requires exactly two indexed gateway-simu identities/sessions"
|
|
},
|
|
{
|
|
name: "hwlab-box-simu",
|
|
serviceId: "hwlab-box-simu",
|
|
replacementKind: "StatefulSet",
|
|
reason: "M3 DEV requires exactly two indexed box-simu resource identities"
|
|
}
|
|
]
|
|
};
|
|
const cleanableLegacySimulatorDeploymentNames = new Set(
|
|
devLegacySimulatorDeploymentCleanupPolicy.allowedDeployments.map((deployment) => deployment.name)
|
|
);
|
|
const devUnmanagedHotfixDeploymentCleanupPolicy = {
|
|
status: "active",
|
|
namespace,
|
|
scope: "DEV-only unmanaged live Deployment hotfix/override fields",
|
|
reason:
|
|
"kubectl apply preserves fields owned by a live hotfix field manager; delete desired Deployments with unmanaged hotfix fields so apply recreates them from source truth",
|
|
matchPatterns: ["hotfix", "override"]
|
|
};
|
|
const requiredValidationCommands = [
|
|
"node --check scripts/validate-dev-gate-report.mjs",
|
|
"node scripts/validate-dev-gate-report.mjs",
|
|
"node --check scripts/dev-deploy-apply.mjs",
|
|
"node --check scripts/src/dev-deploy-apply.mjs",
|
|
"node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked"
|
|
];
|
|
const applyCommand = `node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report ${tempReportPath("dev-cd-apply.json")}`;
|
|
const dryRunPlanCommand = `node scripts/dev-deploy-apply.mjs --dry-run --report ${defaultReportPath}`;
|
|
const blockedDryRunCommand = `node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked --report ${defaultReportPath}`;
|
|
const forbiddenActions = [
|
|
"prod-deploy",
|
|
"secret-read",
|
|
"runtime-substitute",
|
|
"unidesk-restart",
|
|
"code-queue-restart",
|
|
"backend-core-restart",
|
|
"heavy-master-e2e",
|
|
"force-push"
|
|
];
|
|
const runtimeIdentityEnvNames = ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"];
|
|
const digestPattern = /^sha256:[a-f0-9]{64}$/u;
|
|
const defaultCdConcurrency = parseBoundedInteger(process.env.HWLAB_DEV_CD_CONCURRENCY, 4, 1, 8);
|
|
|
|
export function parseArgs(argv) {
|
|
const flags = new Set();
|
|
const errors = [];
|
|
let kubeconfig = null;
|
|
let kubeconfigSpecified = false;
|
|
let registryManifestBaseUrl = null;
|
|
let reportPath = defaultReportPath;
|
|
let rolloutConcurrency = defaultCdConcurrency;
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const arg = argv[index];
|
|
if (!arg.startsWith("--")) continue;
|
|
if (arg === "--kubeconfig") {
|
|
flags.add("--kubeconfig");
|
|
kubeconfigSpecified = true;
|
|
const next = argv[index + 1];
|
|
if (typeof next !== "string" || next.startsWith("--")) {
|
|
errors.push("--kubeconfig requires a non-empty path value");
|
|
} else {
|
|
kubeconfig = next;
|
|
index += 1;
|
|
}
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--kubeconfig=")) {
|
|
flags.add("--kubeconfig");
|
|
kubeconfigSpecified = true;
|
|
kubeconfig = arg.slice("--kubeconfig=".length);
|
|
if (!kubeconfig.trim()) {
|
|
errors.push("--kubeconfig requires a non-empty path value");
|
|
}
|
|
continue;
|
|
}
|
|
if (arg === "--report") {
|
|
flags.add("--report");
|
|
const next = argv[index + 1];
|
|
if (typeof next !== "string" || next.startsWith("--")) {
|
|
errors.push("--report requires a non-empty path value");
|
|
} else {
|
|
reportPath = next;
|
|
flags.add("--report-output");
|
|
index += 1;
|
|
}
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--report=")) {
|
|
flags.add("--report");
|
|
reportPath = arg.slice("--report=".length);
|
|
flags.add("--report-output");
|
|
if (!reportPath.trim()) {
|
|
errors.push("--report requires a non-empty path value");
|
|
}
|
|
continue;
|
|
}
|
|
if (arg === "--rollout-concurrency") {
|
|
flags.add("--rollout-concurrency");
|
|
const next = argv[index + 1];
|
|
if (typeof next !== "string" || next.startsWith("--")) {
|
|
errors.push("--rollout-concurrency requires a numeric value");
|
|
} else {
|
|
rolloutConcurrency = parseBoundedInteger(next, defaultCdConcurrency, 1, 8);
|
|
index += 1;
|
|
}
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--rollout-concurrency=")) {
|
|
flags.add("--rollout-concurrency");
|
|
rolloutConcurrency = parseBoundedInteger(arg.slice("--rollout-concurrency=".length), defaultCdConcurrency, 1, 8);
|
|
continue;
|
|
}
|
|
if (arg === "--registry-manifest-base-url") {
|
|
flags.add("--registry-manifest-base-url");
|
|
const next = argv[index + 1];
|
|
if (typeof next !== "string" || next.startsWith("--")) {
|
|
errors.push("--registry-manifest-base-url requires a non-empty URL value");
|
|
} else {
|
|
registryManifestBaseUrl = next;
|
|
index += 1;
|
|
}
|
|
continue;
|
|
}
|
|
if (arg.startsWith("--registry-manifest-base-url=")) {
|
|
flags.add("--registry-manifest-base-url");
|
|
registryManifestBaseUrl = arg.slice("--registry-manifest-base-url=".length);
|
|
if (!registryManifestBaseUrl.trim()) {
|
|
errors.push("--registry-manifest-base-url requires a non-empty URL value");
|
|
}
|
|
continue;
|
|
}
|
|
flags.add(arg);
|
|
}
|
|
|
|
return {
|
|
apply: flags.has("--apply"),
|
|
dryRun: !flags.has("--apply"),
|
|
expectBlocked: flags.has("--expect-blocked"),
|
|
skipLiveProbe: flags.has("--skip-live-probe"),
|
|
writeReport: flags.has("--report-output"),
|
|
reportPath,
|
|
rolloutConcurrency,
|
|
registryManifestBaseUrl,
|
|
kubeconfig,
|
|
kubeconfigSpecified,
|
|
errors,
|
|
flags
|
|
};
|
|
}
|
|
|
|
function parseBoundedInteger(value, fallback, min, max) {
|
|
const parsed = Number.parseInt(String(value ?? ""), 10);
|
|
if (!Number.isInteger(parsed)) return fallback;
|
|
return Math.min(Math.max(parsed, min), max);
|
|
}
|
|
|
|
function addBlocker(blockers, type, scope, summary) {
|
|
const key = `${type}::${scope}`;
|
|
if (!blockers.some((blocker) => `${blocker.type}::${blocker.scope}` === key)) {
|
|
blockers.push({ type, scope, status: "open", summary: oneLine(summary) });
|
|
}
|
|
}
|
|
|
|
function oneLine(value) {
|
|
return String(value).replace(/\s+/g, " ").trim();
|
|
}
|
|
|
|
async function mapWithConcurrency(items, concurrency, worker) {
|
|
if (items.length === 0) return [];
|
|
const results = new Array(items.length);
|
|
let nextIndex = 0;
|
|
const workerCount = Math.min(Math.max(concurrency, 1), items.length);
|
|
await Promise.all(Array.from({ length: workerCount }, async () => {
|
|
while (nextIndex < items.length) {
|
|
const index = nextIndex;
|
|
nextIndex += 1;
|
|
results[index] = await worker(items[index], index);
|
|
}
|
|
}));
|
|
return results;
|
|
}
|
|
|
|
async function readJson(relativePath, blockers) {
|
|
try {
|
|
return JSON.parse(await readFile(path.join(repoRoot, relativePath), "utf8"));
|
|
} catch (error) {
|
|
addBlocker(blockers, "contract_blocker", relativePath, `Cannot parse ${relativePath}: ${error.message}`);
|
|
return null;
|
|
}
|
|
}
|
|
|
|
async function readText(relativePath) {
|
|
return readFile(path.join(repoRoot, relativePath), "utf8");
|
|
}
|
|
|
|
async function commandResult(command, args, timeoutMs = 15000, options = {}) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn(command, args, {
|
|
cwd: repoRoot,
|
|
env: options.env ?? process.env,
|
|
stdio: ["pipe", "pipe", "pipe"]
|
|
});
|
|
const timer = setTimeout(() => child.kill("SIGTERM"), timeoutMs);
|
|
let stdout = "";
|
|
let stderr = "";
|
|
child.stdin.on("error", () => {});
|
|
child.stdin.end(options.input ?? "");
|
|
child.stdout.on("data", (chunk) => {
|
|
stdout += chunk;
|
|
});
|
|
child.stderr.on("data", (chunk) => {
|
|
stderr += chunk;
|
|
});
|
|
child.on("error", (error) => {
|
|
clearTimeout(timer);
|
|
resolve({ ok: false, code: null, stdout, stderr: error.message });
|
|
});
|
|
child.on("close", (code, signal) => {
|
|
clearTimeout(timer);
|
|
resolve({ ok: code === 0, code, signal, stdout: stdout.trim(), stderr: stderr.trim() });
|
|
});
|
|
});
|
|
}
|
|
|
|
async function findExecutable(name) {
|
|
for (const dir of (process.env.PATH || "").split(path.delimiter)) {
|
|
if (!dir) continue;
|
|
const candidate = path.join(dir, name);
|
|
try {
|
|
await access(candidate, fsConstants.X_OK);
|
|
return candidate;
|
|
} catch {}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
function isNonEmpty(value) {
|
|
return typeof value === "string" && value.trim().length > 0;
|
|
}
|
|
|
|
export function resolveDevKubeconfigSelection({ flagValue = null, env = process.env } = {}) {
|
|
const candidates = [
|
|
{ source: "flag:--kubeconfig", value: flagValue },
|
|
{ source: `env:${devKubeconfigEnvName}`, value: env?.[devKubeconfigEnvName] },
|
|
{ source: `env:${kubeconfigEnvName}`, value: env?.[kubeconfigEnvName] },
|
|
{ source: "default:d601-k3s", value: defaultD601KubeconfigPath }
|
|
];
|
|
const selected = candidates.find((candidate) => isNonEmpty(candidate.value));
|
|
const kubeconfig = selected.value.trim();
|
|
return {
|
|
kubeconfig,
|
|
source: selected.source,
|
|
paths: kubeconfig.split(path.delimiter).filter((item) => item.length > 0)
|
|
};
|
|
}
|
|
|
|
function shellQuote(value) {
|
|
if (/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)) return value;
|
|
return `'${String(value).replaceAll("'", "'\\''")}'`;
|
|
}
|
|
|
|
export function buildKubectlCommandPrefix(kubeconfig) {
|
|
return `KUBECONFIG=${shellQuote(kubeconfig)} kubectl`;
|
|
}
|
|
|
|
export function formatKubeconfigAccessFailure(selection, unreadablePath) {
|
|
return `DEV kubeconfig from ${selection.source} is not readable at ${unreadablePath}; set --kubeconfig, ${devKubeconfigEnvName}, or ${kubeconfigEnvName} to a readable DEV kubeconfig path. Kubeconfig contents and Secret values are never printed.`;
|
|
}
|
|
|
|
async function validateD601NativeKubeconfig(kubectlPath, env) {
|
|
const context = await commandResult(kubectlPath, ["config", "current-context"], 10_000, { env });
|
|
if (!context.ok) {
|
|
return `DEV kubeconfig could not report a kubectl context: ${redactSensitiveText(context.stderr || context.stdout)}`;
|
|
}
|
|
const contextName = context.stdout.trim();
|
|
if (/docker-desktop|desktop-control-plane/iu.test(contextName)) {
|
|
return `DEV kubeconfig resolved to forbidden context ${contextName}; D601 DEV must use native k3s at ${defaultD601KubeconfigPath}.`;
|
|
}
|
|
|
|
const server = await commandResult(kubectlPath, ["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], 10_000, { env });
|
|
if (server.ok && /127\.0\.0\.1:11700|docker-desktop/iu.test(server.stdout)) {
|
|
return "DEV kubeconfig resolved to Docker Desktop Kubernetes server 127.0.0.1:11700; use D601 native k3s kubeconfig instead.";
|
|
}
|
|
|
|
const nodes = await commandResult(kubectlPath, ["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], 15_000, { env });
|
|
if (!nodes.ok) {
|
|
return `DEV kubeconfig could not read native k3s nodes: ${redactSensitiveText(nodes.stderr || nodes.stdout)}`;
|
|
}
|
|
const nodeNames = nodes.stdout.trim().split(/\s+/u).filter(Boolean);
|
|
if (!nodeNames.includes("d601")) {
|
|
return `DEV kubeconfig did not resolve to D601 native k3s; observed nodes=${nodeNames.join(",") || "none"}.`;
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export function redactSensitiveText(value) {
|
|
return String(value)
|
|
.replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]+/giu, "$1<redacted>")
|
|
.replace(/\b(postgres(?:ql)?:\/\/[^:\s/@]+:)([^@\s]+)(@)/giu, "$1<redacted>$3")
|
|
.replace(/\b(token|password|client-key-data|client-certificate-data|certificate-authority-data)\b(\s*[:=]\s*)(["']?)([^"'\s,}]+)/giu, "$1$2$3<redacted>");
|
|
}
|
|
|
|
function redactCommandResult(result) {
|
|
return {
|
|
...result,
|
|
redactedStdout: redactSensitiveText(result.stdout ?? ""),
|
|
redactedStderr: redactSensitiveText(result.stderr ?? "")
|
|
};
|
|
}
|
|
|
|
function commandOutput(result) {
|
|
return result.redactedStderr || result.redactedStdout || result.stderr || result.stdout || "";
|
|
}
|
|
|
|
async function resolveD601Kubectl(args = {}, env = process.env) {
|
|
const kubeconfigSelection = resolveDevKubeconfigSelection({ flagValue: args.kubeconfig, env });
|
|
const kubectlPath = await findExecutable("kubectl");
|
|
const commandPrefix = buildKubectlCommandPrefix(kubeconfigSelection.kubeconfig);
|
|
const base = {
|
|
executor: kubectlPath,
|
|
kubeconfig: kubeconfigSelection.kubeconfig,
|
|
kubeconfigSource: kubeconfigSelection.source,
|
|
kubeconfigPaths: kubeconfigSelection.paths,
|
|
commandPrefix,
|
|
env: {
|
|
...env,
|
|
KUBECONFIG: kubeconfigSelection.kubeconfig
|
|
}
|
|
};
|
|
|
|
if (!kubectlPath) {
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
reason: "kubectl is not installed in the runner"
|
|
};
|
|
}
|
|
|
|
try {
|
|
let readablePathCount = 0;
|
|
let firstUnreadablePath = null;
|
|
for (const kubeconfigPath of kubeconfigSelection.paths) {
|
|
try {
|
|
await access(kubeconfigPath, fsConstants.R_OK);
|
|
readablePathCount += 1;
|
|
} catch (error) {
|
|
firstUnreadablePath ??= error?.path ?? kubeconfigPath;
|
|
}
|
|
}
|
|
if (readablePathCount === 0) {
|
|
throw Object.assign(new Error("no readable kubeconfig paths"), {
|
|
path: firstUnreadablePath ?? kubeconfigSelection.kubeconfig
|
|
});
|
|
}
|
|
} catch (error) {
|
|
const unreadablePath = error?.path ?? kubeconfigSelection.kubeconfig;
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
reason: formatKubeconfigAccessFailure(kubeconfigSelection, unreadablePath)
|
|
};
|
|
}
|
|
|
|
const nativeK3sFailure = await validateD601NativeKubeconfig(kubectlPath, base.env);
|
|
if (nativeK3sFailure !== null) {
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
reason: nativeK3sFailure
|
|
};
|
|
}
|
|
|
|
return {
|
|
...base,
|
|
status: "ready",
|
|
reason: "D601 k3s kubeconfig is readable"
|
|
};
|
|
}
|
|
|
|
function kubectlCommand(kubectl, args) {
|
|
return `${kubectl.commandPrefix} ${args.join(" ")}`;
|
|
}
|
|
|
|
async function kubectlResult(kubectl, args, timeoutMs = 15000, options = {}) {
|
|
if (kubectl.status !== "ready") {
|
|
return {
|
|
ok: false,
|
|
code: null,
|
|
stdout: "",
|
|
stderr: kubectl.reason,
|
|
blocked: true
|
|
};
|
|
}
|
|
const result = await commandResult(kubectl.executor, args, timeoutMs, {
|
|
...options,
|
|
env: kubectl.env
|
|
});
|
|
return redactCommandResult(result);
|
|
}
|
|
|
|
function listItems(document) {
|
|
if (!document) return [];
|
|
return document.kind === "List" ? document.items ?? [] : [document];
|
|
}
|
|
|
|
function containersFor(item) {
|
|
return item?.spec?.template?.spec?.containers ?? [];
|
|
}
|
|
|
|
function serviceIdFor(item, container) {
|
|
return (
|
|
item?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
|
|
item?.spec?.template?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ||
|
|
container?.name
|
|
);
|
|
}
|
|
|
|
function itemNamespace(item) {
|
|
return item?.metadata?.namespace ?? (item?.kind === "Namespace" ? item?.metadata?.name : namespace);
|
|
}
|
|
|
|
function uniqueStrings(values) {
|
|
return [...new Set(values.filter((value) => typeof value === "string" && value.length > 0))];
|
|
}
|
|
|
|
function imageRecordsFor(item) {
|
|
return containersFor(item).map((container) => ({
|
|
name: container.name,
|
|
serviceId: serviceIdFor(item, container),
|
|
image: container.image
|
|
}));
|
|
}
|
|
|
|
function primaryImage(images) {
|
|
if (!images.length) return null;
|
|
if (images.length === 1) return images[0].image ?? null;
|
|
return images.map((image) => `${image.name}=${image.image}`).join(", ");
|
|
}
|
|
|
|
function imagesDiffer(oldImages, newImages) {
|
|
if (oldImages.length !== newImages.length) return true;
|
|
const oldByName = new Map(oldImages.map((image) => [image.name, image.image]));
|
|
return newImages.some((image) => oldByName.get(image.name) !== image.image);
|
|
}
|
|
|
|
function isAllowedDesiredTemplateJob(item) {
|
|
return (
|
|
item?.kind === "Job" &&
|
|
itemNamespace(item) === namespace &&
|
|
item?.spec?.suspend === true &&
|
|
replaceableDevTemplateJobNames.has(item?.metadata?.name)
|
|
);
|
|
}
|
|
|
|
function isDesiredLegacySimulatorReplacement(item) {
|
|
return (
|
|
item?.kind === "StatefulSet" &&
|
|
itemNamespace(item) === namespace &&
|
|
cleanableLegacySimulatorDeploymentNames.has(item?.metadata?.name)
|
|
);
|
|
}
|
|
|
|
function deploymentKey(item) {
|
|
return `${itemNamespace(item)}/${item?.metadata?.name ?? "unknown"}`;
|
|
}
|
|
|
|
function isDesiredDeployment(item) {
|
|
return item?.kind === "Deployment" && itemNamespace(item) === namespace && typeof item?.metadata?.name === "string";
|
|
}
|
|
|
|
function objectTextContainsHotfixMarker(value) {
|
|
if (value === null || value === undefined) return false;
|
|
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
|
|
return /\bhotfix\b|override/iu.test(String(value));
|
|
}
|
|
if (Array.isArray(value)) return value.some((entry) => objectTextContainsHotfixMarker(entry));
|
|
if (typeof value === "object") {
|
|
return Object.entries(value).some(([key, entry]) =>
|
|
objectTextContainsHotfixMarker(key) || objectTextContainsHotfixMarker(entry)
|
|
);
|
|
}
|
|
return false;
|
|
}
|
|
|
|
function volumeIdentity(volume) {
|
|
const parts = [volume?.name ?? ""];
|
|
if (volume?.configMap?.name) parts.push(`configMap:${volume.configMap.name}`);
|
|
if (volume?.secret?.secretName) parts.push(`secret:${volume.secret.secretName}`);
|
|
if (volume?.persistentVolumeClaim?.claimName) parts.push(`pvc:${volume.persistentVolumeClaim.claimName}`);
|
|
return parts.join("|");
|
|
}
|
|
|
|
function volumeMountIdentity(containerName, mount) {
|
|
return [
|
|
containerName ?? "",
|
|
mount?.name ?? "",
|
|
mount?.mountPath ?? "",
|
|
mount?.subPath ?? "",
|
|
mount?.readOnly === true ? "ro" : "rw"
|
|
].join("|");
|
|
}
|
|
|
|
function templateAnnotationIdentity(key, value) {
|
|
return `${key}=${String(value ?? "")}`;
|
|
}
|
|
|
|
function desiredDeploymentIndex(workloads) {
|
|
return new Map(
|
|
listItems(workloads)
|
|
.filter(isDesiredDeployment)
|
|
.map((item) => [deploymentKey(item), item])
|
|
);
|
|
}
|
|
|
|
function deploymentVolumes(item) {
|
|
return item?.spec?.template?.spec?.volumes ?? [];
|
|
}
|
|
|
|
function deploymentContainerMounts(item) {
|
|
return (item?.spec?.template?.spec?.containers ?? []).flatMap((container) =>
|
|
(container.volumeMounts ?? []).map((mount) => ({
|
|
containerName: container.name,
|
|
mount
|
|
}))
|
|
);
|
|
}
|
|
|
|
function deploymentTemplateAnnotations(item) {
|
|
return item?.spec?.template?.metadata?.annotations ?? {};
|
|
}
|
|
|
|
export function decideDevTemplateJobReplacement({
|
|
jobName,
|
|
jobNamespace = namespace,
|
|
desiredSuspended,
|
|
liveExists,
|
|
liveSuspended,
|
|
oldImages = [],
|
|
newImages = []
|
|
}) {
|
|
const base = {
|
|
namespace: jobNamespace,
|
|
jobName,
|
|
allowed: replaceableDevTemplateJobNames.has(jobName),
|
|
desiredSuspended: desiredSuspended === true,
|
|
liveExists: liveExists === true,
|
|
liveSuspended: liveExists === true ? liveSuspended === true : null,
|
|
oldImage: primaryImage(oldImages),
|
|
newImage: primaryImage(newImages),
|
|
oldImages,
|
|
newImages,
|
|
replace: false,
|
|
result: "not_needed",
|
|
reason: "Live suspended template Job already matches the desired image."
|
|
};
|
|
|
|
if (!base.allowed) {
|
|
return {
|
|
...base,
|
|
result: "ignored",
|
|
reason: "Job is not in the DEV template Job replacement allowlist."
|
|
};
|
|
}
|
|
if (jobNamespace !== namespace) {
|
|
return {
|
|
...base,
|
|
result: "blocked",
|
|
reason: `Replacement policy is DEV-only and refuses namespace ${jobNamespace}.`
|
|
};
|
|
}
|
|
if (desiredSuspended !== true) {
|
|
return {
|
|
...base,
|
|
result: "blocked",
|
|
reason: "Desired Job is not suspended, so it is not a template Job replacement candidate."
|
|
};
|
|
}
|
|
if (liveExists !== true) {
|
|
return {
|
|
...base,
|
|
result: "not_found",
|
|
reason: "Live Job is absent; kubectl apply can create it without replacement."
|
|
};
|
|
}
|
|
if (liveSuspended !== true) {
|
|
return {
|
|
...base,
|
|
result: "blocked",
|
|
reason: "Live Job is not suspended; refusing to delete a potentially active Job."
|
|
};
|
|
}
|
|
if (imagesDiffer(oldImages, newImages)) {
|
|
return {
|
|
...base,
|
|
replace: true,
|
|
result: "planned",
|
|
reason: "Live suspended template Job image differs from the desired manifest; delete/recreate is required because spec.template is immutable."
|
|
};
|
|
}
|
|
return base;
|
|
}
|
|
|
|
export function decideDevLegacySimulatorDeploymentCleanup({
|
|
deploymentName,
|
|
deploymentNamespace = namespace,
|
|
desiredReplacementExists,
|
|
desiredReplacementKind,
|
|
liveExists,
|
|
oldImages = [],
|
|
newImages = []
|
|
}) {
|
|
const policy = devLegacySimulatorDeploymentCleanupPolicy.allowedDeployments.find(
|
|
(deployment) => deployment.name === deploymentName
|
|
);
|
|
const base = {
|
|
namespace: deploymentNamespace,
|
|
deploymentName,
|
|
serviceId: policy?.serviceId ?? deploymentName,
|
|
allowed: Boolean(policy),
|
|
desiredReplacementExists: desiredReplacementExists === true,
|
|
desiredReplacementKind: desiredReplacementKind ?? null,
|
|
liveExists: liveExists === true,
|
|
oldImage: primaryImage(oldImages),
|
|
newImage: primaryImage(newImages),
|
|
oldImages,
|
|
newImages,
|
|
cleanup: false,
|
|
result: "not_needed",
|
|
reason: "No stale legacy simulator Deployment exists."
|
|
};
|
|
|
|
if (!policy) {
|
|
return {
|
|
...base,
|
|
result: "ignored",
|
|
reason: "Deployment is not in the DEV stale simulator cleanup allowlist."
|
|
};
|
|
}
|
|
if (deploymentNamespace !== namespace) {
|
|
return {
|
|
...base,
|
|
result: "blocked",
|
|
reason: `Cleanup policy is DEV-only and refuses namespace ${deploymentNamespace}.`
|
|
};
|
|
}
|
|
if (desiredReplacementExists !== true || desiredReplacementKind !== policy.replacementKind) {
|
|
return {
|
|
...base,
|
|
result: "blocked",
|
|
reason: `Refusing to clean up ${deploymentName} without desired ${policy.replacementKind} replacement in source manifests.`
|
|
};
|
|
}
|
|
if (liveExists !== true) {
|
|
return {
|
|
...base,
|
|
result: "not_found",
|
|
reason: "Live legacy Deployment is absent; no cleanup is needed."
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
cleanup: true,
|
|
result: "planned",
|
|
reason: `Live legacy Deployment ${deploymentName} duplicates the desired indexed StatefulSet and must be removed for exact M3 cardinality.`
|
|
};
|
|
}
|
|
|
|
export function decideDevUnmanagedHotfixDeploymentCleanup({
|
|
deploymentName,
|
|
deploymentNamespace = namespace,
|
|
desiredDeployment,
|
|
liveDeployment
|
|
}) {
|
|
const base = {
|
|
namespace: deploymentNamespace,
|
|
deploymentName,
|
|
desiredExists: Boolean(desiredDeployment),
|
|
liveExists: Boolean(liveDeployment),
|
|
cleanup: false,
|
|
result: "not_needed",
|
|
reason: "Live Deployment has no unmanaged hotfix or override fields outside desired source.",
|
|
unmanagedVolumes: [],
|
|
unmanagedVolumeMounts: [],
|
|
unmanagedTemplateAnnotations: []
|
|
};
|
|
|
|
if (deploymentNamespace !== namespace) {
|
|
return {
|
|
...base,
|
|
result: "blocked",
|
|
reason: `Unmanaged hotfix cleanup is DEV-only and refuses namespace ${deploymentNamespace}.`
|
|
};
|
|
}
|
|
if (!desiredDeployment) {
|
|
return {
|
|
...base,
|
|
result: "ignored",
|
|
reason: "No desired Deployment exists in source; cleanup must not delete unknown live Deployments."
|
|
};
|
|
}
|
|
if (!liveDeployment) {
|
|
return {
|
|
...base,
|
|
result: "not_found",
|
|
reason: "Live Deployment is absent; kubectl apply can create it from desired source."
|
|
};
|
|
}
|
|
|
|
const desiredVolumeIdentities = new Set(deploymentVolumes(desiredDeployment).map(volumeIdentity));
|
|
const desiredMountIdentities = new Set(
|
|
deploymentContainerMounts(desiredDeployment).map(({ containerName, mount }) => volumeMountIdentity(containerName, mount))
|
|
);
|
|
const desiredAnnotationIdentities = new Set(
|
|
Object.entries(deploymentTemplateAnnotations(desiredDeployment)).map(([key, value]) =>
|
|
templateAnnotationIdentity(key, value)
|
|
)
|
|
);
|
|
const unmanagedVolumes = deploymentVolumes(liveDeployment)
|
|
.filter((volume) => !desiredVolumeIdentities.has(volumeIdentity(volume)) && objectTextContainsHotfixMarker(volume))
|
|
.map((volume) => ({
|
|
name: volume.name ?? null,
|
|
configMap: volume.configMap?.name ?? null,
|
|
secret: volume.secret?.secretName ?? null,
|
|
identity: volumeIdentity(volume)
|
|
}));
|
|
const unmanagedVolumeMounts = deploymentContainerMounts(liveDeployment)
|
|
.filter(({ containerName, mount }) =>
|
|
!desiredMountIdentities.has(volumeMountIdentity(containerName, mount)) &&
|
|
objectTextContainsHotfixMarker({ containerName, mount })
|
|
)
|
|
.map(({ containerName, mount }) => ({
|
|
containerName,
|
|
name: mount.name ?? null,
|
|
mountPath: mount.mountPath ?? null,
|
|
subPath: mount.subPath ?? null,
|
|
identity: volumeMountIdentity(containerName, mount)
|
|
}));
|
|
const unmanagedTemplateAnnotations = Object.entries(deploymentTemplateAnnotations(liveDeployment))
|
|
.filter(([key, value]) =>
|
|
!desiredAnnotationIdentities.has(templateAnnotationIdentity(key, value)) &&
|
|
objectTextContainsHotfixMarker({ key, value })
|
|
)
|
|
.map(([key, value]) => ({ key, value: String(value ?? "") }));
|
|
|
|
if (unmanagedVolumes.length === 0 && unmanagedVolumeMounts.length === 0 && unmanagedTemplateAnnotations.length === 0) {
|
|
return base;
|
|
}
|
|
|
|
return {
|
|
...base,
|
|
cleanup: true,
|
|
result: "planned",
|
|
reason:
|
|
"Live Deployment contains unmanaged hotfix/override fields that kubectl apply may preserve; delete before apply so desired source recreates it exactly.",
|
|
unmanagedVolumes,
|
|
unmanagedVolumeMounts,
|
|
unmanagedTemplateAnnotations
|
|
};
|
|
}
|
|
|
|
function replicaPlan(item) {
|
|
if (item?.kind === "Job") {
|
|
return item?.spec?.suspend === true ? 0 : 1;
|
|
}
|
|
return item?.spec?.replicas ?? null;
|
|
}
|
|
|
|
function buildWorkloadPlan(workloads) {
|
|
return listItems(workloads).map((item) => {
|
|
const containers = containersFor(item).map((container) => ({
|
|
name: container.name,
|
|
serviceId: serviceIdFor(item, container),
|
|
image: container.image
|
|
}));
|
|
return {
|
|
kind: item?.kind ?? "unknown",
|
|
name: item?.metadata?.name ?? "unknown",
|
|
namespace: item?.metadata?.namespace ?? namespace,
|
|
replicas: replicaPlan(item),
|
|
serviceIds: uniqueStrings(containers.map((container) => container.serviceId)),
|
|
containers
|
|
};
|
|
});
|
|
}
|
|
|
|
function buildServicePlan(services) {
|
|
return listItems(services).map((item) => ({
|
|
kind: item?.kind ?? "unknown",
|
|
name: item?.metadata?.name ?? "unknown",
|
|
namespace: item?.metadata?.namespace ?? namespace,
|
|
serviceId: item?.metadata?.labels?.["hwlab.pikastech.local/service-id"] ?? item?.metadata?.name ?? "unknown",
|
|
type: item?.spec?.type ?? "unknown",
|
|
ports: (item?.spec?.ports ?? []).map((port) => ({
|
|
name: port.name,
|
|
port: port.port,
|
|
targetPort: port.targetPort
|
|
}))
|
|
}));
|
|
}
|
|
|
|
function buildArtifactPlan(deploy, catalog, sourceCommitId, artifactEvidence) {
|
|
const requiredServices = (catalog?.services ?? []).filter((service) => service.artifactRequired !== false);
|
|
const serviceCommitIds = uniqueStrings([
|
|
...(catalog?.services ?? []).map((service) => service.commitId),
|
|
...(artifactEvidence ?? []).map((service) => service.catalogCommitId)
|
|
]);
|
|
const expectedArtifactCommit = deploy?.commitId ?? catalog?.commitId ?? "unknown";
|
|
return {
|
|
expectedArtifactCommit,
|
|
deployCommitId: deploy?.commitId ?? "unknown",
|
|
catalogCommitId: catalog?.commitId ?? "unknown",
|
|
sourceCommitId,
|
|
serviceCommitIds,
|
|
matchesSourceCommit:
|
|
deploy?.commitId && catalog?.commitId
|
|
? commitMatchesSource(deploy.commitId, sourceCommitId) && commitMatchesSource(catalog.commitId, sourceCommitId)
|
|
: false,
|
|
published: catalog?.publish?.ciPublished === true,
|
|
registryVerified: catalog?.publish?.registryVerified === true,
|
|
imageCount: artifactEvidence.length,
|
|
requiredServiceCount: requiredServices.length,
|
|
disabledServiceCount: (catalog?.services ?? []).filter((service) => service.artifactRequired === false).length,
|
|
unpublishedServices: requiredServices
|
|
.filter((service) => service.publishState !== "published" || !service.digest?.startsWith("sha256:"))
|
|
.map((service) => service.serviceId)
|
|
};
|
|
}
|
|
|
|
function artifactIdentityForService(deploy, catalog, serviceId) {
|
|
const deployService = deploy?.services?.find((service) => service.serviceId === serviceId) ?? null;
|
|
const catalogService = catalog?.services?.find((service) => service.serviceId === serviceId) ?? null;
|
|
return {
|
|
serviceId,
|
|
sourceCommitId: catalogService?.commitId ?? deploy?.commitId ?? "unknown",
|
|
image: deployService?.image ?? catalogService?.image ?? null,
|
|
imageTag: catalogService?.imageTag ?? parseImageTag(deployService?.image),
|
|
digest: catalogService?.digest ?? "not_published",
|
|
repositoryDigest: catalogService?.repositoryDigest ?? null,
|
|
publishState: catalogService?.publishState ?? "unknown"
|
|
};
|
|
}
|
|
|
|
function parseImageTag(image) {
|
|
if (typeof image !== "string") return null;
|
|
const slashIndex = image.lastIndexOf("/");
|
|
const colonIndex = image.lastIndexOf(":");
|
|
if (colonIndex <= slashIndex) return null;
|
|
return image.slice(colonIndex + 1);
|
|
}
|
|
|
|
function parseTaggedImageReference(image) {
|
|
if (typeof image !== "string" || image.includes("@")) return null;
|
|
const firstSlash = image.indexOf("/");
|
|
const lastSlash = image.lastIndexOf("/");
|
|
const colonIndex = image.lastIndexOf(":");
|
|
if (firstSlash <= 0 || colonIndex <= lastSlash) return null;
|
|
return {
|
|
image,
|
|
registry: image.slice(0, firstSlash),
|
|
repository: image.slice(firstSlash + 1, colonIndex),
|
|
tag: image.slice(colonIndex + 1)
|
|
};
|
|
}
|
|
|
|
function registryManifestBaseUrlForParsed(parsed, options = {}) {
|
|
const env = options.env ?? process.env;
|
|
const explicitBaseUrl = options.registryManifestBaseUrl ?? env?.[registryManifestBaseUrlEnvName] ?? null;
|
|
if (typeof explicitBaseUrl === "string" && explicitBaseUrl.trim().length > 0) {
|
|
return {
|
|
baseUrl: explicitBaseUrl.trim().replace(/\/+$/u, ""),
|
|
source: options.registryManifestBaseUrl ? "flag:--registry-manifest-base-url" : `env:${registryManifestBaseUrlEnvName}`
|
|
};
|
|
}
|
|
return {
|
|
baseUrl: `http://${parsed.registry}`,
|
|
source: "image-registry"
|
|
};
|
|
}
|
|
|
|
export function registryManifestUrlForImage(image, options = {}) {
|
|
const parsed = parseTaggedImageReference(image);
|
|
if (!parsed) return null;
|
|
const repositoryPath = parsed.repository.split("/").map((part) => encodeURIComponent(part)).join("/");
|
|
const manifestBase = registryManifestBaseUrlForParsed(parsed, options);
|
|
return {
|
|
...parsed,
|
|
url: `${manifestBase.baseUrl}/v2/${repositoryPath}/manifests/${encodeURIComponent(parsed.tag)}`,
|
|
registryAccess: {
|
|
baseUrl: manifestBase.baseUrl,
|
|
source: manifestBase.source
|
|
}
|
|
};
|
|
}
|
|
|
|
function registryAcceptHeader() {
|
|
return [
|
|
"application/vnd.docker.distribution.manifest.v2+json",
|
|
"application/vnd.oci.image.manifest.v1+json",
|
|
"application/vnd.docker.distribution.manifest.list.v2+json",
|
|
"application/vnd.oci.image.index.v1+json",
|
|
"application/vnd.docker.distribution.manifest.v1+json"
|
|
].join(", ");
|
|
}
|
|
|
|
function httpRequestText(url, { method = "GET", headers = {}, timeoutMs = 15000 } = {}) {
|
|
return new Promise((resolve, reject) => {
|
|
const request = httpRequest(url, { method, headers, timeout: timeoutMs }, (response) => {
|
|
response.setEncoding("utf8");
|
|
let body = "";
|
|
response.on("data", (chunk) => {
|
|
body += chunk;
|
|
if (body.length > 4096) {
|
|
request.destroy(new Error("registry manifest response exceeded 4096 bytes during CD verification"));
|
|
}
|
|
});
|
|
response.on("end", () => resolve({
|
|
statusCode: response.statusCode ?? 0,
|
|
headers: response.headers,
|
|
body
|
|
}));
|
|
});
|
|
request.on("timeout", () => {
|
|
request.destroy(new Error(`timeout after ${timeoutMs}ms`));
|
|
});
|
|
request.on("error", reject);
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
function registryVerifyMode(options = {}) {
|
|
const env = options.env ?? process.env;
|
|
const value = String(options.registryVerifyMode ?? env?.[registryVerifyModeEnvName] ?? "auto").trim();
|
|
if (["auto", "direct", "docker-host-network"].includes(value)) return value;
|
|
return "auto";
|
|
}
|
|
|
|
function dockerHostNetworkHelperImage(options = {}) {
|
|
const env = options.env ?? process.env;
|
|
return String(
|
|
options.registryHostNetworkHelperImage ??
|
|
env?.[registryHostNetworkHelperImageEnvName] ??
|
|
defaultRegistryHostNetworkHelperImage
|
|
).trim();
|
|
}
|
|
|
|
async function readRegistryManifestViaDockerHostNetwork(parsed, options = {}) {
|
|
const runCommand = options.runCommand ?? commandResult;
|
|
const helperImage = dockerHostNetworkHelperImage(options);
|
|
const helperScript = `
|
|
const http = require("node:http");
|
|
const url = process.argv[1];
|
|
const accept = process.argv[2];
|
|
const req = http.request(url, {
|
|
method: "HEAD",
|
|
headers: { Accept: accept },
|
|
timeout: 15000
|
|
}, (response) => {
|
|
const digest = response.headers["docker-content-digest"] || null;
|
|
response.resume();
|
|
response.on("end", () => {
|
|
console.log(JSON.stringify({ statusCode: response.statusCode || 0, headers: { "docker-content-digest": digest } }));
|
|
});
|
|
});
|
|
req.on("timeout", () => req.destroy(new Error("timeout after 15000ms")));
|
|
req.on("error", (error) => {
|
|
console.error(error.message);
|
|
process.exitCode = 1;
|
|
});
|
|
req.end();
|
|
`.trim();
|
|
const args = [
|
|
"run",
|
|
"--rm",
|
|
"--pull=never",
|
|
"--network",
|
|
"host",
|
|
helperImage,
|
|
"node",
|
|
"-e",
|
|
helperScript,
|
|
parsed.url,
|
|
registryAcceptHeader()
|
|
];
|
|
const result = await runCommand("docker", args, 60000);
|
|
if (!result.ok) {
|
|
throw new Error(
|
|
`docker host-network registry manifest read failed with ${result.code ?? "unknown"}: ${oneLine(result.stderr || result.stdout || "no output")}`
|
|
);
|
|
}
|
|
const parsedOutput = JSON.parse(result.stdout || "{}");
|
|
return {
|
|
statusCode: Number.parseInt(String(parsedOutput.statusCode ?? 0), 10),
|
|
headers: parsedOutput.headers ?? {},
|
|
body: "",
|
|
registryAccess: {
|
|
mode: "docker-host-network",
|
|
helperImage,
|
|
directUrl: parsed.url,
|
|
directAccessSource: parsed.registryAccess?.source ?? null
|
|
}
|
|
};
|
|
}
|
|
|
|
async function readRegistryManifest(parsed, options = {}) {
|
|
const mode = registryVerifyMode(options);
|
|
const httpReader = options.httpRequestText ?? httpRequestText;
|
|
if (mode === "docker-host-network") {
|
|
return readRegistryManifestViaDockerHostNetwork(parsed, options);
|
|
}
|
|
try {
|
|
const response = await httpReader(parsed.url, {
|
|
method: "HEAD",
|
|
headers: {
|
|
Accept: registryAcceptHeader()
|
|
},
|
|
timeoutMs: 15000
|
|
});
|
|
return {
|
|
...response,
|
|
registryAccess: {
|
|
mode: "direct",
|
|
url: parsed.url,
|
|
source: parsed.registryAccess?.source ?? null
|
|
}
|
|
};
|
|
} catch (error) {
|
|
if (mode === "direct") throw error;
|
|
const fallback = await readRegistryManifestViaDockerHostNetwork(parsed, options);
|
|
return {
|
|
...fallback,
|
|
registryAccess: {
|
|
...fallback.registryAccess,
|
|
directError: oneLine(error.message)
|
|
}
|
|
};
|
|
}
|
|
}
|
|
|
|
export async function verifyCatalogRegistryManifest(service, blockers, options = {}) {
|
|
const image = service.image;
|
|
const expectedDigest = service.digest;
|
|
const parsed = registryManifestUrlForImage(image, options);
|
|
const base = {
|
|
serviceId: service.serviceId,
|
|
image,
|
|
imageTag: service.imageTag ?? parseImageTag(image),
|
|
expectedDigest,
|
|
url: parsed?.url ?? null,
|
|
registryAccess: parsed?.registryAccess ?? null
|
|
};
|
|
|
|
if (!parsed) {
|
|
const reason = `${service.serviceId} image is not a tagged registry image`;
|
|
addBlocker(blockers, "contract_blocker", `registry-manifest-${service.serviceId}`, reason);
|
|
return { ...base, status: "blocked", reason };
|
|
}
|
|
if (!digestPattern.test(expectedDigest ?? "")) {
|
|
const reason = `${service.serviceId} catalog digest is not an immutable sha256 digest`;
|
|
addBlocker(blockers, "contract_blocker", `registry-manifest-${service.serviceId}`, reason);
|
|
return { ...base, status: "blocked", reason };
|
|
}
|
|
|
|
try {
|
|
const response = await readRegistryManifest(parsed, options);
|
|
const observedDigest = String(response.headers["docker-content-digest"] ?? "").trim();
|
|
const okStatus = response.statusCode >= 200 && response.statusCode < 300;
|
|
const digestMatches = observedDigest === expectedDigest;
|
|
if (!okStatus || !digestMatches) {
|
|
const reason = !okStatus
|
|
? `${service.serviceId} registry manifest returned HTTP ${response.statusCode}`
|
|
: `${service.serviceId} registry digest mismatch: expected ${expectedDigest}, observed ${observedDigest || "missing"}`;
|
|
addBlocker(blockers, "environment_blocker", `registry-manifest-${service.serviceId}`, reason);
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
reason,
|
|
httpStatus: response.statusCode,
|
|
observedDigest,
|
|
digestMatches,
|
|
registryAccess: response.registryAccess ?? base.registryAccess
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
status: "pass",
|
|
httpStatus: response.statusCode,
|
|
observedDigest,
|
|
digestMatches: true,
|
|
registryAccess: response.registryAccess ?? base.registryAccess
|
|
};
|
|
} catch (error) {
|
|
const reason = `${service.serviceId} registry manifest read failed: ${oneLine(error.message)}`;
|
|
addBlocker(blockers, "environment_blocker", `registry-manifest-${service.serviceId}`, reason);
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
reason
|
|
};
|
|
}
|
|
}
|
|
|
|
async function verifyCatalogRegistryManifests(catalog, blockers, concurrency = defaultCdConcurrency, options = {}) {
|
|
const services = (catalog?.services ?? []).filter((service) => service.artifactRequired !== false);
|
|
const results = await mapWithConcurrency(services, concurrency, async (service) =>
|
|
verifyCatalogRegistryManifest(service, blockers, options)
|
|
);
|
|
return {
|
|
status: results.every((result) => result.status === "pass") ? "pass" : "blocked",
|
|
source: "registry-manifest",
|
|
releaseGate: true,
|
|
serviceCount: results.length,
|
|
concurrency,
|
|
verifyMode: registryVerifyMode(options),
|
|
results
|
|
};
|
|
}
|
|
|
|
function expectedRuntimeIdentityEnv(artifact) {
|
|
return {
|
|
HWLAB_COMMIT_ID: artifact.sourceCommitId,
|
|
HWLAB_IMAGE: artifact.image,
|
|
HWLAB_IMAGE_TAG: artifact.imageTag
|
|
};
|
|
}
|
|
|
|
function envObjectFromEnvList(envList) {
|
|
const env = {};
|
|
if (!Array.isArray(envList)) return env;
|
|
for (const entry of envList) {
|
|
if (!entry?.name) continue;
|
|
env[entry.name] = Object.hasOwn(entry, "value") ? entry.value : null;
|
|
}
|
|
return env;
|
|
}
|
|
|
|
function runtimeIdentityValueMatches(name, actual, expected) {
|
|
if (typeof actual !== "string" || typeof expected !== "string") return false;
|
|
if (name === "HWLAB_COMMIT_ID") return commitMatchesSource(actual, expected);
|
|
return actual === expected;
|
|
}
|
|
|
|
export function compareRuntimeIdentityEnv(env, expected) {
|
|
const fields = runtimeIdentityEnvNames.map((name) => {
|
|
const actual = Object.hasOwn(env ?? {}, name) ? env[name] : null;
|
|
const expectedValue = expected?.[name] ?? null;
|
|
const status = runtimeIdentityValueMatches(name, actual, expectedValue) ? "match" : "drift";
|
|
return {
|
|
name,
|
|
expected: expectedValue,
|
|
actual,
|
|
status
|
|
};
|
|
});
|
|
return {
|
|
status: fields.every((field) => field.status === "match") ? "match" : "drift",
|
|
matchesDesired: fields.every((field) => field.status === "match"),
|
|
fields,
|
|
drift: fields.filter((field) => field.status === "drift")
|
|
};
|
|
}
|
|
|
|
function formatRuntimeIdentityDrift(drift) {
|
|
return drift.map((field) => `${field.name} expected ${field.expected ?? "missing"} actual ${field.actual ?? "missing"}`).join("; ");
|
|
}
|
|
|
|
function deploymentRevision(deployment) {
|
|
return deployment?.metadata?.annotations?.["deployment.kubernetes.io/revision"] ?? null;
|
|
}
|
|
|
|
function containerImageForDeployment(deployment, containerName) {
|
|
const containers = deployment?.spec?.template?.spec?.containers ?? [];
|
|
return containers.find((container) => container.name === containerName)?.image ?? null;
|
|
}
|
|
|
|
function buildRolloutVerificationCommand(kubectl, serviceId) {
|
|
return kubectlCommand(kubectl, ["-n", namespace, "rollout", "status", `deployment/${serviceId}`, "--timeout=180s"]);
|
|
}
|
|
|
|
function buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase) {
|
|
const artifact = artifactIdentityForService(deploy, catalog, serviceId);
|
|
const commandArgs = ["-n", namespace, "get", "deployment", serviceId, "-o", "json"];
|
|
return {
|
|
serviceId,
|
|
namespace,
|
|
status: "not_evaluated",
|
|
observationPhase,
|
|
sourceCommitId: artifact.sourceCommitId,
|
|
image: artifact.image ?? "unknown",
|
|
imageTag: artifact.imageTag ?? "unknown",
|
|
digest: artifact.digest,
|
|
repositoryDigest: artifact.repositoryDigest,
|
|
publishState: artifact.publishState,
|
|
rolloutRevision: null,
|
|
liveImage: null,
|
|
imageMatchesDesired: false,
|
|
expectedRuntimeEnv: expectedRuntimeIdentityEnv(artifact),
|
|
liveRuntimeEnv: null,
|
|
runtimeEnvMatchesDesired: false,
|
|
runtimeEnvDrift: [],
|
|
verificationCommand: buildRolloutVerificationCommand(kubectl, serviceId),
|
|
readCommand: kubectlCommand(kubectl, commandArgs),
|
|
kubeconfig: kubectl.kubeconfig,
|
|
kubeconfigSource: kubectl.kubeconfigSource
|
|
};
|
|
}
|
|
|
|
function skippedDeploymentRolloutObservation(kubectl, deploy, catalog, serviceId, observationPhase, reason) {
|
|
return {
|
|
...buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase),
|
|
status: "not_evaluated",
|
|
blocker: reason
|
|
};
|
|
}
|
|
|
|
async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blockers, options = {}) {
|
|
const observationPhase = options.observationPhase ?? "read";
|
|
const blockRuntimeEnvDrift = options.blockRuntimeEnvDrift === true;
|
|
const commandArgs = ["-n", namespace, "get", "deployment", serviceId, "-o", "json"];
|
|
const base = buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase);
|
|
|
|
if (kubectl.status !== "ready") {
|
|
addBlocker(blockers, "environment_blocker", `rollout-read-${serviceId}`, kubectl.reason);
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
blocker: kubectl.reason
|
|
};
|
|
}
|
|
|
|
const result = await kubectlResult(kubectl, commandArgs, 15000);
|
|
if (!result.ok) {
|
|
const summary = oneLine(commandOutput(result));
|
|
addBlocker(blockers, "environment_blocker", `rollout-read-${serviceId}`, `Cannot read DEV Deployment ${serviceId} rollout revision: ${summary}`);
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
blocker: summary
|
|
};
|
|
}
|
|
|
|
try {
|
|
const deployment = JSON.parse(result.stdout);
|
|
const liveImage = containerImageForDeployment(deployment, serviceId);
|
|
const liveRuntimeEnv = compareRuntimeIdentityEnv(
|
|
envObjectFromEnvList(deployment?.spec?.template?.spec?.containers?.find((container) => container.name === serviceId)?.env),
|
|
base.expectedRuntimeEnv
|
|
);
|
|
if (blockRuntimeEnvDrift && !liveRuntimeEnv.matchesDesired) {
|
|
addBlocker(
|
|
blockers,
|
|
"runtime_blocker",
|
|
`rollout-runtime-env-${observationPhase}-${serviceId}`,
|
|
`DEV Deployment ${serviceId} runtime identity env drift after apply: ${formatRuntimeIdentityDrift(liveRuntimeEnv.drift)}`
|
|
);
|
|
}
|
|
return {
|
|
...base,
|
|
status: "observed",
|
|
rolloutRevision: deploymentRevision(deployment),
|
|
liveImage,
|
|
imageMatchesDesired: Boolean(base.image && liveImage === base.image),
|
|
liveRuntimeEnv,
|
|
runtimeEnvMatchesDesired: liveRuntimeEnv.matchesDesired,
|
|
runtimeEnvDrift: liveRuntimeEnv.drift
|
|
};
|
|
} catch (error) {
|
|
const summary = oneLine(error.message);
|
|
addBlocker(blockers, "contract_blocker", `rollout-read-${serviceId}`, `Cannot parse DEV Deployment ${serviceId} rollout revision: ${summary}`);
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
blocker: summary
|
|
};
|
|
}
|
|
}
|
|
|
|
function blockerHint(blocker) {
|
|
if (blocker.scope === "artifact-publish") {
|
|
return "Publish DEV images from the intended commit and update the catalog with registry digests and verification evidence.";
|
|
}
|
|
if (blocker.scope === "artifact-source-commit") {
|
|
return "Regenerate deploy/deploy.json and deploy/artifact-catalog.dev.json for the source commit that will be promoted.";
|
|
}
|
|
if (blocker.scope === "kubectl" || blocker.scope === "d601-kubeconfig" || blocker.scope === "kubectl-version") {
|
|
return `Provide kubectl with a readable DEV kubeconfig through --kubeconfig, ${devKubeconfigEnvName}, ${kubeconfigEnvName}, or the D601 default path without exposing token or secret values.`;
|
|
}
|
|
if (blocker.scope === "cluster-read") {
|
|
return "Restore read-only visibility for pods, services, configmaps, deployments, and jobs in hwlab-dev.";
|
|
}
|
|
if (blocker.scope.startsWith("rollout-read-")) {
|
|
return "Restore read-only Deployment visibility in hwlab-dev so rollout revision, image tag, digest, and source commit can be reported after apply.";
|
|
}
|
|
if (blocker.scope.startsWith("code-agent-provider-env-read-")) {
|
|
return "Restore read-only Deployment visibility for hwlab-cloud-api so provider env names and secretKeyRef metadata can be verified without reading Secret values.";
|
|
}
|
|
if (blocker.scope.startsWith("code-agent-provider-env-preservation-") || blocker.scope === "code-agent-provider-desired-state") {
|
|
return "Repair deploy/deploy.json and deploy/k8s/base/workloads.yaml so hwlab-cloud-api preserves OPENAI_API_KEY from hwlab-code-agent-provider/openai-api-key and the DEV egress proxy base URL.";
|
|
}
|
|
if (blocker.scope.startsWith("deploy-runtime-env-") || blocker.scope.startsWith("k8s-runtime-env-")) {
|
|
return "Update deploy/deploy.json and deploy/k8s/base/workloads.yaml so the service owns HWLAB_COMMIT_ID, HWLAB_IMAGE, and HWLAB_IMAGE_TAG.";
|
|
}
|
|
if (blocker.scope.startsWith("rollout-runtime-env-")) {
|
|
return "Reconcile the DEV Deployment runtime identity env to the desired source image and rerun post-apply read-only verification.";
|
|
}
|
|
if (blocker.scope === "dev-health" || blocker.scope === "dev-health-identity") {
|
|
return "Restore the public DEV health path and confirm it identifies HWLAB dev, not a substitute runtime.";
|
|
}
|
|
if (blocker.scope === "dev-health-db" || blocker.scope === "cloud-api-db" || blocker.scope === "cloud-api-db-config") {
|
|
return "Configure and verify the DEV cloud-api DB Secret URL host readiness through health output, without reading secret values.";
|
|
}
|
|
return blocker.summary;
|
|
}
|
|
|
|
function buildRemainingBlockers(blockers) {
|
|
return blockers.map((blocker) => ({
|
|
type: blocker.type,
|
|
scope: blocker.scope,
|
|
status: blocker.status,
|
|
summary: blocker.summary,
|
|
unblockHint: oneLine(blockerHint(blocker))
|
|
}));
|
|
}
|
|
|
|
function buildRollbackHint(kubectl, workloads) {
|
|
const deploymentNames = buildWorkloadPlan(workloads)
|
|
.filter((workload) => workload.kind === "Deployment")
|
|
.map((workload) => workload.name);
|
|
const jobNames = buildWorkloadPlan(workloads)
|
|
.filter((workload) => workload.kind === "Job")
|
|
.map((workload) => workload.name);
|
|
|
|
return {
|
|
namespace,
|
|
strategy: "DEV-only Kubernetes rollback using deployment revision history; do not touch PROD or UniDesk services.",
|
|
captureBeforeApply: [
|
|
kubectlCommand(kubectl, ["-n", namespace, "get", "pods,services,configmaps,deployments,jobs", "-o", "wide"]),
|
|
kubectlCommand(kubectl, ["-n", namespace, "get", "deployments", "-o", "json"])
|
|
],
|
|
deploymentRollbackCommands: deploymentNames.map((name) => kubectlCommand(kubectl, ["-n", namespace, "rollout", "undo", `deployment/${name}`])),
|
|
jobCleanupCommands: jobNames.map((name) => kubectlCommand(kubectl, ["-n", namespace, "delete", "job", name, "--ignore-not-found"])),
|
|
postRollbackChecks: [
|
|
kubectlCommand(kubectl, ["-n", namespace, "rollout", "status", "deployment/hwlab-cloud-api", "--timeout=120s"]),
|
|
`curl -fsS ${new URL(healthPath, DEV_ENDPOINT).href}`
|
|
]
|
|
};
|
|
}
|
|
|
|
function workloadEnvByServiceId(workloads, serviceId) {
|
|
for (const item of listItems(workloads)) {
|
|
for (const container of containersFor(item)) {
|
|
if (serviceIdFor(item, container) !== serviceId) continue;
|
|
const env = new Map((container.env ?? []).filter((entry) => entry?.name).map((entry) => [entry.name, entry]));
|
|
env.__volumeMounts = Array.isArray(container.volumeMounts) ? container.volumeMounts : [];
|
|
env.__volumes = Array.isArray(item?.spec?.template?.spec?.volumes) ? item.spec.template.spec.volumes : [];
|
|
return env;
|
|
}
|
|
}
|
|
return new Map();
|
|
}
|
|
|
|
export function inspectCodeAgentProviderDesiredState(deploy, workloads) {
|
|
const deployEnv = deploy?.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {};
|
|
return inspectCodeAgentProviderManifestRefs({
|
|
deployEnv,
|
|
workloadEnv: workloadEnvByServiceId(workloads, "hwlab-cloud-api")
|
|
});
|
|
}
|
|
|
|
export function inspectCodeAgentProviderLiveDeployment(deployment) {
|
|
const container = containerForDeployment(deployment, "hwlab-cloud-api");
|
|
const workloadEnv = new Map((container?.env ?? []).filter((entry) => entry?.name).map((entry) => [entry.name, entry]));
|
|
workloadEnv.__volumeMounts = Array.isArray(container?.volumeMounts) ? container.volumeMounts : [];
|
|
workloadEnv.__volumes = Array.isArray(deployment?.spec?.template?.spec?.volumes) ? deployment.spec.template.spec.volumes : [];
|
|
const inspection = inspectCodeAgentProviderWorkloadEnv(workloadEnv);
|
|
return {
|
|
...inspection,
|
|
namespace: deployment?.metadata?.namespace ?? namespace,
|
|
deployment: deployment?.metadata?.name ?? "hwlab-cloud-api",
|
|
container: container?.name ?? "hwlab-cloud-api",
|
|
validationScope: "deployment-env-and-secretKeyRef-metadata-only",
|
|
secretValuesRead: false,
|
|
kubernetesSecretDataRead: false
|
|
};
|
|
}
|
|
|
|
function containerForDeployment(deployment, containerName) {
|
|
return (deployment?.spec?.template?.spec?.containers ?? []).find((container) => container.name === containerName) ?? null;
|
|
}
|
|
|
|
function codeAgentProviderBlockerSummary(inspection) {
|
|
const missing = [
|
|
...(inspection.missingDeployEnv ?? []),
|
|
...(inspection.missingWorkloadEnv ?? inspection.missingEnv ?? []),
|
|
...(inspection.deployMismatches ?? []).map((entry) => entry.name),
|
|
...(inspection.workloadMismatches ?? inspection.mismatchedEnv ?? []).map((entry) => entry.name),
|
|
...(inspection.missingSecretRefs ?? []),
|
|
...(inspection.missingEgressContract ?? [])
|
|
];
|
|
const detail = uniqueStrings(missing).join(", ");
|
|
return detail
|
|
? `Code Agent provider env preservation is incomplete; missing or mismatched ${detail}`
|
|
: "Code Agent provider env preservation is incomplete";
|
|
}
|
|
|
|
function addCodeAgentProviderDesiredStateBlocker(blockers, desiredState) {
|
|
if (desiredState.ready) return;
|
|
addBlocker(
|
|
blockers,
|
|
"agent_blocker",
|
|
"code-agent-provider-desired-state",
|
|
codeAgentProviderBlockerSummary(desiredState)
|
|
);
|
|
}
|
|
|
|
async function observeCodeAgentProviderLiveDeployment(kubectl, blockers, { phase, blockOnMismatch }) {
|
|
const commandArgs = ["-n", namespace, "get", "deployment", "hwlab-cloud-api", "-o", "json"];
|
|
const base = {
|
|
phase,
|
|
namespace,
|
|
serviceId: "hwlab-cloud-api",
|
|
status: "not_evaluated",
|
|
readCommand: kubectlCommand(kubectl, commandArgs),
|
|
kubeconfig: kubectl.kubeconfig,
|
|
expectedSecretRef: `${DEV_CODE_AGENT_PROVIDER_CONTRACT.secretRefs[0].secretName}/${DEV_CODE_AGENT_PROVIDER_CONTRACT.secretRefs[0].secretKey}`,
|
|
expectedEgressTarget: DEV_CODE_AGENT_PROVIDER_CONTRACT.egress.target,
|
|
secretValuesRead: false,
|
|
kubernetesSecretDataRead: false,
|
|
valuesRedacted: true
|
|
};
|
|
|
|
if (kubectl.status !== "ready") {
|
|
if (blockOnMismatch) {
|
|
addBlocker(blockers, "environment_blocker", `code-agent-provider-env-read-${phase}`, kubectl.reason);
|
|
}
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
blocker: kubectl.reason
|
|
};
|
|
}
|
|
|
|
const result = await kubectlResult(kubectl, commandArgs, 15000);
|
|
if (!result.ok) {
|
|
const summary = oneLine(result.stderr || result.stdout);
|
|
if (blockOnMismatch) {
|
|
addBlocker(blockers, "environment_blocker", `code-agent-provider-env-read-${phase}`, `Cannot read DEV Deployment hwlab-cloud-api provider env: ${summary}`);
|
|
}
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
blocker: summary
|
|
};
|
|
}
|
|
|
|
try {
|
|
const inspection = inspectCodeAgentProviderLiveDeployment(JSON.parse(result.stdout));
|
|
if (!inspection.ready && blockOnMismatch) {
|
|
addBlocker(blockers, "runtime_blocker", `code-agent-provider-env-preservation-${phase}`, codeAgentProviderBlockerSummary(inspection));
|
|
}
|
|
return {
|
|
...base,
|
|
...inspection,
|
|
phase,
|
|
status: inspection.ready ? "pass" : "blocked"
|
|
};
|
|
} catch (error) {
|
|
const summary = oneLine(error.message);
|
|
if (blockOnMismatch) {
|
|
addBlocker(blockers, "contract_blocker", `code-agent-provider-env-read-${phase}`, `Cannot parse DEV Deployment hwlab-cloud-api provider env: ${summary}`);
|
|
}
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
blocker: summary
|
|
};
|
|
}
|
|
}
|
|
|
|
function buildManualCommands(status) {
|
|
if (status === "blocked") {
|
|
return {
|
|
status: "blocked",
|
|
beforeHumanApproval: [blockedDryRunCommand, "node scripts/validate-dev-gate-report.mjs"],
|
|
afterHumanApproval: [],
|
|
summary: "No apply command is offered while blockers remain open."
|
|
};
|
|
}
|
|
|
|
return {
|
|
status: "ready",
|
|
beforeHumanApproval: [dryRunPlanCommand, "node scripts/validate-dev-gate-report.mjs"],
|
|
afterHumanApproval: [applyCommand],
|
|
summary: "Run the dry-run plan and validator immediately before requesting human approval; live apply still requires the explicit DEV confirmation flags."
|
|
};
|
|
}
|
|
|
|
function buildApplyBoundary(args, status, applyStep, kubectl) {
|
|
return {
|
|
currentMode: args.apply ? "apply-requested" : "dry-run",
|
|
defaultNoWrite: !args.apply,
|
|
mutationAttempted: applyStep.mutationAttempted === true && args.apply,
|
|
mutationAllowed: status === "pass" && args.apply,
|
|
applyRequiresFlags: ["--apply", "--confirm-dev", "--confirmed-non-production"],
|
|
kubeconfigSource: kubectl.kubeconfigSource,
|
|
writeScope: `${kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"])}, namespace hwlab-dev only; delete/recreate is allowed only for explicitly allowlisted suspended DEV template Jobs and stale legacy simulator Deployments`,
|
|
noWriteScope: `source/catalog/k8s validation, optional DEV health probe, ${kubectl.commandPrefix} read, replacement planning, and server-side dry-run only`,
|
|
forbiddenActions
|
|
};
|
|
}
|
|
|
|
function buildPlanConclusion(status, blockers) {
|
|
return {
|
|
status: status === "blocked" ? "blocked" : "ready",
|
|
reason:
|
|
status === "blocked"
|
|
? `${blockers.length} blocker(s) must be cleared before any live DEV apply.`
|
|
: "All DEV-only preconditions passed; human approval is still required before apply.",
|
|
blockerCount: blockers.length
|
|
};
|
|
}
|
|
|
|
function validateArgs(args, blockers) {
|
|
for (const error of args.errors ?? []) {
|
|
const scope = error.startsWith("--report") ? "report-argument" : "kubeconfig-argument";
|
|
addBlocker(blockers, "safety_blocker", scope, error);
|
|
}
|
|
for (const forbidden of ["--prod", "--production", "--read-secret", "--force-push", "--heavyweight-e2e"]) {
|
|
if (args.flags.has(forbidden)) {
|
|
addBlocker(blockers, "safety_blocker", forbidden.slice(2), `${forbidden} is forbidden for DEV deploy apply`);
|
|
}
|
|
}
|
|
if (args.apply && (!args.flags.has("--confirm-dev") || !args.flags.has("--confirmed-non-production"))) {
|
|
addBlocker(
|
|
blockers,
|
|
"safety_blocker",
|
|
"apply-confirmation",
|
|
"Live apply requires --confirm-dev and --confirmed-non-production"
|
|
);
|
|
}
|
|
}
|
|
|
|
function commitMatchesSource(commitId, sourceCommitId) {
|
|
return sourceCommitId.startsWith(commitId) || commitId.startsWith(sourceCommitId);
|
|
}
|
|
|
|
function validateDeployRuntimeIdentityEnv(deploy, catalog, serviceId, blockers) {
|
|
const deployService = deploy?.services?.find((service) => service.serviceId === serviceId) ?? null;
|
|
if (!deployService) return null;
|
|
const artifact = artifactIdentityForService(deploy, catalog, serviceId);
|
|
const expected = expectedRuntimeIdentityEnv(artifact);
|
|
const comparison = compareRuntimeIdentityEnv(deployService.env ?? {}, expected);
|
|
if (!comparison.matchesDesired) {
|
|
addBlocker(
|
|
blockers,
|
|
"contract_blocker",
|
|
`deploy-runtime-env-${serviceId}`,
|
|
`${serviceId} deploy env does not own desired runtime identity: ${formatRuntimeIdentityDrift(comparison.drift)}`
|
|
);
|
|
}
|
|
return {
|
|
serviceId,
|
|
source: "deploy",
|
|
expected,
|
|
...comparison
|
|
};
|
|
}
|
|
|
|
function validateDeployAndCatalog(deploy, catalog, sourceCommitId, blockers) {
|
|
const artifactEvidence = [];
|
|
if (!deploy || !catalog) return artifactEvidence;
|
|
|
|
if (deploy.environment !== ENVIRONMENT_DEV || deploy.namespace !== namespace) {
|
|
addBlocker(blockers, "safety_blocker", "deploy-target", "deploy/deploy.json must target only dev hwlab-dev");
|
|
}
|
|
if (deploy.profiles?.prod?.enabled !== false) {
|
|
addBlocker(blockers, "safety_blocker", "prod-profile", "PROD profile must remain disabled");
|
|
}
|
|
if (catalog.environment !== ENVIRONMENT_DEV || catalog.namespace !== namespace) {
|
|
addBlocker(blockers, "safety_blocker", "catalog-target", "artifact catalog must target only dev hwlab-dev");
|
|
}
|
|
if (!commitMatchesSource(deploy.commitId, sourceCommitId) || !commitMatchesSource(catalog.commitId, sourceCommitId)) {
|
|
addBlocker(
|
|
blockers,
|
|
"observability_blocker",
|
|
"artifact-source-commit",
|
|
`deploy/catalog artifact commit ${deploy.commitId}/${catalog.commitId} does not match source ${sourceCommitId}`
|
|
);
|
|
}
|
|
|
|
const deployById = new Map((deploy.services ?? []).map((service) => [service.serviceId, service]));
|
|
const catalogById = new Map((catalog.services ?? []).map((service) => [service.serviceId, service]));
|
|
for (const serviceId of SERVICE_IDS) {
|
|
const deployService = deployById.get(serviceId);
|
|
const catalogService = catalogById.get(serviceId);
|
|
if (!deployService || !catalogService) {
|
|
addBlocker(blockers, "contract_blocker", `service-${serviceId}`, `${serviceId} missing from deploy or catalog`);
|
|
continue;
|
|
}
|
|
if (deployService.namespace !== namespace || deployService.profile !== ENVIRONMENT_DEV) {
|
|
addBlocker(blockers, "safety_blocker", `service-target-${serviceId}`, `${serviceId} is not DEV-only`);
|
|
}
|
|
if (deployService.image !== catalogService.image) {
|
|
addBlocker(blockers, "contract_blocker", `service-image-${serviceId}`, `${serviceId} image differs in deploy/catalog`);
|
|
}
|
|
artifactEvidence.push({
|
|
serviceId,
|
|
deployCommitId: deploy.commitId,
|
|
catalogCommitId: catalogService.commitId,
|
|
image: deployService.image,
|
|
digest: catalogService.digest,
|
|
publishState: catalogService.publishState
|
|
});
|
|
}
|
|
validateDeployRuntimeIdentityEnv(deploy, catalog, "hwlab-cloud-web", blockers);
|
|
|
|
const requiredServices = (catalog.services ?? []).filter((service) => service.artifactRequired !== false);
|
|
const disabledServices = (catalog.services ?? []).filter((service) => service.artifactRequired === false);
|
|
const hasPublishedCatalog =
|
|
catalog.publish?.ciPublished === true &&
|
|
catalog.publish?.registryVerified === true &&
|
|
requiredServices.every((service) => service.digest?.startsWith("sha256:") && service.publishState === "published") &&
|
|
disabledServices.every((service) => service.digest === "not_published" && service.publishState === "skeleton-only");
|
|
if (!hasPublishedCatalog) {
|
|
addBlocker(
|
|
blockers,
|
|
"observability_blocker",
|
|
"artifact-publish",
|
|
"DEV artifact catalog has no CI publish, registry verification, or required-service registry digests"
|
|
);
|
|
}
|
|
return artifactEvidence;
|
|
}
|
|
|
|
function validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers) {
|
|
const items = [...listItems(namespaceDoc), ...listItems(workloads), ...listItems(services), ...listItems(healthContract)];
|
|
if (devKustomization?.namespace !== namespace) {
|
|
addBlocker(blockers, "safety_blocker", "dev-kustomization", "DEV kustomization must pin namespace hwlab-dev");
|
|
}
|
|
if ((devKustomization?.resources ?? []).some((resource) => resource.includes("prod"))) {
|
|
addBlocker(blockers, "safety_blocker", "dev-kustomization-prod", "DEV kustomization must not reference PROD resources");
|
|
}
|
|
for (const item of items) {
|
|
const itemNamespace = item?.metadata?.namespace ?? (item?.kind === "Namespace" ? item?.metadata?.name : namespace);
|
|
if (itemNamespace !== namespace) {
|
|
addBlocker(blockers, "safety_blocker", `k8s-${item?.kind}-${item?.metadata?.name}`, "Kubernetes resource is not in hwlab-dev");
|
|
}
|
|
}
|
|
|
|
const expectedImages = new Map((deploy?.services ?? []).map((service) => [service.serviceId, service.image]));
|
|
const cloudWebExpectedRuntimeEnv = expectedRuntimeIdentityEnv(artifactIdentityForService(deploy, catalog, "hwlab-cloud-web"));
|
|
const workloadItems = listItems(workloads);
|
|
const images = [];
|
|
const runtimeIdentityEnv = [];
|
|
for (const item of workloadItems) {
|
|
for (const container of containersFor(item)) {
|
|
const serviceId = serviceIdFor(item, container);
|
|
const expectedImage = expectedImages.get(serviceId);
|
|
images.push({ serviceId, image: container.image, kind: item.kind, name: item.metadata?.name });
|
|
if (expectedImage && container.image !== expectedImage) {
|
|
addBlocker(blockers, "contract_blocker", `k8s-image-${serviceId}`, `${serviceId} image differs in k8s workload`);
|
|
}
|
|
if (serviceId === "hwlab-cloud-web") {
|
|
const comparison = compareRuntimeIdentityEnv(envObjectFromEnvList(container.env), cloudWebExpectedRuntimeEnv);
|
|
runtimeIdentityEnv.push({
|
|
serviceId,
|
|
kind: item.kind,
|
|
name: item.metadata?.name,
|
|
container: container.name,
|
|
expected: cloudWebExpectedRuntimeEnv,
|
|
...comparison
|
|
});
|
|
if (!comparison.matchesDesired) {
|
|
addBlocker(
|
|
blockers,
|
|
"contract_blocker",
|
|
`k8s-runtime-env-${serviceId}`,
|
|
`${serviceId} workload env does not own desired runtime identity: ${formatRuntimeIdentityDrift(comparison.drift)}`
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return {
|
|
resourceCounts: {
|
|
workloads: workloadItems.length,
|
|
services: listItems(services).length,
|
|
configMaps: listItems(healthContract).filter((item) => item.kind === "ConfigMap").length
|
|
},
|
|
images,
|
|
runtimeIdentityEnv
|
|
};
|
|
}
|
|
|
|
async function checkCloudApiDb(deploy, workloads, services, blockers) {
|
|
const deployEnv = deploy?.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.env ?? {};
|
|
const envNames = new Set(Object.keys(deployEnv));
|
|
const workloadEnv = new Map();
|
|
for (const item of listItems(workloads)) {
|
|
for (const container of containersFor(item)) {
|
|
if (serviceIdFor(item, container) === "hwlab-cloud-api") {
|
|
for (const env of container.env ?? []) {
|
|
envNames.add(env.name);
|
|
workloadEnv.set(env.name, env);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
const requiredEnv = DEV_DB_ENV_CONTRACT.requiredEnv.map((name) => {
|
|
const secretRef = DEV_DB_ENV_CONTRACT.secretRefs.find((ref) => ref.env === name) ?? null;
|
|
const workloadEntry = workloadEnv.get(name) ?? null;
|
|
const secretRefPresent = secretRef
|
|
? deployEnv[name] === `secretRef:${secretRef.secretName}/${secretRef.secretKey}` &&
|
|
workloadEntry?.valueFrom?.secretKeyRef?.name === secretRef.secretName &&
|
|
workloadEntry?.valueFrom?.secretKeyRef?.key === secretRef.secretKey
|
|
: true;
|
|
return {
|
|
name,
|
|
manifestPresent: Object.hasOwn(deployEnv, name),
|
|
k8sPresent: workloadEnv.has(name),
|
|
redacted: Boolean(secretRef),
|
|
source: secretRef ? "k8s-secret-ref" : "runtime-env",
|
|
secretRef: secretRef
|
|
? {
|
|
secretName: secretRef.secretName,
|
|
secretKey: secretRef.secretKey,
|
|
present: secretRefPresent,
|
|
redacted: true
|
|
}
|
|
: null
|
|
};
|
|
});
|
|
const missingManifest = requiredEnv.filter((env) => !env.manifestPresent).map((env) => env.name);
|
|
const missingK8s = requiredEnv.filter((env) => !env.k8sPresent).map((env) => env.name);
|
|
const missingSecretRefs = requiredEnv
|
|
.filter((env) => env.secretRef && !env.secretRef.present)
|
|
.map((env) => `${env.secretRef.secretName}/${env.secretRef.secretKey}`);
|
|
const expectedSslMode = DEV_DB_ENV_CONTRACT.nonSecretDefaults.HWLAB_CLOUD_DB_SSL_MODE;
|
|
const sslModeMatches =
|
|
deployEnv.HWLAB_CLOUD_DB_SSL_MODE === expectedSslMode &&
|
|
workloadEnv.get("HWLAB_CLOUD_DB_SSL_MODE")?.value === expectedSslMode;
|
|
const mismatchedNonSecretEnv = sslModeMatches
|
|
? []
|
|
: [`HWLAB_CLOUD_DB_SSL_MODE must be ${expectedSslMode}`];
|
|
const dnsContract = inspectCloudApiDbOptionalAliasContract(deployEnv, workloadEnv, services);
|
|
const missingDnsContract = dnsContract.missing;
|
|
if (
|
|
missingManifest.length > 0 ||
|
|
missingK8s.length > 0 ||
|
|
missingSecretRefs.length > 0 ||
|
|
mismatchedNonSecretEnv.length > 0 ||
|
|
missingDnsContract.length > 0
|
|
) {
|
|
addBlocker(
|
|
blockers,
|
|
"runtime_blocker",
|
|
"cloud-api-db-config",
|
|
`cloud-api DEV DB env/authority contract is incomplete; missing ${[
|
|
...missingManifest,
|
|
...missingK8s,
|
|
...missingSecretRefs,
|
|
...mismatchedNonSecretEnv,
|
|
...missingDnsContract
|
|
].join(", ")}`
|
|
);
|
|
}
|
|
|
|
const configReady =
|
|
missingManifest.length === 0 &&
|
|
missingK8s.length === 0 &&
|
|
missingSecretRefs.length === 0 &&
|
|
mismatchedNonSecretEnv.length === 0 &&
|
|
missingDnsContract.length === 0;
|
|
const health = {
|
|
status: configReady ? "degraded" : "blocked",
|
|
ready: false,
|
|
configReady,
|
|
connected: false,
|
|
connectionChecked: false,
|
|
fields: requiredEnv.map((env) => ({
|
|
name: env.name,
|
|
present: env.manifestPresent && env.k8sPresent && (!env.secretRef || env.secretRef.present),
|
|
redacted: env.redacted
|
|
})),
|
|
missingEnv: uniqueStrings([...missingManifest, ...missingK8s]),
|
|
secretRefs: requiredEnv
|
|
.filter((env) => env.secretRef)
|
|
.map((env) => ({
|
|
env: env.name,
|
|
secretName: env.secretRef.secretName,
|
|
secretKey: env.secretRef.secretKey,
|
|
present: env.secretRef.present,
|
|
redacted: true
|
|
})),
|
|
safety: {
|
|
liveDbEvidence: false
|
|
}
|
|
};
|
|
|
|
return {
|
|
status: configReady ? "contract-ready" : "contract-blocked",
|
|
requiredEnv,
|
|
missingManifest,
|
|
missingK8s,
|
|
missingSecretRefs,
|
|
mismatchedNonSecretEnv,
|
|
missingDnsContract,
|
|
dns: dnsContract,
|
|
runtimeHealth: summarizeDbContract(health),
|
|
secretMaterialRead: false,
|
|
valuesRedacted: true,
|
|
liveDbEvidence: false,
|
|
fixtureEvidence: false,
|
|
note: "This check records DB env, Secret reference, and Secret URL host authority only. Optional cloud-api-db alias presence is diagnostic and is not live DB evidence."
|
|
};
|
|
}
|
|
|
|
function inspectCloudApiDbOptionalAliasContract(deployEnv, workloadEnv, services) {
|
|
const alias = DEV_DB_ENV_CONTRACT.dns;
|
|
const service = listItems(services).find((item) => item?.metadata?.name === alias.serviceName) ?? null;
|
|
const actual = {
|
|
deployServiceName: deployEnv.HWLAB_CLOUD_DB_SERVICE_NAME,
|
|
deployServiceNamespace: deployEnv.HWLAB_CLOUD_DB_SERVICE_NAMESPACE,
|
|
deployHost: deployEnv.HWLAB_CLOUD_DB_HOST,
|
|
deployPort: deployEnv.HWLAB_CLOUD_DB_PORT,
|
|
workloadServiceName: workloadEnv.get("HWLAB_CLOUD_DB_SERVICE_NAME")?.value,
|
|
workloadServiceNamespace: workloadEnv.get("HWLAB_CLOUD_DB_SERVICE_NAMESPACE")?.value,
|
|
workloadHost: workloadEnv.get("HWLAB_CLOUD_DB_HOST")?.value,
|
|
workloadPort: workloadEnv.get("HWLAB_CLOUD_DB_PORT")?.value,
|
|
serviceName: service?.metadata?.name
|
|
};
|
|
const missing = [];
|
|
for (const [name, value] of Object.entries(actual)) {
|
|
if (value !== undefined) {
|
|
missing.push(`${name} must be absent because ${alias.serviceName} is not the readiness authority`);
|
|
}
|
|
}
|
|
|
|
return {
|
|
ready: missing.length === 0,
|
|
source: alias.source,
|
|
serviceName: alias.serviceName,
|
|
namespace: alias.namespace,
|
|
port: alias.port,
|
|
portName: alias.portName,
|
|
requiredForReadiness: alias.requiredForReadiness,
|
|
usedForProbe: alias.usedForProbe,
|
|
servicePresent: Boolean(service),
|
|
secretMaterialRead: false,
|
|
liveDbEvidence: false,
|
|
missing,
|
|
actual
|
|
};
|
|
}
|
|
|
|
async function probeDevEndpoint(blockers) {
|
|
const url = new URL(healthPath, DEV_ENDPOINT).href;
|
|
try {
|
|
const response = await httpGet(url, 8000);
|
|
const text = response.body;
|
|
let json = null;
|
|
try {
|
|
json = JSON.parse(text);
|
|
} catch {}
|
|
if (response.statusCode < 200 || response.statusCode >= 300) {
|
|
addBlocker(blockers, "network_blocker", "dev-health", `DEV health returned HTTP ${response.statusCode}`);
|
|
}
|
|
if (json?.serviceId !== "hwlab-cloud-api" || json?.environment !== ENVIRONMENT_DEV) {
|
|
addBlocker(blockers, "runtime_blocker", "dev-health-identity", "DEV health did not identify hwlab-cloud-api in dev");
|
|
}
|
|
if (json?.db?.ready !== true && json?.db?.configReady !== true) {
|
|
addBlocker(blockers, "runtime_blocker", "dev-health-db", "DEV health did not confirm cloud-api DB config readiness");
|
|
}
|
|
return {
|
|
status: response.statusCode >= 200 && response.statusCode < 300 ? "pass" : "blocked",
|
|
url,
|
|
httpStatus: response.statusCode,
|
|
body: text.slice(0, 500)
|
|
};
|
|
} catch (error) {
|
|
const reason = oneLine(error.cause?.message ?? error.message);
|
|
addBlocker(blockers, "network_blocker", "dev-health", `Cannot reach ${url}: ${reason}`);
|
|
return { status: "blocked", url, error: reason };
|
|
}
|
|
}
|
|
|
|
function httpGet(url, timeoutMs) {
|
|
return new Promise((resolve, reject) => {
|
|
const request = httpRequest(url, { method: "GET", timeout: timeoutMs }, (response) => {
|
|
response.setEncoding("utf8");
|
|
let body = "";
|
|
response.on("data", (chunk) => {
|
|
body += chunk;
|
|
});
|
|
response.on("end", () => resolve({ statusCode: response.statusCode ?? 0, body }));
|
|
});
|
|
request.on("timeout", () => {
|
|
request.destroy(new Error(`timeout after ${timeoutMs}ms`));
|
|
});
|
|
request.on("error", reject);
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
async function observeCluster(kubectl, blockers) {
|
|
const blockedBase = {
|
|
status: "blocked",
|
|
executor: kubectl.executor ?? "missing",
|
|
kubeconfig: kubectl.kubeconfig,
|
|
kubeconfigSource: kubectl.kubeconfigSource,
|
|
commandPrefix: kubectl.commandPrefix,
|
|
resources: []
|
|
};
|
|
if (!kubectl.executor) {
|
|
addBlocker(blockers, "environment_blocker", "kubectl", kubectl.reason);
|
|
return blockedBase;
|
|
}
|
|
if (kubectl.status !== "ready") {
|
|
addBlocker(blockers, "environment_blocker", "d601-kubeconfig", kubectl.reason);
|
|
return blockedBase;
|
|
}
|
|
const version = await kubectlResult(kubectl, ["version", "--client=true", "--output=yaml"]);
|
|
if (!version.ok) {
|
|
addBlocker(blockers, "environment_blocker", "kubectl-version", `kubectl client failed: ${version.stderr || version.stdout}`);
|
|
return blockedBase;
|
|
}
|
|
const getArgs = ["get", "pods,services,configmaps,deployments,jobs", "-n", namespace, "-o", "json"];
|
|
const get = await kubectlResult(kubectl, getArgs, 20000);
|
|
if (!get.ok) {
|
|
addBlocker(blockers, "environment_blocker", "cluster-read", `Cannot read hwlab-dev resources: ${get.stderr || get.stdout}`);
|
|
return { ...blockedBase, version: version.stdout, readCommand: kubectlCommand(kubectl, getArgs) };
|
|
}
|
|
const parsed = JSON.parse(get.stdout);
|
|
return {
|
|
status: "pass",
|
|
executor: kubectl.executor,
|
|
kubeconfig: kubectl.kubeconfig,
|
|
kubeconfigSource: kubectl.kubeconfigSource,
|
|
commandPrefix: kubectl.commandPrefix,
|
|
readCommand: kubectlCommand(kubectl, getArgs),
|
|
version: version.stdout,
|
|
resources: (parsed.items ?? []).map((item) => ({
|
|
kind: item.kind,
|
|
name: item.metadata?.name,
|
|
phase: item.status?.phase,
|
|
readyReplicas: item.status?.readyReplicas,
|
|
replicas: item.status?.replicas
|
|
}))
|
|
};
|
|
}
|
|
|
|
function isNotFoundOutput(value) {
|
|
return /\bnot\s+found\b|notfound/iu.test(value);
|
|
}
|
|
|
|
async function readLiveTemplateJob(kubectl, jobName, blockers) {
|
|
if (kubectl.status !== "ready") {
|
|
return { status: "not_evaluated", reason: kubectl.reason };
|
|
}
|
|
const get = await kubectlResult(kubectl, ["-n", namespace, "get", "job", jobName, "-o", "json"], 15000);
|
|
if (!get.ok) {
|
|
const output = commandOutput(get);
|
|
if (isNotFoundOutput(output)) {
|
|
return { status: "not_found", liveJob: null };
|
|
}
|
|
addBlocker(blockers, "environment_blocker", `template-job-read-${jobName}`, `Cannot read live DEV template Job ${jobName}: ${output}`);
|
|
return { status: "blocked", reason: oneLine(output), liveJob: null };
|
|
}
|
|
try {
|
|
return { status: "found", liveJob: JSON.parse(get.stdout) };
|
|
} catch (error) {
|
|
addBlocker(blockers, "contract_blocker", `template-job-read-${jobName}`, `Cannot parse live DEV template Job ${jobName}: ${error.message}`);
|
|
return { status: "blocked", reason: oneLine(error.message), liveJob: null };
|
|
}
|
|
}
|
|
|
|
async function readLiveLegacySimulatorDeployment(kubectl, deploymentName, blockers) {
|
|
if (kubectl.status !== "ready") {
|
|
return { status: "not_evaluated", reason: kubectl.reason };
|
|
}
|
|
const get = await kubectlResult(kubectl, ["-n", namespace, "get", "deployment", deploymentName, "-o", "json"], 15000);
|
|
if (!get.ok) {
|
|
const output = commandOutput(get);
|
|
if (isNotFoundOutput(output)) {
|
|
return { status: "not_found", liveDeployment: null };
|
|
}
|
|
addBlocker(blockers, "environment_blocker", `legacy-simulator-deployment-read-${deploymentName}`, `Cannot read live DEV legacy simulator Deployment ${deploymentName}: ${output}`);
|
|
return { status: "blocked", reason: oneLine(output), liveDeployment: null };
|
|
}
|
|
try {
|
|
return { status: "found", liveDeployment: JSON.parse(get.stdout) };
|
|
} catch (error) {
|
|
addBlocker(blockers, "contract_blocker", `legacy-simulator-deployment-read-${deploymentName}`, `Cannot parse live DEV legacy simulator Deployment ${deploymentName}: ${error.message}`);
|
|
return { status: "blocked", reason: oneLine(error.message), liveDeployment: null };
|
|
}
|
|
}
|
|
|
|
async function readLiveDeployment(kubectl, deploymentName, blockers, scopePrefix) {
|
|
if (kubectl.status !== "ready") {
|
|
return { status: "not_evaluated", reason: kubectl.reason };
|
|
}
|
|
const get = await kubectlResult(kubectl, ["-n", namespace, "get", "deployment", deploymentName, "-o", "json"], 15000);
|
|
if (!get.ok) {
|
|
const output = commandOutput(get);
|
|
if (isNotFoundOutput(output)) {
|
|
return { status: "not_found", liveDeployment: null };
|
|
}
|
|
addBlocker(blockers, "environment_blocker", `${scopePrefix}-${deploymentName}`, `Cannot read live DEV Deployment ${deploymentName}: ${output}`);
|
|
return { status: "blocked", reason: oneLine(output), liveDeployment: null };
|
|
}
|
|
try {
|
|
return { status: "found", liveDeployment: JSON.parse(get.stdout) };
|
|
} catch (error) {
|
|
addBlocker(blockers, "contract_blocker", `${scopePrefix}-${deploymentName}`, `Cannot parse live DEV Deployment ${deploymentName}: ${error.message}`);
|
|
return { status: "blocked", reason: oneLine(error.message), liveDeployment: null };
|
|
}
|
|
}
|
|
|
|
async function buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers) {
|
|
const desiredJobs = listItems(workloads).filter(isAllowedDesiredTemplateJob);
|
|
const replacements = [];
|
|
for (const desiredJob of desiredJobs) {
|
|
const jobName = desiredJob.metadata.name;
|
|
const live = await readLiveTemplateJob(kubectl, jobName, blockers);
|
|
if (live.status === "not_evaluated") {
|
|
replacements.push({
|
|
namespace,
|
|
jobName,
|
|
allowed: true,
|
|
desiredSuspended: true,
|
|
liveExists: null,
|
|
liveSuspended: null,
|
|
oldImage: null,
|
|
newImage: primaryImage(imageRecordsFor(desiredJob)),
|
|
oldImages: [],
|
|
newImages: imageRecordsFor(desiredJob),
|
|
replace: false,
|
|
result: "not_evaluated",
|
|
reason: live.reason
|
|
});
|
|
continue;
|
|
}
|
|
if (live.status === "blocked") {
|
|
replacements.push({
|
|
namespace,
|
|
jobName,
|
|
allowed: true,
|
|
desiredSuspended: true,
|
|
liveExists: null,
|
|
liveSuspended: null,
|
|
oldImage: null,
|
|
newImage: primaryImage(imageRecordsFor(desiredJob)),
|
|
oldImages: [],
|
|
newImages: imageRecordsFor(desiredJob),
|
|
replace: false,
|
|
result: "blocked",
|
|
reason: live.reason
|
|
});
|
|
continue;
|
|
}
|
|
|
|
const decision = decideDevTemplateJobReplacement({
|
|
jobName,
|
|
jobNamespace: itemNamespace(desiredJob),
|
|
desiredSuspended: desiredJob.spec?.suspend,
|
|
liveExists: live.status === "found",
|
|
liveSuspended: live.liveJob?.spec?.suspend,
|
|
oldImages: live.liveJob ? imageRecordsFor(live.liveJob) : [],
|
|
newImages: imageRecordsFor(desiredJob)
|
|
});
|
|
if (decision.result === "blocked") {
|
|
addBlocker(blockers, "safety_blocker", `template-job-replace-${jobName}`, decision.reason);
|
|
}
|
|
replacements.push(decision);
|
|
}
|
|
return replacements;
|
|
}
|
|
|
|
async function buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads, blockers) {
|
|
const desiredReplacements = new Map(
|
|
listItems(workloads)
|
|
.filter(isDesiredLegacySimulatorReplacement)
|
|
.map((item) => [item.metadata.name, item])
|
|
);
|
|
const cleanups = [];
|
|
for (const policy of devLegacySimulatorDeploymentCleanupPolicy.allowedDeployments) {
|
|
const desiredReplacement = desiredReplacements.get(policy.name) ?? null;
|
|
const live = await readLiveLegacySimulatorDeployment(kubectl, policy.name, blockers);
|
|
if (live.status === "not_evaluated") {
|
|
cleanups.push({
|
|
namespace,
|
|
deploymentName: policy.name,
|
|
serviceId: policy.serviceId,
|
|
allowed: true,
|
|
desiredReplacementExists: Boolean(desiredReplacement),
|
|
desiredReplacementKind: desiredReplacement?.kind ?? null,
|
|
liveExists: null,
|
|
oldImage: null,
|
|
newImage: primaryImage(desiredReplacement ? imageRecordsFor(desiredReplacement) : []),
|
|
oldImages: [],
|
|
newImages: desiredReplacement ? imageRecordsFor(desiredReplacement) : [],
|
|
cleanup: false,
|
|
result: "not_evaluated",
|
|
reason: live.reason
|
|
});
|
|
continue;
|
|
}
|
|
if (live.status === "blocked") {
|
|
cleanups.push({
|
|
namespace,
|
|
deploymentName: policy.name,
|
|
serviceId: policy.serviceId,
|
|
allowed: true,
|
|
desiredReplacementExists: Boolean(desiredReplacement),
|
|
desiredReplacementKind: desiredReplacement?.kind ?? null,
|
|
liveExists: null,
|
|
oldImage: null,
|
|
newImage: primaryImage(desiredReplacement ? imageRecordsFor(desiredReplacement) : []),
|
|
oldImages: [],
|
|
newImages: desiredReplacement ? imageRecordsFor(desiredReplacement) : [],
|
|
cleanup: false,
|
|
result: "blocked",
|
|
reason: live.reason
|
|
});
|
|
continue;
|
|
}
|
|
|
|
const decision = decideDevLegacySimulatorDeploymentCleanup({
|
|
deploymentName: policy.name,
|
|
deploymentNamespace: namespace,
|
|
desiredReplacementExists: Boolean(desiredReplacement),
|
|
desiredReplacementKind: desiredReplacement?.kind,
|
|
liveExists: live.status === "found",
|
|
oldImages: live.liveDeployment ? imageRecordsFor(live.liveDeployment) : [],
|
|
newImages: desiredReplacement ? imageRecordsFor(desiredReplacement) : []
|
|
});
|
|
if (decision.result === "blocked") {
|
|
addBlocker(blockers, "safety_blocker", `legacy-simulator-deployment-cleanup-${policy.name}`, decision.reason);
|
|
}
|
|
cleanups.push(decision);
|
|
}
|
|
return cleanups;
|
|
}
|
|
|
|
async function buildDevUnmanagedHotfixDeploymentCleanupPlan(kubectl, workloads, blockers) {
|
|
const desiredDeployments = desiredDeploymentIndex(workloads);
|
|
const cleanups = [];
|
|
for (const [key, desiredDeployment] of desiredDeployments.entries()) {
|
|
const deploymentName = desiredDeployment.metadata.name;
|
|
const live = await readLiveDeployment(kubectl, deploymentName, blockers, "unmanaged-hotfix-deployment-read");
|
|
if (live.status === "not_evaluated") {
|
|
cleanups.push({
|
|
namespace,
|
|
deploymentName,
|
|
desiredExists: true,
|
|
liveExists: null,
|
|
cleanup: false,
|
|
result: "not_evaluated",
|
|
reason: live.reason,
|
|
unmanagedVolumes: [],
|
|
unmanagedVolumeMounts: [],
|
|
unmanagedTemplateAnnotations: []
|
|
});
|
|
continue;
|
|
}
|
|
if (live.status === "blocked") {
|
|
cleanups.push({
|
|
namespace,
|
|
deploymentName,
|
|
desiredExists: true,
|
|
liveExists: null,
|
|
cleanup: false,
|
|
result: "blocked",
|
|
reason: live.reason,
|
|
unmanagedVolumes: [],
|
|
unmanagedVolumeMounts: [],
|
|
unmanagedTemplateAnnotations: []
|
|
});
|
|
continue;
|
|
}
|
|
const decision = decideDevUnmanagedHotfixDeploymentCleanup({
|
|
deploymentName,
|
|
deploymentNamespace: key.split("/")[0],
|
|
desiredDeployment,
|
|
liveDeployment: live.liveDeployment
|
|
});
|
|
if (decision.result === "blocked") {
|
|
addBlocker(blockers, "safety_blocker", `unmanaged-hotfix-deployment-cleanup-${deploymentName}`, decision.reason);
|
|
}
|
|
cleanups.push(decision);
|
|
}
|
|
return cleanups;
|
|
}
|
|
|
|
async function executeDevTemplateJobReplacements(kubectl, replacements, blockers, concurrency = defaultCdConcurrency) {
|
|
const planned = replacements.filter((replacement) => replacement.replace && replacement.result === "planned");
|
|
await mapWithConcurrency(planned, concurrency, async (replacement) => {
|
|
const commandArgs = [
|
|
"-n",
|
|
replacement.namespace,
|
|
"delete",
|
|
"job",
|
|
replacement.jobName,
|
|
"--ignore-not-found=true"
|
|
];
|
|
const result = await kubectlResult(kubectl, commandArgs, 20000);
|
|
replacement.deleteCommand = kubectlCommand(kubectl, commandArgs);
|
|
replacement.deleteStdout = result.redactedStdout ?? result.stdout;
|
|
replacement.deleteStderr = result.redactedStderr ?? result.stderr;
|
|
if (!result.ok) {
|
|
replacement.result = "delete_failed";
|
|
replacement.reason = `Failed to delete live suspended template Job before apply: ${oneLine(commandOutput(result))}`;
|
|
addBlocker(blockers, "environment_blocker", `template-job-replace-${replacement.jobName}`, replacement.reason);
|
|
return;
|
|
}
|
|
replacement.result = "deleted_pending_recreate";
|
|
replacement.reason = "Live suspended template Job was deleted; kubectl apply must recreate it from the desired manifest.";
|
|
});
|
|
return planned.length;
|
|
}
|
|
|
|
async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, blockers, concurrency = defaultCdConcurrency) {
|
|
const planned = cleanups.filter((cleanup) => cleanup.cleanup && cleanup.result === "planned");
|
|
await mapWithConcurrency(planned, concurrency, async (cleanup) => {
|
|
const commandArgs = [
|
|
"-n",
|
|
cleanup.namespace,
|
|
"delete",
|
|
"deployment",
|
|
cleanup.deploymentName,
|
|
"--ignore-not-found=true"
|
|
];
|
|
const result = await kubectlResult(kubectl, commandArgs, 20000);
|
|
cleanup.deleteCommand = kubectlCommand(kubectl, commandArgs);
|
|
cleanup.deleteStdout = result.redactedStdout ?? result.stdout;
|
|
cleanup.deleteStderr = result.redactedStderr ?? result.stderr;
|
|
if (!result.ok) {
|
|
cleanup.result = "delete_failed";
|
|
cleanup.reason = `Failed to delete stale legacy simulator Deployment before apply: ${oneLine(commandOutput(result))}`;
|
|
addBlocker(blockers, "environment_blocker", `legacy-simulator-deployment-cleanup-${cleanup.deploymentName}`, cleanup.reason);
|
|
return;
|
|
}
|
|
cleanup.result = "deleted_pending_apply";
|
|
cleanup.reason = "Live stale legacy simulator Deployment was deleted; kubectl apply must leave the desired indexed StatefulSet in place.";
|
|
});
|
|
return planned.length;
|
|
}
|
|
|
|
async function executeDevUnmanagedHotfixDeploymentCleanups(kubectl, cleanups, blockers, concurrency = defaultCdConcurrency) {
|
|
const planned = cleanups.filter((cleanup) => cleanup.cleanup && cleanup.result === "planned");
|
|
await mapWithConcurrency(planned, concurrency, async (cleanup) => {
|
|
const commandArgs = [
|
|
"-n",
|
|
cleanup.namespace,
|
|
"delete",
|
|
"deployment",
|
|
cleanup.deploymentName,
|
|
"--ignore-not-found=true"
|
|
];
|
|
const result = await kubectlResult(kubectl, commandArgs, 20000);
|
|
cleanup.deleteCommand = kubectlCommand(kubectl, commandArgs);
|
|
cleanup.deleteStdout = result.redactedStdout ?? result.stdout;
|
|
cleanup.deleteStderr = result.redactedStderr ?? result.stderr;
|
|
if (!result.ok) {
|
|
cleanup.result = "delete_failed";
|
|
cleanup.reason = `Failed to delete live Deployment with unmanaged hotfix fields before apply: ${oneLine(commandOutput(result))}`;
|
|
addBlocker(blockers, "environment_blocker", `unmanaged-hotfix-deployment-cleanup-${cleanup.deploymentName}`, cleanup.reason);
|
|
return;
|
|
}
|
|
cleanup.result = "deleted_pending_apply";
|
|
cleanup.reason = "Live Deployment with unmanaged hotfix fields was deleted; kubectl apply must recreate it from desired source.";
|
|
});
|
|
return planned.length;
|
|
}
|
|
|
|
function rolloutApiName(kind) {
|
|
if (kind === "Deployment") return "deployment";
|
|
if (kind === "StatefulSet") return "statefulset";
|
|
return null;
|
|
}
|
|
|
|
function rolloutTargetsFromWorkloads(workloads) {
|
|
return listItems(workloads)
|
|
.map((item) => {
|
|
const apiName = rolloutApiName(item?.kind);
|
|
if (!apiName) return null;
|
|
return {
|
|
kind: item.kind,
|
|
apiName,
|
|
name: item?.metadata?.name ?? "unknown",
|
|
namespace: item?.metadata?.namespace ?? namespace,
|
|
replicas: replicaPlan(item),
|
|
serviceIds: uniqueStrings(containersFor(item).map((container) => serviceIdFor(item, container)))
|
|
};
|
|
})
|
|
.filter(Boolean);
|
|
}
|
|
|
|
async function verifyRolloutTarget(kubectl, target, blockers) {
|
|
const commandArgs = [
|
|
"-n",
|
|
target.namespace,
|
|
"rollout",
|
|
"status",
|
|
`${target.apiName}/${target.name}`,
|
|
"--timeout=180s"
|
|
];
|
|
const base = {
|
|
...target,
|
|
command: kubectlCommand(kubectl, commandArgs)
|
|
};
|
|
if (kubectl.status !== "ready") {
|
|
addBlocker(blockers, "environment_blocker", `rollout-status-${target.name}`, kubectl.reason);
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
reason: kubectl.reason
|
|
};
|
|
}
|
|
const result = await kubectlResult(kubectl, commandArgs, 190000);
|
|
if (!result.ok) {
|
|
const reason = oneLine(commandOutput(result));
|
|
addBlocker(blockers, "runtime_blocker", `rollout-status-${target.name}`, reason);
|
|
return {
|
|
...base,
|
|
status: "blocked",
|
|
reason,
|
|
stdout: result.redactedStdout ?? result.stdout,
|
|
stderr: result.redactedStderr ?? result.stderr
|
|
};
|
|
}
|
|
return {
|
|
...base,
|
|
status: "pass",
|
|
stdout: result.redactedStdout ?? result.stdout,
|
|
stderr: result.redactedStderr ?? result.stderr
|
|
};
|
|
}
|
|
|
|
async function verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep) {
|
|
const targets = rolloutTargetsFromWorkloads(workloads);
|
|
if (!args.apply) {
|
|
return {
|
|
status: "not_run",
|
|
reason: "dry-run mode",
|
|
concurrency: args.rolloutConcurrency,
|
|
targets
|
|
};
|
|
}
|
|
if (applyStep.status !== "pass") {
|
|
return {
|
|
status: "not_run",
|
|
reason: "apply did not complete",
|
|
concurrency: args.rolloutConcurrency,
|
|
targets
|
|
};
|
|
}
|
|
const results = await mapWithConcurrency(targets, args.rolloutConcurrency, async (target) =>
|
|
verifyRolloutTarget(kubectl, target, blockers)
|
|
);
|
|
return {
|
|
status: results.every((result) => result.status === "pass") ? "pass" : "blocked",
|
|
concurrency: args.rolloutConcurrency,
|
|
targetCount: targets.length,
|
|
results
|
|
};
|
|
}
|
|
|
|
function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) {
|
|
const plannedNames = replacementsNeeded.map((replacement) => replacement.jobName);
|
|
if (result.ok || plannedNames.length === 0) return false;
|
|
const redactedOutput = `${result.redactedStdout ?? result.stdout}\n${result.redactedStderr ?? result.stderr}`;
|
|
const effectiveOutput = redactedOutput.trim().length > 0 ? redactedOutput : `${result.stdout}\n${result.stderr}`;
|
|
const errorSections = effectiveOutput.split("Error from server").slice(1);
|
|
const immutableSections = errorSections.filter(
|
|
(section) => section.includes("spec.template") && section.includes("field is immutable")
|
|
);
|
|
if (immutableSections.length === 0 || immutableSections.length !== errorSections.length) return false;
|
|
for (const name of plannedNames) {
|
|
if (!immutableSections.some((section) => section.includes(name))) return false;
|
|
}
|
|
return immutableSections.every(
|
|
(section) =>
|
|
section.includes("Resource=jobs") &&
|
|
section.includes("Kind=Job") &&
|
|
plannedNames.some((name) => section.includes(name))
|
|
);
|
|
}
|
|
|
|
async function runApplyStep(
|
|
args,
|
|
kubectl,
|
|
blockers,
|
|
templateJobReplacements,
|
|
legacySimulatorDeploymentCleanups,
|
|
unmanagedHotfixDeploymentCleanups
|
|
) {
|
|
if (blockers.length > 0) {
|
|
return { status: "not_run", summary: "Skipped because preflight blockers are open" };
|
|
}
|
|
let mutationAttempted = false;
|
|
const replacementsNeeded = templateJobReplacements.filter((replacement) => replacement.replace);
|
|
|
|
if (args.apply) {
|
|
const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers, args.rolloutConcurrency);
|
|
const cleanupCount = await executeDevLegacySimulatorDeploymentCleanups(kubectl, legacySimulatorDeploymentCleanups, blockers, args.rolloutConcurrency);
|
|
const hotfixCleanupCount = await executeDevUnmanagedHotfixDeploymentCleanups(
|
|
kubectl,
|
|
unmanagedHotfixDeploymentCleanups,
|
|
blockers,
|
|
args.rolloutConcurrency
|
|
);
|
|
mutationAttempted = replaceCount > 0 || cleanupCount > 0 || hotfixCleanupCount > 0;
|
|
if (blockers.length > 0) {
|
|
return {
|
|
status: "blocked",
|
|
command: kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"]),
|
|
mutationAttempted,
|
|
summary: "Skipped kubectl apply because DEV pre-apply cleanup failed"
|
|
};
|
|
}
|
|
}
|
|
|
|
const commandArgs = args.apply
|
|
? ["apply", "-k", "deploy/k8s/dev"]
|
|
: ["apply", "--dry-run=server", "-k", "deploy/k8s/dev"];
|
|
const result = await kubectlResult(kubectl, commandArgs, 30000);
|
|
const expectedImmutableDryRun = !args.apply && isExpectedTemplateJobImmutableFailure(result, replacementsNeeded);
|
|
mutationAttempted = mutationAttempted || args.apply;
|
|
if (!result.ok && !expectedImmutableDryRun) {
|
|
addBlocker(blockers, "environment_blocker", args.apply ? "kubectl-apply" : "kubectl-dry-run", commandOutput(result));
|
|
}
|
|
for (const replacement of templateJobReplacements) {
|
|
if (replacement.result === "deleted_pending_recreate") {
|
|
replacement.result = result.ok ? "replaced" : "delete_succeeded_apply_failed";
|
|
replacement.reason = result.ok
|
|
? "Live suspended template Job was deleted and recreated by kubectl apply."
|
|
: "Live suspended template Job was deleted, but kubectl apply failed before confirming recreation.";
|
|
}
|
|
}
|
|
for (const cleanup of legacySimulatorDeploymentCleanups) {
|
|
if (cleanup.result === "deleted_pending_apply") {
|
|
cleanup.result = result.ok ? "deleted" : "delete_succeeded_apply_failed";
|
|
cleanup.reason = result.ok
|
|
? "Live stale legacy simulator Deployment was deleted and the desired indexed StatefulSet apply completed."
|
|
: "Live stale legacy simulator Deployment was deleted, but kubectl apply failed before confirming desired state.";
|
|
}
|
|
}
|
|
for (const cleanup of unmanagedHotfixDeploymentCleanups) {
|
|
if (cleanup.result === "deleted_pending_apply") {
|
|
cleanup.result = result.ok ? "deleted" : "delete_succeeded_apply_failed";
|
|
cleanup.reason = result.ok
|
|
? "Live Deployment with unmanaged hotfix fields was deleted and recreated by kubectl apply."
|
|
: "Live Deployment with unmanaged hotfix fields was deleted, but kubectl apply failed before confirming recreation.";
|
|
}
|
|
}
|
|
return {
|
|
status: result.ok || expectedImmutableDryRun ? "pass" : "blocked",
|
|
command: kubectlCommand(kubectl, commandArgs),
|
|
stdout: result.redactedStdout ?? result.stdout,
|
|
stderr: result.redactedStderr ?? result.stderr,
|
|
mutationAttempted,
|
|
expectedImmutableTemplateJobDryRun: expectedImmutableDryRun,
|
|
expectedImmutableTemplateJobs: expectedImmutableDryRun
|
|
? replacementsNeeded.map((replacement) => replacement.jobName)
|
|
: []
|
|
};
|
|
}
|
|
|
|
async function gitHeadCommit() {
|
|
const result = await commandResult("git", ["rev-parse", "--short=12", "HEAD"]);
|
|
return result.ok ? result.stdout : "unknown";
|
|
}
|
|
|
|
export function resolveApplySourceCommit(deploy, catalog, gitHeadCommitId) {
|
|
const deployCommit = deploy?.commitId;
|
|
const catalogCommit = catalog?.commitId;
|
|
if (
|
|
typeof deployCommit === "string" &&
|
|
typeof catalogCommit === "string" &&
|
|
commitMatchesSource(deployCommit, catalogCommit)
|
|
) {
|
|
return deployCommit.length >= catalogCommit.length ? deployCommit : catalogCommit;
|
|
}
|
|
return gitHeadCommitId;
|
|
}
|
|
|
|
export async function runDevDeployApply(argv, io = {}) {
|
|
const stdout = io.stdout ?? process.stdout;
|
|
const args = parseArgs(argv);
|
|
const blockers = [];
|
|
validateArgs(args, blockers);
|
|
if (args.apply) {
|
|
const transactionGuard = requireDevCdTransactionForSideEffect({
|
|
env: io.env ?? process.env,
|
|
script: "scripts/dev-deploy-apply.mjs",
|
|
mode: "--apply"
|
|
});
|
|
if (transactionGuard) {
|
|
stdout.write(`${JSON.stringify(transactionGuard, null, 2)}\n`);
|
|
return 2;
|
|
}
|
|
}
|
|
const gitHeadCommitIdPromise = gitHeadCommit();
|
|
|
|
const [deploy, catalog, devKustomization, namespaceDoc, workloads, services, healthContract] = await Promise.all([
|
|
readJson("deploy/deploy.json", blockers),
|
|
readJson("deploy/artifact-catalog.dev.json", blockers),
|
|
readJson("deploy/k8s/dev/kustomization.yaml", blockers),
|
|
readJson("deploy/k8s/base/namespace.yaml", blockers),
|
|
readJson("deploy/k8s/base/workloads.yaml", blockers),
|
|
readJson("deploy/k8s/base/services.yaml", blockers),
|
|
readJson("deploy/k8s/dev/health-contract.yaml", blockers)
|
|
]);
|
|
const gitHeadCommitId = await gitHeadCommitIdPromise;
|
|
const commitId = resolveApplySourceCommit(deploy, catalog, gitHeadCommitId);
|
|
|
|
const artifactEvidence = validateDeployAndCatalog(deploy, catalog, commitId, blockers);
|
|
const registryManifests = await verifyCatalogRegistryManifests(catalog, blockers, args.rolloutConcurrency, {
|
|
env: io.env ?? process.env,
|
|
registryManifestBaseUrl: args.registryManifestBaseUrl
|
|
});
|
|
const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers);
|
|
const cloudApiDb = await checkCloudApiDb(deploy, workloads, services, blockers);
|
|
const codeAgentProviderDesiredState = inspectCodeAgentProviderDesiredState(deploy, workloads);
|
|
addCodeAgentProviderDesiredStateBlocker(blockers, codeAgentProviderDesiredState);
|
|
if (!(await readText("internal/cloud/server.mjs")).includes('url.pathname === "/health/live"')) {
|
|
addBlocker(blockers, "contract_blocker", "cloud-api-health-path", "cloud-api does not implement the k8s /health/live path");
|
|
}
|
|
|
|
const kubectl = await resolveD601Kubectl(args);
|
|
const clusterObservation = await observeCluster(kubectl, blockers);
|
|
const cloudWebRolloutBeforeApply = await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
|
|
observationPhase: "before_apply",
|
|
blockRuntimeEnvDrift: false
|
|
});
|
|
const codeAgentProviderLiveBeforeApply = await observeCodeAgentProviderLiveDeployment(kubectl, blockers, {
|
|
phase: "before_apply",
|
|
blockOnMismatch: !args.apply
|
|
});
|
|
const liveProbe = args.skipLiveProbe ? { status: "not_run", reason: "--skip-live-probe" } : await probeDevEndpoint(blockers);
|
|
const templateJobReplacements = await buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers);
|
|
const legacySimulatorDeploymentCleanups = await buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads, blockers);
|
|
const unmanagedHotfixDeploymentCleanups = await buildDevUnmanagedHotfixDeploymentCleanupPlan(kubectl, workloads, blockers);
|
|
const applyStep = await runApplyStep(
|
|
args,
|
|
kubectl,
|
|
blockers,
|
|
templateJobReplacements,
|
|
legacySimulatorDeploymentCleanups,
|
|
unmanagedHotfixDeploymentCleanups
|
|
);
|
|
let rolloutStatusAfterApply;
|
|
let cloudWebRolloutAfterApply;
|
|
let codeAgentProviderLiveAfterApply;
|
|
if (args.apply && applyStep.status === "pass") {
|
|
[rolloutStatusAfterApply, cloudWebRolloutAfterApply, codeAgentProviderLiveAfterApply] = await Promise.all([
|
|
verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep),
|
|
observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
|
|
observationPhase: "after_apply",
|
|
blockRuntimeEnvDrift: true
|
|
}),
|
|
observeCodeAgentProviderLiveDeployment(kubectl, blockers, {
|
|
phase: "after_apply",
|
|
blockOnMismatch: true
|
|
})
|
|
]);
|
|
} else {
|
|
rolloutStatusAfterApply = await verifyRolloutsAfterApply(args, kubectl, workloads, blockers, applyStep);
|
|
cloudWebRolloutAfterApply = skippedDeploymentRolloutObservation(
|
|
kubectl,
|
|
deploy,
|
|
catalog,
|
|
"hwlab-cloud-web",
|
|
"after_apply",
|
|
args.apply ? "apply did not complete successfully; post-apply rollout env verification was not run" : "dry-run mode; no post-apply rollout env verification"
|
|
);
|
|
codeAgentProviderLiveAfterApply = {
|
|
phase: "after_apply",
|
|
status: "not_run",
|
|
reason: args.apply ? "apply did not complete" : "dry-run mode",
|
|
secretValuesRead: false,
|
|
kubernetesSecretDataRead: false,
|
|
valuesRedacted: true
|
|
};
|
|
}
|
|
const cloudWebRollout = args.apply ? cloudWebRolloutAfterApply : cloudWebRolloutBeforeApply;
|
|
const status = blockers.length > 0 ? "blocked" : "pass";
|
|
const artifactPlan = buildArtifactPlan(deploy, catalog, commitId, artifactEvidence);
|
|
const workloadPlan = buildWorkloadPlan(workloads);
|
|
const servicePlan = buildServicePlan(services);
|
|
const planConclusion = buildPlanConclusion(status, blockers);
|
|
const manualCommands = buildManualCommands(status);
|
|
const remainingBlockers = buildRemainingBlockers(blockers);
|
|
|
|
const report = {
|
|
$schema: "https://hwlab.pikastech.local/schemas/dev-gate-report.schema.json",
|
|
$id: "https://hwlab.pikastech.local/dev-cd/dev-deploy-report.json",
|
|
reportVersion: "v1",
|
|
issue: "pikasTech/HWLAB#33",
|
|
supports: ["pikasTech/HWLAB#164", "pikasTech/HWLAB#143", "pikasTech/HWLAB#99", "pikasTech/HWLAB#63", "pikasTech/HWLAB#50", "pikasTech/HWLAB#7", "pikasTech/HWLAB#33", "pikasTech/HWLAB#17", "pikasTech/HWLAB#32", "pikasTech/HWLAB#30"],
|
|
taskId: "dev-deploy-apply",
|
|
commitId,
|
|
acceptanceLevel: "dev_deploy_apply",
|
|
devOnly: true,
|
|
prodDisabled: true,
|
|
reportLifecycle: activeReportLifecycle("Current DEV deploy apply report; active public API/health endpoint is 16667 and browser endpoint is 16666."),
|
|
status,
|
|
generatedAt: new Date().toISOString(),
|
|
namespace,
|
|
endpoint: DEV_ENDPOINT,
|
|
sourceContract: {
|
|
status: "pass",
|
|
documents: [
|
|
"docs/dev-acceptance-matrix.md",
|
|
"docs/m0-contract-audit.md",
|
|
"docs/dev-deploy-apply.md",
|
|
"docs/artifact-catalog.md",
|
|
"deploy/README.md"
|
|
],
|
|
summary: "DEV deploy apply is pinned to hwlab-dev and the frozen HWLAB service IDs"
|
|
},
|
|
validationCommands: requiredValidationCommands,
|
|
localSmoke: {
|
|
status: "not_run",
|
|
commands: ["node scripts/m1-contract-smoke.mjs"],
|
|
evidence: ["This deploy apply task did not run local smoke because artifact/executor blockers stop promotion first."],
|
|
summary: "Local smoke remains available but is not a substitute for DEV artifact publish evidence."
|
|
},
|
|
dryRun: {
|
|
status,
|
|
commands: [
|
|
"node tools/hwlab-cli/bin/hwlab-cli.mjs test e2e --env dev --mvp --dry-run",
|
|
"node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked"
|
|
],
|
|
evidence: [
|
|
`artifact services checked: ${artifactEvidence.length}`,
|
|
`expected artifact commit: ${artifactPlan.expectedArtifactCommit}`,
|
|
`registry manifests: ${registryManifests.status} (${registryManifests.serviceCount} services)`,
|
|
`namespace: ${namespace}`,
|
|
`workloads planned: ${workloadPlan.length}`,
|
|
`kubectl executor: ${kubectl.executor ?? "missing"}`,
|
|
`kubectl kubeconfig source: ${kubectl.kubeconfigSource}`,
|
|
`kubectl kubeconfig: ${kubectl.kubeconfig}`,
|
|
`live health: ${liveProbe.status}`,
|
|
`code agent provider desired-state: ${codeAgentProviderDesiredState.status}`,
|
|
`code agent provider live env before apply: ${codeAgentProviderLiveBeforeApply.status}`,
|
|
`rollout status concurrency: ${args.rolloutConcurrency}`,
|
|
`template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}`,
|
|
`legacy simulator deployment cleanups: ${legacySimulatorDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}`,
|
|
`unmanaged hotfix deployment cleanups: ${unmanagedHotfixDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}`
|
|
],
|
|
summary: status === "blocked" ? "DEV apply is blocked before mutation." : "DEV dry-run preflight passed."
|
|
},
|
|
devPreconditions: {
|
|
status,
|
|
requirements: [
|
|
"DEV artifact catalog must contain CI publish expectations and registry digests; CD must verify those tag/digest pairs directly against registry manifests",
|
|
"kubectl must be available for D601 hwlab-dev and must not target PROD",
|
|
"hwlab-cloud-api /health/live must report serviceId, dev environment, and DB env readiness without exposing secret values",
|
|
"hwlab-cloud-api must declare and preserve OPENAI_API_KEY from hwlab-code-agent-provider/openai-api-key plus the DEV Code Agent egress proxy env without reading Secret values",
|
|
"pods, services, configmaps, and image commit tags must be observable in hwlab-dev"
|
|
],
|
|
summary: status === "blocked" ? "One or more required DEV deploy preconditions are blocked." : "DEV deploy preconditions are satisfied."
|
|
},
|
|
devDeployApply: {
|
|
mode: args.apply ? "apply" : "dry-run",
|
|
conclusion: planConclusion,
|
|
applyBoundary: buildApplyBoundary(args, status, applyStep, kubectl),
|
|
mutationAttempted: applyStep.mutationAttempted === true && args.apply,
|
|
target: {
|
|
environment: ENVIRONMENT_DEV,
|
|
namespace,
|
|
endpoint: DEV_ENDPOINT,
|
|
prodDisabled: true
|
|
},
|
|
artifactPlan,
|
|
workloadPlan,
|
|
servicePlan,
|
|
templateJobReplacementPolicy: devTemplateJobReplacementPolicy,
|
|
templateJobReplacements,
|
|
legacySimulatorDeploymentCleanupPolicy: devLegacySimulatorDeploymentCleanupPolicy,
|
|
legacySimulatorDeploymentCleanups,
|
|
unmanagedHotfixDeploymentCleanupPolicy: devUnmanagedHotfixDeploymentCleanupPolicy,
|
|
unmanagedHotfixDeploymentCleanups,
|
|
cloudWebRollout,
|
|
cloudWebRolloutBeforeApply,
|
|
cloudWebRolloutAfterApply,
|
|
rolloutStatusAfterApply,
|
|
codeAgentProvider: {
|
|
desiredState: codeAgentProviderDesiredState,
|
|
liveBeforeApply: codeAgentProviderLiveBeforeApply,
|
|
liveAfterApply: codeAgentProviderLiveAfterApply,
|
|
validationGate: "no-secret provider env/secretKeyRef preservation",
|
|
providerCallAttempted: false,
|
|
secretValuesRead: false,
|
|
kubernetesSecretDataRead: false,
|
|
valuesRedacted: true
|
|
},
|
|
applyStep,
|
|
manualCommands,
|
|
rollbackHint: buildRollbackHint(kubectl, workloads),
|
|
remainingBlockers,
|
|
artifactEvidence,
|
|
registryManifests,
|
|
cloudApiDb,
|
|
k8sManifest,
|
|
clusterObservation,
|
|
liveProbe
|
|
},
|
|
blockers,
|
|
notes: "DEV-only result. No PROD resources, secret values, force push, heavy e2e, browser e2e, or UniDesk runtime substitute were used."
|
|
};
|
|
|
|
if (args.writeReport) {
|
|
const absoluteReportPath = ensureNotRepoReportsPath(repoRoot, args.reportPath, "--report");
|
|
await mkdir(path.dirname(absoluteReportPath), { recursive: true });
|
|
await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
}
|
|
|
|
stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
return status === "blocked" && !args.expectBlocked ? 2 : 0;
|
|
}
|
|
|
|
export function formatFailure(error) {
|
|
return {
|
|
reportVersion: "v1",
|
|
issue: "pikasTech/HWLAB#33",
|
|
taskId: "dev-deploy-apply",
|
|
status: "failed",
|
|
blockers: [{ type: "observability_blocker", scope: "script", status: "open", summary: oneLine(error.message) }]
|
|
};
|
|
}
|