|
|
|
@@ -22,7 +22,9 @@ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../
|
|
|
|
|
const reportPath = "reports/dev-gate/dev-deploy-report.json";
|
|
|
|
|
const namespace = "hwlab-dev";
|
|
|
|
|
const healthPath = "/health/live";
|
|
|
|
|
const d601KubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
|
|
|
|
|
const defaultD601KubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
|
|
|
|
|
const devKubeconfigEnvName = "HWLAB_DEV_KUBECONFIG";
|
|
|
|
|
const kubeconfigEnvName = "KUBECONFIG";
|
|
|
|
|
const devTemplateJobReplacementPolicy = {
|
|
|
|
|
status: "active",
|
|
|
|
|
namespace,
|
|
|
|
@@ -63,14 +65,48 @@ const forbiddenActions = [
|
|
|
|
|
];
|
|
|
|
|
const runtimeIdentityEnvNames = ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"];
|
|
|
|
|
|
|
|
|
|
function parseArgs(argv) {
|
|
|
|
|
const flags = new Set(argv.filter((arg) => arg.startsWith("--")));
|
|
|
|
|
export function parseArgs(argv) {
|
|
|
|
|
const flags = new Set();
|
|
|
|
|
const errors = [];
|
|
|
|
|
let kubeconfig = null;
|
|
|
|
|
let kubeconfigSpecified = false;
|
|
|
|
|
|
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
|
|
|
const arg = argv[index];
|
|
|
|
|
if (!arg.startsWith("--")) continue;
|
|
|
|
|
if (arg === "--kubeconfig") {
|
|
|
|
|
flags.add("--kubeconfig");
|
|
|
|
|
kubeconfigSpecified = true;
|
|
|
|
|
const next = argv[index + 1];
|
|
|
|
|
if (typeof next !== "string" || next.startsWith("--")) {
|
|
|
|
|
errors.push("--kubeconfig requires a non-empty path value");
|
|
|
|
|
} else {
|
|
|
|
|
kubeconfig = next;
|
|
|
|
|
index += 1;
|
|
|
|
|
}
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
if (arg.startsWith("--kubeconfig=")) {
|
|
|
|
|
flags.add("--kubeconfig");
|
|
|
|
|
kubeconfigSpecified = true;
|
|
|
|
|
kubeconfig = arg.slice("--kubeconfig=".length);
|
|
|
|
|
if (!kubeconfig.trim()) {
|
|
|
|
|
errors.push("--kubeconfig requires a non-empty path value");
|
|
|
|
|
}
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
|
flags.add(arg);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return {
|
|
|
|
|
apply: flags.has("--apply"),
|
|
|
|
|
dryRun: !flags.has("--apply"),
|
|
|
|
|
expectBlocked: flags.has("--expect-blocked"),
|
|
|
|
|
skipLiveProbe: flags.has("--skip-live-probe"),
|
|
|
|
|
writeReport: flags.has("--write-report"),
|
|
|
|
|
kubeconfig,
|
|
|
|
|
kubeconfigSpecified,
|
|
|
|
|
errors,
|
|
|
|
|
flags
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
@@ -140,16 +176,71 @@ async function findExecutable(name) {
|
|
|
|
|
return null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function resolveD601Kubectl() {
|
|
|
|
|
function isNonEmpty(value) {
|
|
|
|
|
return typeof value === "string" && value.trim().length > 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function resolveDevKubeconfigSelection({ flagValue = null, env = process.env } = {}) {
|
|
|
|
|
const candidates = [
|
|
|
|
|
{ source: "flag:--kubeconfig", value: flagValue },
|
|
|
|
|
{ source: `env:${devKubeconfigEnvName}`, value: env?.[devKubeconfigEnvName] },
|
|
|
|
|
{ source: `env:${kubeconfigEnvName}`, value: env?.[kubeconfigEnvName] },
|
|
|
|
|
{ source: "default:d601-k3s", value: defaultD601KubeconfigPath }
|
|
|
|
|
];
|
|
|
|
|
const selected = candidates.find((candidate) => isNonEmpty(candidate.value));
|
|
|
|
|
const kubeconfig = selected.value.trim();
|
|
|
|
|
return {
|
|
|
|
|
kubeconfig,
|
|
|
|
|
source: selected.source,
|
|
|
|
|
paths: kubeconfig.split(path.delimiter).filter((item) => item.length > 0)
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function shellQuote(value) {
|
|
|
|
|
if (/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)) return value;
|
|
|
|
|
return `'${String(value).replaceAll("'", "'\\''")}'`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function buildKubectlCommandPrefix(kubeconfig) {
|
|
|
|
|
return `KUBECONFIG=${shellQuote(kubeconfig)} kubectl`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function formatKubeconfigAccessFailure(selection, unreadablePath) {
|
|
|
|
|
return `DEV kubeconfig from ${selection.source} is not readable at ${unreadablePath}; set --kubeconfig, ${devKubeconfigEnvName}, or ${kubeconfigEnvName} to a readable DEV kubeconfig path. Kubeconfig contents and Secret values are never printed.`;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
export function redactSensitiveText(value) {
|
|
|
|
|
return String(value)
|
|
|
|
|
.replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]+/giu, "$1<redacted>")
|
|
|
|
|
.replace(/\b(postgres(?:ql)?:\/\/[^:\s/@]+:)([^@\s]+)(@)/giu, "$1<redacted>$3")
|
|
|
|
|
.replace(/\b(token|password|client-key-data|client-certificate-data|certificate-authority-data)\b(\s*[:=]\s*)(["']?)([^"'\s,}]+)/giu, "$1$2$3<redacted>");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function redactCommandResult(result) {
|
|
|
|
|
return {
|
|
|
|
|
...result,
|
|
|
|
|
redactedStdout: redactSensitiveText(result.stdout ?? ""),
|
|
|
|
|
redactedStderr: redactSensitiveText(result.stderr ?? "")
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function commandOutput(result) {
|
|
|
|
|
return result.redactedStderr || result.redactedStdout || result.stderr || result.stdout || "";
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
async function resolveD601Kubectl(args = {}, env = process.env) {
|
|
|
|
|
const kubeconfigSelection = resolveDevKubeconfigSelection({ flagValue: args.kubeconfig, env });
|
|
|
|
|
const kubectlPath = await findExecutable("kubectl");
|
|
|
|
|
const commandPrefix = `KUBECONFIG=${d601KubeconfigPath} kubectl`;
|
|
|
|
|
const commandPrefix = buildKubectlCommandPrefix(kubeconfigSelection.kubeconfig);
|
|
|
|
|
const base = {
|
|
|
|
|
executor: kubectlPath,
|
|
|
|
|
kubeconfig: d601KubeconfigPath,
|
|
|
|
|
kubeconfig: kubeconfigSelection.kubeconfig,
|
|
|
|
|
kubeconfigSource: kubeconfigSelection.source,
|
|
|
|
|
kubeconfigPaths: kubeconfigSelection.paths,
|
|
|
|
|
commandPrefix,
|
|
|
|
|
env: {
|
|
|
|
|
...process.env,
|
|
|
|
|
KUBECONFIG: d601KubeconfigPath
|
|
|
|
|
...env,
|
|
|
|
|
KUBECONFIG: kubeconfigSelection.kubeconfig
|
|
|
|
|
}
|
|
|
|
|
};
|
|
|
|
|
|
|
|
|
@@ -162,12 +253,27 @@ async function resolveD601Kubectl() {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
try {
|
|
|
|
|
await access(d601KubeconfigPath, fsConstants.R_OK);
|
|
|
|
|
} catch {
|
|
|
|
|
let readablePathCount = 0;
|
|
|
|
|
let firstUnreadablePath = null;
|
|
|
|
|
for (const kubeconfigPath of kubeconfigSelection.paths) {
|
|
|
|
|
try {
|
|
|
|
|
await access(kubeconfigPath, fsConstants.R_OK);
|
|
|
|
|
readablePathCount += 1;
|
|
|
|
|
} catch (error) {
|
|
|
|
|
firstUnreadablePath ??= error?.path ?? kubeconfigPath;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
if (readablePathCount === 0) {
|
|
|
|
|
throw Object.assign(new Error("no readable kubeconfig paths"), {
|
|
|
|
|
path: firstUnreadablePath ?? kubeconfigSelection.kubeconfig
|
|
|
|
|
});
|
|
|
|
|
}
|
|
|
|
|
} catch (error) {
|
|
|
|
|
const unreadablePath = error?.path ?? kubeconfigSelection.kubeconfig;
|
|
|
|
|
return {
|
|
|
|
|
...base,
|
|
|
|
|
status: "blocked",
|
|
|
|
|
reason: `D601 k3s kubeconfig is not readable at ${d601KubeconfigPath}`
|
|
|
|
|
reason: formatKubeconfigAccessFailure(kubeconfigSelection, unreadablePath)
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -192,10 +298,11 @@ async function kubectlResult(kubectl, args, timeoutMs = 15000, options = {}) {
|
|
|
|
|
blocked: true
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
return commandResult(kubectl.executor, args, timeoutMs, {
|
|
|
|
|
const result = await commandResult(kubectl.executor, args, timeoutMs, {
|
|
|
|
|
...options,
|
|
|
|
|
env: kubectl.env
|
|
|
|
|
});
|
|
|
|
|
return redactCommandResult(result);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function listItems(document) {
|
|
|
|
@@ -470,8 +577,8 @@ function containerImageForDeployment(deployment, containerName) {
|
|
|
|
|
return containers.find((container) => container.name === containerName)?.image ?? null;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function buildRolloutVerificationCommand(serviceId) {
|
|
|
|
|
return `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout status deployment/${serviceId} --timeout=180s`;
|
|
|
|
|
function buildRolloutVerificationCommand(kubectl, serviceId) {
|
|
|
|
|
return kubectlCommand(kubectl, ["-n", namespace, "rollout", "status", `deployment/${serviceId}`, "--timeout=180s"]);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase) {
|
|
|
|
@@ -495,9 +602,10 @@ function buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observa
|
|
|
|
|
liveRuntimeEnv: null,
|
|
|
|
|
runtimeEnvMatchesDesired: false,
|
|
|
|
|
runtimeEnvDrift: [],
|
|
|
|
|
verificationCommand: buildRolloutVerificationCommand(serviceId),
|
|
|
|
|
verificationCommand: buildRolloutVerificationCommand(kubectl, serviceId),
|
|
|
|
|
readCommand: kubectlCommand(kubectl, commandArgs),
|
|
|
|
|
kubeconfig: kubectl.kubeconfig
|
|
|
|
|
kubeconfig: kubectl.kubeconfig,
|
|
|
|
|
kubeconfigSource: kubectl.kubeconfigSource
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
@@ -526,7 +634,7 @@ async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blo
|
|
|
|
|
|
|
|
|
|
const result = await kubectlResult(kubectl, commandArgs, 15000);
|
|
|
|
|
if (!result.ok) {
|
|
|
|
|
const summary = oneLine(result.stderr || result.stdout);
|
|
|
|
|
const summary = oneLine(commandOutput(result));
|
|
|
|
|
addBlocker(blockers, "environment_blocker", `rollout-read-${serviceId}`, `Cannot read DEV Deployment ${serviceId} rollout revision: ${summary}`);
|
|
|
|
|
return {
|
|
|
|
|
...base,
|
|
|
|
@@ -579,7 +687,7 @@ function blockerHint(blocker) {
|
|
|
|
|
return "Regenerate deploy/deploy.json and deploy/artifact-catalog.dev.json for the source commit that will be promoted.";
|
|
|
|
|
}
|
|
|
|
|
if (blocker.scope === "kubectl" || blocker.scope === "d601-kubeconfig" || blocker.scope === "kubectl-version") {
|
|
|
|
|
return `Provide kubectl with readable D601 k3s kubeconfig at ${d601KubeconfigPath} without exposing token or secret values.`;
|
|
|
|
|
return `Provide kubectl with a readable DEV kubeconfig through --kubeconfig, ${devKubeconfigEnvName}, ${kubeconfigEnvName}, or the D601 default path without exposing token or secret values.`;
|
|
|
|
|
}
|
|
|
|
|
if (blocker.scope === "cluster-read") {
|
|
|
|
|
return "Restore read-only visibility for pods, services, configmaps, deployments, and jobs in hwlab-dev.";
|
|
|
|
@@ -618,7 +726,7 @@ function buildRemainingBlockers(blockers) {
|
|
|
|
|
}));
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function buildRollbackHint(workloads) {
|
|
|
|
|
function buildRollbackHint(kubectl, workloads) {
|
|
|
|
|
const deploymentNames = buildWorkloadPlan(workloads)
|
|
|
|
|
.filter((workload) => workload.kind === "Deployment")
|
|
|
|
|
.map((workload) => workload.name);
|
|
|
|
@@ -630,13 +738,13 @@ function buildRollbackHint(workloads) {
|
|
|
|
|
namespace,
|
|
|
|
|
strategy: "DEV-only Kubernetes rollback using deployment revision history; do not touch PROD or UniDesk services.",
|
|
|
|
|
captureBeforeApply: [
|
|
|
|
|
`KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev get pods,services,configmaps,deployments,jobs -o wide`,
|
|
|
|
|
`KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev get deployments -o json`
|
|
|
|
|
kubectlCommand(kubectl, ["-n", namespace, "get", "pods,services,configmaps,deployments,jobs", "-o", "wide"]),
|
|
|
|
|
kubectlCommand(kubectl, ["-n", namespace, "get", "deployments", "-o", "json"])
|
|
|
|
|
],
|
|
|
|
|
deploymentRollbackCommands: deploymentNames.map((name) => `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout undo deployment/${name}`),
|
|
|
|
|
jobCleanupCommands: jobNames.map((name) => `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev delete job ${name} --ignore-not-found`),
|
|
|
|
|
deploymentRollbackCommands: deploymentNames.map((name) => kubectlCommand(kubectl, ["-n", namespace, "rollout", "undo", `deployment/${name}`])),
|
|
|
|
|
jobCleanupCommands: jobNames.map((name) => kubectlCommand(kubectl, ["-n", namespace, "delete", "job", name, "--ignore-not-found"])),
|
|
|
|
|
postRollbackChecks: [
|
|
|
|
|
`KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout status deployment/hwlab-cloud-api --timeout=120s`,
|
|
|
|
|
kubectlCommand(kubectl, ["-n", namespace, "rollout", "status", "deployment/hwlab-cloud-api", "--timeout=120s"]),
|
|
|
|
|
`curl -fsS ${new URL(healthPath, DEV_ENDPOINT).href}`
|
|
|
|
|
]
|
|
|
|
|
};
|
|
|
|
@@ -786,15 +894,16 @@ function buildManualCommands(status) {
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function buildApplyBoundary(args, status, applyStep) {
|
|
|
|
|
function buildApplyBoundary(args, status, applyStep, kubectl) {
|
|
|
|
|
return {
|
|
|
|
|
currentMode: args.apply ? "apply-requested" : "dry-run",
|
|
|
|
|
defaultNoWrite: !args.apply,
|
|
|
|
|
mutationAttempted: applyStep.mutationAttempted === true && args.apply,
|
|
|
|
|
mutationAllowed: status === "pass" && args.apply,
|
|
|
|
|
applyRequiresFlags: ["--apply", "--confirm-dev", "--confirmed-non-production"],
|
|
|
|
|
writeScope: `KUBECONFIG=${d601KubeconfigPath} kubectl apply -k deploy/k8s/dev, namespace hwlab-dev only; delete/recreate is allowed only for explicitly allowlisted suspended DEV template Jobs`,
|
|
|
|
|
noWriteScope: `source/catalog/k8s validation, optional DEV health probe, KUBECONFIG=${d601KubeconfigPath} kubectl read, replacement planning, and server-side dry-run only`,
|
|
|
|
|
kubeconfigSource: kubectl.kubeconfigSource,
|
|
|
|
|
writeScope: `${kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"])}, namespace hwlab-dev only; delete/recreate is allowed only for explicitly allowlisted suspended DEV template Jobs`,
|
|
|
|
|
noWriteScope: `source/catalog/k8s validation, optional DEV health probe, ${kubectl.commandPrefix} read, replacement planning, and server-side dry-run only`,
|
|
|
|
|
forbiddenActions
|
|
|
|
|
};
|
|
|
|
|
}
|
|
|
|
@@ -811,6 +920,9 @@ function buildPlanConclusion(status, blockers) {
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
function validateArgs(args, blockers) {
|
|
|
|
|
for (const error of args.errors ?? []) {
|
|
|
|
|
addBlocker(blockers, "safety_blocker", "kubeconfig-argument", error);
|
|
|
|
|
}
|
|
|
|
|
for (const forbidden of ["--prod", "--production", "--read-secret", "--force-push", "--heavyweight-e2e"]) {
|
|
|
|
|
if (args.flags.has(forbidden)) {
|
|
|
|
|
addBlocker(blockers, "safety_blocker", forbidden.slice(2), `${forbidden} is forbidden for DEV deploy apply`);
|
|
|
|
@@ -1172,6 +1284,7 @@ async function observeCluster(kubectl, blockers) {
|
|
|
|
|
status: "blocked",
|
|
|
|
|
executor: kubectl.executor ?? "missing",
|
|
|
|
|
kubeconfig: kubectl.kubeconfig,
|
|
|
|
|
kubeconfigSource: kubectl.kubeconfigSource,
|
|
|
|
|
commandPrefix: kubectl.commandPrefix,
|
|
|
|
|
resources: []
|
|
|
|
|
};
|
|
|
|
@@ -1199,6 +1312,7 @@ async function observeCluster(kubectl, blockers) {
|
|
|
|
|
status: "pass",
|
|
|
|
|
executor: kubectl.executor,
|
|
|
|
|
kubeconfig: kubectl.kubeconfig,
|
|
|
|
|
kubeconfigSource: kubectl.kubeconfigSource,
|
|
|
|
|
commandPrefix: kubectl.commandPrefix,
|
|
|
|
|
readCommand: kubectlCommand(kubectl, getArgs),
|
|
|
|
|
version: version.stdout,
|
|
|
|
@@ -1222,7 +1336,7 @@ async function readLiveTemplateJob(kubectl, jobName, blockers) {
|
|
|
|
|
}
|
|
|
|
|
const get = await kubectlResult(kubectl, ["-n", namespace, "get", "job", jobName, "-o", "json"], 15000);
|
|
|
|
|
if (!get.ok) {
|
|
|
|
|
const output = get.stderr || get.stdout;
|
|
|
|
|
const output = commandOutput(get);
|
|
|
|
|
if (isNotFoundOutput(output)) {
|
|
|
|
|
return { status: "not_found", liveJob: null };
|
|
|
|
|
}
|
|
|
|
@@ -1310,11 +1424,11 @@ async function executeDevTemplateJobReplacements(kubectl, replacements, blockers
|
|
|
|
|
];
|
|
|
|
|
const result = await kubectlResult(kubectl, commandArgs, 20000);
|
|
|
|
|
replacement.deleteCommand = kubectlCommand(kubectl, commandArgs);
|
|
|
|
|
replacement.deleteStdout = result.stdout;
|
|
|
|
|
replacement.deleteStderr = result.stderr;
|
|
|
|
|
replacement.deleteStdout = result.redactedStdout ?? result.stdout;
|
|
|
|
|
replacement.deleteStderr = result.redactedStderr ?? result.stderr;
|
|
|
|
|
if (!result.ok) {
|
|
|
|
|
replacement.result = "delete_failed";
|
|
|
|
|
replacement.reason = `Failed to delete live suspended template Job before apply: ${oneLine(result.stderr || result.stdout)}`;
|
|
|
|
|
replacement.reason = `Failed to delete live suspended template Job before apply: ${oneLine(commandOutput(result))}`;
|
|
|
|
|
addBlocker(blockers, "environment_blocker", `template-job-replace-${replacement.jobName}`, replacement.reason);
|
|
|
|
|
continue;
|
|
|
|
|
}
|
|
|
|
@@ -1327,8 +1441,9 @@ async function executeDevTemplateJobReplacements(kubectl, replacements, blockers
|
|
|
|
|
function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) {
|
|
|
|
|
const plannedNames = replacementsNeeded.map((replacement) => replacement.jobName);
|
|
|
|
|
if (result.ok || plannedNames.length === 0) return false;
|
|
|
|
|
const output = `${result.stdout}\n${result.stderr}`;
|
|
|
|
|
const errorSections = output.split("Error from server").slice(1);
|
|
|
|
|
const redactedOutput = `${result.redactedStdout ?? result.stdout}\n${result.redactedStderr ?? result.stderr}`;
|
|
|
|
|
const effectiveOutput = redactedOutput.trim().length > 0 ? redactedOutput : `${result.stdout}\n${result.stderr}`;
|
|
|
|
|
const errorSections = effectiveOutput.split("Error from server").slice(1);
|
|
|
|
|
const immutableSections = errorSections.filter(
|
|
|
|
|
(section) => section.includes("spec.template") && section.includes("field is immutable")
|
|
|
|
|
);
|
|
|
|
@@ -1371,7 +1486,7 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements) {
|
|
|
|
|
const expectedImmutableDryRun = !args.apply && isExpectedTemplateJobImmutableFailure(result, replacementsNeeded);
|
|
|
|
|
mutationAttempted = mutationAttempted || args.apply;
|
|
|
|
|
if (!result.ok && !expectedImmutableDryRun) {
|
|
|
|
|
addBlocker(blockers, "environment_blocker", args.apply ? "kubectl-apply" : "kubectl-dry-run", result.stderr || result.stdout);
|
|
|
|
|
addBlocker(blockers, "environment_blocker", args.apply ? "kubectl-apply" : "kubectl-dry-run", commandOutput(result));
|
|
|
|
|
}
|
|
|
|
|
for (const replacement of templateJobReplacements) {
|
|
|
|
|
if (replacement.result === "deleted_pending_recreate") {
|
|
|
|
@@ -1384,8 +1499,8 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements) {
|
|
|
|
|
return {
|
|
|
|
|
status: result.ok || expectedImmutableDryRun ? "pass" : "blocked",
|
|
|
|
|
command: kubectlCommand(kubectl, commandArgs),
|
|
|
|
|
stdout: result.stdout,
|
|
|
|
|
stderr: result.stderr,
|
|
|
|
|
stdout: result.redactedStdout ?? result.stdout,
|
|
|
|
|
stderr: result.redactedStderr ?? result.stderr,
|
|
|
|
|
mutationAttempted,
|
|
|
|
|
expectedImmutableTemplateJobDryRun: expectedImmutableDryRun,
|
|
|
|
|
expectedImmutableTemplateJobs: expectedImmutableDryRun
|
|
|
|
@@ -1440,7 +1555,7 @@ export async function runDevDeployApply(argv, io = {}) {
|
|
|
|
|
addBlocker(blockers, "contract_blocker", "cloud-api-health-path", "cloud-api does not implement the k8s /health/live path");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
const kubectl = await resolveD601Kubectl();
|
|
|
|
|
const kubectl = await resolveD601Kubectl(args);
|
|
|
|
|
const clusterObservation = await observeCluster(kubectl, blockers);
|
|
|
|
|
const cloudWebRolloutBeforeApply = await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
|
|
|
|
|
observationPhase: "before_apply",
|
|
|
|
@@ -1539,6 +1654,7 @@ export async function runDevDeployApply(argv, io = {}) {
|
|
|
|
|
`namespace: ${namespace}`,
|
|
|
|
|
`workloads planned: ${workloadPlan.length}`,
|
|
|
|
|
`kubectl executor: ${kubectl.executor ?? "missing"}`,
|
|
|
|
|
`kubectl kubeconfig source: ${kubectl.kubeconfigSource}`,
|
|
|
|
|
`kubectl kubeconfig: ${kubectl.kubeconfig}`,
|
|
|
|
|
`live health: ${liveProbe.status}`,
|
|
|
|
|
`code agent provider desired-state: ${codeAgentProviderDesiredState.status}`,
|
|
|
|
@@ -1561,7 +1677,7 @@ export async function runDevDeployApply(argv, io = {}) {
|
|
|
|
|
devDeployApply: {
|
|
|
|
|
mode: args.apply ? "apply" : "dry-run",
|
|
|
|
|
conclusion: planConclusion,
|
|
|
|
|
applyBoundary: buildApplyBoundary(args, status, applyStep),
|
|
|
|
|
applyBoundary: buildApplyBoundary(args, status, applyStep, kubectl),
|
|
|
|
|
mutationAttempted: applyStep.mutationAttempted === true && args.apply,
|
|
|
|
|
target: {
|
|
|
|
|
environment: ENVIRONMENT_DEV,
|
|
|
|
@@ -1589,7 +1705,7 @@ export async function runDevDeployApply(argv, io = {}) {
|
|
|
|
|
},
|
|
|
|
|
applyStep,
|
|
|
|
|
manualCommands,
|
|
|
|
|
rollbackHint: buildRollbackHint(workloads),
|
|
|
|
|
rollbackHint: buildRollbackHint(kubectl, workloads),
|
|
|
|
|
remainingBlockers,
|
|
|
|
|
artifactEvidence,
|
|
|
|
|
cloudApiDb,
|
|
|
|
|