fix: add code queue kubeconfig fallback

This commit is contained in:
Code Queue Review
2026-05-23 01:11:11 +00:00
parent b777c5d223
commit 1bff239eab
4 changed files with 301 additions and 42 deletions
+155 -39
View File
@@ -22,7 +22,9 @@ const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../
const reportPath = "reports/dev-gate/dev-deploy-report.json";
const namespace = "hwlab-dev";
const healthPath = "/health/live";
const d601KubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
const defaultD601KubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
const devKubeconfigEnvName = "HWLAB_DEV_KUBECONFIG";
const kubeconfigEnvName = "KUBECONFIG";
const devTemplateJobReplacementPolicy = {
status: "active",
namespace,
@@ -63,14 +65,48 @@ const forbiddenActions = [
];
const runtimeIdentityEnvNames = ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"];
function parseArgs(argv) {
const flags = new Set(argv.filter((arg) => arg.startsWith("--")));
export function parseArgs(argv) {
const flags = new Set();
const errors = [];
let kubeconfig = null;
let kubeconfigSpecified = false;
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (!arg.startsWith("--")) continue;
if (arg === "--kubeconfig") {
flags.add("--kubeconfig");
kubeconfigSpecified = true;
const next = argv[index + 1];
if (typeof next !== "string" || next.startsWith("--")) {
errors.push("--kubeconfig requires a non-empty path value");
} else {
kubeconfig = next;
index += 1;
}
continue;
}
if (arg.startsWith("--kubeconfig=")) {
flags.add("--kubeconfig");
kubeconfigSpecified = true;
kubeconfig = arg.slice("--kubeconfig=".length);
if (!kubeconfig.trim()) {
errors.push("--kubeconfig requires a non-empty path value");
}
continue;
}
flags.add(arg);
}
return {
apply: flags.has("--apply"),
dryRun: !flags.has("--apply"),
expectBlocked: flags.has("--expect-blocked"),
skipLiveProbe: flags.has("--skip-live-probe"),
writeReport: flags.has("--write-report"),
kubeconfig,
kubeconfigSpecified,
errors,
flags
};
}
@@ -140,16 +176,71 @@ async function findExecutable(name) {
return null;
}
async function resolveD601Kubectl() {
function isNonEmpty(value) {
return typeof value === "string" && value.trim().length > 0;
}
export function resolveDevKubeconfigSelection({ flagValue = null, env = process.env } = {}) {
const candidates = [
{ source: "flag:--kubeconfig", value: flagValue },
{ source: `env:${devKubeconfigEnvName}`, value: env?.[devKubeconfigEnvName] },
{ source: `env:${kubeconfigEnvName}`, value: env?.[kubeconfigEnvName] },
{ source: "default:d601-k3s", value: defaultD601KubeconfigPath }
];
const selected = candidates.find((candidate) => isNonEmpty(candidate.value));
const kubeconfig = selected.value.trim();
return {
kubeconfig,
source: selected.source,
paths: kubeconfig.split(path.delimiter).filter((item) => item.length > 0)
};
}
function shellQuote(value) {
if (/^[A-Za-z0-9_@%+=:,./-]+$/u.test(value)) return value;
return `'${String(value).replaceAll("'", "'\\''")}'`;
}
export function buildKubectlCommandPrefix(kubeconfig) {
return `KUBECONFIG=${shellQuote(kubeconfig)} kubectl`;
}
export function formatKubeconfigAccessFailure(selection, unreadablePath) {
return `DEV kubeconfig from ${selection.source} is not readable at ${unreadablePath}; set --kubeconfig, ${devKubeconfigEnvName}, or ${kubeconfigEnvName} to a readable DEV kubeconfig path. Kubeconfig contents and Secret values are never printed.`;
}
export function redactSensitiveText(value) {
return String(value)
.replace(/\b(Bearer\s+)[A-Za-z0-9._~+/=-]+/giu, "$1<redacted>")
.replace(/\b(postgres(?:ql)?:\/\/[^:\s/@]+:)([^@\s]+)(@)/giu, "$1<redacted>$3")
.replace(/\b(token|password|client-key-data|client-certificate-data|certificate-authority-data)\b(\s*[:=]\s*)(["']?)([^"'\s,}]+)/giu, "$1$2$3<redacted>");
}
function redactCommandResult(result) {
return {
...result,
redactedStdout: redactSensitiveText(result.stdout ?? ""),
redactedStderr: redactSensitiveText(result.stderr ?? "")
};
}
function commandOutput(result) {
return result.redactedStderr || result.redactedStdout || result.stderr || result.stdout || "";
}
async function resolveD601Kubectl(args = {}, env = process.env) {
const kubeconfigSelection = resolveDevKubeconfigSelection({ flagValue: args.kubeconfig, env });
const kubectlPath = await findExecutable("kubectl");
const commandPrefix = `KUBECONFIG=${d601KubeconfigPath} kubectl`;
const commandPrefix = buildKubectlCommandPrefix(kubeconfigSelection.kubeconfig);
const base = {
executor: kubectlPath,
kubeconfig: d601KubeconfigPath,
kubeconfig: kubeconfigSelection.kubeconfig,
kubeconfigSource: kubeconfigSelection.source,
kubeconfigPaths: kubeconfigSelection.paths,
commandPrefix,
env: {
...process.env,
KUBECONFIG: d601KubeconfigPath
...env,
KUBECONFIG: kubeconfigSelection.kubeconfig
}
};
@@ -162,12 +253,27 @@ async function resolveD601Kubectl() {
}
try {
await access(d601KubeconfigPath, fsConstants.R_OK);
} catch {
let readablePathCount = 0;
let firstUnreadablePath = null;
for (const kubeconfigPath of kubeconfigSelection.paths) {
try {
await access(kubeconfigPath, fsConstants.R_OK);
readablePathCount += 1;
} catch (error) {
firstUnreadablePath ??= error?.path ?? kubeconfigPath;
}
}
if (readablePathCount === 0) {
throw Object.assign(new Error("no readable kubeconfig paths"), {
path: firstUnreadablePath ?? kubeconfigSelection.kubeconfig
});
}
} catch (error) {
const unreadablePath = error?.path ?? kubeconfigSelection.kubeconfig;
return {
...base,
status: "blocked",
reason: `D601 k3s kubeconfig is not readable at ${d601KubeconfigPath}`
reason: formatKubeconfigAccessFailure(kubeconfigSelection, unreadablePath)
};
}
@@ -192,10 +298,11 @@ async function kubectlResult(kubectl, args, timeoutMs = 15000, options = {}) {
blocked: true
};
}
return commandResult(kubectl.executor, args, timeoutMs, {
const result = await commandResult(kubectl.executor, args, timeoutMs, {
...options,
env: kubectl.env
});
return redactCommandResult(result);
}
function listItems(document) {
@@ -470,8 +577,8 @@ function containerImageForDeployment(deployment, containerName) {
return containers.find((container) => container.name === containerName)?.image ?? null;
}
function buildRolloutVerificationCommand(serviceId) {
return `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout status deployment/${serviceId} --timeout=180s`;
function buildRolloutVerificationCommand(kubectl, serviceId) {
return kubectlCommand(kubectl, ["-n", namespace, "rollout", "status", `deployment/${serviceId}`, "--timeout=180s"]);
}
function buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase) {
@@ -495,9 +602,10 @@ function buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observa
liveRuntimeEnv: null,
runtimeEnvMatchesDesired: false,
runtimeEnvDrift: [],
verificationCommand: buildRolloutVerificationCommand(serviceId),
verificationCommand: buildRolloutVerificationCommand(kubectl, serviceId),
readCommand: kubectlCommand(kubectl, commandArgs),
kubeconfig: kubectl.kubeconfig
kubeconfig: kubectl.kubeconfig,
kubeconfigSource: kubectl.kubeconfigSource
};
}
@@ -526,7 +634,7 @@ async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blo
const result = await kubectlResult(kubectl, commandArgs, 15000);
if (!result.ok) {
const summary = oneLine(result.stderr || result.stdout);
const summary = oneLine(commandOutput(result));
addBlocker(blockers, "environment_blocker", `rollout-read-${serviceId}`, `Cannot read DEV Deployment ${serviceId} rollout revision: ${summary}`);
return {
...base,
@@ -579,7 +687,7 @@ function blockerHint(blocker) {
return "Regenerate deploy/deploy.json and deploy/artifact-catalog.dev.json for the source commit that will be promoted.";
}
if (blocker.scope === "kubectl" || blocker.scope === "d601-kubeconfig" || blocker.scope === "kubectl-version") {
return `Provide kubectl with readable D601 k3s kubeconfig at ${d601KubeconfigPath} without exposing token or secret values.`;
return `Provide kubectl with a readable DEV kubeconfig through --kubeconfig, ${devKubeconfigEnvName}, ${kubeconfigEnvName}, or the D601 default path without exposing token or secret values.`;
}
if (blocker.scope === "cluster-read") {
return "Restore read-only visibility for pods, services, configmaps, deployments, and jobs in hwlab-dev.";
@@ -618,7 +726,7 @@ function buildRemainingBlockers(blockers) {
}));
}
function buildRollbackHint(workloads) {
function buildRollbackHint(kubectl, workloads) {
const deploymentNames = buildWorkloadPlan(workloads)
.filter((workload) => workload.kind === "Deployment")
.map((workload) => workload.name);
@@ -630,13 +738,13 @@ function buildRollbackHint(workloads) {
namespace,
strategy: "DEV-only Kubernetes rollback using deployment revision history; do not touch PROD or UniDesk services.",
captureBeforeApply: [
`KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev get pods,services,configmaps,deployments,jobs -o wide`,
`KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev get deployments -o json`
kubectlCommand(kubectl, ["-n", namespace, "get", "pods,services,configmaps,deployments,jobs", "-o", "wide"]),
kubectlCommand(kubectl, ["-n", namespace, "get", "deployments", "-o", "json"])
],
deploymentRollbackCommands: deploymentNames.map((name) => `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout undo deployment/${name}`),
jobCleanupCommands: jobNames.map((name) => `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev delete job ${name} --ignore-not-found`),
deploymentRollbackCommands: deploymentNames.map((name) => kubectlCommand(kubectl, ["-n", namespace, "rollout", "undo", `deployment/${name}`])),
jobCleanupCommands: jobNames.map((name) => kubectlCommand(kubectl, ["-n", namespace, "delete", "job", name, "--ignore-not-found"])),
postRollbackChecks: [
`KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout status deployment/hwlab-cloud-api --timeout=120s`,
kubectlCommand(kubectl, ["-n", namespace, "rollout", "status", "deployment/hwlab-cloud-api", "--timeout=120s"]),
`curl -fsS ${new URL(healthPath, DEV_ENDPOINT).href}`
]
};
@@ -786,15 +894,16 @@ function buildManualCommands(status) {
};
}
function buildApplyBoundary(args, status, applyStep) {
function buildApplyBoundary(args, status, applyStep, kubectl) {
return {
currentMode: args.apply ? "apply-requested" : "dry-run",
defaultNoWrite: !args.apply,
mutationAttempted: applyStep.mutationAttempted === true && args.apply,
mutationAllowed: status === "pass" && args.apply,
applyRequiresFlags: ["--apply", "--confirm-dev", "--confirmed-non-production"],
writeScope: `KUBECONFIG=${d601KubeconfigPath} kubectl apply -k deploy/k8s/dev, namespace hwlab-dev only; delete/recreate is allowed only for explicitly allowlisted suspended DEV template Jobs`,
noWriteScope: `source/catalog/k8s validation, optional DEV health probe, KUBECONFIG=${d601KubeconfigPath} kubectl read, replacement planning, and server-side dry-run only`,
kubeconfigSource: kubectl.kubeconfigSource,
writeScope: `${kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"])}, namespace hwlab-dev only; delete/recreate is allowed only for explicitly allowlisted suspended DEV template Jobs`,
noWriteScope: `source/catalog/k8s validation, optional DEV health probe, ${kubectl.commandPrefix} read, replacement planning, and server-side dry-run only`,
forbiddenActions
};
}
@@ -811,6 +920,9 @@ function buildPlanConclusion(status, blockers) {
}
function validateArgs(args, blockers) {
for (const error of args.errors ?? []) {
addBlocker(blockers, "safety_blocker", "kubeconfig-argument", error);
}
for (const forbidden of ["--prod", "--production", "--read-secret", "--force-push", "--heavyweight-e2e"]) {
if (args.flags.has(forbidden)) {
addBlocker(blockers, "safety_blocker", forbidden.slice(2), `${forbidden} is forbidden for DEV deploy apply`);
@@ -1172,6 +1284,7 @@ async function observeCluster(kubectl, blockers) {
status: "blocked",
executor: kubectl.executor ?? "missing",
kubeconfig: kubectl.kubeconfig,
kubeconfigSource: kubectl.kubeconfigSource,
commandPrefix: kubectl.commandPrefix,
resources: []
};
@@ -1199,6 +1312,7 @@ async function observeCluster(kubectl, blockers) {
status: "pass",
executor: kubectl.executor,
kubeconfig: kubectl.kubeconfig,
kubeconfigSource: kubectl.kubeconfigSource,
commandPrefix: kubectl.commandPrefix,
readCommand: kubectlCommand(kubectl, getArgs),
version: version.stdout,
@@ -1222,7 +1336,7 @@ async function readLiveTemplateJob(kubectl, jobName, blockers) {
}
const get = await kubectlResult(kubectl, ["-n", namespace, "get", "job", jobName, "-o", "json"], 15000);
if (!get.ok) {
const output = get.stderr || get.stdout;
const output = commandOutput(get);
if (isNotFoundOutput(output)) {
return { status: "not_found", liveJob: null };
}
@@ -1310,11 +1424,11 @@ async function executeDevTemplateJobReplacements(kubectl, replacements, blockers
];
const result = await kubectlResult(kubectl, commandArgs, 20000);
replacement.deleteCommand = kubectlCommand(kubectl, commandArgs);
replacement.deleteStdout = result.stdout;
replacement.deleteStderr = result.stderr;
replacement.deleteStdout = result.redactedStdout ?? result.stdout;
replacement.deleteStderr = result.redactedStderr ?? result.stderr;
if (!result.ok) {
replacement.result = "delete_failed";
replacement.reason = `Failed to delete live suspended template Job before apply: ${oneLine(result.stderr || result.stdout)}`;
replacement.reason = `Failed to delete live suspended template Job before apply: ${oneLine(commandOutput(result))}`;
addBlocker(blockers, "environment_blocker", `template-job-replace-${replacement.jobName}`, replacement.reason);
continue;
}
@@ -1327,8 +1441,9 @@ async function executeDevTemplateJobReplacements(kubectl, replacements, blockers
function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) {
const plannedNames = replacementsNeeded.map((replacement) => replacement.jobName);
if (result.ok || plannedNames.length === 0) return false;
const output = `${result.stdout}\n${result.stderr}`;
const errorSections = output.split("Error from server").slice(1);
const redactedOutput = `${result.redactedStdout ?? result.stdout}\n${result.redactedStderr ?? result.stderr}`;
const effectiveOutput = redactedOutput.trim().length > 0 ? redactedOutput : `${result.stdout}\n${result.stderr}`;
const errorSections = effectiveOutput.split("Error from server").slice(1);
const immutableSections = errorSections.filter(
(section) => section.includes("spec.template") && section.includes("field is immutable")
);
@@ -1371,7 +1486,7 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements) {
const expectedImmutableDryRun = !args.apply && isExpectedTemplateJobImmutableFailure(result, replacementsNeeded);
mutationAttempted = mutationAttempted || args.apply;
if (!result.ok && !expectedImmutableDryRun) {
addBlocker(blockers, "environment_blocker", args.apply ? "kubectl-apply" : "kubectl-dry-run", result.stderr || result.stdout);
addBlocker(blockers, "environment_blocker", args.apply ? "kubectl-apply" : "kubectl-dry-run", commandOutput(result));
}
for (const replacement of templateJobReplacements) {
if (replacement.result === "deleted_pending_recreate") {
@@ -1384,8 +1499,8 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements) {
return {
status: result.ok || expectedImmutableDryRun ? "pass" : "blocked",
command: kubectlCommand(kubectl, commandArgs),
stdout: result.stdout,
stderr: result.stderr,
stdout: result.redactedStdout ?? result.stdout,
stderr: result.redactedStderr ?? result.stderr,
mutationAttempted,
expectedImmutableTemplateJobDryRun: expectedImmutableDryRun,
expectedImmutableTemplateJobs: expectedImmutableDryRun
@@ -1440,7 +1555,7 @@ export async function runDevDeployApply(argv, io = {}) {
addBlocker(blockers, "contract_blocker", "cloud-api-health-path", "cloud-api does not implement the k8s /health/live path");
}
const kubectl = await resolveD601Kubectl();
const kubectl = await resolveD601Kubectl(args);
const clusterObservation = await observeCluster(kubectl, blockers);
const cloudWebRolloutBeforeApply = await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
observationPhase: "before_apply",
@@ -1539,6 +1654,7 @@ export async function runDevDeployApply(argv, io = {}) {
`namespace: ${namespace}`,
`workloads planned: ${workloadPlan.length}`,
`kubectl executor: ${kubectl.executor ?? "missing"}`,
`kubectl kubeconfig source: ${kubectl.kubeconfigSource}`,
`kubectl kubeconfig: ${kubectl.kubeconfig}`,
`live health: ${liveProbe.status}`,
`code agent provider desired-state: ${codeAgentProviderDesiredState.status}`,
@@ -1561,7 +1677,7 @@ export async function runDevDeployApply(argv, io = {}) {
devDeployApply: {
mode: args.apply ? "apply" : "dry-run",
conclusion: planConclusion,
applyBoundary: buildApplyBoundary(args, status, applyStep),
applyBoundary: buildApplyBoundary(args, status, applyStep, kubectl),
mutationAttempted: applyStep.mutationAttempted === true && args.apply,
target: {
environment: ENVIRONMENT_DEV,
@@ -1589,7 +1705,7 @@ export async function runDevDeployApply(argv, io = {}) {
},
applyStep,
manualCommands,
rollbackHint: buildRollbackHint(workloads),
rollbackHint: buildRollbackHint(kubectl, workloads),
remainingBlockers,
artifactEvidence,
cloudApiDb,
+125 -1
View File
@@ -2,11 +2,16 @@ import assert from "node:assert/strict";
import test from "node:test";
import {
buildKubectlCommandPrefix,
compareRuntimeIdentityEnv,
decideDevTemplateJobReplacement,
formatKubeconfigAccessFailure,
inspectCodeAgentProviderDesiredState,
inspectCodeAgentProviderLiveDeployment,
resolveApplySourceCommit
parseArgs,
redactSensitiveText,
resolveApplySourceCommit,
resolveDevKubeconfigSelection
} from "./dev-deploy-apply.mjs";
const codeAgentBaseUrl = "http://172.26.26.227:17680/v1/responses";
@@ -232,3 +237,122 @@ test("Code Agent live Deployment inspection reads only env metadata", () => {
assert.equal(inspection.kubernetesSecretDataRead, false);
assert.equal(inspection.validationScope, "deployment-env-and-secretKeyRef-metadata-only");
});
test("DEV kubeconfig selection prefers flag then HWLAB_DEV_KUBECONFIG then KUBECONFIG then D601 default", () => {
assert.deepEqual(
resolveDevKubeconfigSelection({
flagValue: "/tmp/from-flag",
env: {
HWLAB_DEV_KUBECONFIG: "/tmp/from-hwlab",
KUBECONFIG: "/tmp/from-kubeconfig"
}
}),
{
kubeconfig: "/tmp/from-flag",
source: "flag:--kubeconfig",
paths: ["/tmp/from-flag"]
}
);
assert.deepEqual(
resolveDevKubeconfigSelection({
env: {
HWLAB_DEV_KUBECONFIG: "/tmp/from-hwlab",
KUBECONFIG: "/tmp/from-kubeconfig"
}
}),
{
kubeconfig: "/tmp/from-hwlab",
source: "env:HWLAB_DEV_KUBECONFIG",
paths: ["/tmp/from-hwlab"]
}
);
assert.deepEqual(
resolveDevKubeconfigSelection({
env: {
KUBECONFIG: "/tmp/from-kubeconfig"
}
}),
{
kubeconfig: "/tmp/from-kubeconfig",
source: "env:KUBECONFIG",
paths: ["/tmp/from-kubeconfig"]
}
);
assert.deepEqual(
resolveDevKubeconfigSelection({ env: {} }),
{
kubeconfig: "/etc/rancher/k3s/k3s.yaml",
source: "default:d601-k3s",
paths: ["/etc/rancher/k3s/k3s.yaml"]
}
);
});
test("DEV kubeconfig selection keeps multi-path KUBECONFIG readable checks explicit", () => {
const delimiter = process.platform === "win32" ? ";" : ":";
const selection = resolveDevKubeconfigSelection({
env: {
KUBECONFIG: [`/tmp/one`, `/tmp/two`].join(delimiter)
}
});
assert.equal(selection.kubeconfig, [`/tmp/one`, `/tmp/two`].join(delimiter));
assert.equal(selection.source, "env:KUBECONFIG");
assert.deepEqual(selection.paths, ["/tmp/one", "/tmp/two"]);
});
test("kubectl command prefix quotes kubeconfig paths without exposing contents", () => {
assert.equal(
buildKubectlCommandPrefix("/var/lib/unidesk/code-queue/kubeconfig"),
"KUBECONFIG=/var/lib/unidesk/code-queue/kubeconfig kubectl"
);
assert.equal(
buildKubectlCommandPrefix("/tmp/path with space/kubeconfig"),
"KUBECONFIG='/tmp/path with space/kubeconfig' kubectl"
);
});
test("kubeconfig access failure names source and path without secret material", () => {
const message = formatKubeconfigAccessFailure(
{
source: "env:HWLAB_DEV_KUBECONFIG"
},
"/missing/kubeconfig"
);
assert.match(message, /env:HWLAB_DEV_KUBECONFIG/u);
assert.match(message, /\/missing\/kubeconfig/u);
assert.match(message, /Secret values are never printed/u);
assert.equal(message.includes("client-key-data:"), false);
});
test("parseArgs reports a missing kubeconfig path as a blocking error", () => {
const args = parseArgs(["--apply", "--kubeconfig"]);
assert.equal(args.apply, true);
assert.equal(args.kubeconfigSpecified, true);
assert.deepEqual(args.errors, ["--kubeconfig requires a non-empty path value"]);
});
test("redaction removes common token and password material from kubectl output", () => {
const output = redactSensitiveText(
[
"Authorization: Bearer abc.def-123",
"token: abcdef123456",
"password=plain-secret",
"client-key-data: LS0tPRIVATE",
"postgresql://hwlab:db-secret@example.invalid:5432/hwlab"
].join("\n")
);
assert.equal(output.includes("abc.def-123"), false);
assert.equal(output.includes("plain-secret"), false);
assert.equal(output.includes("LS0tPRIVATE"), false);
assert.equal(output.includes("db-secret"), false);
assert.match(output, /Bearer <redacted>/u);
assert.match(output, /password=<redacted>/u);
assert.match(output, /client-key-data: <redacted>/u);
assert.match(output, /postgresql:\/\/hwlab:<redacted>@example.invalid/u);
});
+2 -2
View File
@@ -906,9 +906,9 @@ function assertDevDeployApplyReport(report, label) {
}
assert.equal(typeof plan.cloudWebRollout.imageMatchesDesired, "boolean", `${label}.devDeployApply.cloudWebRollout.imageMatchesDesired`);
assert.match(plan.cloudWebRollout.verificationCommand, /rollout status deployment\/hwlab-cloud-web/u, `${label}.devDeployApply.cloudWebRollout.verificationCommand`);
assert.match(plan.cloudWebRollout.verificationCommand, /^KUBECONFIG=\/etc\/rancher\/k3s\/k3s\.yaml kubectl /u, `${label}.devDeployApply.cloudWebRollout.verificationCommand`);
assert.match(plan.cloudWebRollout.verificationCommand, /^KUBECONFIG=(?:[A-Za-z0-9_@%+=:,./-]+|'[^']+') kubectl /u, `${label}.devDeployApply.cloudWebRollout.verificationCommand`);
assert.match(plan.cloudWebRollout.readCommand, /get deployment hwlab-cloud-web/u, `${label}.devDeployApply.cloudWebRollout.readCommand`);
assert.match(plan.cloudWebRollout.readCommand, /^KUBECONFIG=\/etc\/rancher\/k3s\/k3s\.yaml kubectl /u, `${label}.devDeployApply.cloudWebRollout.readCommand`);
assert.match(plan.cloudWebRollout.readCommand, /^KUBECONFIG=(?:[A-Za-z0-9_@%+=:,./-]+|'[^']+') kubectl /u, `${label}.devDeployApply.cloudWebRollout.readCommand`);
if (plan.cloudWebRollout.status === "blocked") {
assertString(plan.cloudWebRollout.blocker, `${label}.devDeployApply.cloudWebRollout.blocker`);
}