mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: remove unmanaged hotfix overrides during dev cd

Browse Source
This commit is contained in:
Codex
2026-05-24 17:52:34 +00:00
parent 0463bb8b8e
commit 3ebbbbf625
4 changed files with 369 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/dev-deploy-apply.mjs
+298 -4
View File
@@ -71,6 +71,14 @@ const devLegacySimulatorDeploymentCleanupPolicy = {
const cleanableLegacySimulatorDeploymentNames = new Set(
devLegacySimulatorDeploymentCleanupPolicy.allowedDeployments.map((deployment) => deployment.name)
);
const devUnmanagedHotfixDeploymentCleanupPolicy = {
status: "active",
namespace,
scope: "DEV-only unmanaged live Deployment hotfix/override fields",
reason:
"kubectl apply preserves fields owned by a live hotfix field manager; delete desired Deployments with unmanaged hotfix fields so apply recreates them from source truth",
matchPatterns: ["hotfix", "override"]
};
const requiredValidationCommands = [
"node --check scripts/validate-dev-gate-report.mjs",
"node scripts/validate-dev-gate-report.mjs",
@@ -515,6 +523,75 @@ function isDesiredLegacySimulatorReplacement(item) {
);
}
function deploymentKey(item) {
return `${itemNamespace(item)}/${item?.metadata?.name ?? "unknown"}`;
}
function isDesiredDeployment(item) {
return item?.kind === "Deployment" && itemNamespace(item) === namespace && typeof item?.metadata?.name === "string";
}
function objectTextContainsHotfixMarker(value) {
if (value === null || value === undefined) return false;
if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") {
return /\bhotfix\b|override/iu.test(String(value));
}
if (Array.isArray(value)) return value.some((entry) => objectTextContainsHotfixMarker(entry));
if (typeof value === "object") {
return Object.entries(value).some(([key, entry]) =>
objectTextContainsHotfixMarker(key) || objectTextContainsHotfixMarker(entry)
);
}
return false;
}
function volumeIdentity(volume) {
const parts = [volume?.name ?? ""];
if (volume?.configMap?.name) parts.push(`configMap:${volume.configMap.name}`);
if (volume?.secret?.secretName) parts.push(`secret:${volume.secret.secretName}`);
if (volume?.persistentVolumeClaim?.claimName) parts.push(`pvc:${volume.persistentVolumeClaim.claimName}`);
return parts.join("|");
}
function volumeMountIdentity(containerName, mount) {
return [
containerName ?? "",
mount?.name ?? "",
mount?.mountPath ?? "",
mount?.subPath ?? "",
mount?.readOnly === true ? "ro" : "rw"
].join("|");
}
function templateAnnotationIdentity(key, value) {
return `${key}=${String(value ?? "")}`;
}
function desiredDeploymentIndex(workloads) {
return new Map(
listItems(workloads)
.filter(isDesiredDeployment)
.map((item) => [deploymentKey(item), item])
);
}
function deploymentVolumes(item) {
return item?.spec?.template?.spec?.volumes ?? [];
}
function deploymentContainerMounts(item) {
return (item?.spec?.template?.spec?.containers ?? []).flatMap((container) =>
(container.volumeMounts ?? []).map((mount) => ({
containerName: container.name,
mount
}))
);
}
function deploymentTemplateAnnotations(item) {
return item?.spec?.template?.metadata?.annotations ?? {};
}
export function decideDevTemplateJobReplacement({
jobName,
jobNamespace = namespace,
@@ -651,6 +728,99 @@ export function decideDevLegacySimulatorDeploymentCleanup({
};
}
export function decideDevUnmanagedHotfixDeploymentCleanup({
deploymentName,
deploymentNamespace = namespace,
desiredDeployment,
liveDeployment
}) {
const base = {
namespace: deploymentNamespace,
deploymentName,
desiredExists: Boolean(desiredDeployment),
liveExists: Boolean(liveDeployment),
cleanup: false,
result: "not_needed",
reason: "Live Deployment has no unmanaged hotfix or override fields outside desired source.",
unmanagedVolumes: [],
unmanagedVolumeMounts: [],
unmanagedTemplateAnnotations: []
};
if (deploymentNamespace !== namespace) {
return {
...base,
result: "blocked",
reason: `Unmanaged hotfix cleanup is DEV-only and refuses namespace ${deploymentNamespace}.`
};
}
if (!desiredDeployment) {
return {
...base,
result: "ignored",
reason: "No desired Deployment exists in source; cleanup must not delete unknown live Deployments."
};
}
if (!liveDeployment) {
return {
...base,
result: "not_found",
reason: "Live Deployment is absent; kubectl apply can create it from desired source."
};
}
const desiredVolumeIdentities = new Set(deploymentVolumes(desiredDeployment).map(volumeIdentity));
const desiredMountIdentities = new Set(
deploymentContainerMounts(desiredDeployment).map(({ containerName, mount }) => volumeMountIdentity(containerName, mount))
);
const desiredAnnotationIdentities = new Set(
Object.entries(deploymentTemplateAnnotations(desiredDeployment)).map(([key, value]) =>
templateAnnotationIdentity(key, value)
)
);
const unmanagedVolumes = deploymentVolumes(liveDeployment)
.filter((volume) => !desiredVolumeIdentities.has(volumeIdentity(volume)) && objectTextContainsHotfixMarker(volume))
.map((volume) => ({
name: volume.name ?? null,
configMap: volume.configMap?.name ?? null,
secret: volume.secret?.secretName ?? null,
identity: volumeIdentity(volume)
}));
const unmanagedVolumeMounts = deploymentContainerMounts(liveDeployment)
.filter(({ containerName, mount }) =>
!desiredMountIdentities.has(volumeMountIdentity(containerName, mount)) &&
objectTextContainsHotfixMarker({ containerName, mount })
)
.map(({ containerName, mount }) => ({
containerName,
name: mount.name ?? null,
mountPath: mount.mountPath ?? null,
subPath: mount.subPath ?? null,
identity: volumeMountIdentity(containerName, mount)
}));
const unmanagedTemplateAnnotations = Object.entries(deploymentTemplateAnnotations(liveDeployment))
.filter(([key, value]) =>
!desiredAnnotationIdentities.has(templateAnnotationIdentity(key, value)) &&
objectTextContainsHotfixMarker({ key, value })
)
.map(([key, value]) => ({ key, value: String(value ?? "") }));
if (unmanagedVolumes.length === 0 && unmanagedVolumeMounts.length === 0 && unmanagedTemplateAnnotations.length === 0) {
return base;
}
return {
...base,
cleanup: true,
result: "planned",
reason:
"Live Deployment contains unmanaged hotfix/override fields that kubectl apply may preserve; delete before apply so desired source recreates it exactly.",
unmanagedVolumes,
unmanagedVolumeMounts,
unmanagedTemplateAnnotations
};
}
function replicaPlan(item) {
if (item?.kind === "Job") {
return item?.spec?.suspend === true ? 0 : 1;
@@ -1887,6 +2057,27 @@ async function readLiveLegacySimulatorDeployment(kubectl, deploymentName, blocke
}
}
async function readLiveDeployment(kubectl, deploymentName, blockers, scopePrefix) {
if (kubectl.status !== "ready") {
return { status: "not_evaluated", reason: kubectl.reason };
}
const get = await kubectlResult(kubectl, ["-n", namespace, "get", "deployment", deploymentName, "-o", "json"], 15000);
if (!get.ok) {
const output = commandOutput(get);
if (isNotFoundOutput(output)) {
return { status: "not_found", liveDeployment: null };
}
addBlocker(blockers, "environment_blocker", `${scopePrefix}-${deploymentName}`, `Cannot read live DEV Deployment ${deploymentName}: ${output}`);
return { status: "blocked", reason: oneLine(output), liveDeployment: null };
}
try {
return { status: "found", liveDeployment: JSON.parse(get.stdout) };
} catch (error) {
addBlocker(blockers, "contract_blocker", `${scopePrefix}-${deploymentName}`, `Cannot parse live DEV Deployment ${deploymentName}: ${error.message}`);
return { status: "blocked", reason: oneLine(error.message), liveDeployment: null };
}
}
async function buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers) {
const desiredJobs = listItems(workloads).filter(isAllowedDesiredTemplateJob);
const replacements = [];
@@ -2013,6 +2204,56 @@ async function buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads,
return cleanups;
}
async function buildDevUnmanagedHotfixDeploymentCleanupPlan(kubectl, workloads, blockers) {
const desiredDeployments = desiredDeploymentIndex(workloads);
const cleanups = [];
for (const [key, desiredDeployment] of desiredDeployments.entries()) {
const deploymentName = desiredDeployment.metadata.name;
const live = await readLiveDeployment(kubectl, deploymentName, blockers, "unmanaged-hotfix-deployment-read");
if (live.status === "not_evaluated") {
cleanups.push({
namespace,
deploymentName,
desiredExists: true,
liveExists: null,
cleanup: false,
result: "not_evaluated",
reason: live.reason,
unmanagedVolumes: [],
unmanagedVolumeMounts: [],
unmanagedTemplateAnnotations: []
});
continue;
}
if (live.status === "blocked") {
cleanups.push({
namespace,
deploymentName,
desiredExists: true,
liveExists: null,
cleanup: false,
result: "blocked",
reason: live.reason,
unmanagedVolumes: [],
unmanagedVolumeMounts: [],
unmanagedTemplateAnnotations: []
});
continue;
}
const decision = decideDevUnmanagedHotfixDeploymentCleanup({
deploymentName,
deploymentNamespace: key.split("/")[0],
desiredDeployment,
liveDeployment: live.liveDeployment
});
if (decision.result === "blocked") {
addBlocker(blockers, "safety_blocker", `unmanaged-hotfix-deployment-cleanup-${deploymentName}`, decision.reason);
}
cleanups.push(decision);
}
return cleanups;
}
async function executeDevTemplateJobReplacements(kubectl, replacements, blockers, concurrency = defaultCdConcurrency) {
const planned = replacements.filter((replacement) => replacement.replace && replacement.result === "planned");
await mapWithConcurrency(planned, concurrency, async (replacement) => {
@@ -2067,6 +2308,33 @@ async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, bl
return planned.length;
}
async function executeDevUnmanagedHotfixDeploymentCleanups(kubectl, cleanups, blockers, concurrency = defaultCdConcurrency) {
const planned = cleanups.filter((cleanup) => cleanup.cleanup && cleanup.result === "planned");
await mapWithConcurrency(planned, concurrency, async (cleanup) => {
const commandArgs = [
"-n",
cleanup.namespace,
"delete",
"deployment",
cleanup.deploymentName,
"--ignore-not-found=true"
];
const result = await kubectlResult(kubectl, commandArgs, 20000);
cleanup.deleteCommand = kubectlCommand(kubectl, commandArgs);
cleanup.deleteStdout = result.redactedStdout ?? result.stdout;
cleanup.deleteStderr = result.redactedStderr ?? result.stderr;
if (!result.ok) {
cleanup.result = "delete_failed";
cleanup.reason = `Failed to delete live Deployment with unmanaged hotfix fields before apply: ${oneLine(commandOutput(result))}`;
addBlocker(blockers, "environment_blocker", `unmanaged-hotfix-deployment-cleanup-${cleanup.deploymentName}`, cleanup.reason);
return;
}
cleanup.result = "deleted_pending_apply";
cleanup.reason = "Live Deployment with unmanaged hotfix fields was deleted; kubectl apply must recreate it from desired source.";
});
return planned.length;
}
function rolloutApiName(kind) {
if (kind === "Deployment") return "deployment";
if (kind === "StatefulSet") return "statefulset";
@@ -2181,7 +2449,14 @@ function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) {
);
}
async function runApplyStep(args, kubectl, blockers, templateJobReplacements, legacySimulatorDeploymentCleanups) {
async function runApplyStep(
args,
kubectl,
blockers,
templateJobReplacements,
legacySimulatorDeploymentCleanups,
unmanagedHotfixDeploymentCleanups
) {
if (blockers.length > 0) {
return { status: "not_run", summary: "Skipped because preflight blockers are open" };
}
@@ -2191,7 +2466,13 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements, le
if (args.apply) {
const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers, args.rolloutConcurrency);
const cleanupCount = await executeDevLegacySimulatorDeploymentCleanups(kubectl, legacySimulatorDeploymentCleanups, blockers, args.rolloutConcurrency);
mutationAttempted = replaceCount > 0 || cleanupCount > 0;
const hotfixCleanupCount = await executeDevUnmanagedHotfixDeploymentCleanups(
kubectl,
unmanagedHotfixDeploymentCleanups,
blockers,
args.rolloutConcurrency
);
mutationAttempted = replaceCount > 0 || cleanupCount > 0 || hotfixCleanupCount > 0;
if (blockers.length > 0) {
return {
status: "blocked",
@@ -2227,6 +2508,14 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements, le
: "Live stale legacy simulator Deployment was deleted, but kubectl apply failed before confirming desired state.";
}
}
for (const cleanup of unmanagedHotfixDeploymentCleanups) {
if (cleanup.result === "deleted_pending_apply") {
cleanup.result = result.ok ? "deleted" : "delete_succeeded_apply_failed";
cleanup.reason = result.ok
? "Live Deployment with unmanaged hotfix fields was deleted and recreated by kubectl apply."
: "Live Deployment with unmanaged hotfix fields was deleted, but kubectl apply failed before confirming recreation.";
}
}
return {
status: result.ok || expectedImmutableDryRun ? "pass" : "blocked",
command: kubectlCommand(kubectl, commandArgs),
@@ -2314,12 +2603,14 @@ export async function runDevDeployApply(argv, io = {}) {
const liveProbe = args.skipLiveProbe ? { status: "not_run", reason: "--skip-live-probe" } : await probeDevEndpoint(blockers);
const templateJobReplacements = await buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers);
const legacySimulatorDeploymentCleanups = await buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads, blockers);
const unmanagedHotfixDeploymentCleanups = await buildDevUnmanagedHotfixDeploymentCleanupPlan(kubectl, workloads, blockers);
const applyStep = await runApplyStep(
args,
kubectl,
blockers,
templateJobReplacements,
legacySimulatorDeploymentCleanups
legacySimulatorDeploymentCleanups,
unmanagedHotfixDeploymentCleanups
);
let rolloutStatusAfterApply;
let cloudWebRolloutAfterApply;
@@ -2418,7 +2709,8 @@ export async function runDevDeployApply(argv, io = {}) {
`code agent provider live env before apply: ${codeAgentProviderLiveBeforeApply.status}`,
`rollout status concurrency: ${args.rolloutConcurrency}`,
`template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}`,
`legacy simulator deployment cleanups: ${legacySimulatorDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}`
`legacy simulator deployment cleanups: ${legacySimulatorDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}`,
`unmanaged hotfix deployment cleanups: ${unmanagedHotfixDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}`
],
summary: status === "blocked" ? "DEV apply is blocked before mutation." : "DEV dry-run preflight passed."
},
@@ -2451,6 +2743,8 @@ export async function runDevDeployApply(argv, io = {}) {
templateJobReplacements,
legacySimulatorDeploymentCleanupPolicy: devLegacySimulatorDeploymentCleanupPolicy,
legacySimulatorDeploymentCleanups,
unmanagedHotfixDeploymentCleanupPolicy: devUnmanagedHotfixDeploymentCleanupPolicy,
unmanagedHotfixDeploymentCleanups,
cloudWebRollout,
cloudWebRolloutBeforeApply,
cloudWebRolloutAfterApply,
scripts/src/dev-deploy-apply.test.mjs
+59
View File
@@ -6,6 +6,7 @@ import {
compareRuntimeIdentityEnv,
decideDevLegacySimulatorDeploymentCleanup,
decideDevTemplateJobReplacement,
decideDevUnmanagedHotfixDeploymentCleanup,
formatKubeconfigAccessFailure,
inspectCodeAgentProviderDesiredState,
inspectCodeAgentProviderLiveDeployment,
@@ -321,6 +322,64 @@ test("legacy simulator cleanup refuses non-DEV or missing replacement cleanup",
assert.equal(ignored.result, "ignored");
});
test("unmanaged hotfix Deployment cleanup plans recreation when live has extra hotfix mounts", () => {
const desired = codeAgentWorkloads().items[0];
const live = structuredClone(desired);
live.spec.template.metadata = {
annotations: {
"hwlab.pikastech.local/runtime-hotfix": "2026-05-24"
}
};
live.spec.template.spec.volumes.unshift({
name: "hwlab-code-agent-hotfix-20260524",
configMap: { name: "hwlab-code-agent-hotfix-20260524" }
});
live.spec.template.spec.containers[0].volumeMounts.unshift({
name: "hwlab-code-agent-hotfix-20260524",
mountPath: "/app/internal/cloud/server.mjs",
subPath: "server.mjs",
readOnly: true
});
const decision = decideDevUnmanagedHotfixDeploymentCleanup({
deploymentName: "hwlab-cloud-api",
deploymentNamespace: "hwlab-dev",
desiredDeployment: desired,
liveDeployment: live
});
assert.equal(decision.cleanup, true);
assert.equal(decision.result, "planned");
assert.equal(decision.unmanagedVolumes[0].configMap, "hwlab-code-agent-hotfix-20260524");
assert.equal(decision.unmanagedVolumeMounts[0].mountPath, "/app/internal/cloud/server.mjs");
assert.equal(decision.unmanagedTemplateAnnotations[0].key, "hwlab.pikastech.local/runtime-hotfix");
});
test("unmanaged hotfix Deployment cleanup ignores fields already present in desired source", () => {
const desired = codeAgentWorkloads().items[0];
desired.spec.template.spec.volumes.unshift({
name: "hwlab-code-agent-hotfix-20260524",
configMap: { name: "hwlab-code-agent-hotfix-20260524" }
});
desired.spec.template.spec.containers[0].volumeMounts.unshift({
name: "hwlab-code-agent-hotfix-20260524",
mountPath: "/app/internal/cloud/server.mjs",
subPath: "server.mjs",
readOnly: true
});
const live = structuredClone(desired);
const decision = decideDevUnmanagedHotfixDeploymentCleanup({
deploymentName: "hwlab-cloud-api",
deploymentNamespace: "hwlab-dev",
desiredDeployment: desired,
liveDeployment: live
});
assert.equal(decision.cleanup, false);
assert.equal(decision.result, "not_needed");
});
test("apply source commit follows deploy/catalog artifact identity", () => {
assert.equal(
resolveApplySourceCommit({ commitId: "73b379f" }, { commitId: "73b379f" }, "cb35ada68606"),