diff --git a/docs/reference/deployment-publish.md b/docs/reference/deployment-publish.md index 5488f149..9449ac3a 100644 --- a/docs/reference/deployment-publish.md +++ b/docs/reference/deployment-publish.md @@ -169,6 +169,15 @@ node tools/hwlab-cli/bin/hwlab-cli.mjs cicd report 非阻塞 DEV CD job,产出通过报告。 - CI 产物事实以 registry manifest 和 artifact catalog 为准;临时 `/tmp/hwlab-dev-gate/*.json` 只用于审计和耗时排障,不得成为 CD 放行硬依赖。 +- `kubectl apply -k` 不会自动移除由运行面 `kubectl patch`、临时 ConfigMap + 热修或其他 field manager 写入、且不属于 desired manifest / last-applied 的 + Deployment `volumes`、`volumeMounts` 或 template annotations。正式 CD 不能把 + “镜像 tag 已更新”误判为“热修覆盖已撤销”。DEV CD apply 必须在 apply 前比对 + desired Deployment 与 live Deployment:若 live 存在 desired 不包含、且明确带 + `hotfix`/`override` 语义的额外覆盖字段,应在 `hwlab-dev` 内删除该 desired + Deployment,再由同一次 `kubectl apply -k deploy/k8s/dev` 从源码 desired-state + 重新创建;报告中记录 cleanup 计划和实际动作。不要把这种 cleanup 做成某个服务 + 或某个 skill 的特例。 ## DEV CD Transaction diff --git a/docs/reference/dev-runtime-hotfix-runbook.md b/docs/reference/dev-runtime-hotfix-runbook.md index 80e6a70e..c621e581 100644 --- a/docs/reference/dev-runtime-hotfix-runbook.md +++ b/docs/reference/dev-runtime-hotfix-runbook.md @@ -112,7 +112,9 @@ node scripts/dev-runtime-hotfix-audit.mjs --collect-readonly --pretty ## 回滚口径 -优先回滚方式是用源码化 artifact/CD 覆盖 runtime hotfix:#460/#461 合并并发布后,Deployment 应消费正式镜像,不再通过 ConfigMap 覆盖 `/app/internal/cloud/code-agent-chat.mjs`。只读确认口径是: +优先回滚方式是用源码化 artifact/CD 覆盖 runtime hotfix:#460/#461 合并并发布后,Deployment 应消费正式镜像,不再通过 ConfigMap 覆盖 `/app/internal/cloud/code-agent-chat.mjs`。注意 `kubectl apply -k` 可能保留运行面 patch 写入、且源码 desired-state 不拥有的 hotfix `volumes`、`volumeMounts` 或 template annotations;正式 DEV CD apply 应先识别这种 unmanaged hotfix 覆盖,删除对应 desired Deployment,再由同一次 apply 从源码重新创建。 + +只读确认口径是: ```sh export KUBECONFIG=/etc/rancher/k3s/k3s.yaml diff --git a/scripts/src/dev-deploy-apply.mjs b/scripts/src/dev-deploy-apply.mjs index 773757a0..d6745a2a 100755 --- a/scripts/src/dev-deploy-apply.mjs +++ b/scripts/src/dev-deploy-apply.mjs @@ -71,6 +71,14 @@ const devLegacySimulatorDeploymentCleanupPolicy = { const cleanableLegacySimulatorDeploymentNames = new Set( devLegacySimulatorDeploymentCleanupPolicy.allowedDeployments.map((deployment) => deployment.name) ); +const devUnmanagedHotfixDeploymentCleanupPolicy = { + status: "active", + namespace, + scope: "DEV-only unmanaged live Deployment hotfix/override fields", + reason: + "kubectl apply preserves fields owned by a live hotfix field manager; delete desired Deployments with unmanaged hotfix fields so apply recreates them from source truth", + matchPatterns: ["hotfix", "override"] +}; const requiredValidationCommands = [ "node --check scripts/validate-dev-gate-report.mjs", "node scripts/validate-dev-gate-report.mjs", @@ -515,6 +523,75 @@ function isDesiredLegacySimulatorReplacement(item) { ); } +function deploymentKey(item) { + return `${itemNamespace(item)}/${item?.metadata?.name ?? "unknown"}`; +} + +function isDesiredDeployment(item) { + return item?.kind === "Deployment" && itemNamespace(item) === namespace && typeof item?.metadata?.name === "string"; +} + +function objectTextContainsHotfixMarker(value) { + if (value === null || value === undefined) return false; + if (typeof value === "string" || typeof value === "number" || typeof value === "boolean") { + return /\bhotfix\b|override/iu.test(String(value)); + } + if (Array.isArray(value)) return value.some((entry) => objectTextContainsHotfixMarker(entry)); + if (typeof value === "object") { + return Object.entries(value).some(([key, entry]) => + objectTextContainsHotfixMarker(key) || objectTextContainsHotfixMarker(entry) + ); + } + return false; +} + +function volumeIdentity(volume) { + const parts = [volume?.name ?? ""]; + if (volume?.configMap?.name) parts.push(`configMap:${volume.configMap.name}`); + if (volume?.secret?.secretName) parts.push(`secret:${volume.secret.secretName}`); + if (volume?.persistentVolumeClaim?.claimName) parts.push(`pvc:${volume.persistentVolumeClaim.claimName}`); + return parts.join("|"); +} + +function volumeMountIdentity(containerName, mount) { + return [ + containerName ?? "", + mount?.name ?? "", + mount?.mountPath ?? "", + mount?.subPath ?? "", + mount?.readOnly === true ? "ro" : "rw" + ].join("|"); +} + +function templateAnnotationIdentity(key, value) { + return `${key}=${String(value ?? "")}`; +} + +function desiredDeploymentIndex(workloads) { + return new Map( + listItems(workloads) + .filter(isDesiredDeployment) + .map((item) => [deploymentKey(item), item]) + ); +} + +function deploymentVolumes(item) { + return item?.spec?.template?.spec?.volumes ?? []; +} + +function deploymentContainerMounts(item) { + return (item?.spec?.template?.spec?.containers ?? []).flatMap((container) => + (container.volumeMounts ?? []).map((mount) => ({ + containerName: container.name, + mount + })) + ); +} + +function deploymentTemplateAnnotations(item) { + return item?.spec?.template?.metadata?.annotations ?? {}; +} + export function decideDevTemplateJobReplacement({ jobName, jobNamespace = namespace, @@ -651,6 +728,99 @@ export function decideDevLegacySimulatorDeploymentCleanup({ }; } +export function decideDevUnmanagedHotfixDeploymentCleanup({ + deploymentName, + deploymentNamespace = namespace, + desiredDeployment, + liveDeployment +}) { + const base = { + namespace: deploymentNamespace, + deploymentName, + desiredExists: Boolean(desiredDeployment), + liveExists: Boolean(liveDeployment), + cleanup: false, + result: "not_needed", + reason: "Live Deployment has no unmanaged hotfix or override fields outside desired source.", + unmanagedVolumes: [], + unmanagedVolumeMounts: [], + unmanagedTemplateAnnotations: [] + }; + + if (deploymentNamespace !== namespace) { + return { + ...base, + result: "blocked", + reason: `Unmanaged hotfix cleanup is DEV-only and refuses namespace ${deploymentNamespace}.` + }; + } + if (!desiredDeployment) { + return { + ...base, + result: "ignored", + reason: "No desired Deployment exists in source; cleanup must not delete unknown live Deployments." + }; + } + if (!liveDeployment) { + return { + ...base, + result: "not_found", + reason: "Live Deployment is absent; kubectl apply can create it from desired source." + }; + } + + const desiredVolumeIdentities = new Set(deploymentVolumes(desiredDeployment).map(volumeIdentity)); + const desiredMountIdentities = new Set( + deploymentContainerMounts(desiredDeployment).map(({ containerName, mount }) => volumeMountIdentity(containerName, mount)) + ); + const desiredAnnotationIdentities = new Set( + Object.entries(deploymentTemplateAnnotations(desiredDeployment)).map(([key, value]) => + templateAnnotationIdentity(key, value) + ) + ); + const unmanagedVolumes = deploymentVolumes(liveDeployment) + .filter((volume) => !desiredVolumeIdentities.has(volumeIdentity(volume)) && objectTextContainsHotfixMarker(volume)) + .map((volume) => ({ + name: volume.name ?? null, + configMap: volume.configMap?.name ?? null, + secret: volume.secret?.secretName ?? null, + identity: volumeIdentity(volume) + })); + const unmanagedVolumeMounts = deploymentContainerMounts(liveDeployment) + .filter(({ containerName, mount }) => + !desiredMountIdentities.has(volumeMountIdentity(containerName, mount)) && + objectTextContainsHotfixMarker({ containerName, mount }) + ) + .map(({ containerName, mount }) => ({ + containerName, + name: mount.name ?? null, + mountPath: mount.mountPath ?? null, + subPath: mount.subPath ?? null, + identity: volumeMountIdentity(containerName, mount) + })); + const unmanagedTemplateAnnotations = Object.entries(deploymentTemplateAnnotations(liveDeployment)) + .filter(([key, value]) => + !desiredAnnotationIdentities.has(templateAnnotationIdentity(key, value)) && + objectTextContainsHotfixMarker({ key, value }) + ) + .map(([key, value]) => ({ key, value: String(value ?? "") })); + + if (unmanagedVolumes.length === 0 && unmanagedVolumeMounts.length === 0 && unmanagedTemplateAnnotations.length === 0) { + return base; + } + + return { + ...base, + cleanup: true, + result: "planned", + reason: + "Live Deployment contains unmanaged hotfix/override fields that kubectl apply may preserve; delete before apply so desired source recreates it exactly.", + unmanagedVolumes, + unmanagedVolumeMounts, + unmanagedTemplateAnnotations + }; +} + function replicaPlan(item) { if (item?.kind === "Job") { return item?.spec?.suspend === true ? 0 : 1; @@ -1887,6 +2057,27 @@ async function readLiveLegacySimulatorDeployment(kubectl, deploymentName, blocke } } +async function readLiveDeployment(kubectl, deploymentName, blockers, scopePrefix) { + if (kubectl.status !== "ready") { + return { status: "not_evaluated", reason: kubectl.reason }; + } + const get = await kubectlResult(kubectl, ["-n", namespace, "get", "deployment", deploymentName, "-o", "json"], 15000); + if (!get.ok) { + const output = commandOutput(get); + if (isNotFoundOutput(output)) { + return { status: "not_found", liveDeployment: null }; + } + addBlocker(blockers, "environment_blocker", `${scopePrefix}-${deploymentName}`, `Cannot read live DEV Deployment ${deploymentName}: ${output}`); + return { status: "blocked", reason: oneLine(output), liveDeployment: null }; + } + try { + return { status: "found", liveDeployment: JSON.parse(get.stdout) }; + } catch (error) { + addBlocker(blockers, "contract_blocker", `${scopePrefix}-${deploymentName}`, `Cannot parse live DEV Deployment ${deploymentName}: ${error.message}`); + return { status: "blocked", reason: oneLine(error.message), liveDeployment: null }; + } +} + async function buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers) { const desiredJobs = listItems(workloads).filter(isAllowedDesiredTemplateJob); const replacements = []; @@ -2013,6 +2204,56 @@ async function buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads, return cleanups; } +async function buildDevUnmanagedHotfixDeploymentCleanupPlan(kubectl, workloads, blockers) { + const desiredDeployments = desiredDeploymentIndex(workloads); + const cleanups = []; + for (const [key, desiredDeployment] of desiredDeployments.entries()) { + const deploymentName = desiredDeployment.metadata.name; + const live = await readLiveDeployment(kubectl, deploymentName, blockers, "unmanaged-hotfix-deployment-read"); + if (live.status === "not_evaluated") { + cleanups.push({ + namespace, + deploymentName, + desiredExists: true, + liveExists: null, + cleanup: false, + result: "not_evaluated", + reason: live.reason, + unmanagedVolumes: [], + unmanagedVolumeMounts: [], + unmanagedTemplateAnnotations: [] + }); + continue; + } + if (live.status === "blocked") { + cleanups.push({ + namespace, + deploymentName, + desiredExists: true, + liveExists: null, + cleanup: false, + result: "blocked", + reason: live.reason, + unmanagedVolumes: [], + unmanagedVolumeMounts: [], + unmanagedTemplateAnnotations: [] + }); + continue; + } + const decision = decideDevUnmanagedHotfixDeploymentCleanup({ + deploymentName, + deploymentNamespace: key.split("/")[0], + desiredDeployment, + liveDeployment: live.liveDeployment + }); + if (decision.result === "blocked") { + addBlocker(blockers, "safety_blocker", `unmanaged-hotfix-deployment-cleanup-${deploymentName}`, decision.reason); + } + cleanups.push(decision); + } + return cleanups; +} + async function executeDevTemplateJobReplacements(kubectl, replacements, blockers, concurrency = defaultCdConcurrency) { const planned = replacements.filter((replacement) => replacement.replace && replacement.result === "planned"); await mapWithConcurrency(planned, concurrency, async (replacement) => { @@ -2067,6 +2308,33 @@ async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, bl return planned.length; } +async function executeDevUnmanagedHotfixDeploymentCleanups(kubectl, cleanups, blockers, concurrency = defaultCdConcurrency) { + const planned = cleanups.filter((cleanup) => cleanup.cleanup && cleanup.result === "planned"); + await mapWithConcurrency(planned, concurrency, async (cleanup) => { + const commandArgs = [ + "-n", + cleanup.namespace, + "delete", + "deployment", + cleanup.deploymentName, + "--ignore-not-found=true" + ]; + const result = await kubectlResult(kubectl, commandArgs, 20000); + cleanup.deleteCommand = kubectlCommand(kubectl, commandArgs); + cleanup.deleteStdout = result.redactedStdout ?? result.stdout; + cleanup.deleteStderr = result.redactedStderr ?? result.stderr; + if (!result.ok) { + cleanup.result = "delete_failed"; + cleanup.reason = `Failed to delete live Deployment with unmanaged hotfix fields before apply: ${oneLine(commandOutput(result))}`; + addBlocker(blockers, "environment_blocker", `unmanaged-hotfix-deployment-cleanup-${cleanup.deploymentName}`, cleanup.reason); + return; + } + cleanup.result = "deleted_pending_apply"; + cleanup.reason = "Live Deployment with unmanaged hotfix fields was deleted; kubectl apply must recreate it from desired source."; + }); + return planned.length; +} + function rolloutApiName(kind) { if (kind === "Deployment") return "deployment"; if (kind === "StatefulSet") return "statefulset"; @@ -2181,7 +2449,14 @@ function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) { ); } -async function runApplyStep(args, kubectl, blockers, templateJobReplacements, legacySimulatorDeploymentCleanups) { +async function runApplyStep( + args, + kubectl, + blockers, + templateJobReplacements, + legacySimulatorDeploymentCleanups, + unmanagedHotfixDeploymentCleanups +) { if (blockers.length > 0) { return { status: "not_run", summary: "Skipped because preflight blockers are open" }; } @@ -2191,7 +2466,13 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements, le if (args.apply) { const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers, args.rolloutConcurrency); const cleanupCount = await executeDevLegacySimulatorDeploymentCleanups(kubectl, legacySimulatorDeploymentCleanups, blockers, args.rolloutConcurrency); - mutationAttempted = replaceCount > 0 || cleanupCount > 0; + const hotfixCleanupCount = await executeDevUnmanagedHotfixDeploymentCleanups( + kubectl, + unmanagedHotfixDeploymentCleanups, + blockers, + args.rolloutConcurrency + ); + mutationAttempted = replaceCount > 0 || cleanupCount > 0 || hotfixCleanupCount > 0; if (blockers.length > 0) { return { status: "blocked", @@ -2227,6 +2508,14 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements, le : "Live stale legacy simulator Deployment was deleted, but kubectl apply failed before confirming desired state."; } } + for (const cleanup of unmanagedHotfixDeploymentCleanups) { + if (cleanup.result === "deleted_pending_apply") { + cleanup.result = result.ok ? "deleted" : "delete_succeeded_apply_failed"; + cleanup.reason = result.ok + ? "Live Deployment with unmanaged hotfix fields was deleted and recreated by kubectl apply." + : "Live Deployment with unmanaged hotfix fields was deleted, but kubectl apply failed before confirming recreation."; + } + } return { status: result.ok || expectedImmutableDryRun ? "pass" : "blocked", command: kubectlCommand(kubectl, commandArgs), @@ -2314,12 +2603,14 @@ export async function runDevDeployApply(argv, io = {}) { const liveProbe = args.skipLiveProbe ? { status: "not_run", reason: "--skip-live-probe" } : await probeDevEndpoint(blockers); const templateJobReplacements = await buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers); const legacySimulatorDeploymentCleanups = await buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads, blockers); + const unmanagedHotfixDeploymentCleanups = await buildDevUnmanagedHotfixDeploymentCleanupPlan(kubectl, workloads, blockers); const applyStep = await runApplyStep( args, kubectl, blockers, templateJobReplacements, - legacySimulatorDeploymentCleanups + legacySimulatorDeploymentCleanups, + unmanagedHotfixDeploymentCleanups ); let rolloutStatusAfterApply; let cloudWebRolloutAfterApply; @@ -2418,7 +2709,8 @@ export async function runDevDeployApply(argv, io = {}) { `code agent provider live env before apply: ${codeAgentProviderLiveBeforeApply.status}`, `rollout status concurrency: ${args.rolloutConcurrency}`, `template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}`, - `legacy simulator deployment cleanups: ${legacySimulatorDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}` + `legacy simulator deployment cleanups: ${legacySimulatorDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}`, + `unmanaged hotfix deployment cleanups: ${unmanagedHotfixDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}` ], summary: status === "blocked" ? "DEV apply is blocked before mutation." : "DEV dry-run preflight passed." }, @@ -2451,6 +2743,8 @@ export async function runDevDeployApply(argv, io = {}) { templateJobReplacements, legacySimulatorDeploymentCleanupPolicy: devLegacySimulatorDeploymentCleanupPolicy, legacySimulatorDeploymentCleanups, + unmanagedHotfixDeploymentCleanupPolicy: devUnmanagedHotfixDeploymentCleanupPolicy, + unmanagedHotfixDeploymentCleanups, cloudWebRollout, cloudWebRolloutBeforeApply, cloudWebRolloutAfterApply, diff --git a/scripts/src/dev-deploy-apply.test.mjs b/scripts/src/dev-deploy-apply.test.mjs index 6acc9c82..af6deddd 100644 --- a/scripts/src/dev-deploy-apply.test.mjs +++ b/scripts/src/dev-deploy-apply.test.mjs @@ -6,6 +6,7 @@ import { compareRuntimeIdentityEnv, decideDevLegacySimulatorDeploymentCleanup, decideDevTemplateJobReplacement, + decideDevUnmanagedHotfixDeploymentCleanup, formatKubeconfigAccessFailure, inspectCodeAgentProviderDesiredState, inspectCodeAgentProviderLiveDeployment, @@ -321,6 +322,64 @@ test("legacy simulator cleanup refuses non-DEV or missing replacement cleanup", assert.equal(ignored.result, "ignored"); }); +test("unmanaged hotfix Deployment cleanup plans recreation when live has extra hotfix mounts", () => { + const desired = codeAgentWorkloads().items[0]; + const live = structuredClone(desired); + live.spec.template.metadata = { + annotations: { + "hwlab.pikastech.local/runtime-hotfix": "2026-05-24" + } + }; + live.spec.template.spec.volumes.unshift({ + name: "hwlab-code-agent-hotfix-20260524", + configMap: { name: "hwlab-code-agent-hotfix-20260524" } + }); + live.spec.template.spec.containers[0].volumeMounts.unshift({ + name: "hwlab-code-agent-hotfix-20260524", + mountPath: "/app/internal/cloud/server.mjs", + subPath: "server.mjs", + readOnly: true + }); + + const decision = decideDevUnmanagedHotfixDeploymentCleanup({ + deploymentName: "hwlab-cloud-api", + deploymentNamespace: "hwlab-dev", + desiredDeployment: desired, + liveDeployment: live + }); + + assert.equal(decision.cleanup, true); + assert.equal(decision.result, "planned"); + assert.equal(decision.unmanagedVolumes[0].configMap, "hwlab-code-agent-hotfix-20260524"); + assert.equal(decision.unmanagedVolumeMounts[0].mountPath, "/app/internal/cloud/server.mjs"); + assert.equal(decision.unmanagedTemplateAnnotations[0].key, "hwlab.pikastech.local/runtime-hotfix"); +}); + +test("unmanaged hotfix Deployment cleanup ignores fields already present in desired source", () => { + const desired = codeAgentWorkloads().items[0]; + desired.spec.template.spec.volumes.unshift({ + name: "hwlab-code-agent-hotfix-20260524", + configMap: { name: "hwlab-code-agent-hotfix-20260524" } + }); + desired.spec.template.spec.containers[0].volumeMounts.unshift({ + name: "hwlab-code-agent-hotfix-20260524", + mountPath: "/app/internal/cloud/server.mjs", + subPath: "server.mjs", + readOnly: true + }); + const live = structuredClone(desired); + + const decision = decideDevUnmanagedHotfixDeploymentCleanup({ + deploymentName: "hwlab-cloud-api", + deploymentNamespace: "hwlab-dev", + desiredDeployment: desired, + liveDeployment: live + }); + + assert.equal(decision.cleanup, false); + assert.equal(decision.result, "not_needed"); +}); + test("apply source commit follows deploy/catalog artifact identity", () => { assert.equal( resolveApplySourceCommit({ commitId: "73b379f" }, { commitId: "73b379f" }, "cb35ada68606"),