diff --git a/docs/dev-deploy-apply.md b/docs/dev-deploy-apply.md index 0523946b..dbcfcef8 100644 --- a/docs/dev-deploy-apply.md +++ b/docs/dev-deploy-apply.md @@ -25,6 +25,10 @@ not be used for PROD. - Plan and, in apply mode, execute a scoped replacement for DEV suspended template Jobs whose image changed but whose Kubernetes `spec.template` is immutable. +- Plan and, in apply mode, execute a scoped cleanup for stale DEV simulator + Deployments `hwlab-gateway-simu` and `hwlab-box-simu` only when the source + manifest contains their indexed StatefulSet replacements. This keeps the M3 + DEV runtime at exactly two gateway-simu sessions and two box-simu resources. - Stop before mutation when artifact publish evidence, registry digests, `kubectl`, DEV health, cloud-api DB connectivity, or Code Agent provider env preservation are missing. @@ -133,6 +137,13 @@ operator fields: - `devDeployApply.templateJobReplacements`: per-Job replacement decisions with namespace, Job name, old image, new image, and result (`planned`, `replaced`, `not_needed`, `not_found`, or blocked/failure states). +- `devDeployApply.legacySimulatorDeploymentCleanupPolicy`: the DEV-only + allowlist for stale simulator Deployment cleanup after the M3 simulator + runtime moved to indexed StatefulSets. +- `devDeployApply.legacySimulatorDeploymentCleanups`: per-Deployment cleanup + decisions for `hwlab-gateway-simu` and `hwlab-box-simu`, including old/new + images and result (`planned`, `deleted`, `not_found`, or blocked/failure + states). - `devDeployApply.cloudWebRollout`: Cloud Web source commit, image tag, image reference, digest, live image, and Kubernetes rollout revision when read-only Deployment access is available. If access is missing, this field records the diff --git a/scripts/src/dev-deploy-apply.mjs b/scripts/src/dev-deploy-apply.mjs index 86243d6f..b05aae05 100755 --- a/scripts/src/dev-deploy-apply.mjs +++ b/scripts/src/dev-deploy-apply.mjs @@ -43,6 +43,28 @@ const devTemplateJobReplacementPolicy = { ] }; const replaceableDevTemplateJobNames = new Set(devTemplateJobReplacementPolicy.allowedJobs.map((job) => job.name)); +const devLegacySimulatorDeploymentCleanupPolicy = { + status: "active", + namespace, + scope: "DEV-only stale simulator Deployments replaced by indexed StatefulSets", + allowedDeployments: [ + { + name: "hwlab-gateway-simu", + serviceId: "hwlab-gateway-simu", + replacementKind: "StatefulSet", + reason: "M3 DEV requires exactly two indexed gateway-simu identities/sessions" + }, + { + name: "hwlab-box-simu", + serviceId: "hwlab-box-simu", + replacementKind: "StatefulSet", + reason: "M3 DEV requires exactly two indexed box-simu resource identities" + } + ] +}; +const cleanableLegacySimulatorDeploymentNames = new Set( + devLegacySimulatorDeploymentCleanupPolicy.allowedDeployments.map((deployment) => deployment.name) +); const requiredValidationCommands = [ "node --check scripts/validate-dev-gate-report.mjs", "node scripts/validate-dev-gate-report.mjs", @@ -359,6 +381,14 @@ function isAllowedDesiredTemplateJob(item) { ); } +function isDesiredLegacySimulatorReplacement(item) { + return ( + item?.kind === "StatefulSet" && + itemNamespace(item) === namespace && + cleanableLegacySimulatorDeploymentNames.has(item?.metadata?.name) + ); +} + export function decideDevTemplateJobReplacement({ jobName, jobNamespace = namespace, @@ -430,6 +460,71 @@ export function decideDevTemplateJobReplacement({ return base; } +export function decideDevLegacySimulatorDeploymentCleanup({ + deploymentName, + deploymentNamespace = namespace, + desiredReplacementExists, + desiredReplacementKind, + liveExists, + oldImages = [], + newImages = [] +}) { + const policy = devLegacySimulatorDeploymentCleanupPolicy.allowedDeployments.find( + (deployment) => deployment.name === deploymentName + ); + const base = { + namespace: deploymentNamespace, + deploymentName, + serviceId: policy?.serviceId ?? deploymentName, + allowed: Boolean(policy), + desiredReplacementExists: desiredReplacementExists === true, + desiredReplacementKind: desiredReplacementKind ?? null, + liveExists: liveExists === true, + oldImage: primaryImage(oldImages), + newImage: primaryImage(newImages), + oldImages, + newImages, + cleanup: false, + result: "not_needed", + reason: "No stale legacy simulator Deployment exists." + }; + + if (!policy) { + return { + ...base, + result: "ignored", + reason: "Deployment is not in the DEV stale simulator cleanup allowlist." + }; + } + if (deploymentNamespace !== namespace) { + return { + ...base, + result: "blocked", + reason: `Cleanup policy is DEV-only and refuses namespace ${deploymentNamespace}.` + }; + } + if (desiredReplacementExists !== true || desiredReplacementKind !== policy.replacementKind) { + return { + ...base, + result: "blocked", + reason: `Refusing to clean up ${deploymentName} without desired ${policy.replacementKind} replacement in source manifests.` + }; + } + if (liveExists !== true) { + return { + ...base, + result: "not_found", + reason: "Live legacy Deployment is absent; no cleanup is needed." + }; + } + return { + ...base, + cleanup: true, + result: "planned", + reason: `Live legacy Deployment ${deploymentName} duplicates the desired indexed StatefulSet and must be removed for exact M3 cardinality.` + }; +} + function replicaPlan(item) { if (item?.kind === "Job") { return item?.spec?.suspend === true ? 0 : 1; @@ -902,7 +997,7 @@ function buildApplyBoundary(args, status, applyStep, kubectl) { mutationAllowed: status === "pass" && args.apply, applyRequiresFlags: ["--apply", "--confirm-dev", "--confirmed-non-production"], kubeconfigSource: kubectl.kubeconfigSource, - writeScope: `${kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"])}, namespace hwlab-dev only; delete/recreate is allowed only for explicitly allowlisted suspended DEV template Jobs`, + writeScope: `${kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"])}, namespace hwlab-dev only; delete/recreate is allowed only for explicitly allowlisted suspended DEV template Jobs and stale legacy simulator Deployments`, noWriteScope: `source/catalog/k8s validation, optional DEV health probe, ${kubectl.commandPrefix} read, replacement planning, and server-side dry-run only`, forbiddenActions }; @@ -1372,6 +1467,27 @@ async function readLiveTemplateJob(kubectl, jobName, blockers) { } } +async function readLiveLegacySimulatorDeployment(kubectl, deploymentName, blockers) { + if (kubectl.status !== "ready") { + return { status: "not_evaluated", reason: kubectl.reason }; + } + const get = await kubectlResult(kubectl, ["-n", namespace, "get", "deployment", deploymentName, "-o", "json"], 15000); + if (!get.ok) { + const output = commandOutput(get); + if (isNotFoundOutput(output)) { + return { status: "not_found", liveDeployment: null }; + } + addBlocker(blockers, "environment_blocker", `legacy-simulator-deployment-read-${deploymentName}`, `Cannot read live DEV legacy simulator Deployment ${deploymentName}: ${output}`); + return { status: "blocked", reason: oneLine(output), liveDeployment: null }; + } + try { + return { status: "found", liveDeployment: JSON.parse(get.stdout) }; + } catch (error) { + addBlocker(blockers, "contract_blocker", `legacy-simulator-deployment-read-${deploymentName}`, `Cannot parse live DEV legacy simulator Deployment ${deploymentName}: ${error.message}`); + return { status: "blocked", reason: oneLine(error.message), liveDeployment: null }; + } +} + async function buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers) { const desiredJobs = listItems(workloads).filter(isAllowedDesiredTemplateJob); const replacements = []; @@ -1432,6 +1548,72 @@ async function buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers) return replacements; } +async function buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads, blockers) { + const desiredReplacements = new Map( + listItems(workloads) + .filter(isDesiredLegacySimulatorReplacement) + .map((item) => [item.metadata.name, item]) + ); + const cleanups = []; + for (const policy of devLegacySimulatorDeploymentCleanupPolicy.allowedDeployments) { + const desiredReplacement = desiredReplacements.get(policy.name) ?? null; + const live = await readLiveLegacySimulatorDeployment(kubectl, policy.name, blockers); + if (live.status === "not_evaluated") { + cleanups.push({ + namespace, + deploymentName: policy.name, + serviceId: policy.serviceId, + allowed: true, + desiredReplacementExists: Boolean(desiredReplacement), + desiredReplacementKind: desiredReplacement?.kind ?? null, + liveExists: null, + oldImage: null, + newImage: primaryImage(desiredReplacement ? imageRecordsFor(desiredReplacement) : []), + oldImages: [], + newImages: desiredReplacement ? imageRecordsFor(desiredReplacement) : [], + cleanup: false, + result: "not_evaluated", + reason: live.reason + }); + continue; + } + if (live.status === "blocked") { + cleanups.push({ + namespace, + deploymentName: policy.name, + serviceId: policy.serviceId, + allowed: true, + desiredReplacementExists: Boolean(desiredReplacement), + desiredReplacementKind: desiredReplacement?.kind ?? null, + liveExists: null, + oldImage: null, + newImage: primaryImage(desiredReplacement ? imageRecordsFor(desiredReplacement) : []), + oldImages: [], + newImages: desiredReplacement ? imageRecordsFor(desiredReplacement) : [], + cleanup: false, + result: "blocked", + reason: live.reason + }); + continue; + } + + const decision = decideDevLegacySimulatorDeploymentCleanup({ + deploymentName: policy.name, + deploymentNamespace: namespace, + desiredReplacementExists: Boolean(desiredReplacement), + desiredReplacementKind: desiredReplacement?.kind, + liveExists: live.status === "found", + oldImages: live.liveDeployment ? imageRecordsFor(live.liveDeployment) : [], + newImages: desiredReplacement ? imageRecordsFor(desiredReplacement) : [] + }); + if (decision.result === "blocked") { + addBlocker(blockers, "safety_blocker", `legacy-simulator-deployment-cleanup-${policy.name}`, decision.reason); + } + cleanups.push(decision); + } + return cleanups; +} + async function executeDevTemplateJobReplacements(kubectl, replacements, blockers) { const planned = replacements.filter((replacement) => replacement.replace && replacement.result === "planned"); for (const replacement of planned) { @@ -1459,6 +1641,33 @@ async function executeDevTemplateJobReplacements(kubectl, replacements, blockers return planned.length; } +async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, blockers) { + const planned = cleanups.filter((cleanup) => cleanup.cleanup && cleanup.result === "planned"); + for (const cleanup of planned) { + const commandArgs = [ + "-n", + cleanup.namespace, + "delete", + "deployment", + cleanup.deploymentName, + "--ignore-not-found=true" + ]; + const result = await kubectlResult(kubectl, commandArgs, 20000); + cleanup.deleteCommand = kubectlCommand(kubectl, commandArgs); + cleanup.deleteStdout = result.redactedStdout ?? result.stdout; + cleanup.deleteStderr = result.redactedStderr ?? result.stderr; + if (!result.ok) { + cleanup.result = "delete_failed"; + cleanup.reason = `Failed to delete stale legacy simulator Deployment before apply: ${oneLine(commandOutput(result))}`; + addBlocker(blockers, "environment_blocker", `legacy-simulator-deployment-cleanup-${cleanup.deploymentName}`, cleanup.reason); + continue; + } + cleanup.result = "deleted_pending_apply"; + cleanup.reason = "Live stale legacy simulator Deployment was deleted; kubectl apply must leave the desired indexed StatefulSet in place."; + } + return planned.length; +} + function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) { const plannedNames = replacementsNeeded.map((replacement) => replacement.jobName); if (result.ok || plannedNames.length === 0) return false; @@ -1480,7 +1689,7 @@ function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) { ); } -async function runApplyStep(args, kubectl, blockers, templateJobReplacements) { +async function runApplyStep(args, kubectl, blockers, templateJobReplacements, legacySimulatorDeploymentCleanups) { if (blockers.length > 0) { return { status: "not_run", summary: "Skipped because preflight blockers are open" }; } @@ -1489,13 +1698,14 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements) { if (args.apply) { const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers); - mutationAttempted = replaceCount > 0; + const cleanupCount = await executeDevLegacySimulatorDeploymentCleanups(kubectl, legacySimulatorDeploymentCleanups, blockers); + mutationAttempted = replaceCount > 0 || cleanupCount > 0; if (blockers.length > 0) { return { status: "blocked", command: kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"]), mutationAttempted, - summary: "Skipped kubectl apply because DEV template Job replacement failed" + summary: "Skipped kubectl apply because DEV pre-apply cleanup failed" }; } } @@ -1517,6 +1727,14 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements) { : "Live suspended template Job was deleted, but kubectl apply failed before confirming recreation."; } } + for (const cleanup of legacySimulatorDeploymentCleanups) { + if (cleanup.result === "deleted_pending_apply") { + cleanup.result = result.ok ? "deleted" : "delete_succeeded_apply_failed"; + cleanup.reason = result.ok + ? "Live stale legacy simulator Deployment was deleted and the desired indexed StatefulSet apply completed." + : "Live stale legacy simulator Deployment was deleted, but kubectl apply failed before confirming desired state."; + } + } return { status: result.ok || expectedImmutableDryRun ? "pass" : "blocked", command: kubectlCommand(kubectl, commandArgs), @@ -1588,11 +1806,13 @@ export async function runDevDeployApply(argv, io = {}) { }); const liveProbe = args.skipLiveProbe ? { status: "not_run", reason: "--skip-live-probe" } : await probeDevEndpoint(blockers); const templateJobReplacements = await buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers); + const legacySimulatorDeploymentCleanups = await buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads, blockers); const applyStep = await runApplyStep( args, kubectl, blockers, - templateJobReplacements + templateJobReplacements, + legacySimulatorDeploymentCleanups ); const cloudWebRolloutAfterApply = args.apply && applyStep.status === "pass" ? await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, { @@ -1680,7 +1900,8 @@ export async function runDevDeployApply(argv, io = {}) { `live health: ${liveProbe.status}`, `code agent provider desired-state: ${codeAgentProviderDesiredState.status}`, `code agent provider live env before apply: ${codeAgentProviderLiveBeforeApply.status}`, - `template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}` + `template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}`, + `legacy simulator deployment cleanups: ${legacySimulatorDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}` ], summary: status === "blocked" ? "DEV apply is blocked before mutation." : "DEV dry-run preflight passed." }, @@ -1711,6 +1932,8 @@ export async function runDevDeployApply(argv, io = {}) { servicePlan, templateJobReplacementPolicy: devTemplateJobReplacementPolicy, templateJobReplacements, + legacySimulatorDeploymentCleanupPolicy: devLegacySimulatorDeploymentCleanupPolicy, + legacySimulatorDeploymentCleanups, cloudWebRollout, cloudWebRolloutBeforeApply, cloudWebRolloutAfterApply, diff --git a/scripts/src/dev-deploy-apply.test.mjs b/scripts/src/dev-deploy-apply.test.mjs index 2d12df13..65e9845a 100644 --- a/scripts/src/dev-deploy-apply.test.mjs +++ b/scripts/src/dev-deploy-apply.test.mjs @@ -4,6 +4,7 @@ import test from "node:test"; import { buildKubectlCommandPrefix, compareRuntimeIdentityEnv, + decideDevLegacySimulatorDeploymentCleanup, decideDevTemplateJobReplacement, formatKubeconfigAccessFailure, inspectCodeAgentProviderDesiredState, @@ -150,6 +151,71 @@ test("replacement policy refuses non-allowlisted or active Jobs", () => { assert.equal(active.result, "blocked"); }); +test("allowlisted stale simulator Deployment plans cleanup only with StatefulSet replacement", () => { + const decision = decideDevLegacySimulatorDeploymentCleanup({ + deploymentName: "hwlab-box-simu", + deploymentNamespace: "hwlab-dev", + desiredReplacementExists: true, + desiredReplacementKind: "StatefulSet", + liveExists: true, + oldImages: [ + { + name: "hwlab-box-simu", + serviceId: "hwlab-box-simu", + image: "127.0.0.1:5000/hwlab/hwlab-box-simu:old1234" + } + ], + newImages: [ + { + name: "hwlab-box-simu", + serviceId: "hwlab-box-simu", + image: "127.0.0.1:5000/hwlab/hwlab-box-simu:new5678" + } + ] + }); + + assert.equal(decision.cleanup, true); + assert.equal(decision.result, "planned"); + assert.equal(decision.namespace, "hwlab-dev"); + assert.equal(decision.deploymentName, "hwlab-box-simu"); + assert.equal(decision.serviceId, "hwlab-box-simu"); + assert.equal(decision.desiredReplacementKind, "StatefulSet"); + assert.equal(decision.oldImage, "127.0.0.1:5000/hwlab/hwlab-box-simu:old1234"); + assert.equal(decision.newImage, "127.0.0.1:5000/hwlab/hwlab-box-simu:new5678"); +}); + +test("legacy simulator cleanup refuses non-DEV or missing replacement cleanup", () => { + const wrongNamespace = decideDevLegacySimulatorDeploymentCleanup({ + deploymentName: "hwlab-gateway-simu", + deploymentNamespace: "hwlab-prod", + desiredReplacementExists: true, + desiredReplacementKind: "StatefulSet", + liveExists: true + }); + assert.equal(wrongNamespace.cleanup, false); + assert.equal(wrongNamespace.result, "blocked"); + + const missingReplacement = decideDevLegacySimulatorDeploymentCleanup({ + deploymentName: "hwlab-gateway-simu", + deploymentNamespace: "hwlab-dev", + desiredReplacementExists: false, + desiredReplacementKind: null, + liveExists: true + }); + assert.equal(missingReplacement.cleanup, false); + assert.equal(missingReplacement.result, "blocked"); + + const ignored = decideDevLegacySimulatorDeploymentCleanup({ + deploymentName: "hwlab-cloud-api", + deploymentNamespace: "hwlab-dev", + desiredReplacementExists: true, + desiredReplacementKind: "StatefulSet", + liveExists: true + }); + assert.equal(ignored.cleanup, false); + assert.equal(ignored.result, "ignored"); +}); + test("apply source commit follows deploy/catalog artifact identity", () => { assert.equal( resolveApplySourceCommit({ commitId: "73b379f" }, { commitId: "73b379f" }, "cb35ada68606"),