mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: clean stale M3 simulator deployments (#285)

Browse Source 
Co-authored-by: Code Queue Review <code-queue-review@localhost>
This commit is contained in:
Lyon
2026-05-23 15:34:31 +08:00
committed by GitHub
parent 881594eba1
commit 333cfbe3b7
3 changed files with 306 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/dev-deploy-apply.mjs
+229 -6
View File
@@ -43,6 +43,28 @@ const devTemplateJobReplacementPolicy = {
]
};
const replaceableDevTemplateJobNames = new Set(devTemplateJobReplacementPolicy.allowedJobs.map((job) => job.name));
const devLegacySimulatorDeploymentCleanupPolicy = {
status: "active",
namespace,
scope: "DEV-only stale simulator Deployments replaced by indexed StatefulSets",
allowedDeployments: [
{
name: "hwlab-gateway-simu",
serviceId: "hwlab-gateway-simu",
replacementKind: "StatefulSet",
reason: "M3 DEV requires exactly two indexed gateway-simu identities/sessions"
},
{
name: "hwlab-box-simu",
serviceId: "hwlab-box-simu",
replacementKind: "StatefulSet",
reason: "M3 DEV requires exactly two indexed box-simu resource identities"
}
]
};
const cleanableLegacySimulatorDeploymentNames = new Set(
devLegacySimulatorDeploymentCleanupPolicy.allowedDeployments.map((deployment) => deployment.name)
);
const requiredValidationCommands = [
"node --check scripts/validate-dev-gate-report.mjs",
"node scripts/validate-dev-gate-report.mjs",
@@ -359,6 +381,14 @@ function isAllowedDesiredTemplateJob(item) {
);
}
function isDesiredLegacySimulatorReplacement(item) {
return (
item?.kind === "StatefulSet" &&
itemNamespace(item) === namespace &&
cleanableLegacySimulatorDeploymentNames.has(item?.metadata?.name)
);
}
export function decideDevTemplateJobReplacement({
jobName,
jobNamespace = namespace,
@@ -430,6 +460,71 @@ export function decideDevTemplateJobReplacement({
return base;
}
export function decideDevLegacySimulatorDeploymentCleanup({
deploymentName,
deploymentNamespace = namespace,
desiredReplacementExists,
desiredReplacementKind,
liveExists,
oldImages = [],
newImages = []
}) {
const policy = devLegacySimulatorDeploymentCleanupPolicy.allowedDeployments.find(
(deployment) => deployment.name === deploymentName
);
const base = {
namespace: deploymentNamespace,
deploymentName,
serviceId: policy?.serviceId ?? deploymentName,
allowed: Boolean(policy),
desiredReplacementExists: desiredReplacementExists === true,
desiredReplacementKind: desiredReplacementKind ?? null,
liveExists: liveExists === true,
oldImage: primaryImage(oldImages),
newImage: primaryImage(newImages),
oldImages,
newImages,
cleanup: false,
result: "not_needed",
reason: "No stale legacy simulator Deployment exists."
};
if (!policy) {
return {
...base,
result: "ignored",
reason: "Deployment is not in the DEV stale simulator cleanup allowlist."
};
}
if (deploymentNamespace !== namespace) {
return {
...base,
result: "blocked",
reason: `Cleanup policy is DEV-only and refuses namespace ${deploymentNamespace}.`
};
}
if (desiredReplacementExists !== true || desiredReplacementKind !== policy.replacementKind) {
return {
...base,
result: "blocked",
reason: `Refusing to clean up ${deploymentName} without desired ${policy.replacementKind} replacement in source manifests.`
};
}
if (liveExists !== true) {
return {
...base,
result: "not_found",
reason: "Live legacy Deployment is absent; no cleanup is needed."
};
}
return {
...base,
cleanup: true,
result: "planned",
reason: `Live legacy Deployment ${deploymentName} duplicates the desired indexed StatefulSet and must be removed for exact M3 cardinality.`
};
}
function replicaPlan(item) {
if (item?.kind === "Job") {
return item?.spec?.suspend === true ? 0 : 1;
@@ -902,7 +997,7 @@ function buildApplyBoundary(args, status, applyStep, kubectl) {
mutationAllowed: status === "pass" && args.apply,
applyRequiresFlags: ["--apply", "--confirm-dev", "--confirmed-non-production"],
kubeconfigSource: kubectl.kubeconfigSource,
writeScope: `${kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"])}, namespace hwlab-dev only; delete/recreate is allowed only for explicitly allowlisted suspended DEV template Jobs`,
writeScope: `${kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"])}, namespace hwlab-dev only; delete/recreate is allowed only for explicitly allowlisted suspended DEV template Jobs and stale legacy simulator Deployments`,
noWriteScope: `source/catalog/k8s validation, optional DEV health probe, ${kubectl.commandPrefix} read, replacement planning, and server-side dry-run only`,
forbiddenActions
};
@@ -1372,6 +1467,27 @@ async function readLiveTemplateJob(kubectl, jobName, blockers) {
}
}
async function readLiveLegacySimulatorDeployment(kubectl, deploymentName, blockers) {
if (kubectl.status !== "ready") {
return { status: "not_evaluated", reason: kubectl.reason };
}
const get = await kubectlResult(kubectl, ["-n", namespace, "get", "deployment", deploymentName, "-o", "json"], 15000);
if (!get.ok) {
const output = commandOutput(get);
if (isNotFoundOutput(output)) {
return { status: "not_found", liveDeployment: null };
}
addBlocker(blockers, "environment_blocker", `legacy-simulator-deployment-read-${deploymentName}`, `Cannot read live DEV legacy simulator Deployment ${deploymentName}: ${output}`);
return { status: "blocked", reason: oneLine(output), liveDeployment: null };
}
try {
return { status: "found", liveDeployment: JSON.parse(get.stdout) };
} catch (error) {
addBlocker(blockers, "contract_blocker", `legacy-simulator-deployment-read-${deploymentName}`, `Cannot parse live DEV legacy simulator Deployment ${deploymentName}: ${error.message}`);
return { status: "blocked", reason: oneLine(error.message), liveDeployment: null };
}
}
async function buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers) {
const desiredJobs = listItems(workloads).filter(isAllowedDesiredTemplateJob);
const replacements = [];
@@ -1432,6 +1548,72 @@ async function buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers)
return replacements;
}
async function buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads, blockers) {
const desiredReplacements = new Map(
listItems(workloads)
.filter(isDesiredLegacySimulatorReplacement)
.map((item) => [item.metadata.name, item])
);
const cleanups = [];
for (const policy of devLegacySimulatorDeploymentCleanupPolicy.allowedDeployments) {
const desiredReplacement = desiredReplacements.get(policy.name) ?? null;
const live = await readLiveLegacySimulatorDeployment(kubectl, policy.name, blockers);
if (live.status === "not_evaluated") {
cleanups.push({
namespace,
deploymentName: policy.name,
serviceId: policy.serviceId,
allowed: true,
desiredReplacementExists: Boolean(desiredReplacement),
desiredReplacementKind: desiredReplacement?.kind ?? null,
liveExists: null,
oldImage: null,
newImage: primaryImage(desiredReplacement ? imageRecordsFor(desiredReplacement) : []),
oldImages: [],
newImages: desiredReplacement ? imageRecordsFor(desiredReplacement) : [],
cleanup: false,
result: "not_evaluated",
reason: live.reason
});
continue;
}
if (live.status === "blocked") {
cleanups.push({
namespace,
deploymentName: policy.name,
serviceId: policy.serviceId,
allowed: true,
desiredReplacementExists: Boolean(desiredReplacement),
desiredReplacementKind: desiredReplacement?.kind ?? null,
liveExists: null,
oldImage: null,
newImage: primaryImage(desiredReplacement ? imageRecordsFor(desiredReplacement) : []),
oldImages: [],
newImages: desiredReplacement ? imageRecordsFor(desiredReplacement) : [],
cleanup: false,
result: "blocked",
reason: live.reason
});
continue;
}
const decision = decideDevLegacySimulatorDeploymentCleanup({
deploymentName: policy.name,
deploymentNamespace: namespace,
desiredReplacementExists: Boolean(desiredReplacement),
desiredReplacementKind: desiredReplacement?.kind,
liveExists: live.status === "found",
oldImages: live.liveDeployment ? imageRecordsFor(live.liveDeployment) : [],
newImages: desiredReplacement ? imageRecordsFor(desiredReplacement) : []
});
if (decision.result === "blocked") {
addBlocker(blockers, "safety_blocker", `legacy-simulator-deployment-cleanup-${policy.name}`, decision.reason);
}
cleanups.push(decision);
}
return cleanups;
}
async function executeDevTemplateJobReplacements(kubectl, replacements, blockers) {
const planned = replacements.filter((replacement) => replacement.replace && replacement.result === "planned");
for (const replacement of planned) {
@@ -1459,6 +1641,33 @@ async function executeDevTemplateJobReplacements(kubectl, replacements, blockers
return planned.length;
}
async function executeDevLegacySimulatorDeploymentCleanups(kubectl, cleanups, blockers) {
const planned = cleanups.filter((cleanup) => cleanup.cleanup && cleanup.result === "planned");
for (const cleanup of planned) {
const commandArgs = [
"-n",
cleanup.namespace,
"delete",
"deployment",
cleanup.deploymentName,
"--ignore-not-found=true"
];
const result = await kubectlResult(kubectl, commandArgs, 20000);
cleanup.deleteCommand = kubectlCommand(kubectl, commandArgs);
cleanup.deleteStdout = result.redactedStdout ?? result.stdout;
cleanup.deleteStderr = result.redactedStderr ?? result.stderr;
if (!result.ok) {
cleanup.result = "delete_failed";
cleanup.reason = `Failed to delete stale legacy simulator Deployment before apply: ${oneLine(commandOutput(result))}`;
addBlocker(blockers, "environment_blocker", `legacy-simulator-deployment-cleanup-${cleanup.deploymentName}`, cleanup.reason);
continue;
}
cleanup.result = "deleted_pending_apply";
cleanup.reason = "Live stale legacy simulator Deployment was deleted; kubectl apply must leave the desired indexed StatefulSet in place.";
}
return planned.length;
}
function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) {
const plannedNames = replacementsNeeded.map((replacement) => replacement.jobName);
if (result.ok || plannedNames.length === 0) return false;
@@ -1480,7 +1689,7 @@ function isExpectedTemplateJobImmutableFailure(result, replacementsNeeded) {
);
}
async function runApplyStep(args, kubectl, blockers, templateJobReplacements) {
async function runApplyStep(args, kubectl, blockers, templateJobReplacements, legacySimulatorDeploymentCleanups) {
if (blockers.length > 0) {
return { status: "not_run", summary: "Skipped because preflight blockers are open" };
}
@@ -1489,13 +1698,14 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements) {
if (args.apply) {
const replaceCount = await executeDevTemplateJobReplacements(kubectl, templateJobReplacements, blockers);
mutationAttempted = replaceCount > 0;
const cleanupCount = await executeDevLegacySimulatorDeploymentCleanups(kubectl, legacySimulatorDeploymentCleanups, blockers);
mutationAttempted = replaceCount > 0 || cleanupCount > 0;
if (blockers.length > 0) {
return {
status: "blocked",
command: kubectlCommand(kubectl, ["apply", "-k", "deploy/k8s/dev"]),
mutationAttempted,
summary: "Skipped kubectl apply because DEV template Job replacement failed"
summary: "Skipped kubectl apply because DEV pre-apply cleanup failed"
};
}
}
@@ -1517,6 +1727,14 @@ async function runApplyStep(args, kubectl, blockers, templateJobReplacements) {
: "Live suspended template Job was deleted, but kubectl apply failed before confirming recreation.";
}
}
for (const cleanup of legacySimulatorDeploymentCleanups) {
if (cleanup.result === "deleted_pending_apply") {
cleanup.result = result.ok ? "deleted" : "delete_succeeded_apply_failed";
cleanup.reason = result.ok
? "Live stale legacy simulator Deployment was deleted and the desired indexed StatefulSet apply completed."
: "Live stale legacy simulator Deployment was deleted, but kubectl apply failed before confirming desired state.";
}
}
return {
status: result.ok || expectedImmutableDryRun ? "pass" : "blocked",
command: kubectlCommand(kubectl, commandArgs),
@@ -1588,11 +1806,13 @@ export async function runDevDeployApply(argv, io = {}) {
});
const liveProbe = args.skipLiveProbe ? { status: "not_run", reason: "--skip-live-probe" } : await probeDevEndpoint(blockers);
const templateJobReplacements = await buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers);
const legacySimulatorDeploymentCleanups = await buildDevLegacySimulatorDeploymentCleanupPlan(kubectl, workloads, blockers);
const applyStep = await runApplyStep(
args,
kubectl,
blockers,
templateJobReplacements
templateJobReplacements,
legacySimulatorDeploymentCleanups
);
const cloudWebRolloutAfterApply = args.apply && applyStep.status === "pass"
? await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
@@ -1680,7 +1900,8 @@ export async function runDevDeployApply(argv, io = {}) {
`live health: ${liveProbe.status}`,
`code agent provider desired-state: ${codeAgentProviderDesiredState.status}`,
`code agent provider live env before apply: ${codeAgentProviderLiveBeforeApply.status}`,
`template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}`
`template job replacements: ${templateJobReplacements.filter((replacement) => replacement.replace).length}`,
`legacy simulator deployment cleanups: ${legacySimulatorDeploymentCleanups.filter((cleanup) => cleanup.cleanup).length}`
],
summary: status === "blocked" ? "DEV apply is blocked before mutation." : "DEV dry-run preflight passed."
},
@@ -1711,6 +1932,8 @@ export async function runDevDeployApply(argv, io = {}) {
servicePlan,
templateJobReplacementPolicy: devTemplateJobReplacementPolicy,
templateJobReplacements,
legacySimulatorDeploymentCleanupPolicy: devLegacySimulatorDeploymentCleanupPolicy,
legacySimulatorDeploymentCleanups,
cloudWebRollout,
cloudWebRolloutBeforeApply,
cloudWebRolloutAfterApply,
scripts/src/dev-deploy-apply.test.mjs
+66
View File
@@ -4,6 +4,7 @@ import test from "node:test";
import {
buildKubectlCommandPrefix,
compareRuntimeIdentityEnv,
decideDevLegacySimulatorDeploymentCleanup,
decideDevTemplateJobReplacement,
formatKubeconfigAccessFailure,
inspectCodeAgentProviderDesiredState,
@@ -150,6 +151,71 @@ test("replacement policy refuses non-allowlisted or active Jobs", () => {
assert.equal(active.result, "blocked");
});
test("allowlisted stale simulator Deployment plans cleanup only with StatefulSet replacement", () => {
const decision = decideDevLegacySimulatorDeploymentCleanup({
deploymentName: "hwlab-box-simu",
deploymentNamespace: "hwlab-dev",
desiredReplacementExists: true,
desiredReplacementKind: "StatefulSet",
liveExists: true,
oldImages: [
{
name: "hwlab-box-simu",
serviceId: "hwlab-box-simu",
image: "127.0.0.1:5000/hwlab/hwlab-box-simu:old1234"
}
],
newImages: [
{
name: "hwlab-box-simu",
serviceId: "hwlab-box-simu",
image: "127.0.0.1:5000/hwlab/hwlab-box-simu:new5678"
}
]
});
assert.equal(decision.cleanup, true);
assert.equal(decision.result, "planned");
assert.equal(decision.namespace, "hwlab-dev");
assert.equal(decision.deploymentName, "hwlab-box-simu");
assert.equal(decision.serviceId, "hwlab-box-simu");
assert.equal(decision.desiredReplacementKind, "StatefulSet");
assert.equal(decision.oldImage, "127.0.0.1:5000/hwlab/hwlab-box-simu:old1234");
assert.equal(decision.newImage, "127.0.0.1:5000/hwlab/hwlab-box-simu:new5678");
});
test("legacy simulator cleanup refuses non-DEV or missing replacement cleanup", () => {
const wrongNamespace = decideDevLegacySimulatorDeploymentCleanup({
deploymentName: "hwlab-gateway-simu",
deploymentNamespace: "hwlab-prod",
desiredReplacementExists: true,
desiredReplacementKind: "StatefulSet",
liveExists: true
});
assert.equal(wrongNamespace.cleanup, false);
assert.equal(wrongNamespace.result, "blocked");
const missingReplacement = decideDevLegacySimulatorDeploymentCleanup({
deploymentName: "hwlab-gateway-simu",
deploymentNamespace: "hwlab-dev",
desiredReplacementExists: false,
desiredReplacementKind: null,
liveExists: true
});
assert.equal(missingReplacement.cleanup, false);
assert.equal(missingReplacement.result, "blocked");
const ignored = decideDevLegacySimulatorDeploymentCleanup({
deploymentName: "hwlab-cloud-api",
deploymentNamespace: "hwlab-dev",
desiredReplacementExists: true,
desiredReplacementKind: "StatefulSet",
liveExists: true
});
assert.equal(ignored.cleanup, false);
assert.equal(ignored.result, "ignored");
});
test("apply source commit follows deploy/catalog artifact identity", () => {
assert.equal(
resolveApplySourceCommit({ commitId: "73b379f" }, { commitId: "73b379f" }, "cb35ada68606"),