Merge pull request #208 from pikasTech/fix/cloud-web-env-drift-source
fix: guard cloud-web runtime image env drift
This commit is contained in:
+4
-1
@@ -213,7 +213,10 @@
|
||||
"replicas": 1,
|
||||
"env": {
|
||||
"HWLAB_ENVIRONMENT": "dev",
|
||||
"HWLAB_API_BASE_URL": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667"
|
||||
"HWLAB_API_BASE_URL": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667",
|
||||
"HWLAB_COMMIT_ID": "7de6edd",
|
||||
"HWLAB_IMAGE": "127.0.0.1:5000/hwlab/hwlab-cloud-web:7de6edd",
|
||||
"HWLAB_IMAGE_TAG": "7de6edd"
|
||||
}
|
||||
},
|
||||
{
|
||||
|
||||
@@ -170,6 +170,18 @@
|
||||
{
|
||||
"name": "HWLAB_API_BASE_URL",
|
||||
"value": "http://hwlab-cloud-api.hwlab-dev.svc.cluster.local:6667"
|
||||
},
|
||||
{
|
||||
"name": "HWLAB_COMMIT_ID",
|
||||
"value": "7de6edd"
|
||||
},
|
||||
{
|
||||
"name": "HWLAB_IMAGE",
|
||||
"value": "127.0.0.1:5000/hwlab/hwlab-cloud-web:7de6edd"
|
||||
},
|
||||
{
|
||||
"name": "HWLAB_IMAGE_TAG",
|
||||
"value": "7de6edd"
|
||||
}
|
||||
],
|
||||
"readinessProbe": {
|
||||
|
||||
@@ -7,6 +7,7 @@ import test from "node:test";
|
||||
import { buildDesiredStatePlan } from "./src/deploy-desired-state-plan.mjs";
|
||||
|
||||
async function makeFixture({
|
||||
serviceId = "hwlab-cloud-api",
|
||||
commitId = "abc1234",
|
||||
catalogCommitId = commitId,
|
||||
catalogServiceCommitId = catalogCommitId,
|
||||
@@ -14,37 +15,41 @@ async function makeFixture({
|
||||
deployEnvCommitId = commitId,
|
||||
deployEnvImageTag = commitId,
|
||||
deployEnvImage = null,
|
||||
deployEnvMirrors = true,
|
||||
deploySkillsCommitId = null,
|
||||
workloadTag = commitId,
|
||||
workloadEnvCommitId = commitId,
|
||||
workloadEnvImageTag = commitId,
|
||||
workloadEnvImage = null,
|
||||
workloadEnvMirrors = true,
|
||||
workloadSkillsCommitId = null
|
||||
} = {}) {
|
||||
const root = await mkdtemp(path.join(os.tmpdir(), "hwlab-desired-state-"));
|
||||
await mkdir(path.join(root, "deploy/k8s/base"), { recursive: true });
|
||||
await mkdir(path.join(root, "reports/dev-gate"), { recursive: true });
|
||||
const image = `127.0.0.1:5000/hwlab/hwlab-cloud-api:${commitId}`;
|
||||
const catalogImage = `127.0.0.1:5000/hwlab/hwlab-cloud-api:${catalogImageTag}`;
|
||||
const workloadImage = `127.0.0.1:5000/hwlab/hwlab-cloud-api:${workloadTag}`;
|
||||
const image = `127.0.0.1:5000/hwlab/${serviceId}:${commitId}`;
|
||||
const catalogImage = `127.0.0.1:5000/hwlab/${serviceId}:${catalogImageTag}`;
|
||||
const workloadImage = `127.0.0.1:5000/hwlab/${serviceId}:${workloadTag}`;
|
||||
const deploy = {
|
||||
manifestVersion: "v1",
|
||||
environment: "dev",
|
||||
commitId,
|
||||
services: [
|
||||
{
|
||||
serviceId: "hwlab-cloud-api",
|
||||
serviceId,
|
||||
image,
|
||||
namespace: "hwlab-dev",
|
||||
healthPath: "/health/live",
|
||||
profile: "dev",
|
||||
replicas: 1,
|
||||
env: {
|
||||
HWLAB_COMMIT_ID: deployEnvCommitId,
|
||||
HWLAB_IMAGE: deployEnvImage ?? image,
|
||||
HWLAB_IMAGE_TAG: deployEnvImageTag,
|
||||
...(deploySkillsCommitId ? { HWLAB_SKILLS_COMMIT_ID: deploySkillsCommitId } : {})
|
||||
}
|
||||
env: deployEnvMirrors
|
||||
? {
|
||||
HWLAB_COMMIT_ID: deployEnvCommitId,
|
||||
HWLAB_IMAGE: deployEnvImage ?? image,
|
||||
HWLAB_IMAGE_TAG: deployEnvImageTag,
|
||||
...(deploySkillsCommitId ? { HWLAB_SKILLS_COMMIT_ID: deploySkillsCommitId } : {})
|
||||
}
|
||||
: {}
|
||||
}
|
||||
]
|
||||
};
|
||||
@@ -64,7 +69,7 @@ async function makeFixture({
|
||||
},
|
||||
services: [
|
||||
{
|
||||
serviceId: "hwlab-cloud-api",
|
||||
serviceId,
|
||||
commitId: catalogServiceCommitId,
|
||||
image: catalogImage,
|
||||
imageTag: catalogImageTag,
|
||||
@@ -82,10 +87,10 @@ async function makeFixture({
|
||||
apiVersion: "apps/v1",
|
||||
kind: "Deployment",
|
||||
metadata: {
|
||||
name: "hwlab-cloud-api",
|
||||
name: serviceId,
|
||||
namespace: "hwlab-dev",
|
||||
labels: {
|
||||
"hwlab.pikastech.local/service-id": "hwlab-cloud-api"
|
||||
"hwlab.pikastech.local/service-id": serviceId
|
||||
}
|
||||
},
|
||||
spec: {
|
||||
@@ -93,14 +98,16 @@ async function makeFixture({
|
||||
spec: {
|
||||
containers: [
|
||||
{
|
||||
name: "hwlab-cloud-api",
|
||||
name: serviceId,
|
||||
image: workloadImage,
|
||||
env: [
|
||||
{ name: "HWLAB_COMMIT_ID", value: workloadEnvCommitId },
|
||||
{ name: "HWLAB_IMAGE", value: workloadEnvImage ?? image },
|
||||
{ name: "HWLAB_IMAGE_TAG", value: workloadEnvImageTag },
|
||||
...(workloadSkillsCommitId ? [{ name: "HWLAB_SKILLS_COMMIT_ID", value: workloadSkillsCommitId }] : [])
|
||||
]
|
||||
env: workloadEnvMirrors
|
||||
? [
|
||||
{ name: "HWLAB_COMMIT_ID", value: workloadEnvCommitId },
|
||||
{ name: "HWLAB_IMAGE", value: workloadEnvImage ?? image },
|
||||
{ name: "HWLAB_IMAGE_TAG", value: workloadEnvImageTag },
|
||||
...(workloadSkillsCommitId ? [{ name: "HWLAB_SKILLS_COMMIT_ID", value: workloadSkillsCommitId }] : [])
|
||||
]
|
||||
: []
|
||||
}
|
||||
]
|
||||
}
|
||||
@@ -124,6 +131,33 @@ test("passes internally consistent desired-state", async () => {
|
||||
assert.deepEqual(plan.diagnostics, []);
|
||||
});
|
||||
|
||||
test("passes when cloud-web owns runtime identity mirrors", async () => {
|
||||
const repoRoot = await makeFixture({ serviceId: "hwlab-cloud-web" });
|
||||
const plan = await buildDesiredStatePlan({ repoRoot });
|
||||
assert.equal(plan.status, "pass");
|
||||
assert.equal(plan.summary.presentMirrorCount, 6);
|
||||
assert.deepEqual(plan.diagnostics, []);
|
||||
});
|
||||
|
||||
test("blocks when cloud-web runtime identity mirrors are missing", async () => {
|
||||
const repoRoot = await makeFixture({
|
||||
serviceId: "hwlab-cloud-web",
|
||||
deployEnvMirrors: false,
|
||||
workloadEnvMirrors: false
|
||||
});
|
||||
const plan = await buildDesiredStatePlan({ repoRoot });
|
||||
assert.equal(plan.status, "blocked");
|
||||
assert.equal(plan.summary.blockers, 6);
|
||||
assert.ok(plan.diagnostics.some((diagnostic) =>
|
||||
diagnostic.code === "missing_mirror" &&
|
||||
diagnostic.path === "deploy/deploy.json.services.hwlab-cloud-web.env.HWLAB_COMMIT_ID"
|
||||
));
|
||||
assert.ok(plan.diagnostics.some((diagnostic) =>
|
||||
diagnostic.code === "missing_mirror" &&
|
||||
diagnostic.path.endsWith(".env.HWLAB_IMAGE_TAG")
|
||||
));
|
||||
});
|
||||
|
||||
test("target tag review is read-only promotion_pending when current state is uniformly older", async () => {
|
||||
const repoRoot = await makeFixture();
|
||||
const plan = await buildDesiredStatePlan({ repoRoot, targetTag: "def5678" });
|
||||
|
||||
@@ -23,6 +23,9 @@ const commitMirrorEnvNames = new Set(["HWLAB_COMMIT_ID", "HWLAB_SKILLS_COMMIT_ID
|
||||
const imageMirrorEnvNames = new Set(["HWLAB_IMAGE"]);
|
||||
const tagMirrorEnvNames = new Set(["HWLAB_IMAGE_TAG"]);
|
||||
const digestMirrorEnvNames = new Set(["HWLAB_IMAGE_DIGEST"]);
|
||||
const requiredMirrorEnvNamesByService = new Map([
|
||||
["hwlab-cloud-web", ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"]]
|
||||
]);
|
||||
const tagPattern = /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/;
|
||||
const commitPattern = /^[a-f0-9]{7,40}$/;
|
||||
|
||||
@@ -212,14 +215,28 @@ function valueMatchesExpected(kind, actual, expected) {
|
||||
return Object.is(actual, expected);
|
||||
}
|
||||
|
||||
function inspectEnvMirrors(ctx, { source, basePath, serviceId, env, service, desiredCommitId }) {
|
||||
function inspectEnvMirrors(ctx, { source, basePath, serviceId, env, service, desiredCommitId, requiredMirrorNames = [] }) {
|
||||
const present = {};
|
||||
const missing = [];
|
||||
const diagnostics = [];
|
||||
const required = new Set(requiredMirrorNames);
|
||||
|
||||
for (const name of mirrorEnvNames) {
|
||||
if (!Object.hasOwn(env ?? {}, name)) {
|
||||
missing.push(name);
|
||||
if (required.has(name)) {
|
||||
const kind = observationKindForEnv(name);
|
||||
const expected = expectedEnvValue(name, service, desiredCommitId);
|
||||
const diagnostic = {
|
||||
code: "missing_mirror",
|
||||
path: `${basePath}.${name}`,
|
||||
message: `${source} ${serviceId} must include ${name} so DEV applies own runtime ${kind} identity`,
|
||||
expected,
|
||||
actual: null
|
||||
};
|
||||
diagnostics.push(diagnostic);
|
||||
addDiagnostic(ctx, diagnostic);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
const actual = env[name];
|
||||
@@ -600,7 +617,8 @@ export async function buildDesiredStatePlan(options = {}) {
|
||||
serviceId,
|
||||
env: deployService.env ?? {},
|
||||
service: expectedService,
|
||||
desiredCommitId
|
||||
desiredCommitId,
|
||||
requiredMirrorNames: requiredMirrorEnvNamesByService.get(serviceId) ?? []
|
||||
})
|
||||
};
|
||||
}
|
||||
@@ -691,7 +709,8 @@ export async function buildDesiredStatePlan(options = {}) {
|
||||
serviceId,
|
||||
env: workload.env,
|
||||
service: expectedService,
|
||||
desiredCommitId
|
||||
desiredCommitId,
|
||||
requiredMirrorNames: requiredMirrorEnvNamesByService.get(serviceId) ?? []
|
||||
})
|
||||
});
|
||||
}
|
||||
|
||||
@@ -56,6 +56,7 @@ const forbiddenActions = [
|
||||
"heavy-master-e2e",
|
||||
"force-push"
|
||||
];
|
||||
const runtimeIdentityEnvNames = ["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"];
|
||||
|
||||
function parseArgs(argv) {
|
||||
const flags = new Set(argv.filter((arg) => arg.startsWith("--")));
|
||||
@@ -407,6 +408,54 @@ function parseImageTag(image) {
|
||||
return image.slice(colonIndex + 1);
|
||||
}
|
||||
|
||||
function expectedRuntimeIdentityEnv(artifact) {
|
||||
return {
|
||||
HWLAB_COMMIT_ID: artifact.sourceCommitId,
|
||||
HWLAB_IMAGE: artifact.image,
|
||||
HWLAB_IMAGE_TAG: artifact.imageTag
|
||||
};
|
||||
}
|
||||
|
||||
function envObjectFromEnvList(envList) {
|
||||
const env = {};
|
||||
if (!Array.isArray(envList)) return env;
|
||||
for (const entry of envList) {
|
||||
if (!entry?.name) continue;
|
||||
env[entry.name] = Object.hasOwn(entry, "value") ? entry.value : null;
|
||||
}
|
||||
return env;
|
||||
}
|
||||
|
||||
function runtimeIdentityValueMatches(name, actual, expected) {
|
||||
if (typeof actual !== "string" || typeof expected !== "string") return false;
|
||||
if (name === "HWLAB_COMMIT_ID") return commitMatchesSource(actual, expected);
|
||||
return actual === expected;
|
||||
}
|
||||
|
||||
export function compareRuntimeIdentityEnv(env, expected) {
|
||||
const fields = runtimeIdentityEnvNames.map((name) => {
|
||||
const actual = Object.hasOwn(env ?? {}, name) ? env[name] : null;
|
||||
const expectedValue = expected?.[name] ?? null;
|
||||
const status = runtimeIdentityValueMatches(name, actual, expectedValue) ? "match" : "drift";
|
||||
return {
|
||||
name,
|
||||
expected: expectedValue,
|
||||
actual,
|
||||
status
|
||||
};
|
||||
});
|
||||
return {
|
||||
status: fields.every((field) => field.status === "match") ? "match" : "drift",
|
||||
matchesDesired: fields.every((field) => field.status === "match"),
|
||||
fields,
|
||||
drift: fields.filter((field) => field.status === "drift")
|
||||
};
|
||||
}
|
||||
|
||||
function formatRuntimeIdentityDrift(drift) {
|
||||
return drift.map((field) => `${field.name} expected ${field.expected ?? "missing"} actual ${field.actual ?? "missing"}`).join("; ");
|
||||
}
|
||||
|
||||
function deploymentRevision(deployment) {
|
||||
return deployment?.metadata?.annotations?.["deployment.kubernetes.io/revision"] ?? null;
|
||||
}
|
||||
@@ -420,13 +469,14 @@ function buildRolloutVerificationCommand(serviceId) {
|
||||
return `KUBECONFIG=${d601KubeconfigPath} kubectl -n hwlab-dev rollout status deployment/${serviceId} --timeout=180s`;
|
||||
}
|
||||
|
||||
async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blockers) {
|
||||
function buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase) {
|
||||
const artifact = artifactIdentityForService(deploy, catalog, serviceId);
|
||||
const commandArgs = ["-n", namespace, "get", "deployment", serviceId, "-o", "json"];
|
||||
const base = {
|
||||
return {
|
||||
serviceId,
|
||||
namespace,
|
||||
status: "not_evaluated",
|
||||
observationPhase,
|
||||
sourceCommitId: artifact.sourceCommitId,
|
||||
image: artifact.image ?? "unknown",
|
||||
imageTag: artifact.imageTag ?? "unknown",
|
||||
@@ -436,10 +486,29 @@ async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blo
|
||||
rolloutRevision: null,
|
||||
liveImage: null,
|
||||
imageMatchesDesired: false,
|
||||
expectedRuntimeEnv: expectedRuntimeIdentityEnv(artifact),
|
||||
liveRuntimeEnv: null,
|
||||
runtimeEnvMatchesDesired: false,
|
||||
runtimeEnvDrift: [],
|
||||
verificationCommand: buildRolloutVerificationCommand(serviceId),
|
||||
readCommand: kubectlCommand(kubectl, commandArgs),
|
||||
kubeconfig: kubectl.kubeconfig
|
||||
};
|
||||
}
|
||||
|
||||
function skippedDeploymentRolloutObservation(kubectl, deploy, catalog, serviceId, observationPhase, reason) {
|
||||
return {
|
||||
...buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase),
|
||||
status: "not_evaluated",
|
||||
blocker: reason
|
||||
};
|
||||
}
|
||||
|
||||
async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blockers, options = {}) {
|
||||
const observationPhase = options.observationPhase ?? "read";
|
||||
const blockRuntimeEnvDrift = options.blockRuntimeEnvDrift === true;
|
||||
const commandArgs = ["-n", namespace, "get", "deployment", serviceId, "-o", "json"];
|
||||
const base = buildDeploymentRolloutBase(kubectl, deploy, catalog, serviceId, observationPhase);
|
||||
|
||||
if (kubectl.status !== "ready") {
|
||||
addBlocker(blockers, "environment_blocker", `rollout-read-${serviceId}`, kubectl.reason);
|
||||
@@ -464,12 +533,27 @@ async function observeDeploymentRollout(kubectl, deploy, catalog, serviceId, blo
|
||||
try {
|
||||
const deployment = JSON.parse(result.stdout);
|
||||
const liveImage = containerImageForDeployment(deployment, serviceId);
|
||||
const liveRuntimeEnv = compareRuntimeIdentityEnv(
|
||||
envObjectFromEnvList(deployment?.spec?.template?.spec?.containers?.find((container) => container.name === serviceId)?.env),
|
||||
base.expectedRuntimeEnv
|
||||
);
|
||||
if (blockRuntimeEnvDrift && !liveRuntimeEnv.matchesDesired) {
|
||||
addBlocker(
|
||||
blockers,
|
||||
"runtime_blocker",
|
||||
`rollout-runtime-env-${observationPhase}-${serviceId}`,
|
||||
`DEV Deployment ${serviceId} runtime identity env drift after apply: ${formatRuntimeIdentityDrift(liveRuntimeEnv.drift)}`
|
||||
);
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
status: "observed",
|
||||
rolloutRevision: deploymentRevision(deployment),
|
||||
liveImage,
|
||||
imageMatchesDesired: Boolean(artifact.image && liveImage === artifact.image)
|
||||
imageMatchesDesired: Boolean(base.image && liveImage === base.image),
|
||||
liveRuntimeEnv,
|
||||
runtimeEnvMatchesDesired: liveRuntimeEnv.matchesDesired,
|
||||
runtimeEnvDrift: liveRuntimeEnv.drift
|
||||
};
|
||||
} catch (error) {
|
||||
const summary = oneLine(error.message);
|
||||
@@ -498,6 +582,12 @@ function blockerHint(blocker) {
|
||||
if (blocker.scope.startsWith("rollout-read-")) {
|
||||
return "Restore read-only Deployment visibility in hwlab-dev so rollout revision, image tag, digest, and source commit can be reported after apply.";
|
||||
}
|
||||
if (blocker.scope.startsWith("deploy-runtime-env-") || blocker.scope.startsWith("k8s-runtime-env-")) {
|
||||
return "Update deploy/deploy.json and deploy/k8s/base/workloads.yaml so the service owns HWLAB_COMMIT_ID, HWLAB_IMAGE, and HWLAB_IMAGE_TAG.";
|
||||
}
|
||||
if (blocker.scope.startsWith("rollout-runtime-env-")) {
|
||||
return "Reconcile the DEV Deployment runtime identity env to the desired source image and rerun post-apply read-only verification.";
|
||||
}
|
||||
if (blocker.scope === "dev-health" || blocker.scope === "dev-health-identity") {
|
||||
return "Restore the public DEV health path and confirm it identifies HWLAB dev, not a substitute runtime.";
|
||||
}
|
||||
@@ -603,6 +693,28 @@ function commitMatchesSource(commitId, sourceCommitId) {
|
||||
return sourceCommitId.startsWith(commitId) || commitId.startsWith(sourceCommitId);
|
||||
}
|
||||
|
||||
function validateDeployRuntimeIdentityEnv(deploy, catalog, serviceId, blockers) {
|
||||
const deployService = deploy?.services?.find((service) => service.serviceId === serviceId) ?? null;
|
||||
if (!deployService) return null;
|
||||
const artifact = artifactIdentityForService(deploy, catalog, serviceId);
|
||||
const expected = expectedRuntimeIdentityEnv(artifact);
|
||||
const comparison = compareRuntimeIdentityEnv(deployService.env ?? {}, expected);
|
||||
if (!comparison.matchesDesired) {
|
||||
addBlocker(
|
||||
blockers,
|
||||
"contract_blocker",
|
||||
`deploy-runtime-env-${serviceId}`,
|
||||
`${serviceId} deploy env does not own desired runtime identity: ${formatRuntimeIdentityDrift(comparison.drift)}`
|
||||
);
|
||||
}
|
||||
return {
|
||||
serviceId,
|
||||
source: "deploy",
|
||||
expected,
|
||||
...comparison
|
||||
};
|
||||
}
|
||||
|
||||
function validateDeployAndCatalog(deploy, catalog, sourceCommitId, blockers) {
|
||||
const artifactEvidence = [];
|
||||
if (!deploy || !catalog) return artifactEvidence;
|
||||
@@ -649,6 +761,7 @@ function validateDeployAndCatalog(deploy, catalog, sourceCommitId, blockers) {
|
||||
publishState: catalogService.publishState
|
||||
});
|
||||
}
|
||||
validateDeployRuntimeIdentityEnv(deploy, catalog, "hwlab-cloud-web", blockers);
|
||||
|
||||
const requiredServices = (catalog.services ?? []).filter((service) => service.artifactRequired !== false);
|
||||
const disabledServices = (catalog.services ?? []).filter((service) => service.artifactRequired === false);
|
||||
@@ -668,7 +781,7 @@ function validateDeployAndCatalog(deploy, catalog, sourceCommitId, blockers) {
|
||||
return artifactEvidence;
|
||||
}
|
||||
|
||||
function validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, blockers) {
|
||||
function validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers) {
|
||||
const items = [...listItems(namespaceDoc), ...listItems(workloads), ...listItems(services), ...listItems(healthContract)];
|
||||
if (devKustomization?.namespace !== namespace) {
|
||||
addBlocker(blockers, "safety_blocker", "dev-kustomization", "DEV kustomization must pin namespace hwlab-dev");
|
||||
@@ -684,8 +797,10 @@ function validateK8s(devKustomization, namespaceDoc, workloads, services, health
|
||||
}
|
||||
|
||||
const expectedImages = new Map((deploy?.services ?? []).map((service) => [service.serviceId, service.image]));
|
||||
const cloudWebExpectedRuntimeEnv = expectedRuntimeIdentityEnv(artifactIdentityForService(deploy, catalog, "hwlab-cloud-web"));
|
||||
const workloadItems = listItems(workloads);
|
||||
const images = [];
|
||||
const runtimeIdentityEnv = [];
|
||||
for (const item of workloadItems) {
|
||||
for (const container of containersFor(item)) {
|
||||
const serviceId = serviceIdFor(item, container);
|
||||
@@ -694,6 +809,25 @@ function validateK8s(devKustomization, namespaceDoc, workloads, services, health
|
||||
if (expectedImage && container.image !== expectedImage) {
|
||||
addBlocker(blockers, "contract_blocker", `k8s-image-${serviceId}`, `${serviceId} image differs in k8s workload`);
|
||||
}
|
||||
if (serviceId === "hwlab-cloud-web") {
|
||||
const comparison = compareRuntimeIdentityEnv(envObjectFromEnvList(container.env), cloudWebExpectedRuntimeEnv);
|
||||
runtimeIdentityEnv.push({
|
||||
serviceId,
|
||||
kind: item.kind,
|
||||
name: item.metadata?.name,
|
||||
container: container.name,
|
||||
expected: cloudWebExpectedRuntimeEnv,
|
||||
...comparison
|
||||
});
|
||||
if (!comparison.matchesDesired) {
|
||||
addBlocker(
|
||||
blockers,
|
||||
"contract_blocker",
|
||||
`k8s-runtime-env-${serviceId}`,
|
||||
`${serviceId} workload env does not own desired runtime identity: ${formatRuntimeIdentityDrift(comparison.drift)}`
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
@@ -702,7 +836,8 @@ function validateK8s(devKustomization, namespaceDoc, workloads, services, health
|
||||
services: listItems(services).length,
|
||||
configMaps: listItems(healthContract).filter((item) => item.kind === "ConfigMap").length
|
||||
},
|
||||
images
|
||||
images,
|
||||
runtimeIdentityEnv
|
||||
};
|
||||
}
|
||||
|
||||
@@ -1160,7 +1295,7 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
const commitId = resolveApplySourceCommit(deploy, catalog, gitHeadCommitId);
|
||||
|
||||
const artifactEvidence = validateDeployAndCatalog(deploy, catalog, commitId, blockers);
|
||||
const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, blockers);
|
||||
const k8sManifest = validateK8s(devKustomization, namespaceDoc, workloads, services, healthContract, deploy, catalog, blockers);
|
||||
const cloudApiDb = await checkCloudApiDb(deploy, workloads, services, blockers);
|
||||
if (!(await readText("internal/cloud/server.mjs")).includes('url.pathname === "/health/live"')) {
|
||||
addBlocker(blockers, "contract_blocker", "cloud-api-health-path", "cloud-api does not implement the k8s /health/live path");
|
||||
@@ -1168,7 +1303,10 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
|
||||
const kubectl = await resolveD601Kubectl();
|
||||
const clusterObservation = await observeCluster(kubectl, blockers);
|
||||
const cloudWebRollout = await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers);
|
||||
const cloudWebRolloutBeforeApply = await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
|
||||
observationPhase: "before_apply",
|
||||
blockRuntimeEnvDrift: false
|
||||
});
|
||||
const liveProbe = args.skipLiveProbe ? { status: "not_run", reason: "--skip-live-probe" } : await probeDevEndpoint(blockers);
|
||||
const templateJobReplacements = await buildDevTemplateJobReplacementPlan(kubectl, workloads, blockers);
|
||||
const applyStep = await runApplyStep(
|
||||
@@ -1177,6 +1315,20 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
blockers,
|
||||
templateJobReplacements
|
||||
);
|
||||
const cloudWebRolloutAfterApply = args.apply && applyStep.status === "pass"
|
||||
? await observeDeploymentRollout(kubectl, deploy, catalog, "hwlab-cloud-web", blockers, {
|
||||
observationPhase: "after_apply",
|
||||
blockRuntimeEnvDrift: true
|
||||
})
|
||||
: skippedDeploymentRolloutObservation(
|
||||
kubectl,
|
||||
deploy,
|
||||
catalog,
|
||||
"hwlab-cloud-web",
|
||||
"after_apply",
|
||||
args.apply ? "apply did not complete successfully; post-apply rollout env verification was not run" : "dry-run mode; no post-apply rollout env verification"
|
||||
);
|
||||
const cloudWebRollout = args.apply ? cloudWebRolloutAfterApply : cloudWebRolloutBeforeApply;
|
||||
const status = blockers.length > 0 ? "blocked" : "pass";
|
||||
const artifactPlan = buildArtifactPlan(deploy, catalog, commitId, artifactEvidence);
|
||||
const workloadPlan = buildWorkloadPlan(workloads);
|
||||
@@ -1264,6 +1416,8 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
templateJobReplacementPolicy: devTemplateJobReplacementPolicy,
|
||||
templateJobReplacements,
|
||||
cloudWebRollout,
|
||||
cloudWebRolloutBeforeApply,
|
||||
cloudWebRolloutAfterApply,
|
||||
applyStep,
|
||||
manualCommands,
|
||||
rollbackHint: buildRollbackHint(workloads),
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
|
||||
import { decideDevTemplateJobReplacement, resolveApplySourceCommit } from "./dev-deploy-apply.mjs";
|
||||
import { compareRuntimeIdentityEnv, decideDevTemplateJobReplacement, resolveApplySourceCommit } from "./dev-deploy-apply.mjs";
|
||||
|
||||
test("allowlisted suspended template Job image change plans replacement", () => {
|
||||
const decision = decideDevTemplateJobReplacement({
|
||||
@@ -86,3 +86,44 @@ test("apply source commit follows deploy/catalog artifact identity", () => {
|
||||
"cb35ada68606"
|
||||
);
|
||||
});
|
||||
|
||||
test("runtime identity env comparator accepts matching commit image and tag", () => {
|
||||
const comparison = compareRuntimeIdentityEnv(
|
||||
{
|
||||
HWLAB_COMMIT_ID: "7de6edd",
|
||||
HWLAB_IMAGE: "127.0.0.1:5000/hwlab/hwlab-cloud-web:7de6edd",
|
||||
HWLAB_IMAGE_TAG: "7de6edd"
|
||||
},
|
||||
{
|
||||
HWLAB_COMMIT_ID: "7de6edd2c41fb50dcb007353780338be8832b27e",
|
||||
HWLAB_IMAGE: "127.0.0.1:5000/hwlab/hwlab-cloud-web:7de6edd",
|
||||
HWLAB_IMAGE_TAG: "7de6edd"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(comparison.status, "match");
|
||||
assert.equal(comparison.matchesDesired, true);
|
||||
assert.deepEqual(comparison.drift, []);
|
||||
});
|
||||
|
||||
test("runtime identity env comparator detects stale cloud-web env drift", () => {
|
||||
const comparison = compareRuntimeIdentityEnv(
|
||||
{
|
||||
HWLAB_COMMIT_ID: "c7de474",
|
||||
HWLAB_IMAGE: "127.0.0.1:5000/hwlab/hwlab-cloud-web:c7de474",
|
||||
HWLAB_IMAGE_TAG: "c7de474"
|
||||
},
|
||||
{
|
||||
HWLAB_COMMIT_ID: "7de6edd",
|
||||
HWLAB_IMAGE: "127.0.0.1:5000/hwlab/hwlab-cloud-web:7de6edd",
|
||||
HWLAB_IMAGE_TAG: "7de6edd"
|
||||
}
|
||||
);
|
||||
|
||||
assert.equal(comparison.status, "drift");
|
||||
assert.equal(comparison.matchesDesired, false);
|
||||
assert.deepEqual(
|
||||
comparison.drift.map((field) => field.name),
|
||||
["HWLAB_COMMIT_ID", "HWLAB_IMAGE", "HWLAB_IMAGE_TAG"]
|
||||
);
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user