fix: serialize DEV CD apply
This commit is contained in:
Code Queue Review
@@ -0,0 +1,382 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import {
|
||||
buildLockAnnotations,
|
||||
classifyDeployLock,
|
||||
deployLockHeldFailure,
|
||||
parseDeployLock,
|
||||
runDevCdApply,
|
||||
verifyDevLive
|
||||
} from "./dev-cd-apply.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), "../..");
|
||||
|
||||
function iso(offsetMs = 0) {
|
||||
return new Date(Date.parse("2026-05-23T08:00:00.000Z") + offsetMs).toISOString();
|
||||
}
|
||||
|
||||
function leaseFromLock(lock) {
|
||||
return {
|
||||
apiVersion: "coordination.k8s.io/v1",
|
||||
kind: "Lease",
|
||||
metadata: {
|
||||
name: "hwlab-dev-cd-lock",
|
||||
namespace: "hwlab-dev",
|
||||
resourceVersion: "1",
|
||||
annotations: buildLockAnnotations(lock)
|
||||
},
|
||||
spec: {
|
||||
holderIdentity: `${lock.ownerTaskId}/${lock.transactionId}`,
|
||||
leaseDurationSeconds: lock.ttlSeconds,
|
||||
acquireTime: lock.startedAt,
|
||||
renewTime: lock.updatedAt
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function parsePatch(args) {
|
||||
const index = args.indexOf("-p");
|
||||
assert.notEqual(index, -1, "kubectl patch must include -p");
|
||||
return JSON.parse(args[index + 1]);
|
||||
}
|
||||
|
||||
async function makeRepo() {
|
||||
const repoRoot = await fsTempDir();
|
||||
await mkdir(path.join(repoRoot, "deploy"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(repoRoot, "deploy/deploy.json"),
|
||||
`${JSON.stringify({
|
||||
manifestVersion: "v1",
|
||||
environment: "dev",
|
||||
namespace: "hwlab-dev",
|
||||
endpoint: "http://74.48.78.17:16667",
|
||||
commitId: "abc1234"
|
||||
}, null, 2)}\n`
|
||||
);
|
||||
return repoRoot;
|
||||
}
|
||||
|
||||
async function fsTempDir() {
|
||||
return await import("node:fs/promises").then((fs) => fs.mkdtemp(path.join(os.tmpdir(), "hwlab-cd-")));
|
||||
}
|
||||
|
||||
function makeHttpGetJson() {
|
||||
return async (url) => {
|
||||
if (url.includes("16666")) {
|
||||
return {
|
||||
statusCode: 200,
|
||||
json: {
|
||||
serviceId: "hwlab-cloud-web",
|
||||
environment: "dev",
|
||||
status: "ok",
|
||||
revision: "abc1234",
|
||||
image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-web:abc1234", tag: "abc1234" },
|
||||
observedAt: iso()
|
||||
}
|
||||
};
|
||||
}
|
||||
return {
|
||||
statusCode: 200,
|
||||
json: {
|
||||
serviceId: "hwlab-cloud-api",
|
||||
environment: "dev",
|
||||
status: "degraded",
|
||||
ready: false,
|
||||
commit: { id: "abc1234", source: "runtime-env" },
|
||||
image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", tag: "abc1234" },
|
||||
blockerCodes: ["runtime_durable_adapter_auth_blocked"],
|
||||
observedAt: iso()
|
||||
}
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
function makeRunCommand({ heldLock = null, commandLog = [] } = {}) {
|
||||
let lease = heldLock ? leaseFromLock(heldLock) : null;
|
||||
return async (command, args, options = {}) => {
|
||||
commandLog.push({ command, args, env: options.env ?? {}, input: options.input ?? "" });
|
||||
if (command === "git" && args[0] === "rev-parse" && args.includes("origin/main^{commit}")) {
|
||||
return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" };
|
||||
}
|
||||
if (command === "git" && args[0] === "rev-parse" && args.includes("HEAD^{commit}")) {
|
||||
return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" };
|
||||
}
|
||||
if (command === "which" && args[0] === "kubectl") {
|
||||
return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) {
|
||||
if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" };
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("create")) {
|
||||
lease = JSON.parse(options.input);
|
||||
lease.metadata.resourceVersion = "2";
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("replace")) {
|
||||
const nextLease = JSON.parse(options.input);
|
||||
assert.equal(nextLease.metadata?.resourceVersion, lease?.metadata?.resourceVersion);
|
||||
lease = nextLease;
|
||||
lease.metadata.resourceVersion = String(Number(lease.metadata.resourceVersion ?? "2") + 1);
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("patch")) {
|
||||
assert.ok(lease, "lease must exist before patch");
|
||||
const patch = parsePatch(args);
|
||||
lease.metadata.annotations = {
|
||||
...lease.metadata.annotations,
|
||||
...(patch.metadata?.annotations ?? {})
|
||||
};
|
||||
lease.spec = {
|
||||
...lease.spec,
|
||||
...(patch.spec ?? {})
|
||||
};
|
||||
lease.metadata.resourceVersion = String(Number(lease.metadata.resourceVersion ?? "2") + 1);
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
}
|
||||
if (command === process.execPath || command.endsWith("/node")) {
|
||||
return { code: 0, stdout: JSON.stringify({ ok: true, command: args.join(" ") }), stderr: "" };
|
||||
}
|
||||
return { code: 0, stdout: "", stderr: "" };
|
||||
};
|
||||
}
|
||||
|
||||
test("lock classifier reports held and stale states with retryAfterSeconds", () => {
|
||||
const lock = {
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
promotionCommit: "abc1234",
|
||||
deployJsonHash: "sha256:a",
|
||||
phase: "applying",
|
||||
startedAt: iso(0),
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
targetNamespace: "hwlab-dev"
|
||||
};
|
||||
const parsed = parseDeployLock(leaseFromLock(lock));
|
||||
const held = classifyDeployLock(parsed, new Date(iso(120000)));
|
||||
assert.equal(held.held, true);
|
||||
assert.equal(held.stale, false);
|
||||
assert.equal(held.retryAfterSeconds, 180);
|
||||
|
||||
const stale = classifyDeployLock(parsed, new Date(iso(301000)));
|
||||
assert.equal(stale.held, false);
|
||||
assert.equal(stale.stale, true);
|
||||
assert.equal(stale.retryAfterSeconds, 0);
|
||||
});
|
||||
|
||||
test("deploy-lock-held failure is structured before publish/apply", () => {
|
||||
const failure = deployLockHeldFailure({
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
promotionCommit: "abc1234",
|
||||
deployJsonHash: "sha256:a",
|
||||
phase: "applying",
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
targetNamespace: "hwlab-dev",
|
||||
lockName: "hwlab-dev-cd-lock",
|
||||
lockBackend: "Lease"
|
||||
}, new Date(iso(100000)));
|
||||
|
||||
assert.equal(failure.ok, false);
|
||||
assert.equal(failure.error, "deploy-lock-held");
|
||||
assert.equal(failure.holder, "task-a");
|
||||
assert.equal(failure.promotionCommit, "abc1234");
|
||||
assert.equal(failure.phase, "applying");
|
||||
assert.equal(failure.retryAfterSeconds, 200);
|
||||
assert.equal(failure.mutationAttempted, false);
|
||||
});
|
||||
|
||||
test("second transaction exits deploy-lock-held without running side effects", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-b",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
commandLog,
|
||||
heldLock: {
|
||||
promotionCommit: "abc1234",
|
||||
deployJsonHash: "sha256:a",
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
phase: "publishing",
|
||||
startedAt: iso(0),
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
liveBefore: { status: "pass" },
|
||||
targetNamespace: "hwlab-dev",
|
||||
targetRef: "origin/main",
|
||||
lockBackend: "Lease"
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const failure = JSON.parse(output);
|
||||
assert.equal(code, 2);
|
||||
assert.equal(failure.error, "deploy-lock-held");
|
||||
assert.equal(failure.holder, "task-a");
|
||||
assert.equal(failure.retryAfterSeconds, 200);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
|
||||
});
|
||||
|
||||
test("stale lock requires explicit break-stale-lock confirmation", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-b",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
heldLock: {
|
||||
promotionCommit: "abc1234",
|
||||
deployJsonHash: "sha256:a",
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
phase: "verifying",
|
||||
startedAt: iso(0),
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
liveBefore: { status: "pass" },
|
||||
targetNamespace: "hwlab-dev",
|
||||
targetRef: "origin/main",
|
||||
lockBackend: "Lease"
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(301000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const failure = JSON.parse(output);
|
||||
assert.equal(code, 2);
|
||||
assert.equal(failure.error, "deploy-lock-held");
|
||||
assert.equal(failure.stale, true);
|
||||
assert.equal(failure.requiresBreakStaleLock, true);
|
||||
});
|
||||
|
||||
test("transaction runs phases, allows internal side-effect env, releases lock, and reports live verify", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--break-stale-lock",
|
||||
"--owner-task-id",
|
||||
"task-b",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig",
|
||||
"--write-report"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
commandLog,
|
||||
heldLock: {
|
||||
promotionCommit: "old1234",
|
||||
deployJsonHash: "sha256:old",
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
phase: "verifying",
|
||||
startedAt: iso(0),
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
liveBefore: { status: "pass" },
|
||||
targetNamespace: "hwlab-dev",
|
||||
targetRef: "origin/main",
|
||||
lockBackend: "Lease"
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(301000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const report = JSON.parse(output);
|
||||
assert.equal(code, 0);
|
||||
assert.equal(report.status, "pass");
|
||||
assert.deepEqual(
|
||||
report.transaction.phases.map((phase) => phase.phase),
|
||||
["publishing", "applying", "verifying", "released"]
|
||||
);
|
||||
assert.equal(report.devCdApply.lock.staleBreak.ownerTaskId, "task-a");
|
||||
assert.equal(report.transaction.release.status, "released");
|
||||
assert.equal(report.devCdApply.liveVerify.status, "pass");
|
||||
assert.equal(report.devCdApply.liveVerify.summary.checked, 2);
|
||||
assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-web-16666"), true);
|
||||
assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-api-16667"), true);
|
||||
assert.equal(report.devCdApply.reportPaths.transaction, "reports/dev-gate/dev-cd-apply.json");
|
||||
|
||||
const publishCall = commandLog.find((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs"));
|
||||
const applyCall = commandLog.find((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs"));
|
||||
assert.ok(publishCall?.env.HWLAB_CD_TRANSACTION_ID);
|
||||
assert.equal(applyCall?.env.HWLAB_CD_TRANSACTION_ID, publishCall.env.HWLAB_CD_TRANSACTION_ID);
|
||||
|
||||
const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
|
||||
assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2);
|
||||
});
|
||||
|
||||
test("live verify summarizes 16666 and 16667 health identity", async () => {
|
||||
const result = await verifyDevLive({
|
||||
now: () => new Date(iso()),
|
||||
httpGetJson: makeHttpGetJson()
|
||||
}, { expectedCommit: "abc1234" });
|
||||
|
||||
assert.equal(result.status, "pass");
|
||||
assert.equal(result.summary.checked, 2);
|
||||
assert.deepEqual(result.endpoints.map((endpoint) => endpoint.url), [
|
||||
"http://74.48.78.17:16666/health/live",
|
||||
"http://74.48.78.17:16667/health/live"
|
||||
]);
|
||||
assert.equal(result.endpoints.every((endpoint) => endpoint.commitMatches), true);
|
||||
});
|
||||
|
||||
test("legacy artifact publish CLI rejects side effects without transaction env", async () => {
|
||||
await assert.rejects(
|
||||
execFileAsync(process.execPath, ["scripts/dev-artifact-publish.mjs", "--publish", "--no-report"], {
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
HWLAB_CD_TRANSACTION_ID: ""
|
||||
}
|
||||
}),
|
||||
(error) => {
|
||||
assert.equal(error.code, 2);
|
||||
const payload = JSON.parse(error.stdout);
|
||||
assert.equal(payload.error, "cd-transaction-required");
|
||||
assert.equal(payload.script, "scripts/dev-artifact-publish.mjs");
|
||||
assert.equal(payload.mutationAttempted, false);
|
||||
return true;
|
||||
}
|
||||
);
|
||||
});
|
||||
Reference in New Issue
Block a user