Code Queue Review
390 lines
13 KiB
JavaScript
390 lines
13 KiB
JavaScript
import assert from "node:assert/strict";
|
|
import test from "node:test";
|
|
import { execFile } from "node:child_process";
|
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import os from "node:os";
|
|
import path from "node:path";
|
|
import { promisify } from "node:util";
|
|
|
|
import {
|
|
buildLockAnnotations,
|
|
classifyDeployLock,
|
|
deployLockHeldFailure,
|
|
parseDeployLock,
|
|
runDevCdApply,
|
|
verifyDevLive
|
|
} from "./dev-cd-apply.mjs";
|
|
|
|
const execFileAsync = promisify(execFile);
|
|
const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), "../..");
|
|
|
|
function iso(offsetMs = 0) {
|
|
return new Date(Date.parse("2026-05-23T08:00:00.000Z") + offsetMs).toISOString();
|
|
}
|
|
|
|
function leaseFromLock(lock) {
|
|
return {
|
|
apiVersion: "coordination.k8s.io/v1",
|
|
kind: "Lease",
|
|
metadata: {
|
|
name: "hwlab-dev-cd-lock",
|
|
namespace: "hwlab-dev",
|
|
resourceVersion: "1",
|
|
annotations: buildLockAnnotations(lock)
|
|
},
|
|
spec: {
|
|
holderIdentity: `${lock.ownerTaskId}/${lock.transactionId}`,
|
|
leaseDurationSeconds: lock.ttlSeconds,
|
|
acquireTime: lock.startedAt,
|
|
renewTime: lock.updatedAt
|
|
}
|
|
};
|
|
}
|
|
|
|
function parsePatch(args) {
|
|
const index = args.indexOf("-p");
|
|
assert.notEqual(index, -1, "kubectl patch must include -p");
|
|
return JSON.parse(args[index + 1]);
|
|
}
|
|
|
|
async function makeRepo() {
|
|
const repoRoot = await fsTempDir();
|
|
await mkdir(path.join(repoRoot, "deploy"), { recursive: true });
|
|
await writeFile(
|
|
path.join(repoRoot, "deploy/deploy.json"),
|
|
`${JSON.stringify({
|
|
manifestVersion: "v1",
|
|
environment: "dev",
|
|
namespace: "hwlab-dev",
|
|
endpoint: "http://74.48.78.17:16667",
|
|
commitId: "abc1234"
|
|
}, null, 2)}\n`
|
|
);
|
|
return repoRoot;
|
|
}
|
|
|
|
async function fsTempDir() {
|
|
return await import("node:fs/promises").then((fs) => fs.mkdtemp(path.join(os.tmpdir(), "hwlab-cd-")));
|
|
}
|
|
|
|
function makeHttpGetJson() {
|
|
return async (url) => {
|
|
if (url.includes("16666")) {
|
|
return {
|
|
statusCode: 200,
|
|
json: {
|
|
serviceId: "hwlab-cloud-web",
|
|
environment: "dev",
|
|
status: "ok",
|
|
revision: "abc1234",
|
|
image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-web:abc1234", tag: "abc1234" },
|
|
observedAt: iso()
|
|
}
|
|
};
|
|
}
|
|
return {
|
|
statusCode: 200,
|
|
json: {
|
|
serviceId: "hwlab-cloud-api",
|
|
environment: "dev",
|
|
status: "degraded",
|
|
ready: false,
|
|
commit: { id: "abc1234", source: "runtime-env" },
|
|
image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", tag: "abc1234" },
|
|
blockerCodes: ["runtime_durable_adapter_auth_blocked"],
|
|
observedAt: iso()
|
|
}
|
|
};
|
|
};
|
|
}
|
|
|
|
function makeRunCommand({ heldLock = null, commandLog = [] } = {}) {
|
|
let lease = heldLock ? leaseFromLock(heldLock) : null;
|
|
const assertLeaseMicroTime = (value) => {
|
|
assert.match(value, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/u);
|
|
};
|
|
return async (command, args, options = {}) => {
|
|
commandLog.push({ command, args, env: options.env ?? {}, input: options.input ?? "" });
|
|
if (command === "git" && args[0] === "rev-parse" && args.includes("origin/main^{commit}")) {
|
|
return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" };
|
|
}
|
|
if (command === "git" && args[0] === "rev-parse" && args.includes("HEAD^{commit}")) {
|
|
return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" };
|
|
}
|
|
if (command === "which" && args[0] === "kubectl") {
|
|
return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" };
|
|
}
|
|
if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) {
|
|
if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" };
|
|
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
|
}
|
|
if (command.includes("kubectl") && args.includes("create")) {
|
|
lease = JSON.parse(options.input);
|
|
assertLeaseMicroTime(lease.spec.acquireTime);
|
|
assertLeaseMicroTime(lease.spec.renewTime);
|
|
lease.metadata.resourceVersion = "2";
|
|
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
|
}
|
|
if (command.includes("kubectl") && args.includes("replace")) {
|
|
const nextLease = JSON.parse(options.input);
|
|
assert.equal(nextLease.metadata?.resourceVersion, lease?.metadata?.resourceVersion);
|
|
if (nextLease.spec?.acquireTime) assertLeaseMicroTime(nextLease.spec.acquireTime);
|
|
if (nextLease.spec?.renewTime) assertLeaseMicroTime(nextLease.spec.renewTime);
|
|
lease = nextLease;
|
|
lease.metadata.resourceVersion = String(Number(lease.metadata.resourceVersion ?? "2") + 1);
|
|
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
|
}
|
|
if (command.includes("kubectl") && args.includes("patch")) {
|
|
assert.ok(lease, "lease must exist before patch");
|
|
const patch = parsePatch(args);
|
|
lease.metadata.annotations = {
|
|
...lease.metadata.annotations,
|
|
...(patch.metadata?.annotations ?? {})
|
|
};
|
|
lease.spec = {
|
|
...lease.spec,
|
|
...(patch.spec ?? {})
|
|
};
|
|
lease.metadata.resourceVersion = String(Number(lease.metadata.resourceVersion ?? "2") + 1);
|
|
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
|
}
|
|
if (command === process.execPath || command.endsWith("/node")) {
|
|
return { code: 0, stdout: JSON.stringify({ ok: true, command: args.join(" ") }), stderr: "" };
|
|
}
|
|
return { code: 0, stdout: "", stderr: "" };
|
|
};
|
|
}
|
|
|
|
test("lock classifier reports held and stale states with retryAfterSeconds", () => {
|
|
const lock = {
|
|
ownerTaskId: "task-a",
|
|
transactionId: "tx-a",
|
|
promotionCommit: "abc1234",
|
|
deployJsonHash: "sha256:a",
|
|
phase: "applying",
|
|
startedAt: iso(0),
|
|
updatedAt: iso(0),
|
|
ttlSeconds: 300,
|
|
targetNamespace: "hwlab-dev"
|
|
};
|
|
const parsed = parseDeployLock(leaseFromLock(lock));
|
|
const held = classifyDeployLock(parsed, new Date(iso(120000)));
|
|
assert.equal(held.held, true);
|
|
assert.equal(held.stale, false);
|
|
assert.equal(held.retryAfterSeconds, 180);
|
|
|
|
const stale = classifyDeployLock(parsed, new Date(iso(301000)));
|
|
assert.equal(stale.held, false);
|
|
assert.equal(stale.stale, true);
|
|
assert.equal(stale.retryAfterSeconds, 0);
|
|
});
|
|
|
|
test("deploy-lock-held failure is structured before publish/apply", () => {
|
|
const failure = deployLockHeldFailure({
|
|
ownerTaskId: "task-a",
|
|
transactionId: "tx-a",
|
|
promotionCommit: "abc1234",
|
|
deployJsonHash: "sha256:a",
|
|
phase: "applying",
|
|
updatedAt: iso(0),
|
|
ttlSeconds: 300,
|
|
targetNamespace: "hwlab-dev",
|
|
lockName: "hwlab-dev-cd-lock",
|
|
lockBackend: "Lease"
|
|
}, new Date(iso(100000)));
|
|
|
|
assert.equal(failure.ok, false);
|
|
assert.equal(failure.error, "deploy-lock-held");
|
|
assert.equal(failure.holder, "task-a");
|
|
assert.equal(failure.promotionCommit, "abc1234");
|
|
assert.equal(failure.phase, "applying");
|
|
assert.equal(failure.retryAfterSeconds, 200);
|
|
assert.equal(failure.mutationAttempted, false);
|
|
});
|
|
|
|
test("second transaction exits deploy-lock-held without running side effects", async () => {
|
|
const repoRoot = await makeRepo();
|
|
const commandLog = [];
|
|
let output = "";
|
|
const code = await runDevCdApply([
|
|
"--apply",
|
|
"--confirm-dev",
|
|
"--confirmed-non-production",
|
|
"--owner-task-id",
|
|
"task-b",
|
|
"--kubeconfig",
|
|
"/tmp/kubeconfig"
|
|
], {
|
|
repoRoot,
|
|
env: {},
|
|
runCommand: makeRunCommand({
|
|
commandLog,
|
|
heldLock: {
|
|
promotionCommit: "abc1234",
|
|
deployJsonHash: "sha256:a",
|
|
ownerTaskId: "task-a",
|
|
transactionId: "tx-a",
|
|
phase: "publishing",
|
|
startedAt: iso(0),
|
|
updatedAt: iso(0),
|
|
ttlSeconds: 300,
|
|
liveBefore: { status: "pass" },
|
|
targetNamespace: "hwlab-dev",
|
|
targetRef: "origin/main",
|
|
lockBackend: "Lease"
|
|
}
|
|
}),
|
|
httpGetJson: makeHttpGetJson(),
|
|
now: () => new Date(iso(100000)),
|
|
stdout: { write: (chunk) => { output += chunk; } }
|
|
});
|
|
|
|
const failure = JSON.parse(output);
|
|
assert.equal(code, 2);
|
|
assert.equal(failure.error, "deploy-lock-held");
|
|
assert.equal(failure.holder, "task-a");
|
|
assert.equal(failure.retryAfterSeconds, 200);
|
|
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
|
|
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
|
|
});
|
|
|
|
test("stale lock requires explicit break-stale-lock confirmation", async () => {
|
|
const repoRoot = await makeRepo();
|
|
let output = "";
|
|
const code = await runDevCdApply([
|
|
"--apply",
|
|
"--confirm-dev",
|
|
"--confirmed-non-production",
|
|
"--owner-task-id",
|
|
"task-b",
|
|
"--kubeconfig",
|
|
"/tmp/kubeconfig"
|
|
], {
|
|
repoRoot,
|
|
env: {},
|
|
runCommand: makeRunCommand({
|
|
heldLock: {
|
|
promotionCommit: "abc1234",
|
|
deployJsonHash: "sha256:a",
|
|
ownerTaskId: "task-a",
|
|
transactionId: "tx-a",
|
|
phase: "verifying",
|
|
startedAt: iso(0),
|
|
updatedAt: iso(0),
|
|
ttlSeconds: 300,
|
|
liveBefore: { status: "pass" },
|
|
targetNamespace: "hwlab-dev",
|
|
targetRef: "origin/main",
|
|
lockBackend: "Lease"
|
|
}
|
|
}),
|
|
httpGetJson: makeHttpGetJson(),
|
|
now: () => new Date(iso(301000)),
|
|
stdout: { write: (chunk) => { output += chunk; } }
|
|
});
|
|
|
|
const failure = JSON.parse(output);
|
|
assert.equal(code, 2);
|
|
assert.equal(failure.error, "deploy-lock-held");
|
|
assert.equal(failure.stale, true);
|
|
assert.equal(failure.requiresBreakStaleLock, true);
|
|
});
|
|
|
|
test("transaction runs phases, allows internal side-effect env, releases lock, and reports live verify", async () => {
|
|
const repoRoot = await makeRepo();
|
|
const commandLog = [];
|
|
let output = "";
|
|
const code = await runDevCdApply([
|
|
"--apply",
|
|
"--confirm-dev",
|
|
"--confirmed-non-production",
|
|
"--break-stale-lock",
|
|
"--owner-task-id",
|
|
"task-b",
|
|
"--kubeconfig",
|
|
"/tmp/kubeconfig",
|
|
"--write-report"
|
|
], {
|
|
repoRoot,
|
|
env: {},
|
|
runCommand: makeRunCommand({
|
|
commandLog,
|
|
heldLock: {
|
|
promotionCommit: "old1234",
|
|
deployJsonHash: "sha256:old",
|
|
ownerTaskId: "task-a",
|
|
transactionId: "tx-a",
|
|
phase: "verifying",
|
|
startedAt: iso(0),
|
|
updatedAt: iso(0),
|
|
ttlSeconds: 300,
|
|
liveBefore: { status: "pass" },
|
|
targetNamespace: "hwlab-dev",
|
|
targetRef: "origin/main",
|
|
lockBackend: "Lease"
|
|
}
|
|
}),
|
|
httpGetJson: makeHttpGetJson(),
|
|
now: () => new Date(iso(301000)),
|
|
stdout: { write: (chunk) => { output += chunk; } }
|
|
});
|
|
|
|
const report = JSON.parse(output);
|
|
assert.equal(code, 0);
|
|
assert.equal(report.status, "pass");
|
|
assert.deepEqual(
|
|
report.transaction.phases.map((phase) => phase.phase),
|
|
["publishing", "applying", "verifying", "released"]
|
|
);
|
|
assert.equal(report.devCdApply.lock.staleBreak.ownerTaskId, "task-a");
|
|
assert.equal(report.transaction.release.status, "released");
|
|
assert.equal(report.devCdApply.liveVerify.status, "pass");
|
|
assert.equal(report.devCdApply.liveVerify.summary.checked, 2);
|
|
assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-web-16666"), true);
|
|
assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-api-16667"), true);
|
|
assert.equal(report.devCdApply.reportPaths.transaction, "reports/dev-gate/dev-cd-apply.json");
|
|
|
|
const publishCall = commandLog.find((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs"));
|
|
const applyCall = commandLog.find((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs"));
|
|
assert.ok(publishCall?.env.HWLAB_CD_TRANSACTION_ID);
|
|
assert.equal(applyCall?.env.HWLAB_CD_TRANSACTION_ID, publishCall.env.HWLAB_CD_TRANSACTION_ID);
|
|
|
|
const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
|
|
assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2);
|
|
});
|
|
|
|
test("live verify summarizes 16666 and 16667 health identity", async () => {
|
|
const result = await verifyDevLive({
|
|
now: () => new Date(iso()),
|
|
httpGetJson: makeHttpGetJson()
|
|
}, { expectedCommit: "abc1234" });
|
|
|
|
assert.equal(result.status, "pass");
|
|
assert.equal(result.summary.checked, 2);
|
|
assert.deepEqual(result.endpoints.map((endpoint) => endpoint.url), [
|
|
"http://74.48.78.17:16666/health/live",
|
|
"http://74.48.78.17:16667/health/live"
|
|
]);
|
|
assert.equal(result.endpoints.every((endpoint) => endpoint.commitMatches), true);
|
|
});
|
|
|
|
test("legacy artifact publish CLI rejects side effects without transaction env", async () => {
|
|
await assert.rejects(
|
|
execFileAsync(process.execPath, ["scripts/dev-artifact-publish.mjs", "--publish", "--no-report"], {
|
|
cwd: repoRoot,
|
|
env: {
|
|
...process.env,
|
|
HWLAB_CD_TRANSACTION_ID: ""
|
|
}
|
|
}),
|
|
(error) => {
|
|
assert.equal(error.code, 2);
|
|
const payload = JSON.parse(error.stdout);
|
|
assert.equal(payload.error, "cd-transaction-required");
|
|
assert.equal(payload.script, "scripts/dev-artifact-publish.mjs");
|
|
assert.equal(payload.mutationAttempted, false);
|
|
return true;
|
|
}
|
|
);
|
|
});
|