fix: serialize DEV CD apply
This commit is contained in:
Code Queue Review
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,382 @@
|
||||
import assert from "node:assert/strict";
|
||||
import test from "node:test";
|
||||
import { execFile } from "node:child_process";
|
||||
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
||||
import os from "node:os";
|
||||
import path from "node:path";
|
||||
import { promisify } from "node:util";
|
||||
|
||||
import {
|
||||
buildLockAnnotations,
|
||||
classifyDeployLock,
|
||||
deployLockHeldFailure,
|
||||
parseDeployLock,
|
||||
runDevCdApply,
|
||||
verifyDevLive
|
||||
} from "./dev-cd-apply.mjs";
|
||||
|
||||
const execFileAsync = promisify(execFile);
|
||||
const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), "../..");
|
||||
|
||||
function iso(offsetMs = 0) {
|
||||
return new Date(Date.parse("2026-05-23T08:00:00.000Z") + offsetMs).toISOString();
|
||||
}
|
||||
|
||||
function leaseFromLock(lock) {
|
||||
return {
|
||||
apiVersion: "coordination.k8s.io/v1",
|
||||
kind: "Lease",
|
||||
metadata: {
|
||||
name: "hwlab-dev-cd-lock",
|
||||
namespace: "hwlab-dev",
|
||||
resourceVersion: "1",
|
||||
annotations: buildLockAnnotations(lock)
|
||||
},
|
||||
spec: {
|
||||
holderIdentity: `${lock.ownerTaskId}/${lock.transactionId}`,
|
||||
leaseDurationSeconds: lock.ttlSeconds,
|
||||
acquireTime: lock.startedAt,
|
||||
renewTime: lock.updatedAt
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function parsePatch(args) {
|
||||
const index = args.indexOf("-p");
|
||||
assert.notEqual(index, -1, "kubectl patch must include -p");
|
||||
return JSON.parse(args[index + 1]);
|
||||
}
|
||||
|
||||
async function makeRepo() {
|
||||
const repoRoot = await fsTempDir();
|
||||
await mkdir(path.join(repoRoot, "deploy"), { recursive: true });
|
||||
await writeFile(
|
||||
path.join(repoRoot, "deploy/deploy.json"),
|
||||
`${JSON.stringify({
|
||||
manifestVersion: "v1",
|
||||
environment: "dev",
|
||||
namespace: "hwlab-dev",
|
||||
endpoint: "http://74.48.78.17:16667",
|
||||
commitId: "abc1234"
|
||||
}, null, 2)}\n`
|
||||
);
|
||||
return repoRoot;
|
||||
}
|
||||
|
||||
async function fsTempDir() {
|
||||
return await import("node:fs/promises").then((fs) => fs.mkdtemp(path.join(os.tmpdir(), "hwlab-cd-")));
|
||||
}
|
||||
|
||||
function makeHttpGetJson() {
|
||||
return async (url) => {
|
||||
if (url.includes("16666")) {
|
||||
return {
|
||||
statusCode: 200,
|
||||
json: {
|
||||
serviceId: "hwlab-cloud-web",
|
||||
environment: "dev",
|
||||
status: "ok",
|
||||
revision: "abc1234",
|
||||
image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-web:abc1234", tag: "abc1234" },
|
||||
observedAt: iso()
|
||||
}
|
||||
};
|
||||
}
|
||||
return {
|
||||
statusCode: 200,
|
||||
json: {
|
||||
serviceId: "hwlab-cloud-api",
|
||||
environment: "dev",
|
||||
status: "degraded",
|
||||
ready: false,
|
||||
commit: { id: "abc1234", source: "runtime-env" },
|
||||
image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", tag: "abc1234" },
|
||||
blockerCodes: ["runtime_durable_adapter_auth_blocked"],
|
||||
observedAt: iso()
|
||||
}
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
function makeRunCommand({ heldLock = null, commandLog = [] } = {}) {
|
||||
let lease = heldLock ? leaseFromLock(heldLock) : null;
|
||||
return async (command, args, options = {}) => {
|
||||
commandLog.push({ command, args, env: options.env ?? {}, input: options.input ?? "" });
|
||||
if (command === "git" && args[0] === "rev-parse" && args.includes("origin/main^{commit}")) {
|
||||
return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" };
|
||||
}
|
||||
if (command === "git" && args[0] === "rev-parse" && args.includes("HEAD^{commit}")) {
|
||||
return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" };
|
||||
}
|
||||
if (command === "which" && args[0] === "kubectl") {
|
||||
return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) {
|
||||
if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" };
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("create")) {
|
||||
lease = JSON.parse(options.input);
|
||||
lease.metadata.resourceVersion = "2";
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("replace")) {
|
||||
const nextLease = JSON.parse(options.input);
|
||||
assert.equal(nextLease.metadata?.resourceVersion, lease?.metadata?.resourceVersion);
|
||||
lease = nextLease;
|
||||
lease.metadata.resourceVersion = String(Number(lease.metadata.resourceVersion ?? "2") + 1);
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("patch")) {
|
||||
assert.ok(lease, "lease must exist before patch");
|
||||
const patch = parsePatch(args);
|
||||
lease.metadata.annotations = {
|
||||
...lease.metadata.annotations,
|
||||
...(patch.metadata?.annotations ?? {})
|
||||
};
|
||||
lease.spec = {
|
||||
...lease.spec,
|
||||
...(patch.spec ?? {})
|
||||
};
|
||||
lease.metadata.resourceVersion = String(Number(lease.metadata.resourceVersion ?? "2") + 1);
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
}
|
||||
if (command === process.execPath || command.endsWith("/node")) {
|
||||
return { code: 0, stdout: JSON.stringify({ ok: true, command: args.join(" ") }), stderr: "" };
|
||||
}
|
||||
return { code: 0, stdout: "", stderr: "" };
|
||||
};
|
||||
}
|
||||
|
||||
test("lock classifier reports held and stale states with retryAfterSeconds", () => {
|
||||
const lock = {
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
promotionCommit: "abc1234",
|
||||
deployJsonHash: "sha256:a",
|
||||
phase: "applying",
|
||||
startedAt: iso(0),
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
targetNamespace: "hwlab-dev"
|
||||
};
|
||||
const parsed = parseDeployLock(leaseFromLock(lock));
|
||||
const held = classifyDeployLock(parsed, new Date(iso(120000)));
|
||||
assert.equal(held.held, true);
|
||||
assert.equal(held.stale, false);
|
||||
assert.equal(held.retryAfterSeconds, 180);
|
||||
|
||||
const stale = classifyDeployLock(parsed, new Date(iso(301000)));
|
||||
assert.equal(stale.held, false);
|
||||
assert.equal(stale.stale, true);
|
||||
assert.equal(stale.retryAfterSeconds, 0);
|
||||
});
|
||||
|
||||
test("deploy-lock-held failure is structured before publish/apply", () => {
|
||||
const failure = deployLockHeldFailure({
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
promotionCommit: "abc1234",
|
||||
deployJsonHash: "sha256:a",
|
||||
phase: "applying",
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
targetNamespace: "hwlab-dev",
|
||||
lockName: "hwlab-dev-cd-lock",
|
||||
lockBackend: "Lease"
|
||||
}, new Date(iso(100000)));
|
||||
|
||||
assert.equal(failure.ok, false);
|
||||
assert.equal(failure.error, "deploy-lock-held");
|
||||
assert.equal(failure.holder, "task-a");
|
||||
assert.equal(failure.promotionCommit, "abc1234");
|
||||
assert.equal(failure.phase, "applying");
|
||||
assert.equal(failure.retryAfterSeconds, 200);
|
||||
assert.equal(failure.mutationAttempted, false);
|
||||
});
|
||||
|
||||
test("second transaction exits deploy-lock-held without running side effects", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-b",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
commandLog,
|
||||
heldLock: {
|
||||
promotionCommit: "abc1234",
|
||||
deployJsonHash: "sha256:a",
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
phase: "publishing",
|
||||
startedAt: iso(0),
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
liveBefore: { status: "pass" },
|
||||
targetNamespace: "hwlab-dev",
|
||||
targetRef: "origin/main",
|
||||
lockBackend: "Lease"
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const failure = JSON.parse(output);
|
||||
assert.equal(code, 2);
|
||||
assert.equal(failure.error, "deploy-lock-held");
|
||||
assert.equal(failure.holder, "task-a");
|
||||
assert.equal(failure.retryAfterSeconds, 200);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
|
||||
});
|
||||
|
||||
test("stale lock requires explicit break-stale-lock confirmation", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-b",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
heldLock: {
|
||||
promotionCommit: "abc1234",
|
||||
deployJsonHash: "sha256:a",
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
phase: "verifying",
|
||||
startedAt: iso(0),
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
liveBefore: { status: "pass" },
|
||||
targetNamespace: "hwlab-dev",
|
||||
targetRef: "origin/main",
|
||||
lockBackend: "Lease"
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(301000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const failure = JSON.parse(output);
|
||||
assert.equal(code, 2);
|
||||
assert.equal(failure.error, "deploy-lock-held");
|
||||
assert.equal(failure.stale, true);
|
||||
assert.equal(failure.requiresBreakStaleLock, true);
|
||||
});
|
||||
|
||||
test("transaction runs phases, allows internal side-effect env, releases lock, and reports live verify", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--break-stale-lock",
|
||||
"--owner-task-id",
|
||||
"task-b",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig",
|
||||
"--write-report"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
commandLog,
|
||||
heldLock: {
|
||||
promotionCommit: "old1234",
|
||||
deployJsonHash: "sha256:old",
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
phase: "verifying",
|
||||
startedAt: iso(0),
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
liveBefore: { status: "pass" },
|
||||
targetNamespace: "hwlab-dev",
|
||||
targetRef: "origin/main",
|
||||
lockBackend: "Lease"
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(301000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const report = JSON.parse(output);
|
||||
assert.equal(code, 0);
|
||||
assert.equal(report.status, "pass");
|
||||
assert.deepEqual(
|
||||
report.transaction.phases.map((phase) => phase.phase),
|
||||
["publishing", "applying", "verifying", "released"]
|
||||
);
|
||||
assert.equal(report.devCdApply.lock.staleBreak.ownerTaskId, "task-a");
|
||||
assert.equal(report.transaction.release.status, "released");
|
||||
assert.equal(report.devCdApply.liveVerify.status, "pass");
|
||||
assert.equal(report.devCdApply.liveVerify.summary.checked, 2);
|
||||
assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-web-16666"), true);
|
||||
assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-api-16667"), true);
|
||||
assert.equal(report.devCdApply.reportPaths.transaction, "reports/dev-gate/dev-cd-apply.json");
|
||||
|
||||
const publishCall = commandLog.find((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs"));
|
||||
const applyCall = commandLog.find((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs"));
|
||||
assert.ok(publishCall?.env.HWLAB_CD_TRANSACTION_ID);
|
||||
assert.equal(applyCall?.env.HWLAB_CD_TRANSACTION_ID, publishCall.env.HWLAB_CD_TRANSACTION_ID);
|
||||
|
||||
const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
|
||||
assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2);
|
||||
});
|
||||
|
||||
test("live verify summarizes 16666 and 16667 health identity", async () => {
|
||||
const result = await verifyDevLive({
|
||||
now: () => new Date(iso()),
|
||||
httpGetJson: makeHttpGetJson()
|
||||
}, { expectedCommit: "abc1234" });
|
||||
|
||||
assert.equal(result.status, "pass");
|
||||
assert.equal(result.summary.checked, 2);
|
||||
assert.deepEqual(result.endpoints.map((endpoint) => endpoint.url), [
|
||||
"http://74.48.78.17:16666/health/live",
|
||||
"http://74.48.78.17:16667/health/live"
|
||||
]);
|
||||
assert.equal(result.endpoints.every((endpoint) => endpoint.commitMatches), true);
|
||||
});
|
||||
|
||||
test("legacy artifact publish CLI rejects side effects without transaction env", async () => {
|
||||
await assert.rejects(
|
||||
execFileAsync(process.execPath, ["scripts/dev-artifact-publish.mjs", "--publish", "--no-report"], {
|
||||
cwd: repoRoot,
|
||||
env: {
|
||||
...process.env,
|
||||
HWLAB_CD_TRANSACTION_ID: ""
|
||||
}
|
||||
}),
|
||||
(error) => {
|
||||
assert.equal(error.code, 2);
|
||||
const payload = JSON.parse(error.stdout);
|
||||
assert.equal(payload.error, "cd-transaction-required");
|
||||
assert.equal(payload.script, "scripts/dev-artifact-publish.mjs");
|
||||
assert.equal(payload.mutationAttempted, false);
|
||||
return true;
|
||||
}
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
const transactionEnvNames = [
|
||||
"HWLAB_CD_TRANSACTION_ID",
|
||||
"HWLAB_CD_TRANSACTION_OWNER",
|
||||
"HWLAB_CD_LOCK_NAME"
|
||||
];
|
||||
|
||||
export function hasDevCdTransactionEnv(env = process.env) {
|
||||
return typeof env.HWLAB_CD_TRANSACTION_ID === "string" && env.HWLAB_CD_TRANSACTION_ID.trim().length > 0;
|
||||
}
|
||||
|
||||
export function devCdTransactionGuardFailure({
|
||||
script,
|
||||
mode,
|
||||
requiredEnv = "HWLAB_CD_TRANSACTION_ID"
|
||||
}) {
|
||||
return {
|
||||
ok: false,
|
||||
status: "blocked",
|
||||
error: "cd-transaction-required",
|
||||
code: "cd-transaction-required",
|
||||
script,
|
||||
mode,
|
||||
requiredEnv,
|
||||
acceptedEnv: transactionEnvNames,
|
||||
entrypoint: "node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report",
|
||||
summary: `${script} ${mode} is a DEV CD side-effect step and must run inside scripts/dev-cd-apply.mjs.`,
|
||||
devOnly: true,
|
||||
prodTouched: false,
|
||||
mutationAttempted: false
|
||||
};
|
||||
}
|
||||
|
||||
export function requireDevCdTransactionForSideEffect({ env = process.env, script, mode }) {
|
||||
if (hasDevCdTransactionEnv(env)) return null;
|
||||
return devCdTransactionGuardFailure({ script, mode });
|
||||
}
|
||||
@@ -17,6 +17,7 @@ import {
|
||||
} from "../../internal/cloud/code-agent-contract.mjs";
|
||||
import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs";
|
||||
import { DEV_ENDPOINT, ENVIRONMENT_DEV, SERVICE_IDS } from "../../internal/protocol/index.mjs";
|
||||
import { requireDevCdTransactionForSideEffect } from "./dev-cd-transaction-guard.mjs";
|
||||
|
||||
const repoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
||||
const reportPath = "reports/dev-gate/dev-deploy-report.json";
|
||||
@@ -72,7 +73,7 @@ const requiredValidationCommands = [
|
||||
"node --check scripts/src/dev-deploy-apply.mjs",
|
||||
"node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked"
|
||||
];
|
||||
const applyCommand = "node scripts/dev-deploy-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report";
|
||||
const applyCommand = "node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report";
|
||||
const dryRunPlanCommand = "node scripts/dev-deploy-apply.mjs --dry-run --write-report";
|
||||
const blockedDryRunCommand = "node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked --write-report";
|
||||
const forbiddenActions = [
|
||||
@@ -1771,6 +1772,17 @@ export async function runDevDeployApply(argv, io = {}) {
|
||||
const args = parseArgs(argv);
|
||||
const blockers = [];
|
||||
validateArgs(args, blockers);
|
||||
if (args.apply) {
|
||||
const transactionGuard = requireDevCdTransactionForSideEffect({
|
||||
env: io.env ?? process.env,
|
||||
script: "scripts/dev-deploy-apply.mjs",
|
||||
mode: "--apply"
|
||||
});
|
||||
if (transactionGuard) {
|
||||
stdout.write(`${JSON.stringify(transactionGuard, null, 2)}\n`);
|
||||
return 2;
|
||||
}
|
||||
}
|
||||
const gitHeadCommitIdPromise = gitHeadCommit();
|
||||
|
||||
const [deploy, catalog, devKustomization, namespaceDoc, workloads, services, healthContract] = await Promise.all([
|
||||
|
||||
@@ -14,6 +14,11 @@ import {
|
||||
resolveApplySourceCommit,
|
||||
resolveDevKubeconfigSelection
|
||||
} from "./dev-deploy-apply.mjs";
|
||||
import {
|
||||
devCdTransactionGuardFailure,
|
||||
hasDevCdTransactionEnv,
|
||||
requireDevCdTransactionForSideEffect
|
||||
} from "./dev-cd-transaction-guard.mjs";
|
||||
|
||||
const codeAgentBaseUrl = "http://172.26.26.227:17680/v1/responses";
|
||||
|
||||
@@ -109,6 +114,33 @@ test("allowlisted suspended template Job image change plans replacement", () =>
|
||||
assert.equal(decision.newImage, "127.0.0.1:5000/hwlab/hwlab-agent-worker:new5678");
|
||||
});
|
||||
|
||||
test("DEV CD transaction guard rejects direct legacy side-effect calls", () => {
|
||||
assert.equal(hasDevCdTransactionEnv({}), false);
|
||||
assert.equal(hasDevCdTransactionEnv({ HWLAB_CD_TRANSACTION_ID: "tx-123" }), true);
|
||||
|
||||
const failure = requireDevCdTransactionForSideEffect({
|
||||
env: {},
|
||||
script: "scripts/dev-deploy-apply.mjs",
|
||||
mode: "--apply"
|
||||
});
|
||||
assert.deepEqual(failure, devCdTransactionGuardFailure({
|
||||
script: "scripts/dev-deploy-apply.mjs",
|
||||
mode: "--apply"
|
||||
}));
|
||||
assert.equal(failure.error, "cd-transaction-required");
|
||||
assert.equal(failure.mutationAttempted, false);
|
||||
assert.match(failure.entrypoint, /scripts\/dev-cd-apply\.mjs/u);
|
||||
|
||||
assert.equal(
|
||||
requireDevCdTransactionForSideEffect({
|
||||
env: { HWLAB_CD_TRANSACTION_ID: "tx-123" },
|
||||
script: "scripts/dev-deploy-apply.mjs",
|
||||
mode: "--apply"
|
||||
}),
|
||||
null
|
||||
);
|
||||
});
|
||||
|
||||
test("matching allowlisted suspended template Job image does not replace", () => {
|
||||
const image = "127.0.0.1:5000/hwlab/hwlab-cli:73b379f";
|
||||
const decision = decideDevTemplateJobReplacement({
|
||||
|
||||
Reference in New Issue
Block a user