import assert from "node:assert/strict"; import test from "node:test"; import { execFile } from "node:child_process"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import os from "node:os"; import path from "node:path"; import { promisify } from "node:util"; import { buildLockAnnotations, classifyDeployLock, deployLockHeldFailure, parseDeployLock, runDevCdApply, verifyDevLive } from "./dev-cd-apply.mjs"; const execFileAsync = promisify(execFile); const repoRoot = path.resolve(path.dirname(new URL(import.meta.url).pathname), "../.."); function iso(offsetMs = 0) { return new Date(Date.parse("2026-05-23T08:00:00.000Z") + offsetMs).toISOString(); } function leaseFromLock(lock) { return { apiVersion: "coordination.k8s.io/v1", kind: "Lease", metadata: { name: "hwlab-dev-cd-lock", namespace: "hwlab-dev", resourceVersion: "1", annotations: buildLockAnnotations(lock) }, spec: { holderIdentity: `${lock.ownerTaskId}/${lock.transactionId}`, leaseDurationSeconds: lock.ttlSeconds, acquireTime: lock.startedAt, renewTime: lock.updatedAt } }; } function parsePatch(args) { const index = args.indexOf("-p"); assert.notEqual(index, -1, "kubectl patch must include -p"); return JSON.parse(args[index + 1]); } async function makeRepo() { const repoRoot = await fsTempDir(); await mkdir(path.join(repoRoot, "deploy"), { recursive: true }); await writeFile( path.join(repoRoot, "deploy/deploy.json"), `${JSON.stringify({ manifestVersion: "v1", environment: "dev", namespace: "hwlab-dev", endpoint: "http://74.48.78.17:16667", commitId: "abc1234" }, null, 2)}\n` ); return repoRoot; } async function fsTempDir() { return await import("node:fs/promises").then((fs) => fs.mkdtemp(path.join(os.tmpdir(), "hwlab-cd-"))); } function makeHttpGetJson() { return async (url) => { if (url.includes("16666")) { return { statusCode: 200, json: { serviceId: "hwlab-cloud-web", environment: "dev", status: "ok", revision: "abc1234", image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-web:abc1234", tag: "abc1234" }, observedAt: iso() } }; } return { statusCode: 200, json: { serviceId: "hwlab-cloud-api", environment: "dev", status: "degraded", ready: false, commit: { id: "abc1234", source: "runtime-env" }, image: { reference: "127.0.0.1:5000/hwlab/hwlab-cloud-api:abc1234", tag: "abc1234" }, blockerCodes: ["runtime_durable_adapter_auth_blocked"], observedAt: iso() } }; }; } function makeRunCommand({ heldLock = null, commandLog = [] } = {}) { let lease = heldLock ? leaseFromLock(heldLock) : null; const assertLeaseMicroTime = (value) => { assert.match(value, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/u); }; return async (command, args, options = {}) => { commandLog.push({ command, args, env: options.env ?? {}, input: options.input ?? "" }); if (command === "git" && args[0] === "rev-parse" && args.includes("origin/main^{commit}")) { return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" }; } if (command === "git" && args[0] === "rev-parse" && args.includes("HEAD^{commit}")) { return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" }; } if (command === "which" && args[0] === "kubectl") { return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" }; } if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) { if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" }; return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" }; } if (command.includes("kubectl") && args.includes("create")) { lease = JSON.parse(options.input); assertLeaseMicroTime(lease.spec.acquireTime); assertLeaseMicroTime(lease.spec.renewTime); lease.metadata.resourceVersion = "2"; return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" }; } if (command.includes("kubectl") && args.includes("replace")) { const nextLease = JSON.parse(options.input); assert.equal(nextLease.metadata?.resourceVersion, lease?.metadata?.resourceVersion); if (nextLease.spec?.acquireTime) assertLeaseMicroTime(nextLease.spec.acquireTime); if (nextLease.spec?.renewTime) assertLeaseMicroTime(nextLease.spec.renewTime); lease = nextLease; lease.metadata.resourceVersion = String(Number(lease.metadata.resourceVersion ?? "2") + 1); return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" }; } if (command.includes("kubectl") && args.includes("patch")) { assert.ok(lease, "lease must exist before patch"); const patch = parsePatch(args); lease.metadata.annotations = { ...lease.metadata.annotations, ...(patch.metadata?.annotations ?? {}) }; lease.spec = { ...lease.spec, ...(patch.spec ?? {}) }; lease.metadata.resourceVersion = String(Number(lease.metadata.resourceVersion ?? "2") + 1); return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" }; } if (command === process.execPath || command.endsWith("/node")) { return { code: 0, stdout: JSON.stringify({ ok: true, command: args.join(" ") }), stderr: "" }; } return { code: 0, stdout: "", stderr: "" }; }; } test("lock classifier reports held and stale states with retryAfterSeconds", () => { const lock = { ownerTaskId: "task-a", transactionId: "tx-a", promotionCommit: "abc1234", deployJsonHash: "sha256:a", phase: "applying", startedAt: iso(0), updatedAt: iso(0), ttlSeconds: 300, targetNamespace: "hwlab-dev" }; const parsed = parseDeployLock(leaseFromLock(lock)); const held = classifyDeployLock(parsed, new Date(iso(120000))); assert.equal(held.held, true); assert.equal(held.stale, false); assert.equal(held.retryAfterSeconds, 180); const stale = classifyDeployLock(parsed, new Date(iso(301000))); assert.equal(stale.held, false); assert.equal(stale.stale, true); assert.equal(stale.retryAfterSeconds, 0); }); test("deploy-lock-held failure is structured before publish/apply", () => { const failure = deployLockHeldFailure({ ownerTaskId: "task-a", transactionId: "tx-a", promotionCommit: "abc1234", deployJsonHash: "sha256:a", phase: "applying", updatedAt: iso(0), ttlSeconds: 300, targetNamespace: "hwlab-dev", lockName: "hwlab-dev-cd-lock", lockBackend: "Lease" }, new Date(iso(100000))); assert.equal(failure.ok, false); assert.equal(failure.error, "deploy-lock-held"); assert.equal(failure.holder, "task-a"); assert.equal(failure.promotionCommit, "abc1234"); assert.equal(failure.phase, "applying"); assert.equal(failure.retryAfterSeconds, 200); assert.equal(failure.mutationAttempted, false); }); test("second transaction exits deploy-lock-held without running side effects", async () => { const repoRoot = await makeRepo(); const commandLog = []; let output = ""; const code = await runDevCdApply([ "--apply", "--confirm-dev", "--confirmed-non-production", "--owner-task-id", "task-b", "--kubeconfig", "/tmp/kubeconfig" ], { repoRoot, env: {}, runCommand: makeRunCommand({ commandLog, heldLock: { promotionCommit: "abc1234", deployJsonHash: "sha256:a", ownerTaskId: "task-a", transactionId: "tx-a", phase: "publishing", startedAt: iso(0), updatedAt: iso(0), ttlSeconds: 300, liveBefore: { status: "pass" }, targetNamespace: "hwlab-dev", targetRef: "origin/main", lockBackend: "Lease" } }), httpGetJson: makeHttpGetJson(), now: () => new Date(iso(100000)), stdout: { write: (chunk) => { output += chunk; } } }); const failure = JSON.parse(output); assert.equal(code, 2); assert.equal(failure.error, "deploy-lock-held"); assert.equal(failure.holder, "task-a"); assert.equal(failure.retryAfterSeconds, 200); assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false); assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false); }); test("stale lock requires explicit break-stale-lock confirmation", async () => { const repoRoot = await makeRepo(); let output = ""; const code = await runDevCdApply([ "--apply", "--confirm-dev", "--confirmed-non-production", "--owner-task-id", "task-b", "--kubeconfig", "/tmp/kubeconfig" ], { repoRoot, env: {}, runCommand: makeRunCommand({ heldLock: { promotionCommit: "abc1234", deployJsonHash: "sha256:a", ownerTaskId: "task-a", transactionId: "tx-a", phase: "verifying", startedAt: iso(0), updatedAt: iso(0), ttlSeconds: 300, liveBefore: { status: "pass" }, targetNamespace: "hwlab-dev", targetRef: "origin/main", lockBackend: "Lease" } }), httpGetJson: makeHttpGetJson(), now: () => new Date(iso(301000)), stdout: { write: (chunk) => { output += chunk; } } }); const failure = JSON.parse(output); assert.equal(code, 2); assert.equal(failure.error, "deploy-lock-held"); assert.equal(failure.stale, true); assert.equal(failure.requiresBreakStaleLock, true); }); test("transaction runs phases, allows internal side-effect env, releases lock, and reports live verify", async () => { const repoRoot = await makeRepo(); const commandLog = []; let output = ""; const code = await runDevCdApply([ "--apply", "--confirm-dev", "--confirmed-non-production", "--break-stale-lock", "--owner-task-id", "task-b", "--kubeconfig", "/tmp/kubeconfig", "--write-report" ], { repoRoot, env: {}, runCommand: makeRunCommand({ commandLog, heldLock: { promotionCommit: "old1234", deployJsonHash: "sha256:old", ownerTaskId: "task-a", transactionId: "tx-a", phase: "verifying", startedAt: iso(0), updatedAt: iso(0), ttlSeconds: 300, liveBefore: { status: "pass" }, targetNamespace: "hwlab-dev", targetRef: "origin/main", lockBackend: "Lease" } }), httpGetJson: makeHttpGetJson(), now: () => new Date(iso(301000)), stdout: { write: (chunk) => { output += chunk; } } }); const report = JSON.parse(output); assert.equal(code, 0); assert.equal(report.status, "pass"); assert.deepEqual( report.transaction.phases.map((phase) => phase.phase), ["publishing", "applying", "verifying", "released"] ); assert.equal(report.devCdApply.lock.staleBreak.ownerTaskId, "task-a"); assert.equal(report.transaction.release.status, "released"); assert.equal(report.devCdApply.liveVerify.status, "pass"); assert.equal(report.devCdApply.liveVerify.summary.checked, 2); assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-web-16666"), true); assert.equal(report.devCdApply.liveVerify.endpoints.some((endpoint) => endpoint.id === "cloud-api-16667"), true); assert.equal(report.devCdApply.reportPaths.transaction, "reports/dev-gate/dev-cd-apply.json"); const publishCall = commandLog.find((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")); const applyCall = commandLog.find((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")); assert.ok(publishCall?.env.HWLAB_CD_TRANSACTION_ID); assert.equal(applyCall?.env.HWLAB_CD_TRANSACTION_ID, publishCall.env.HWLAB_CD_TRANSACTION_ID); const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8")); assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2); }); test("live verify summarizes 16666 and 16667 health identity", async () => { const result = await verifyDevLive({ now: () => new Date(iso()), httpGetJson: makeHttpGetJson() }, { expectedCommit: "abc1234" }); assert.equal(result.status, "pass"); assert.equal(result.summary.checked, 2); assert.deepEqual(result.endpoints.map((endpoint) => endpoint.url), [ "http://74.48.78.17:16666/health/live", "http://74.48.78.17:16667/health/live" ]); assert.equal(result.endpoints.every((endpoint) => endpoint.commitMatches), true); }); test("legacy artifact publish CLI rejects side effects without transaction env", async () => { await assert.rejects( execFileAsync(process.execPath, ["scripts/dev-artifact-publish.mjs", "--publish", "--no-report"], { cwd: repoRoot, env: { ...process.env, HWLAB_CD_TRANSACTION_ID: "" } }), (error) => { assert.equal(error.code, 2); const payload = JSON.parse(error.stdout); assert.equal(payload.error, "cd-transaction-required"); assert.equal(payload.script, "scripts/dev-artifact-publish.mjs"); assert.equal(payload.mutationAttempted, false); return true; } ); });