feat: add read-only dev cd status surface

This commit is contained in:
Code Queue Review
2026-05-23 10:43:52 +00:00
parent 3de0b62eec
commit 4c12985068
3 changed files with 281 additions and 267 deletions
+59 -156
View File
@@ -42,11 +42,7 @@ const lockAnnotationFields = {
releasedAt: `${lockAnnotationPrefix}/releasedAt`,
releaseStatus: `${lockAnnotationPrefix}/releaseStatus`,
staleLockBrokenAt: `${lockAnnotationPrefix}/staleLockBrokenAt`,
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`,
forceInterruptedAt: `${lockAnnotationPrefix}/forceInterruptedAt`,
forceInterruptedBy: `${lockAnnotationPrefix}/forceInterruptedBy`,
forceInterruptStatus: `${lockAnnotationPrefix}/forceInterruptStatus`,
forceInterruptedPrevious: `${lockAnnotationPrefix}/forceInterruptedPrevious`
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`
};
class DevCdApplyError extends Error {
@@ -72,7 +68,6 @@ export function parseArgs(argv) {
ttlSeconds: defaultTtlSeconds,
ownerTaskId: null,
breakStaleLock: false,
forceInterruptLock: false,
skipLiveVerify: false,
status: false,
dryRun: false,
@@ -98,9 +93,6 @@ export function parseArgs(argv) {
} else if (arg === "--break-stale-lock") {
args.breakStaleLock = true;
args.flags.add(arg);
} else if (arg === "--force-interrupt-lock") {
args.forceInterruptLock = true;
args.flags.add(arg);
} else if (arg === "--dry-run" || arg === "--status") {
args.status = true;
args.dryRun = arg === "--dry-run";
@@ -140,7 +132,15 @@ export function parseArgs(argv) {
} else if (arg === "--help" || arg === "-h") {
args.help = true;
} else {
throw new Error(`unknown argument ${arg}`);
throw new DevCdApplyError(`unknown argument ${arg}`, {
code: "unknown-argument",
blockers: [{
type: "safety_blocker",
scope: "unknown-argument",
status: "open",
summary: `${arg} is not a supported DEV CD option. Use --status/--dry-run for read-only observability or --apply with explicit DEV confirmations.`
}]
});
}
}
@@ -163,21 +163,19 @@ export function usage() {
examples: [
"node scripts/dev-cd-apply.mjs --status",
"node scripts/dev-cd-apply.mjs --dry-run",
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report",
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --force-interrupt-lock --write-report"
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report"
],
options: {
"--status": "Read-only status: target ref, deploy.json, current Lease, and public live health.",
"--dry-run": "Read-only alias for --status; does not acquire the Lease or run side effects.",
"--apply": "Run the DEV CD transaction side effects.",
"--confirm-dev": "Required for DEV mutation and stale/force lock takeover.",
"--confirm-dev": "Required for DEV mutation and explicit stale-lock recovery.",
"--confirmed-non-production": "Required for DEV mutation.",
"--target-ref REF": "git ref to publish/apply; default: origin/main",
"--kubeconfig PATH": "DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG",
"--ttl-seconds SECONDS": "lock TTL; default: 3600",
"--owner-task-id ID": "lock holder; default: Code Queue/env/user derived",
"--break-stale-lock": "May take over an expired lock only with --confirm-dev.",
"--force-interrupt-lock": "DEV-only explicit takeover: try to interrupt the current local dev-cd apply process, then acquire the Lease.",
"--report PATH": `default: ${defaultReportPath}`,
"--write-report": "Write the full report to --report.",
"--full-output": "Print the full report to stdout instead of the concise summary.",
@@ -241,15 +239,17 @@ function defaultOwnerTaskId(env = process.env) {
function defaultRunCommand(command, args, options = {}) {
return new Promise((resolve) => {
const abortController = options.timeoutMs ? new AbortController() : null;
const child = spawn(command, args, {
cwd: options.cwd ?? defaultRepoRoot,
env: options.env ?? process.env,
stdio: ["pipe", "pipe", "pipe"]
stdio: ["pipe", "pipe", "pipe"],
signal: abortController?.signal
});
let stdout = "";
let stderr = "";
const timeout = options.timeoutMs
? setTimeout(() => child.kill("SIGTERM"), options.timeoutMs)
? setTimeout(() => abortController.abort(), options.timeoutMs)
: null;
child.stdin.on("error", () => {});
@@ -262,7 +262,12 @@ function defaultRunCommand(command, args, options = {}) {
});
child.on("error", (error) => {
if (timeout) clearTimeout(timeout);
resolve({ code: 127, signal: null, stdout, stderr: error.message });
resolve({
code: error.name === "AbortError" ? 124 : 127,
signal: null,
stdout,
stderr: error.message
});
});
child.on("close", (code, signal) => {
if (timeout) clearTimeout(timeout);
@@ -346,22 +351,6 @@ function validateArgs(args) {
summary: "Breaking a stale DEV CD lock requires --break-stale-lock and --confirm-dev."
});
}
if (args.forceInterruptLock && !args.confirmDev) {
blockers.push({
type: "safety_blocker",
scope: "force-interrupt-confirmation",
status: "open",
summary: "Force-interrupting a DEV CD lock requires --force-interrupt-lock and --confirm-dev."
});
}
if (args.forceInterruptLock && !args.apply) {
blockers.push({
type: "safety_blocker",
scope: "force-interrupt-mode",
status: "open",
summary: "--force-interrupt-lock is only valid with --apply."
});
}
if (!Number.isInteger(args.ttlSeconds) || args.ttlSeconds < 60 || args.ttlSeconds > 24 * 60 * 60) {
blockers.push({
type: "safety_blocker",
@@ -446,11 +435,7 @@ export function parseDeployLock(lease) {
releasedAt: readAnnotation(annotations, "releasedAt"),
releaseStatus: readAnnotation(annotations, "releaseStatus"),
staleLockBrokenAt: readAnnotation(annotations, "staleLockBrokenAt"),
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious"),
forceInterruptedAt: readAnnotation(annotations, "forceInterruptedAt"),
forceInterruptedBy: readAnnotation(annotations, "forceInterruptedBy"),
forceInterruptStatus: readAnnotation(annotations, "forceInterruptStatus"),
forceInterruptedPrevious: readJsonAnnotation(annotations, "forceInterruptedPrevious")
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious")
};
}
@@ -512,7 +497,7 @@ function isConflict(result) {
return /conflict|object has been modified|operation cannot be fulfilled/i.test(`${result.stderr}\n${result.stdout}`);
}
function compactInterruptedLock(lock) {
function compactLockSummary(lock) {
if (!lock) return null;
return {
lockName: lock.lockName ?? defaultLockName,
@@ -526,39 +511,14 @@ function compactInterruptedLock(lock) {
updatedAt: lock.updatedAt ?? null,
ttlSeconds: lock.ttlSeconds ?? null,
targetNamespace: lock.targetNamespace ?? defaultNamespace,
targetRef: lock.targetRef ?? null
targetRef: lock.targetRef ?? null,
releasedAt: lock.releasedAt ?? null,
releaseStatus: lock.releaseStatus ?? null,
staleLockBrokenAt: lock.staleLockBrokenAt ?? null
};
}
function compactForceInterruptRecord(record) {
if (!record) return null;
return {
status: record.status,
warning: record.warning ?? null,
processCandidates: (record.processCandidates ?? []).map((candidate) => ({
pid: candidate.pid,
command: candidate.command
})),
attempts: (record.attempts ?? []).map((attempt) => ({
pid: attempt.pid,
signal: attempt.signal,
status: attempt.status
}))
};
}
function attachForceInterruptAnnotations(lock, forceInterrupt, timestamp, ownerTaskId) {
if (!forceInterrupt) return;
lock.forceInterruptedAt = timestamp;
lock.forceInterruptedBy = ownerTaskId;
lock.forceInterruptStatus = forceInterrupt.status;
lock.forceInterruptedPrevious = {
previousLock: compactInterruptedLock(forceInterrupt.previousLock),
interrupt: compactForceInterruptRecord(forceInterrupt)
};
}
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null, forceInterrupt = null }) {
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null }) {
const lock = {
promotionCommit: target.shortCommitId,
deployJsonHash: deployBefore.hash,
@@ -577,7 +537,6 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa
lock.staleLockBrokenAt = startedAt;
lock.staleLockPrevious = staleLockPrevious;
}
attachForceInterruptAnnotations(lock, forceInterrupt, startedAt, ownerTaskId);
return {
apiVersion: "coordination.k8s.io/v1",
kind: "Lease",
@@ -600,7 +559,7 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa
};
}
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null, forceInterrupt = null }) {
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null }) {
const lock = {
promotionCommit: target.shortCommitId,
deployJsonHash: deployBefore.hash,
@@ -619,7 +578,6 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha
lock.staleLockBrokenAt = updatedAt;
lock.staleLockPrevious = staleLockPrevious;
}
attachForceInterruptAnnotations(lock, forceInterrupt, updatedAt, ownerTaskId);
return {
metadata: {
labels: {
@@ -638,66 +596,6 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha
};
}
export function findDevCdApplyProcessCandidates(psOutput, currentPid = process.pid) {
return String(psOutput ?? "")
.split(/\r?\n/u)
.map((line) => line.trim())
.filter(Boolean)
.map((line) => {
const match = line.match(/^(\d+)\s+(.+)$/u);
if (!match) return null;
return {
pid: Number.parseInt(match[1], 10),
command: oneLine(redactSensitiveText(match[2])).slice(0, 500)
};
})
.filter((entry) => {
if (!entry || !Number.isInteger(entry.pid) || entry.pid === currentPid) return false;
return entry.command.includes("scripts/dev-cd-apply.mjs") && entry.command.includes("--apply");
});
}
async function forceInterruptCurrentApplyProcess(ctx, existingLock) {
const ps = await ctx.runCommand("ps", ["-eo", "pid=,args="], {
cwd: ctx.repoRoot,
timeoutMs: 5000
});
if (ps.code !== 0) {
return {
status: "ps-failed",
previousLock: compactInterruptedLock(existingLock),
processCandidates: [],
attempts: [],
warning: oneLine(redactSensitiveText(ps.stderr || ps.stdout || "ps failed"))
};
}
const processCandidates = findDevCdApplyProcessCandidates(ps.stdout, process.pid);
const attempts = [];
for (const candidate of processCandidates) {
const result = await ctx.runCommand("kill", ["-TERM", String(candidate.pid)], {
cwd: ctx.repoRoot,
timeoutMs: 5000
});
attempts.push({
pid: candidate.pid,
signal: "TERM",
status: result.code === 0 ? "sent" : "failed",
stderrTail: redactSensitiveText(result.stderr || result.stdout || "").trim().slice(-500)
});
}
return {
status: processCandidates.length > 0 ? "attempted" : "no-local-process-found",
previousLock: compactInterruptedLock(existingLock),
processCandidates,
attempts,
warning: processCandidates.length > 0
? null
: "No local scripts/dev-cd-apply.mjs --apply process was found; the Lease will still be taken over so the previous holder cannot continue lock-owned updates."
};
}
async function kubectl(ctx, kubectlContext, args, options = {}) {
const result = await ctx.runCommand(kubectlContext.executor, args, {
cwd: ctx.repoRoot,
@@ -728,7 +626,7 @@ async function replaceLease(ctx, kubectlContext, args, lease, timeoutMs = 30000)
function mergeLease(lease, patch, { resetLockAnnotations = false } = {}) {
const baseAnnotations = { ...(lease.metadata?.annotations ?? {}) };
if (resetLockAnnotations) {
for (const key of Object.values(lockAnnotationFields)) {
for (const key of Object.keys(baseAnnotations).filter((name) => name.startsWith(`${lockAnnotationPrefix}/`))) {
delete baseAnnotations[key];
}
}
@@ -760,14 +658,14 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
const lease = JSON.parse(get.stdout);
const existing = parseDeployLock(lease);
const classification = classifyDeployLock(existing, now);
if (classification.held && !args.forceInterruptLock) {
if (classification.held) {
const failure = deployLockHeldFailure(existing, now);
throw new DevCdApplyError(failure.summary, {
code: "deploy-lock-held",
lockFailure: failure
});
}
if (classification.stale && !args.breakStaleLock && !args.forceInterruptLock) {
if (classification.stale && !args.breakStaleLock) {
const failure = deployLockHeldFailure(existing, now);
throw new DevCdApplyError(failure.summary, {
code: "deploy-lock-held",
@@ -775,10 +673,6 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
});
}
const forceInterrupt = args.forceInterruptLock
? await forceInterruptCurrentApplyProcess(ctx, existing)
: null;
const patch = lockPatch({
args,
target,
@@ -789,8 +683,7 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
startedAt,
updatedAt: startedAt,
liveBefore,
staleLockPrevious: classification.stale ? existing : null,
forceInterrupt
staleLockPrevious: classification.stale ? existing : null
});
const replaced = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch, { resetLockAnnotations: true }));
if (isConflict(replaced)) {
@@ -806,7 +699,6 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
acquired: true,
lock: parseDeployLock(JSON.parse(replaced.stdout)),
staleBreak: classification.stale ? existing : null,
forceInterrupt,
command: replaced.command
};
}
@@ -837,7 +729,6 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
acquired: true,
lock: parseDeployLock(JSON.parse(created.stdout)),
staleBreak: null,
forceInterrupt: null,
command: created.command
};
}
@@ -1410,6 +1301,10 @@ function summarizeDeployJson(deploy) {
};
}
function deployJsonMatchesTarget(deploy, target) {
return deploy.commitId === target.shortCommitId || deploy.commitId === target.commitId;
}
function compactLiveEndpoint(endpoint) {
return {
id: endpoint.id,
@@ -1481,7 +1376,8 @@ function summarizeLockStatus(lockRead, now) {
current: null,
retryAfterSeconds: null,
expiresAt: null,
error: oneLine(lockRead.error)
reason: "kubectl or DEV kubeconfig unavailable; command output is redacted",
redactedReason: oneLine(lockRead.error).slice(0, 500)
};
}
const classification = classifyDeployLock(lockRead.lock, now);
@@ -1494,12 +1390,12 @@ function summarizeLockStatus(lockRead, now) {
: "unknown";
return {
status,
current: compactInterruptedLock(lockRead.lock),
lockName: lockRead.lock?.lockName ?? lockRead.lockName ?? defaultLockName,
current: compactLockSummary(lockRead.lock),
retryAfterSeconds: classification.retryAfterSeconds,
expiresAt: classification.expiresAt,
takeover: {
stale: classification.stale ? "--apply --confirm-dev --confirmed-non-production --break-stale-lock" : null,
forceInterrupt: status === "held" ? "--apply --confirm-dev --confirmed-non-production --force-interrupt-lock" : null
allowedLockAction: {
breakStale: classification.stale ? "--apply --confirm-dev --confirmed-non-production --break-stale-lock" : null
}
};
}
@@ -1513,7 +1409,7 @@ function buildStatusNextActions({ target, deployBefore, lock, liveVerify }) {
actions.push(`deploy/deploy.json commitId is ${deployBefore.commitId}; target ref is ${target.shortCommitId}.`);
}
if (lock.status === "held") {
actions.push(`Wait ${lock.retryAfterSeconds}s or use explicit --force-interrupt-lock if the current apply must be stopped.`);
actions.push(`Wait for the current DEV CD holder to release the Lease or for the ${lock.retryAfterSeconds}s TTL to expire before retrying apply.`);
} else if (lock.status === "stale") {
actions.push("Use --break-stale-lock only after confirming the holder is inactive.");
} else if (lock.status === "unavailable") {
@@ -1538,7 +1434,13 @@ async function runDevCdStatus(args, ctx, stdout) {
const liveVerify = args.skipLiveVerify
? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } }
: await verifyDevLive(ctx, { expectedCommit: null });
const status = lock.status === "unavailable" || liveVerify.status !== "pass" ? "degraded" : "pass";
const deployMatchesTarget = deployJsonMatchesTarget(deployBefore, target);
const status = (
lock.status === "unavailable" ||
liveVerify.status !== "pass" ||
!target.headMatchesTarget ||
!deployMatchesTarget
) ? "degraded" : "pass";
const report = {
ok: true,
status,
@@ -1555,7 +1457,10 @@ async function runDevCdStatus(args, ctx, stdout) {
headMatchesTarget: target.headMatchesTarget,
namespace: args.targetNamespace
},
deployJson: summarizeDeployJson(deployBefore),
deployJson: {
...summarizeDeployJson(deployBefore),
matchesTarget: deployMatchesTarget
},
kubectl: {
kubeconfigSource: kubectlContext.kubeconfigSource,
kubeconfig: kubectlContext.kubeconfig ? "<set>" : "<unset>",
@@ -1592,9 +1497,8 @@ function summarizeDevCdApplyReport(report) {
target: apply.target ?? null,
lock: {
acquired: Boolean(apply.lock?.acquired),
current: compactInterruptedLock(apply.lock?.current),
staleBreak: compactInterruptedLock(apply.lock?.staleBreak),
forceInterrupt: compactForceInterruptRecord(apply.lock?.forceInterrupt)
current: compactLockSummary(apply.lock?.current),
staleBreak: compactLockSummary(apply.lock?.staleBreak)
},
steps: (apply.steps ?? []).map((step) => ({
id: step.id,
@@ -1726,8 +1630,7 @@ function buildReport({
lock: {
acquired: Boolean(lockState?.acquired),
current: lockState?.lock ?? null,
staleBreak: lockState?.staleBreak ?? null,
forceInterrupt: lockState?.forceInterrupt ?? null
staleBreak: lockState?.staleBreak ?? null
},
steps,
liveBefore,
+185 -72
View File
@@ -10,7 +10,6 @@ import {
buildLockAnnotations,
classifyDeployLock,
deployLockHeldFailure,
findDevCdApplyProcessCandidates,
parseDeployLock,
runDevCdApply,
verifyDevLive
@@ -99,7 +98,7 @@ function makeHttpGetJson() {
};
}
function makeRunCommand({ heldLock = null, commandLog = [], psOutput = "", killLog = [] } = {}) {
function makeRunCommand({ heldLock = null, commandLog = [] } = {}) {
let lease = heldLock ? leaseFromLock(heldLock) : null;
const assertLeaseMicroTime = (value) => {
assert.match(value, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/u);
@@ -115,13 +114,6 @@ function makeRunCommand({ heldLock = null, commandLog = [], psOutput = "", killL
if (command === "which" && args[0] === "kubectl") {
return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" };
}
if (command === "ps") {
return { code: 0, stdout: psOutput, stderr: "" };
}
if (command === "kill") {
killLog.push({ signal: args[0], pid: args[1] });
return { code: 0, stdout: "", stderr: "" };
}
if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) {
if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" };
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
@@ -191,6 +183,18 @@ function makeRunCommand({ heldLock = null, commandLog = [], psOutput = "", killL
};
}
function assertNoReadOnlySideEffects(commandLog) {
const mutatingKubectlVerbs = new Set(["apply", "create", "replace", "patch", "delete", "rollout", "wait", "logs"]);
assert.equal(commandLog.some((entry) => entry.command === "ps"), false);
assert.equal(commandLog.some((entry) => entry.command === "kill"), false);
assert.equal(commandLog.some((entry) =>
entry.command.includes("kubectl") && entry.args.some((arg) => mutatingKubectlVerbs.has(arg))
), false);
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/refresh-artifact-catalog.mjs")), false);
}
test("lock classifier reports held and stale states with retryAfterSeconds", () => {
const lock = {
ownerTaskId: "task-a",
@@ -222,7 +226,8 @@ test("dev-cd status and dry-run are read-only compact JSON", async () => {
const code = await runDevCdApply([
"--dry-run",
"--kubeconfig",
"/tmp/kubeconfig"
"/tmp/kubeconfig",
"--write-report"
], {
repoRoot,
env: {},
@@ -239,22 +244,179 @@ test("dev-cd status and dry-run are read-only compact JSON", async () => {
assert.equal(status.prodTouched, false);
assert.equal(status.lock.status, "absent");
assert.equal(status.deployJson.manifest, undefined);
assert.equal(status.deployJson.matchesTarget, true);
assert.equal(status.live.summary.checked, 2);
assert.equal(commandLog.some((entry) => entry.args.includes("create")), false);
assert.equal(commandLog.some((entry) => entry.args.includes("replace")), false);
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
assertNoReadOnlySideEffects(commandLog);
await assert.rejects(
readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"),
{ code: "ENOENT" }
);
});
test("process candidate discovery only returns dev-cd apply processes", () => {
const psOutput = [
"101 node scripts/dev-cd-apply.mjs --apply --confirm-dev",
"202 node scripts/dev-cd-apply.mjs --status",
"303 node scripts/dev-deploy-apply.mjs --apply",
"404 node scripts/dev-cd-apply.mjs --apply --confirm-dev"
].join("\n");
const candidates = findDevCdApplyProcessCandidates(psOutput, 404);
assert.deepEqual(candidates.map((candidate) => candidate.pid), [101]);
test("dev-cd default status is read-only compact JSON", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const code = await runDevCdApply([], {
repoRoot,
env: {},
runCommand: makeRunCommand({ commandLog }),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const status = JSON.parse(output);
assert.equal(code, 0);
assert.equal(status.mode, "status");
assert.equal(status.mutationAttempted, false);
assert.equal(status.prodTouched, false);
assert.equal(status.target.shortCommitId, "abc1234");
assert.equal(status.deployJson.hash.startsWith("sha256:"), true);
assert.equal(status.deployJson.matchesTarget, true);
assert.equal(status.lock.status, "absent");
assertNoReadOnlySideEffects(commandLog);
});
test("dev-cd status summarizes held, released, and stale locks without unsafe takeover hints", async () => {
const cases = [
{
name: "held",
phase: "applying",
now: iso(100000),
expectedStatus: "held",
expectedRetryAfterSeconds: 200
},
{
name: "released",
phase: "released",
releasedAt: iso(50000),
releaseStatus: "pass",
now: iso(100000),
expectedStatus: "released",
expectedRetryAfterSeconds: 0
},
{
name: "stale",
phase: "verifying",
now: iso(301000),
expectedStatus: "stale",
expectedRetryAfterSeconds: 0
}
];
for (const item of cases) {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], {
repoRoot,
env: {},
runCommand: makeRunCommand({
commandLog,
heldLock: {
promotionCommit: "abc1234",
deployJsonHash: "sha256:a",
ownerTaskId: `task-${item.name}`,
transactionId: `tx-${item.name}`,
phase: item.phase,
startedAt: iso(0),
updatedAt: iso(0),
ttlSeconds: 300,
liveBefore: { status: "pass" },
targetNamespace: "hwlab-dev",
targetRef: "origin/main",
lockBackend: "Lease",
...(item.releasedAt ? { releasedAt: item.releasedAt } : {}),
...(item.releaseStatus ? { releaseStatus: item.releaseStatus } : {})
}
}),
httpGetJson: makeHttpGetJson(),
now: () => new Date(item.now),
stdout: { write: (chunk) => { output += chunk; } }
});
const status = JSON.parse(output);
assert.equal(code, 0);
assert.equal(status.lock.status, item.expectedStatus);
assert.equal(status.lock.lockName, "hwlab-dev-cd-lock");
assert.equal(status.lock.current.ownerTaskId, `task-${item.name}`);
assert.equal(status.lock.retryAfterSeconds, item.expectedRetryAfterSeconds);
assert.equal(JSON.stringify(status).includes("force-interrupt"), false);
assert.equal(JSON.stringify(status.nextActions).includes("force"), false);
if (item.expectedStatus === "stale") {
assert.match(status.lock.allowedLockAction.breakStale, /--break-stale-lock/u);
}
assertNoReadOnlySideEffects(commandLog);
}
});
test("dev-cd status reports unavailable kubectl with redacted reason", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const runCommand = async (command, args, options = {}) => {
commandLog.push({ command, args, env: options.env ?? {}, input: options.input ?? "" });
if (command === "git" && args[0] === "rev-parse" && args.includes("origin/main^{commit}")) {
return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" };
}
if (command === "git" && args[0] === "rev-parse" && args.includes("HEAD^{commit}")) {
return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" };
}
if (command === "which" && args[0] === "kubectl") {
return { code: 1, stdout: "", stderr: "kubectl missing" };
}
if (command === "kubectl") {
return {
code: 1,
stdout: "",
stderr: "connection failed token=SECRET_TOKEN password=SECRET_PASSWORD postgresql://user:SECRET_DB@db"
};
}
return { code: 0, stdout: "", stderr: "" };
};
const code = await runDevCdApply(["--status"], {
repoRoot,
env: {},
runCommand,
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const status = JSON.parse(output);
const text = JSON.stringify(status);
assert.equal(code, 0);
assert.equal(status.status, "degraded");
assert.equal(status.lock.status, "unavailable");
assert.equal(status.lock.reason, "kubectl or DEV kubeconfig unavailable; command output is redacted");
assert.equal(text.includes("SECRET_TOKEN"), false);
assert.equal(text.includes("SECRET_PASSWORD"), false);
assert.equal(text.includes("SECRET_DB"), false);
assert.match(status.lock.redactedReason, /<redacted>/u);
assertNoReadOnlySideEffects(commandLog);
});
test("force-interrupt option is rejected before process discovery or lease mutation", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const code = await runDevCdApply(["--force-interrupt-lock"], {
repoRoot,
env: {},
runCommand: makeRunCommand({ commandLog }),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const failure = JSON.parse(output);
assert.equal(code, 2);
assert.equal(failure.error, "unknown-argument");
assert.equal(failure.mutationAttempted, false);
assert.equal(failure.prodTouched, false);
assert.deepEqual(commandLog, []);
});
test("deploy-lock-held failure is structured before publish/apply", () => {
@@ -489,55 +651,6 @@ test("apply stdout is concise by default while write-report keeps the full repor
assert.equal(writtenReport.devCdApply.deployJson.before.manifest.environment, "dev");
});
test("force-interrupt-lock terminates local apply candidate and takes a held lock", async () => {
const repoRoot = await makeRepo();
const killLog = [];
let output = "";
const code = await runDevCdApply([
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--force-interrupt-lock",
"--owner-task-id",
"task-force",
"--kubeconfig",
"/tmp/kubeconfig",
"--full-output"
], {
repoRoot,
env: {},
runCommand: makeRunCommand({
killLog,
psOutput: "555 node scripts/dev-cd-apply.mjs --apply --confirm-dev\n666 node scripts/dev-cd-apply.mjs --status\n",
heldLock: {
promotionCommit: "abc1234",
deployJsonHash: "sha256:a",
ownerTaskId: "task-a",
transactionId: "tx-a",
phase: "applying",
startedAt: iso(0),
updatedAt: iso(0),
ttlSeconds: 300,
liveBefore: { status: "pass" },
targetNamespace: "hwlab-dev",
targetRef: "origin/main",
lockBackend: "Lease"
}
}),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const report = JSON.parse(output);
assert.equal(code, 0);
assert.deepEqual(killLog, [{ signal: "-TERM", pid: "555" }]);
assert.equal(report.devCdApply.lock.forceInterrupt.status, "attempted");
assert.equal(report.devCdApply.lock.forceInterrupt.processCandidates[0].pid, 555);
assert.equal(report.devCdApply.lock.current.forceInterruptStatus, "attempted");
assert.equal(report.devCdApply.lock.current.forceInterruptedBy, "task-force");
});
test("live verify summarizes 16666 and 16667 health identity", async () => {
const result = await verifyDevLive({
now: () => new Date(iso()),