feat: add read-only dev cd status surface
This commit is contained in:
+59
-156
@@ -42,11 +42,7 @@ const lockAnnotationFields = {
|
||||
releasedAt: `${lockAnnotationPrefix}/releasedAt`,
|
||||
releaseStatus: `${lockAnnotationPrefix}/releaseStatus`,
|
||||
staleLockBrokenAt: `${lockAnnotationPrefix}/staleLockBrokenAt`,
|
||||
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`,
|
||||
forceInterruptedAt: `${lockAnnotationPrefix}/forceInterruptedAt`,
|
||||
forceInterruptedBy: `${lockAnnotationPrefix}/forceInterruptedBy`,
|
||||
forceInterruptStatus: `${lockAnnotationPrefix}/forceInterruptStatus`,
|
||||
forceInterruptedPrevious: `${lockAnnotationPrefix}/forceInterruptedPrevious`
|
||||
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`
|
||||
};
|
||||
|
||||
class DevCdApplyError extends Error {
|
||||
@@ -72,7 +68,6 @@ export function parseArgs(argv) {
|
||||
ttlSeconds: defaultTtlSeconds,
|
||||
ownerTaskId: null,
|
||||
breakStaleLock: false,
|
||||
forceInterruptLock: false,
|
||||
skipLiveVerify: false,
|
||||
status: false,
|
||||
dryRun: false,
|
||||
@@ -98,9 +93,6 @@ export function parseArgs(argv) {
|
||||
} else if (arg === "--break-stale-lock") {
|
||||
args.breakStaleLock = true;
|
||||
args.flags.add(arg);
|
||||
} else if (arg === "--force-interrupt-lock") {
|
||||
args.forceInterruptLock = true;
|
||||
args.flags.add(arg);
|
||||
} else if (arg === "--dry-run" || arg === "--status") {
|
||||
args.status = true;
|
||||
args.dryRun = arg === "--dry-run";
|
||||
@@ -140,7 +132,15 @@ export function parseArgs(argv) {
|
||||
} else if (arg === "--help" || arg === "-h") {
|
||||
args.help = true;
|
||||
} else {
|
||||
throw new Error(`unknown argument ${arg}`);
|
||||
throw new DevCdApplyError(`unknown argument ${arg}`, {
|
||||
code: "unknown-argument",
|
||||
blockers: [{
|
||||
type: "safety_blocker",
|
||||
scope: "unknown-argument",
|
||||
status: "open",
|
||||
summary: `${arg} is not a supported DEV CD option. Use --status/--dry-run for read-only observability or --apply with explicit DEV confirmations.`
|
||||
}]
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -163,21 +163,19 @@ export function usage() {
|
||||
examples: [
|
||||
"node scripts/dev-cd-apply.mjs --status",
|
||||
"node scripts/dev-cd-apply.mjs --dry-run",
|
||||
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report",
|
||||
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --force-interrupt-lock --write-report"
|
||||
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report"
|
||||
],
|
||||
options: {
|
||||
"--status": "Read-only status: target ref, deploy.json, current Lease, and public live health.",
|
||||
"--dry-run": "Read-only alias for --status; does not acquire the Lease or run side effects.",
|
||||
"--apply": "Run the DEV CD transaction side effects.",
|
||||
"--confirm-dev": "Required for DEV mutation and stale/force lock takeover.",
|
||||
"--confirm-dev": "Required for DEV mutation and explicit stale-lock recovery.",
|
||||
"--confirmed-non-production": "Required for DEV mutation.",
|
||||
"--target-ref REF": "git ref to publish/apply; default: origin/main",
|
||||
"--kubeconfig PATH": "DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG",
|
||||
"--ttl-seconds SECONDS": "lock TTL; default: 3600",
|
||||
"--owner-task-id ID": "lock holder; default: Code Queue/env/user derived",
|
||||
"--break-stale-lock": "May take over an expired lock only with --confirm-dev.",
|
||||
"--force-interrupt-lock": "DEV-only explicit takeover: try to interrupt the current local dev-cd apply process, then acquire the Lease.",
|
||||
"--report PATH": `default: ${defaultReportPath}`,
|
||||
"--write-report": "Write the full report to --report.",
|
||||
"--full-output": "Print the full report to stdout instead of the concise summary.",
|
||||
@@ -241,15 +239,17 @@ function defaultOwnerTaskId(env = process.env) {
|
||||
|
||||
function defaultRunCommand(command, args, options = {}) {
|
||||
return new Promise((resolve) => {
|
||||
const abortController = options.timeoutMs ? new AbortController() : null;
|
||||
const child = spawn(command, args, {
|
||||
cwd: options.cwd ?? defaultRepoRoot,
|
||||
env: options.env ?? process.env,
|
||||
stdio: ["pipe", "pipe", "pipe"]
|
||||
stdio: ["pipe", "pipe", "pipe"],
|
||||
signal: abortController?.signal
|
||||
});
|
||||
let stdout = "";
|
||||
let stderr = "";
|
||||
const timeout = options.timeoutMs
|
||||
? setTimeout(() => child.kill("SIGTERM"), options.timeoutMs)
|
||||
? setTimeout(() => abortController.abort(), options.timeoutMs)
|
||||
: null;
|
||||
|
||||
child.stdin.on("error", () => {});
|
||||
@@ -262,7 +262,12 @@ function defaultRunCommand(command, args, options = {}) {
|
||||
});
|
||||
child.on("error", (error) => {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
resolve({ code: 127, signal: null, stdout, stderr: error.message });
|
||||
resolve({
|
||||
code: error.name === "AbortError" ? 124 : 127,
|
||||
signal: null,
|
||||
stdout,
|
||||
stderr: error.message
|
||||
});
|
||||
});
|
||||
child.on("close", (code, signal) => {
|
||||
if (timeout) clearTimeout(timeout);
|
||||
@@ -346,22 +351,6 @@ function validateArgs(args) {
|
||||
summary: "Breaking a stale DEV CD lock requires --break-stale-lock and --confirm-dev."
|
||||
});
|
||||
}
|
||||
if (args.forceInterruptLock && !args.confirmDev) {
|
||||
blockers.push({
|
||||
type: "safety_blocker",
|
||||
scope: "force-interrupt-confirmation",
|
||||
status: "open",
|
||||
summary: "Force-interrupting a DEV CD lock requires --force-interrupt-lock and --confirm-dev."
|
||||
});
|
||||
}
|
||||
if (args.forceInterruptLock && !args.apply) {
|
||||
blockers.push({
|
||||
type: "safety_blocker",
|
||||
scope: "force-interrupt-mode",
|
||||
status: "open",
|
||||
summary: "--force-interrupt-lock is only valid with --apply."
|
||||
});
|
||||
}
|
||||
if (!Number.isInteger(args.ttlSeconds) || args.ttlSeconds < 60 || args.ttlSeconds > 24 * 60 * 60) {
|
||||
blockers.push({
|
||||
type: "safety_blocker",
|
||||
@@ -446,11 +435,7 @@ export function parseDeployLock(lease) {
|
||||
releasedAt: readAnnotation(annotations, "releasedAt"),
|
||||
releaseStatus: readAnnotation(annotations, "releaseStatus"),
|
||||
staleLockBrokenAt: readAnnotation(annotations, "staleLockBrokenAt"),
|
||||
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious"),
|
||||
forceInterruptedAt: readAnnotation(annotations, "forceInterruptedAt"),
|
||||
forceInterruptedBy: readAnnotation(annotations, "forceInterruptedBy"),
|
||||
forceInterruptStatus: readAnnotation(annotations, "forceInterruptStatus"),
|
||||
forceInterruptedPrevious: readJsonAnnotation(annotations, "forceInterruptedPrevious")
|
||||
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious")
|
||||
};
|
||||
}
|
||||
|
||||
@@ -512,7 +497,7 @@ function isConflict(result) {
|
||||
return /conflict|object has been modified|operation cannot be fulfilled/i.test(`${result.stderr}\n${result.stdout}`);
|
||||
}
|
||||
|
||||
function compactInterruptedLock(lock) {
|
||||
function compactLockSummary(lock) {
|
||||
if (!lock) return null;
|
||||
return {
|
||||
lockName: lock.lockName ?? defaultLockName,
|
||||
@@ -526,39 +511,14 @@ function compactInterruptedLock(lock) {
|
||||
updatedAt: lock.updatedAt ?? null,
|
||||
ttlSeconds: lock.ttlSeconds ?? null,
|
||||
targetNamespace: lock.targetNamespace ?? defaultNamespace,
|
||||
targetRef: lock.targetRef ?? null
|
||||
targetRef: lock.targetRef ?? null,
|
||||
releasedAt: lock.releasedAt ?? null,
|
||||
releaseStatus: lock.releaseStatus ?? null,
|
||||
staleLockBrokenAt: lock.staleLockBrokenAt ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function compactForceInterruptRecord(record) {
|
||||
if (!record) return null;
|
||||
return {
|
||||
status: record.status,
|
||||
warning: record.warning ?? null,
|
||||
processCandidates: (record.processCandidates ?? []).map((candidate) => ({
|
||||
pid: candidate.pid,
|
||||
command: candidate.command
|
||||
})),
|
||||
attempts: (record.attempts ?? []).map((attempt) => ({
|
||||
pid: attempt.pid,
|
||||
signal: attempt.signal,
|
||||
status: attempt.status
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
function attachForceInterruptAnnotations(lock, forceInterrupt, timestamp, ownerTaskId) {
|
||||
if (!forceInterrupt) return;
|
||||
lock.forceInterruptedAt = timestamp;
|
||||
lock.forceInterruptedBy = ownerTaskId;
|
||||
lock.forceInterruptStatus = forceInterrupt.status;
|
||||
lock.forceInterruptedPrevious = {
|
||||
previousLock: compactInterruptedLock(forceInterrupt.previousLock),
|
||||
interrupt: compactForceInterruptRecord(forceInterrupt)
|
||||
};
|
||||
}
|
||||
|
||||
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null, forceInterrupt = null }) {
|
||||
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null }) {
|
||||
const lock = {
|
||||
promotionCommit: target.shortCommitId,
|
||||
deployJsonHash: deployBefore.hash,
|
||||
@@ -577,7 +537,6 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa
|
||||
lock.staleLockBrokenAt = startedAt;
|
||||
lock.staleLockPrevious = staleLockPrevious;
|
||||
}
|
||||
attachForceInterruptAnnotations(lock, forceInterrupt, startedAt, ownerTaskId);
|
||||
return {
|
||||
apiVersion: "coordination.k8s.io/v1",
|
||||
kind: "Lease",
|
||||
@@ -600,7 +559,7 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa
|
||||
};
|
||||
}
|
||||
|
||||
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null, forceInterrupt = null }) {
|
||||
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null }) {
|
||||
const lock = {
|
||||
promotionCommit: target.shortCommitId,
|
||||
deployJsonHash: deployBefore.hash,
|
||||
@@ -619,7 +578,6 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha
|
||||
lock.staleLockBrokenAt = updatedAt;
|
||||
lock.staleLockPrevious = staleLockPrevious;
|
||||
}
|
||||
attachForceInterruptAnnotations(lock, forceInterrupt, updatedAt, ownerTaskId);
|
||||
return {
|
||||
metadata: {
|
||||
labels: {
|
||||
@@ -638,66 +596,6 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha
|
||||
};
|
||||
}
|
||||
|
||||
export function findDevCdApplyProcessCandidates(psOutput, currentPid = process.pid) {
|
||||
return String(psOutput ?? "")
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const match = line.match(/^(\d+)\s+(.+)$/u);
|
||||
if (!match) return null;
|
||||
return {
|
||||
pid: Number.parseInt(match[1], 10),
|
||||
command: oneLine(redactSensitiveText(match[2])).slice(0, 500)
|
||||
};
|
||||
})
|
||||
.filter((entry) => {
|
||||
if (!entry || !Number.isInteger(entry.pid) || entry.pid === currentPid) return false;
|
||||
return entry.command.includes("scripts/dev-cd-apply.mjs") && entry.command.includes("--apply");
|
||||
});
|
||||
}
|
||||
|
||||
async function forceInterruptCurrentApplyProcess(ctx, existingLock) {
|
||||
const ps = await ctx.runCommand("ps", ["-eo", "pid=,args="], {
|
||||
cwd: ctx.repoRoot,
|
||||
timeoutMs: 5000
|
||||
});
|
||||
if (ps.code !== 0) {
|
||||
return {
|
||||
status: "ps-failed",
|
||||
previousLock: compactInterruptedLock(existingLock),
|
||||
processCandidates: [],
|
||||
attempts: [],
|
||||
warning: oneLine(redactSensitiveText(ps.stderr || ps.stdout || "ps failed"))
|
||||
};
|
||||
}
|
||||
|
||||
const processCandidates = findDevCdApplyProcessCandidates(ps.stdout, process.pid);
|
||||
const attempts = [];
|
||||
for (const candidate of processCandidates) {
|
||||
const result = await ctx.runCommand("kill", ["-TERM", String(candidate.pid)], {
|
||||
cwd: ctx.repoRoot,
|
||||
timeoutMs: 5000
|
||||
});
|
||||
attempts.push({
|
||||
pid: candidate.pid,
|
||||
signal: "TERM",
|
||||
status: result.code === 0 ? "sent" : "failed",
|
||||
stderrTail: redactSensitiveText(result.stderr || result.stdout || "").trim().slice(-500)
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
status: processCandidates.length > 0 ? "attempted" : "no-local-process-found",
|
||||
previousLock: compactInterruptedLock(existingLock),
|
||||
processCandidates,
|
||||
attempts,
|
||||
warning: processCandidates.length > 0
|
||||
? null
|
||||
: "No local scripts/dev-cd-apply.mjs --apply process was found; the Lease will still be taken over so the previous holder cannot continue lock-owned updates."
|
||||
};
|
||||
}
|
||||
|
||||
async function kubectl(ctx, kubectlContext, args, options = {}) {
|
||||
const result = await ctx.runCommand(kubectlContext.executor, args, {
|
||||
cwd: ctx.repoRoot,
|
||||
@@ -728,7 +626,7 @@ async function replaceLease(ctx, kubectlContext, args, lease, timeoutMs = 30000)
|
||||
function mergeLease(lease, patch, { resetLockAnnotations = false } = {}) {
|
||||
const baseAnnotations = { ...(lease.metadata?.annotations ?? {}) };
|
||||
if (resetLockAnnotations) {
|
||||
for (const key of Object.values(lockAnnotationFields)) {
|
||||
for (const key of Object.keys(baseAnnotations).filter((name) => name.startsWith(`${lockAnnotationPrefix}/`))) {
|
||||
delete baseAnnotations[key];
|
||||
}
|
||||
}
|
||||
@@ -760,14 +658,14 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
|
||||
const lease = JSON.parse(get.stdout);
|
||||
const existing = parseDeployLock(lease);
|
||||
const classification = classifyDeployLock(existing, now);
|
||||
if (classification.held && !args.forceInterruptLock) {
|
||||
if (classification.held) {
|
||||
const failure = deployLockHeldFailure(existing, now);
|
||||
throw new DevCdApplyError(failure.summary, {
|
||||
code: "deploy-lock-held",
|
||||
lockFailure: failure
|
||||
});
|
||||
}
|
||||
if (classification.stale && !args.breakStaleLock && !args.forceInterruptLock) {
|
||||
if (classification.stale && !args.breakStaleLock) {
|
||||
const failure = deployLockHeldFailure(existing, now);
|
||||
throw new DevCdApplyError(failure.summary, {
|
||||
code: "deploy-lock-held",
|
||||
@@ -775,10 +673,6 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
|
||||
});
|
||||
}
|
||||
|
||||
const forceInterrupt = args.forceInterruptLock
|
||||
? await forceInterruptCurrentApplyProcess(ctx, existing)
|
||||
: null;
|
||||
|
||||
const patch = lockPatch({
|
||||
args,
|
||||
target,
|
||||
@@ -789,8 +683,7 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
|
||||
startedAt,
|
||||
updatedAt: startedAt,
|
||||
liveBefore,
|
||||
staleLockPrevious: classification.stale ? existing : null,
|
||||
forceInterrupt
|
||||
staleLockPrevious: classification.stale ? existing : null
|
||||
});
|
||||
const replaced = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch, { resetLockAnnotations: true }));
|
||||
if (isConflict(replaced)) {
|
||||
@@ -806,7 +699,6 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
|
||||
acquired: true,
|
||||
lock: parseDeployLock(JSON.parse(replaced.stdout)),
|
||||
staleBreak: classification.stale ? existing : null,
|
||||
forceInterrupt,
|
||||
command: replaced.command
|
||||
};
|
||||
}
|
||||
@@ -837,7 +729,6 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
|
||||
acquired: true,
|
||||
lock: parseDeployLock(JSON.parse(created.stdout)),
|
||||
staleBreak: null,
|
||||
forceInterrupt: null,
|
||||
command: created.command
|
||||
};
|
||||
}
|
||||
@@ -1410,6 +1301,10 @@ function summarizeDeployJson(deploy) {
|
||||
};
|
||||
}
|
||||
|
||||
function deployJsonMatchesTarget(deploy, target) {
|
||||
return deploy.commitId === target.shortCommitId || deploy.commitId === target.commitId;
|
||||
}
|
||||
|
||||
function compactLiveEndpoint(endpoint) {
|
||||
return {
|
||||
id: endpoint.id,
|
||||
@@ -1481,7 +1376,8 @@ function summarizeLockStatus(lockRead, now) {
|
||||
current: null,
|
||||
retryAfterSeconds: null,
|
||||
expiresAt: null,
|
||||
error: oneLine(lockRead.error)
|
||||
reason: "kubectl or DEV kubeconfig unavailable; command output is redacted",
|
||||
redactedReason: oneLine(lockRead.error).slice(0, 500)
|
||||
};
|
||||
}
|
||||
const classification = classifyDeployLock(lockRead.lock, now);
|
||||
@@ -1494,12 +1390,12 @@ function summarizeLockStatus(lockRead, now) {
|
||||
: "unknown";
|
||||
return {
|
||||
status,
|
||||
current: compactInterruptedLock(lockRead.lock),
|
||||
lockName: lockRead.lock?.lockName ?? lockRead.lockName ?? defaultLockName,
|
||||
current: compactLockSummary(lockRead.lock),
|
||||
retryAfterSeconds: classification.retryAfterSeconds,
|
||||
expiresAt: classification.expiresAt,
|
||||
takeover: {
|
||||
stale: classification.stale ? "--apply --confirm-dev --confirmed-non-production --break-stale-lock" : null,
|
||||
forceInterrupt: status === "held" ? "--apply --confirm-dev --confirmed-non-production --force-interrupt-lock" : null
|
||||
allowedLockAction: {
|
||||
breakStale: classification.stale ? "--apply --confirm-dev --confirmed-non-production --break-stale-lock" : null
|
||||
}
|
||||
};
|
||||
}
|
||||
@@ -1513,7 +1409,7 @@ function buildStatusNextActions({ target, deployBefore, lock, liveVerify }) {
|
||||
actions.push(`deploy/deploy.json commitId is ${deployBefore.commitId}; target ref is ${target.shortCommitId}.`);
|
||||
}
|
||||
if (lock.status === "held") {
|
||||
actions.push(`Wait ${lock.retryAfterSeconds}s or use explicit --force-interrupt-lock if the current apply must be stopped.`);
|
||||
actions.push(`Wait for the current DEV CD holder to release the Lease or for the ${lock.retryAfterSeconds}s TTL to expire before retrying apply.`);
|
||||
} else if (lock.status === "stale") {
|
||||
actions.push("Use --break-stale-lock only after confirming the holder is inactive.");
|
||||
} else if (lock.status === "unavailable") {
|
||||
@@ -1538,7 +1434,13 @@ async function runDevCdStatus(args, ctx, stdout) {
|
||||
const liveVerify = args.skipLiveVerify
|
||||
? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } }
|
||||
: await verifyDevLive(ctx, { expectedCommit: null });
|
||||
const status = lock.status === "unavailable" || liveVerify.status !== "pass" ? "degraded" : "pass";
|
||||
const deployMatchesTarget = deployJsonMatchesTarget(deployBefore, target);
|
||||
const status = (
|
||||
lock.status === "unavailable" ||
|
||||
liveVerify.status !== "pass" ||
|
||||
!target.headMatchesTarget ||
|
||||
!deployMatchesTarget
|
||||
) ? "degraded" : "pass";
|
||||
const report = {
|
||||
ok: true,
|
||||
status,
|
||||
@@ -1555,7 +1457,10 @@ async function runDevCdStatus(args, ctx, stdout) {
|
||||
headMatchesTarget: target.headMatchesTarget,
|
||||
namespace: args.targetNamespace
|
||||
},
|
||||
deployJson: summarizeDeployJson(deployBefore),
|
||||
deployJson: {
|
||||
...summarizeDeployJson(deployBefore),
|
||||
matchesTarget: deployMatchesTarget
|
||||
},
|
||||
kubectl: {
|
||||
kubeconfigSource: kubectlContext.kubeconfigSource,
|
||||
kubeconfig: kubectlContext.kubeconfig ? "<set>" : "<unset>",
|
||||
@@ -1592,9 +1497,8 @@ function summarizeDevCdApplyReport(report) {
|
||||
target: apply.target ?? null,
|
||||
lock: {
|
||||
acquired: Boolean(apply.lock?.acquired),
|
||||
current: compactInterruptedLock(apply.lock?.current),
|
||||
staleBreak: compactInterruptedLock(apply.lock?.staleBreak),
|
||||
forceInterrupt: compactForceInterruptRecord(apply.lock?.forceInterrupt)
|
||||
current: compactLockSummary(apply.lock?.current),
|
||||
staleBreak: compactLockSummary(apply.lock?.staleBreak)
|
||||
},
|
||||
steps: (apply.steps ?? []).map((step) => ({
|
||||
id: step.id,
|
||||
@@ -1726,8 +1630,7 @@ function buildReport({
|
||||
lock: {
|
||||
acquired: Boolean(lockState?.acquired),
|
||||
current: lockState?.lock ?? null,
|
||||
staleBreak: lockState?.staleBreak ?? null,
|
||||
forceInterrupt: lockState?.forceInterrupt ?? null
|
||||
staleBreak: lockState?.staleBreak ?? null
|
||||
},
|
||||
steps,
|
||||
liveBefore,
|
||||
|
||||
@@ -10,7 +10,6 @@ import {
|
||||
buildLockAnnotations,
|
||||
classifyDeployLock,
|
||||
deployLockHeldFailure,
|
||||
findDevCdApplyProcessCandidates,
|
||||
parseDeployLock,
|
||||
runDevCdApply,
|
||||
verifyDevLive
|
||||
@@ -99,7 +98,7 @@ function makeHttpGetJson() {
|
||||
};
|
||||
}
|
||||
|
||||
function makeRunCommand({ heldLock = null, commandLog = [], psOutput = "", killLog = [] } = {}) {
|
||||
function makeRunCommand({ heldLock = null, commandLog = [] } = {}) {
|
||||
let lease = heldLock ? leaseFromLock(heldLock) : null;
|
||||
const assertLeaseMicroTime = (value) => {
|
||||
assert.match(value, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/u);
|
||||
@@ -115,13 +114,6 @@ function makeRunCommand({ heldLock = null, commandLog = [], psOutput = "", killL
|
||||
if (command === "which" && args[0] === "kubectl") {
|
||||
return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" };
|
||||
}
|
||||
if (command === "ps") {
|
||||
return { code: 0, stdout: psOutput, stderr: "" };
|
||||
}
|
||||
if (command === "kill") {
|
||||
killLog.push({ signal: args[0], pid: args[1] });
|
||||
return { code: 0, stdout: "", stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) {
|
||||
if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" };
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
@@ -191,6 +183,18 @@ function makeRunCommand({ heldLock = null, commandLog = [], psOutput = "", killL
|
||||
};
|
||||
}
|
||||
|
||||
function assertNoReadOnlySideEffects(commandLog) {
|
||||
const mutatingKubectlVerbs = new Set(["apply", "create", "replace", "patch", "delete", "rollout", "wait", "logs"]);
|
||||
assert.equal(commandLog.some((entry) => entry.command === "ps"), false);
|
||||
assert.equal(commandLog.some((entry) => entry.command === "kill"), false);
|
||||
assert.equal(commandLog.some((entry) =>
|
||||
entry.command.includes("kubectl") && entry.args.some((arg) => mutatingKubectlVerbs.has(arg))
|
||||
), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/refresh-artifact-catalog.mjs")), false);
|
||||
}
|
||||
|
||||
test("lock classifier reports held and stale states with retryAfterSeconds", () => {
|
||||
const lock = {
|
||||
ownerTaskId: "task-a",
|
||||
@@ -222,7 +226,8 @@ test("dev-cd status and dry-run are read-only compact JSON", async () => {
|
||||
const code = await runDevCdApply([
|
||||
"--dry-run",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig"
|
||||
"/tmp/kubeconfig",
|
||||
"--write-report"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
@@ -239,22 +244,179 @@ test("dev-cd status and dry-run are read-only compact JSON", async () => {
|
||||
assert.equal(status.prodTouched, false);
|
||||
assert.equal(status.lock.status, "absent");
|
||||
assert.equal(status.deployJson.manifest, undefined);
|
||||
assert.equal(status.deployJson.matchesTarget, true);
|
||||
assert.equal(status.live.summary.checked, 2);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("create")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("replace")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
|
||||
assertNoReadOnlySideEffects(commandLog);
|
||||
await assert.rejects(
|
||||
readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"),
|
||||
{ code: "ENOENT" }
|
||||
);
|
||||
});
|
||||
|
||||
test("process candidate discovery only returns dev-cd apply processes", () => {
|
||||
const psOutput = [
|
||||
"101 node scripts/dev-cd-apply.mjs --apply --confirm-dev",
|
||||
"202 node scripts/dev-cd-apply.mjs --status",
|
||||
"303 node scripts/dev-deploy-apply.mjs --apply",
|
||||
"404 node scripts/dev-cd-apply.mjs --apply --confirm-dev"
|
||||
].join("\n");
|
||||
const candidates = findDevCdApplyProcessCandidates(psOutput, 404);
|
||||
assert.deepEqual(candidates.map((candidate) => candidate.pid), [101]);
|
||||
test("dev-cd default status is read-only compact JSON", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({ commandLog }),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const status = JSON.parse(output);
|
||||
assert.equal(code, 0);
|
||||
assert.equal(status.mode, "status");
|
||||
assert.equal(status.mutationAttempted, false);
|
||||
assert.equal(status.prodTouched, false);
|
||||
assert.equal(status.target.shortCommitId, "abc1234");
|
||||
assert.equal(status.deployJson.hash.startsWith("sha256:"), true);
|
||||
assert.equal(status.deployJson.matchesTarget, true);
|
||||
assert.equal(status.lock.status, "absent");
|
||||
assertNoReadOnlySideEffects(commandLog);
|
||||
});
|
||||
|
||||
test("dev-cd status summarizes held, released, and stale locks without unsafe takeover hints", async () => {
|
||||
const cases = [
|
||||
{
|
||||
name: "held",
|
||||
phase: "applying",
|
||||
now: iso(100000),
|
||||
expectedStatus: "held",
|
||||
expectedRetryAfterSeconds: 200
|
||||
},
|
||||
{
|
||||
name: "released",
|
||||
phase: "released",
|
||||
releasedAt: iso(50000),
|
||||
releaseStatus: "pass",
|
||||
now: iso(100000),
|
||||
expectedStatus: "released",
|
||||
expectedRetryAfterSeconds: 0
|
||||
},
|
||||
{
|
||||
name: "stale",
|
||||
phase: "verifying",
|
||||
now: iso(301000),
|
||||
expectedStatus: "stale",
|
||||
expectedRetryAfterSeconds: 0
|
||||
}
|
||||
];
|
||||
|
||||
for (const item of cases) {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
commandLog,
|
||||
heldLock: {
|
||||
promotionCommit: "abc1234",
|
||||
deployJsonHash: "sha256:a",
|
||||
ownerTaskId: `task-${item.name}`,
|
||||
transactionId: `tx-${item.name}`,
|
||||
phase: item.phase,
|
||||
startedAt: iso(0),
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
liveBefore: { status: "pass" },
|
||||
targetNamespace: "hwlab-dev",
|
||||
targetRef: "origin/main",
|
||||
lockBackend: "Lease",
|
||||
...(item.releasedAt ? { releasedAt: item.releasedAt } : {}),
|
||||
...(item.releaseStatus ? { releaseStatus: item.releaseStatus } : {})
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(item.now),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const status = JSON.parse(output);
|
||||
assert.equal(code, 0);
|
||||
assert.equal(status.lock.status, item.expectedStatus);
|
||||
assert.equal(status.lock.lockName, "hwlab-dev-cd-lock");
|
||||
assert.equal(status.lock.current.ownerTaskId, `task-${item.name}`);
|
||||
assert.equal(status.lock.retryAfterSeconds, item.expectedRetryAfterSeconds);
|
||||
assert.equal(JSON.stringify(status).includes("force-interrupt"), false);
|
||||
assert.equal(JSON.stringify(status.nextActions).includes("force"), false);
|
||||
if (item.expectedStatus === "stale") {
|
||||
assert.match(status.lock.allowedLockAction.breakStale, /--break-stale-lock/u);
|
||||
}
|
||||
assertNoReadOnlySideEffects(commandLog);
|
||||
}
|
||||
});
|
||||
|
||||
test("dev-cd status reports unavailable kubectl with redacted reason", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const runCommand = async (command, args, options = {}) => {
|
||||
commandLog.push({ command, args, env: options.env ?? {}, input: options.input ?? "" });
|
||||
if (command === "git" && args[0] === "rev-parse" && args.includes("origin/main^{commit}")) {
|
||||
return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" };
|
||||
}
|
||||
if (command === "git" && args[0] === "rev-parse" && args.includes("HEAD^{commit}")) {
|
||||
return { code: 0, stdout: "abc1234abc1234abc1234abc1234abc1234abc1\n", stderr: "" };
|
||||
}
|
||||
if (command === "which" && args[0] === "kubectl") {
|
||||
return { code: 1, stdout: "", stderr: "kubectl missing" };
|
||||
}
|
||||
if (command === "kubectl") {
|
||||
return {
|
||||
code: 1,
|
||||
stdout: "",
|
||||
stderr: "connection failed token=SECRET_TOKEN password=SECRET_PASSWORD postgresql://user:SECRET_DB@db"
|
||||
};
|
||||
}
|
||||
return { code: 0, stdout: "", stderr: "" };
|
||||
};
|
||||
|
||||
const code = await runDevCdApply(["--status"], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand,
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const status = JSON.parse(output);
|
||||
const text = JSON.stringify(status);
|
||||
assert.equal(code, 0);
|
||||
assert.equal(status.status, "degraded");
|
||||
assert.equal(status.lock.status, "unavailable");
|
||||
assert.equal(status.lock.reason, "kubectl or DEV kubeconfig unavailable; command output is redacted");
|
||||
assert.equal(text.includes("SECRET_TOKEN"), false);
|
||||
assert.equal(text.includes("SECRET_PASSWORD"), false);
|
||||
assert.equal(text.includes("SECRET_DB"), false);
|
||||
assert.match(status.lock.redactedReason, /<redacted>/u);
|
||||
assertNoReadOnlySideEffects(commandLog);
|
||||
});
|
||||
|
||||
test("force-interrupt option is rejected before process discovery or lease mutation", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply(["--force-interrupt-lock"], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({ commandLog }),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const failure = JSON.parse(output);
|
||||
assert.equal(code, 2);
|
||||
assert.equal(failure.error, "unknown-argument");
|
||||
assert.equal(failure.mutationAttempted, false);
|
||||
assert.equal(failure.prodTouched, false);
|
||||
assert.deepEqual(commandLog, []);
|
||||
});
|
||||
|
||||
test("deploy-lock-held failure is structured before publish/apply", () => {
|
||||
@@ -489,55 +651,6 @@ test("apply stdout is concise by default while write-report keeps the full repor
|
||||
assert.equal(writtenReport.devCdApply.deployJson.before.manifest.environment, "dev");
|
||||
});
|
||||
|
||||
test("force-interrupt-lock terminates local apply candidate and takes a held lock", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const killLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--force-interrupt-lock",
|
||||
"--owner-task-id",
|
||||
"task-force",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig",
|
||||
"--full-output"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
killLog,
|
||||
psOutput: "555 node scripts/dev-cd-apply.mjs --apply --confirm-dev\n666 node scripts/dev-cd-apply.mjs --status\n",
|
||||
heldLock: {
|
||||
promotionCommit: "abc1234",
|
||||
deployJsonHash: "sha256:a",
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
phase: "applying",
|
||||
startedAt: iso(0),
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
liveBefore: { status: "pass" },
|
||||
targetNamespace: "hwlab-dev",
|
||||
targetRef: "origin/main",
|
||||
lockBackend: "Lease"
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const report = JSON.parse(output);
|
||||
assert.equal(code, 0);
|
||||
assert.deepEqual(killLog, [{ signal: "-TERM", pid: "555" }]);
|
||||
assert.equal(report.devCdApply.lock.forceInterrupt.status, "attempted");
|
||||
assert.equal(report.devCdApply.lock.forceInterrupt.processCandidates[0].pid, 555);
|
||||
assert.equal(report.devCdApply.lock.current.forceInterruptStatus, "attempted");
|
||||
assert.equal(report.devCdApply.lock.current.forceInterruptedBy, "task-force");
|
||||
});
|
||||
|
||||
test("live verify summarizes 16666 and 16667 health identity", async () => {
|
||||
const result = await verifyDevLive({
|
||||
now: () => new Date(iso()),
|
||||
|
||||
Reference in New Issue
Block a user