feat: improve dev cd apply observability
Add compact DEV CD status/dry-run output, concise apply summaries, and explicit DEV-only force-interrupt lock takeover.
This commit is contained in:
@@ -57,6 +57,23 @@ not silently overwritten; taking over a stale lock requires both
|
||||
`--break-stale-lock` and `--confirm-dev`, and the previous holder is retained
|
||||
in audit annotations/report fields.
|
||||
|
||||
`scripts/dev-cd-apply.mjs` also owns the read-only status surface. Running it
|
||||
without arguments, with `--status`, or with `--dry-run` must return compact JSON
|
||||
for the target ref, `deploy/deploy.json`, current Lease, and public
|
||||
`16666/16667` live health. These modes must not acquire the Lease, publish
|
||||
artifacts, apply Kubernetes workloads, or write reports. Normal `--apply`
|
||||
stdout is also compact by default; the complete transaction record is preserved
|
||||
through `--write-report`, or printed only when `--full-output` is explicit.
|
||||
|
||||
When the current DEV apply must be stopped, the operator may use
|
||||
`--force-interrupt-lock` together with `--apply --confirm-dev
|
||||
--confirmed-non-production`. This is a DEV-only takeover path: it first tries
|
||||
to send `TERM` to local `scripts/dev-cd-apply.mjs --apply` processes, then
|
||||
replaces the Lease holder and records the interrupted holder and interrupt
|
||||
attempt in Lease annotations and the transaction report. Do not use this flag
|
||||
for ordinary contention; prefer waiting for the holder or breaking only a
|
||||
confirmed stale lock.
|
||||
|
||||
The old side-effect commands are now internal transaction steps. Direct manual
|
||||
use of `scripts/dev-artifact-publish.mjs --publish` or
|
||||
`scripts/dev-deploy-apply.mjs --apply` without `HWLAB_CD_TRANSACTION_ID` is
|
||||
|
||||
+427
-25
@@ -38,7 +38,11 @@ const lockAnnotationFields = {
|
||||
releasedAt: `${lockAnnotationPrefix}/releasedAt`,
|
||||
releaseStatus: `${lockAnnotationPrefix}/releaseStatus`,
|
||||
staleLockBrokenAt: `${lockAnnotationPrefix}/staleLockBrokenAt`,
|
||||
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`
|
||||
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`,
|
||||
forceInterruptedAt: `${lockAnnotationPrefix}/forceInterruptedAt`,
|
||||
forceInterruptedBy: `${lockAnnotationPrefix}/forceInterruptedBy`,
|
||||
forceInterruptStatus: `${lockAnnotationPrefix}/forceInterruptStatus`,
|
||||
forceInterruptedPrevious: `${lockAnnotationPrefix}/forceInterruptedPrevious`
|
||||
};
|
||||
|
||||
class DevCdApplyError extends Error {
|
||||
@@ -64,7 +68,11 @@ export function parseArgs(argv) {
|
||||
ttlSeconds: defaultTtlSeconds,
|
||||
ownerTaskId: null,
|
||||
breakStaleLock: false,
|
||||
forceInterruptLock: false,
|
||||
skipLiveVerify: false,
|
||||
status: false,
|
||||
dryRun: false,
|
||||
fullOutput: false,
|
||||
help: false,
|
||||
flags: new Set()
|
||||
};
|
||||
@@ -86,6 +94,16 @@ export function parseArgs(argv) {
|
||||
} else if (arg === "--break-stale-lock") {
|
||||
args.breakStaleLock = true;
|
||||
args.flags.add(arg);
|
||||
} else if (arg === "--force-interrupt-lock") {
|
||||
args.forceInterruptLock = true;
|
||||
args.flags.add(arg);
|
||||
} else if (arg === "--dry-run" || arg === "--status") {
|
||||
args.status = true;
|
||||
args.dryRun = arg === "--dry-run";
|
||||
args.flags.add(arg);
|
||||
} else if (arg === "--full-output") {
|
||||
args.fullOutput = true;
|
||||
args.flags.add(arg);
|
||||
} else if (arg === "--skip-live-verify") {
|
||||
args.skipLiveVerify = true;
|
||||
args.flags.add(arg);
|
||||
@@ -134,20 +152,34 @@ function readOption(argv, index, name) {
|
||||
}
|
||||
|
||||
export function usage() {
|
||||
return [
|
||||
"usage: node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production [--write-report]",
|
||||
"",
|
||||
"Single DEV CD transaction for latest-main publish, desired-state refresh, apply, and live verify.",
|
||||
"",
|
||||
"options:",
|
||||
" --target-ref REF git ref to publish/apply; default: origin/main",
|
||||
" --kubeconfig PATH DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG",
|
||||
" --ttl-seconds SECONDS lock TTL; default: 3600",
|
||||
" --owner-task-id ID lock holder; default: Code Queue/env/user derived",
|
||||
" --break-stale-lock may take over an expired lock only with --confirm-dev",
|
||||
" --report PATH default: reports/dev-gate/dev-cd-apply.json",
|
||||
" --skip-live-verify test-only escape hatch; do not use for DEV acceptance"
|
||||
].join("\n");
|
||||
return {
|
||||
command: "node scripts/dev-cd-apply.mjs",
|
||||
summary: "Single DEV CD transaction for latest-main publish, desired-state refresh, apply, and live verify.",
|
||||
defaultMode: "status",
|
||||
examples: [
|
||||
"node scripts/dev-cd-apply.mjs --status",
|
||||
"node scripts/dev-cd-apply.mjs --dry-run",
|
||||
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report",
|
||||
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --force-interrupt-lock --write-report"
|
||||
],
|
||||
options: {
|
||||
"--status": "Read-only status: target ref, deploy.json, current Lease, and public live health.",
|
||||
"--dry-run": "Read-only alias for --status; does not acquire the Lease or run side effects.",
|
||||
"--apply": "Run the DEV CD transaction side effects.",
|
||||
"--confirm-dev": "Required for DEV mutation and stale/force lock takeover.",
|
||||
"--confirmed-non-production": "Required for DEV mutation.",
|
||||
"--target-ref REF": "git ref to publish/apply; default: origin/main",
|
||||
"--kubeconfig PATH": "DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG",
|
||||
"--ttl-seconds SECONDS": "lock TTL; default: 3600",
|
||||
"--owner-task-id ID": "lock holder; default: Code Queue/env/user derived",
|
||||
"--break-stale-lock": "May take over an expired lock only with --confirm-dev.",
|
||||
"--force-interrupt-lock": "DEV-only explicit takeover: try to interrupt the current local dev-cd apply process, then acquire the Lease.",
|
||||
"--report PATH": `default: ${defaultReportPath}`,
|
||||
"--write-report": "Write the full report to --report.",
|
||||
"--full-output": "Print the full report to stdout instead of the concise summary.",
|
||||
"--skip-live-verify": "test-only escape hatch; do not use for DEV acceptance"
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function nowIso() {
|
||||
@@ -286,12 +318,12 @@ async function resolveTargetRef(ctx, targetRef) {
|
||||
|
||||
function validateArgs(args) {
|
||||
const blockers = [];
|
||||
if (!args.apply) {
|
||||
if (!args.apply && !args.status) {
|
||||
blockers.push({
|
||||
type: "safety_blocker",
|
||||
scope: "apply-mode",
|
||||
status: "open",
|
||||
summary: "DEV CD transaction side effects require --apply."
|
||||
summary: "DEV CD side effects require --apply; use --status or --dry-run for read-only observability."
|
||||
});
|
||||
}
|
||||
if (args.apply && (!args.confirmDev || !args.confirmedNonProduction)) {
|
||||
@@ -310,6 +342,22 @@ function validateArgs(args) {
|
||||
summary: "Breaking a stale DEV CD lock requires --break-stale-lock and --confirm-dev."
|
||||
});
|
||||
}
|
||||
if (args.forceInterruptLock && !args.confirmDev) {
|
||||
blockers.push({
|
||||
type: "safety_blocker",
|
||||
scope: "force-interrupt-confirmation",
|
||||
status: "open",
|
||||
summary: "Force-interrupting a DEV CD lock requires --force-interrupt-lock and --confirm-dev."
|
||||
});
|
||||
}
|
||||
if (args.forceInterruptLock && !args.apply) {
|
||||
blockers.push({
|
||||
type: "safety_blocker",
|
||||
scope: "force-interrupt-mode",
|
||||
status: "open",
|
||||
summary: "--force-interrupt-lock is only valid with --apply."
|
||||
});
|
||||
}
|
||||
if (!Number.isInteger(args.ttlSeconds) || args.ttlSeconds < 60 || args.ttlSeconds > 24 * 60 * 60) {
|
||||
blockers.push({
|
||||
type: "safety_blocker",
|
||||
@@ -394,7 +442,11 @@ export function parseDeployLock(lease) {
|
||||
releasedAt: readAnnotation(annotations, "releasedAt"),
|
||||
releaseStatus: readAnnotation(annotations, "releaseStatus"),
|
||||
staleLockBrokenAt: readAnnotation(annotations, "staleLockBrokenAt"),
|
||||
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious")
|
||||
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious"),
|
||||
forceInterruptedAt: readAnnotation(annotations, "forceInterruptedAt"),
|
||||
forceInterruptedBy: readAnnotation(annotations, "forceInterruptedBy"),
|
||||
forceInterruptStatus: readAnnotation(annotations, "forceInterruptStatus"),
|
||||
forceInterruptedPrevious: readJsonAnnotation(annotations, "forceInterruptedPrevious")
|
||||
};
|
||||
}
|
||||
|
||||
@@ -456,7 +508,53 @@ function isConflict(result) {
|
||||
return /conflict|object has been modified|operation cannot be fulfilled/i.test(`${result.stderr}\n${result.stdout}`);
|
||||
}
|
||||
|
||||
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null }) {
|
||||
function compactInterruptedLock(lock) {
|
||||
if (!lock) return null;
|
||||
return {
|
||||
lockName: lock.lockName ?? defaultLockName,
|
||||
lockBackend: lock.lockBackend ?? "Lease",
|
||||
ownerTaskId: lock.ownerTaskId ?? null,
|
||||
transactionId: lock.transactionId ?? null,
|
||||
holderIdentity: lock.holderIdentity ?? null,
|
||||
phase: lock.phase ?? null,
|
||||
promotionCommit: lock.promotionCommit ?? null,
|
||||
deployJsonHash: lock.deployJsonHash ?? null,
|
||||
updatedAt: lock.updatedAt ?? null,
|
||||
ttlSeconds: lock.ttlSeconds ?? null,
|
||||
targetNamespace: lock.targetNamespace ?? defaultNamespace,
|
||||
targetRef: lock.targetRef ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function compactForceInterruptRecord(record) {
|
||||
if (!record) return null;
|
||||
return {
|
||||
status: record.status,
|
||||
warning: record.warning ?? null,
|
||||
processCandidates: (record.processCandidates ?? []).map((candidate) => ({
|
||||
pid: candidate.pid,
|
||||
command: candidate.command
|
||||
})),
|
||||
attempts: (record.attempts ?? []).map((attempt) => ({
|
||||
pid: attempt.pid,
|
||||
signal: attempt.signal,
|
||||
status: attempt.status
|
||||
}))
|
||||
};
|
||||
}
|
||||
|
||||
function attachForceInterruptAnnotations(lock, forceInterrupt, timestamp, ownerTaskId) {
|
||||
if (!forceInterrupt) return;
|
||||
lock.forceInterruptedAt = timestamp;
|
||||
lock.forceInterruptedBy = ownerTaskId;
|
||||
lock.forceInterruptStatus = forceInterrupt.status;
|
||||
lock.forceInterruptedPrevious = {
|
||||
previousLock: compactInterruptedLock(forceInterrupt.previousLock),
|
||||
interrupt: compactForceInterruptRecord(forceInterrupt)
|
||||
};
|
||||
}
|
||||
|
||||
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null, forceInterrupt = null }) {
|
||||
const lock = {
|
||||
promotionCommit: target.shortCommitId,
|
||||
deployJsonHash: deployBefore.hash,
|
||||
@@ -475,6 +573,7 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa
|
||||
lock.staleLockBrokenAt = startedAt;
|
||||
lock.staleLockPrevious = staleLockPrevious;
|
||||
}
|
||||
attachForceInterruptAnnotations(lock, forceInterrupt, startedAt, ownerTaskId);
|
||||
return {
|
||||
apiVersion: "coordination.k8s.io/v1",
|
||||
kind: "Lease",
|
||||
@@ -497,7 +596,7 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa
|
||||
};
|
||||
}
|
||||
|
||||
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null }) {
|
||||
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null, forceInterrupt = null }) {
|
||||
const lock = {
|
||||
promotionCommit: target.shortCommitId,
|
||||
deployJsonHash: deployBefore.hash,
|
||||
@@ -516,6 +615,7 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha
|
||||
lock.staleLockBrokenAt = updatedAt;
|
||||
lock.staleLockPrevious = staleLockPrevious;
|
||||
}
|
||||
attachForceInterruptAnnotations(lock, forceInterrupt, updatedAt, ownerTaskId);
|
||||
return {
|
||||
metadata: {
|
||||
labels: {
|
||||
@@ -534,6 +634,66 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha
|
||||
};
|
||||
}
|
||||
|
||||
export function findDevCdApplyProcessCandidates(psOutput, currentPid = process.pid) {
|
||||
return String(psOutput ?? "")
|
||||
.split(/\r?\n/u)
|
||||
.map((line) => line.trim())
|
||||
.filter(Boolean)
|
||||
.map((line) => {
|
||||
const match = line.match(/^(\d+)\s+(.+)$/u);
|
||||
if (!match) return null;
|
||||
return {
|
||||
pid: Number.parseInt(match[1], 10),
|
||||
command: oneLine(redactSensitiveText(match[2])).slice(0, 500)
|
||||
};
|
||||
})
|
||||
.filter((entry) => {
|
||||
if (!entry || !Number.isInteger(entry.pid) || entry.pid === currentPid) return false;
|
||||
return entry.command.includes("scripts/dev-cd-apply.mjs") && entry.command.includes("--apply");
|
||||
});
|
||||
}
|
||||
|
||||
async function forceInterruptCurrentApplyProcess(ctx, existingLock) {
|
||||
const ps = await ctx.runCommand("ps", ["-eo", "pid=,args="], {
|
||||
cwd: ctx.repoRoot,
|
||||
timeoutMs: 5000
|
||||
});
|
||||
if (ps.code !== 0) {
|
||||
return {
|
||||
status: "ps-failed",
|
||||
previousLock: compactInterruptedLock(existingLock),
|
||||
processCandidates: [],
|
||||
attempts: [],
|
||||
warning: oneLine(redactSensitiveText(ps.stderr || ps.stdout || "ps failed"))
|
||||
};
|
||||
}
|
||||
|
||||
const processCandidates = findDevCdApplyProcessCandidates(ps.stdout, process.pid);
|
||||
const attempts = [];
|
||||
for (const candidate of processCandidates) {
|
||||
const result = await ctx.runCommand("kill", ["-TERM", String(candidate.pid)], {
|
||||
cwd: ctx.repoRoot,
|
||||
timeoutMs: 5000
|
||||
});
|
||||
attempts.push({
|
||||
pid: candidate.pid,
|
||||
signal: "TERM",
|
||||
status: result.code === 0 ? "sent" : "failed",
|
||||
stderrTail: redactSensitiveText(result.stderr || result.stdout || "").trim().slice(-500)
|
||||
});
|
||||
}
|
||||
|
||||
return {
|
||||
status: processCandidates.length > 0 ? "attempted" : "no-local-process-found",
|
||||
previousLock: compactInterruptedLock(existingLock),
|
||||
processCandidates,
|
||||
attempts,
|
||||
warning: processCandidates.length > 0
|
||||
? null
|
||||
: "No local scripts/dev-cd-apply.mjs --apply process was found; the Lease will still be taken over so the previous holder cannot continue lock-owned updates."
|
||||
};
|
||||
}
|
||||
|
||||
async function kubectl(ctx, kubectlContext, args, options = {}) {
|
||||
const result = await ctx.runCommand(kubectlContext.executor, args, {
|
||||
cwd: ctx.repoRoot,
|
||||
@@ -596,13 +756,24 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
|
||||
const lease = JSON.parse(get.stdout);
|
||||
const existing = parseDeployLock(lease);
|
||||
const classification = classifyDeployLock(existing, now);
|
||||
if (classification.held || (classification.stale && !args.breakStaleLock)) {
|
||||
if (classification.held && !args.forceInterruptLock) {
|
||||
const failure = deployLockHeldFailure(existing, now);
|
||||
throw new DevCdApplyError(failure.summary, {
|
||||
code: "deploy-lock-held",
|
||||
lockFailure: failure
|
||||
});
|
||||
}
|
||||
if (classification.stale && !args.breakStaleLock && !args.forceInterruptLock) {
|
||||
const failure = deployLockHeldFailure(existing, now);
|
||||
throw new DevCdApplyError(failure.summary, {
|
||||
code: "deploy-lock-held",
|
||||
lockFailure: failure
|
||||
});
|
||||
}
|
||||
|
||||
const forceInterrupt = args.forceInterruptLock
|
||||
? await forceInterruptCurrentApplyProcess(ctx, existing)
|
||||
: null;
|
||||
|
||||
const patch = lockPatch({
|
||||
args,
|
||||
@@ -614,7 +785,8 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
|
||||
startedAt,
|
||||
updatedAt: startedAt,
|
||||
liveBefore,
|
||||
staleLockPrevious: classification.stale ? existing : null
|
||||
staleLockPrevious: classification.stale ? existing : null,
|
||||
forceInterrupt
|
||||
});
|
||||
const replaced = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch, { resetLockAnnotations: true }));
|
||||
if (isConflict(replaced)) {
|
||||
@@ -630,6 +802,7 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
|
||||
acquired: true,
|
||||
lock: parseDeployLock(JSON.parse(replaced.stdout)),
|
||||
staleBreak: classification.stale ? existing : null,
|
||||
forceInterrupt,
|
||||
command: replaced.command
|
||||
};
|
||||
}
|
||||
@@ -660,6 +833,7 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
|
||||
acquired: true,
|
||||
lock: parseDeployLock(JSON.parse(created.stdout)),
|
||||
staleBreak: null,
|
||||
forceInterrupt: null,
|
||||
command: created.command
|
||||
};
|
||||
}
|
||||
@@ -1006,6 +1180,226 @@ export async function verifyDevLive(ctx, { expectedCommit = null } = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function deployServiceCount(manifest) {
|
||||
const services = manifest?.services;
|
||||
if (Array.isArray(services)) return services.length;
|
||||
if (services && typeof services === "object") return Object.keys(services).length;
|
||||
return null;
|
||||
}
|
||||
|
||||
function summarizeDeployJson(deploy) {
|
||||
return {
|
||||
path: deploy.path,
|
||||
hash: deploy.hash,
|
||||
commitId: deploy.commitId,
|
||||
namespace: deploy.namespace,
|
||||
environment: deploy.environment,
|
||||
endpoint: deploy.endpoint,
|
||||
serviceCount: deployServiceCount(deploy.manifest)
|
||||
};
|
||||
}
|
||||
|
||||
function compactLiveEndpoint(endpoint) {
|
||||
return {
|
||||
id: endpoint.id,
|
||||
url: endpoint.url,
|
||||
status: endpoint.status,
|
||||
httpStatus: endpoint.httpStatus ?? null,
|
||||
reachable: endpoint.reachable,
|
||||
identityMatches: endpoint.identityMatches,
|
||||
serviceId: endpoint.serviceId,
|
||||
environment: endpoint.environment,
|
||||
observedCommit: endpoint.observedCommit,
|
||||
commitMatches: endpoint.commitMatches,
|
||||
blockerCodes: endpoint.blockerCodes ?? [],
|
||||
error: endpoint.error ?? null
|
||||
};
|
||||
}
|
||||
|
||||
function compactLiveReport(live) {
|
||||
return {
|
||||
status: live.status,
|
||||
expectedCommit: live.expectedCommit ?? null,
|
||||
summary: live.summary ?? { checked: 0 },
|
||||
endpoints: (live.endpoints ?? []).map(compactLiveEndpoint)
|
||||
};
|
||||
}
|
||||
|
||||
async function readDeployLockStatus({ ctx, args, kubectlContext }) {
|
||||
const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], {
|
||||
timeoutMs: 15000
|
||||
});
|
||||
if (get.code === 0) {
|
||||
return {
|
||||
status: "read",
|
||||
command: get.command,
|
||||
lock: parseDeployLock(JSON.parse(get.stdout))
|
||||
};
|
||||
}
|
||||
if (isNotFound(get)) {
|
||||
return {
|
||||
status: "absent",
|
||||
command: get.command,
|
||||
lockName: args.lockName,
|
||||
lock: null
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: "unavailable",
|
||||
command: get.command,
|
||||
lockName: args.lockName,
|
||||
lock: null,
|
||||
error: redactSensitiveText(get.stderr || get.stdout)
|
||||
};
|
||||
}
|
||||
|
||||
function summarizeLockStatus(lockRead, now) {
|
||||
if (lockRead.status === "absent") {
|
||||
return {
|
||||
status: "absent",
|
||||
lockName: lockRead.lockName ?? defaultLockName,
|
||||
current: null,
|
||||
retryAfterSeconds: 0,
|
||||
expiresAt: null
|
||||
};
|
||||
}
|
||||
if (lockRead.status === "unavailable") {
|
||||
return {
|
||||
status: "unavailable",
|
||||
lockName: lockRead.lockName ?? defaultLockName,
|
||||
current: null,
|
||||
retryAfterSeconds: null,
|
||||
expiresAt: null,
|
||||
error: oneLine(lockRead.error)
|
||||
};
|
||||
}
|
||||
const classification = classifyDeployLock(lockRead.lock, now);
|
||||
const status = lockRead.lock?.phase === "released"
|
||||
? "released"
|
||||
: classification.held
|
||||
? "held"
|
||||
: classification.stale
|
||||
? "stale"
|
||||
: "unknown";
|
||||
return {
|
||||
status,
|
||||
current: compactInterruptedLock(lockRead.lock),
|
||||
retryAfterSeconds: classification.retryAfterSeconds,
|
||||
expiresAt: classification.expiresAt,
|
||||
takeover: {
|
||||
stale: classification.stale ? "--apply --confirm-dev --confirmed-non-production --break-stale-lock" : null,
|
||||
forceInterrupt: status === "held" ? "--apply --confirm-dev --confirmed-non-production --force-interrupt-lock" : null
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
function buildStatusNextActions({ target, deployBefore, lock, liveVerify }) {
|
||||
const actions = [];
|
||||
if (!target.headMatchesTarget) {
|
||||
actions.push(`Checkout or fast-forward to ${target.ref} ${target.shortCommitId} before apply.`);
|
||||
}
|
||||
if (deployBefore.commitId !== target.shortCommitId && deployBefore.commitId !== target.commitId) {
|
||||
actions.push(`deploy/deploy.json commitId is ${deployBefore.commitId}; target ref is ${target.shortCommitId}.`);
|
||||
}
|
||||
if (lock.status === "held") {
|
||||
actions.push(`Wait ${lock.retryAfterSeconds}s or use explicit --force-interrupt-lock if the current apply must be stopped.`);
|
||||
} else if (lock.status === "stale") {
|
||||
actions.push("Use --break-stale-lock only after confirming the holder is inactive.");
|
||||
} else if (lock.status === "unavailable") {
|
||||
actions.push("Fix kubectl/KUBECONFIG access before applying.");
|
||||
}
|
||||
if (liveVerify.status !== "pass") {
|
||||
actions.push("Inspect live endpoint identity/health before treating DEV as converged.");
|
||||
}
|
||||
if (actions.length === 0) {
|
||||
actions.push("Run --apply --confirm-dev --confirmed-non-production --write-report when ready to publish/apply DEV.");
|
||||
}
|
||||
return actions;
|
||||
}
|
||||
|
||||
async function runDevCdStatus(args, ctx, stdout) {
|
||||
const generatedAt = ctx.now().toISOString();
|
||||
const target = await resolveTargetRef(ctx, args.targetRef);
|
||||
const deployBefore = await readDeployJson(ctx.repoRoot);
|
||||
const kubectlContext = await resolveKubectlContext(args, ctx.env, ctx.runCommand);
|
||||
const lockRead = await readDeployLockStatus({ ctx, args, kubectlContext });
|
||||
const lock = summarizeLockStatus(lockRead, ctx.now());
|
||||
const liveVerify = args.skipLiveVerify
|
||||
? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } }
|
||||
: await verifyDevLive(ctx, { expectedCommit: null });
|
||||
const status = lock.status === "unavailable" || liveVerify.status !== "pass" ? "degraded" : "pass";
|
||||
const report = {
|
||||
ok: true,
|
||||
status,
|
||||
mode: args.dryRun ? "dry-run" : "status",
|
||||
command: "dev-cd-apply",
|
||||
generatedAt,
|
||||
mutationAttempted: false,
|
||||
prodTouched: false,
|
||||
target: {
|
||||
ref: target.ref,
|
||||
promotionCommit: target.commitId,
|
||||
shortCommitId: target.shortCommitId,
|
||||
headCommitId: target.headCommitId,
|
||||
headMatchesTarget: target.headMatchesTarget,
|
||||
namespace: args.targetNamespace
|
||||
},
|
||||
deployJson: summarizeDeployJson(deployBefore),
|
||||
kubectl: {
|
||||
kubeconfigSource: kubectlContext.kubeconfigSource,
|
||||
kubeconfig: kubectlContext.kubeconfig ? "<set>" : "<unset>",
|
||||
lockReadCommand: lockRead.command
|
||||
},
|
||||
lock,
|
||||
live: compactLiveReport(liveVerify),
|
||||
nextActions: buildStatusNextActions({ target, deployBefore, lock, liveVerify }),
|
||||
fullOutputHint: "Use --apply --confirm-dev --confirmed-non-production --write-report for the full transaction report; use --full-output only when stdout needs the full JSON."
|
||||
};
|
||||
stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
return 0;
|
||||
}
|
||||
|
||||
function summarizeDevCdApplyReport(report) {
|
||||
const apply = report.devCdApply ?? {};
|
||||
return {
|
||||
ok: report.status === "pass",
|
||||
status: report.status,
|
||||
mode: apply.mode ?? "apply",
|
||||
command: "dev-cd-apply",
|
||||
commitId: report.commitId,
|
||||
generatedAt: report.generatedAt,
|
||||
mutationAttempted: apply.safety?.mutationAttempted ?? false,
|
||||
prodTouched: false,
|
||||
transaction: {
|
||||
transactionId: report.transaction?.transactionId ?? null,
|
||||
ownerTaskId: report.transaction?.ownerTaskId ?? null,
|
||||
targetNamespace: report.transaction?.targetNamespace ?? defaultNamespace,
|
||||
lockName: report.transaction?.lockName ?? defaultLockName,
|
||||
phases: report.transaction?.phases ?? [],
|
||||
release: report.transaction?.release ?? null
|
||||
},
|
||||
target: apply.target ?? null,
|
||||
lock: {
|
||||
acquired: Boolean(apply.lock?.acquired),
|
||||
current: compactInterruptedLock(apply.lock?.current),
|
||||
staleBreak: compactInterruptedLock(apply.lock?.staleBreak),
|
||||
forceInterrupt: compactForceInterruptRecord(apply.lock?.forceInterrupt)
|
||||
},
|
||||
steps: (apply.steps ?? []).map((step) => ({
|
||||
id: step.id,
|
||||
phase: step.phase,
|
||||
status: step.status,
|
||||
code: step.code,
|
||||
reportPath: step.reportPath
|
||||
})),
|
||||
liveBefore: apply.liveBefore ? compactLiveReport(apply.liveBefore) : null,
|
||||
liveVerify: apply.liveVerify ? compactLiveReport(apply.liveVerify) : null,
|
||||
blockers: report.blockers ?? [],
|
||||
reportPaths: apply.reportPaths ?? {},
|
||||
fullOutputHint: "Use --full-output to print the full report to stdout. Use --write-report to persist the full report."
|
||||
};
|
||||
}
|
||||
|
||||
function buildReport({
|
||||
args,
|
||||
transaction,
|
||||
@@ -1120,7 +1514,8 @@ function buildReport({
|
||||
lock: {
|
||||
acquired: Boolean(lockState?.acquired),
|
||||
current: lockState?.lock ?? null,
|
||||
staleBreak: lockState?.staleBreak ?? null
|
||||
staleBreak: lockState?.staleBreak ?? null,
|
||||
forceInterrupt: lockState?.forceInterrupt ?? null
|
||||
},
|
||||
steps,
|
||||
liveBefore,
|
||||
@@ -1190,10 +1585,16 @@ export async function runDevCdApply(argv, io = {}) {
|
||||
try {
|
||||
args = parseArgs(argv);
|
||||
if (args.help) {
|
||||
stdout.write(`${usage()}\n`);
|
||||
stdout.write(`${JSON.stringify({ ok: true, status: "pass", help: usage() }, null, 2)}\n`);
|
||||
return 0;
|
||||
}
|
||||
if (!args.apply && !args.status) {
|
||||
args.status = true;
|
||||
}
|
||||
validateArgs(args);
|
||||
if (args.status) {
|
||||
return await runDevCdStatus(args, ctx, stdout);
|
||||
}
|
||||
} catch (error) {
|
||||
const failure = formatDevCdApplyFailure(error);
|
||||
stdout.write(`${JSON.stringify(failure, null, 2)}\n`);
|
||||
@@ -1437,6 +1838,7 @@ export async function runDevCdApply(argv, io = {}) {
|
||||
await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`);
|
||||
}
|
||||
|
||||
stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
||||
const stdoutPayload = args.fullOutput ? report : summarizeDevCdApplyReport(report);
|
||||
stdout.write(`${JSON.stringify(stdoutPayload, null, 2)}\n`);
|
||||
return status === "pass" ? 0 : 2;
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ import {
|
||||
buildLockAnnotations,
|
||||
classifyDeployLock,
|
||||
deployLockHeldFailure,
|
||||
findDevCdApplyProcessCandidates,
|
||||
parseDeployLock,
|
||||
runDevCdApply,
|
||||
verifyDevLive
|
||||
@@ -98,7 +99,7 @@ function makeHttpGetJson() {
|
||||
};
|
||||
}
|
||||
|
||||
function makeRunCommand({ heldLock = null, commandLog = [] } = {}) {
|
||||
function makeRunCommand({ heldLock = null, commandLog = [], psOutput = "", killLog = [] } = {}) {
|
||||
let lease = heldLock ? leaseFromLock(heldLock) : null;
|
||||
const assertLeaseMicroTime = (value) => {
|
||||
assert.match(value, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/u);
|
||||
@@ -114,6 +115,13 @@ function makeRunCommand({ heldLock = null, commandLog = [] } = {}) {
|
||||
if (command === "which" && args[0] === "kubectl") {
|
||||
return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" };
|
||||
}
|
||||
if (command === "ps") {
|
||||
return { code: 0, stdout: psOutput, stderr: "" };
|
||||
}
|
||||
if (command === "kill") {
|
||||
killLog.push({ signal: args[0], pid: args[1] });
|
||||
return { code: 0, stdout: "", stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) {
|
||||
if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" };
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
@@ -179,6 +187,48 @@ test("lock classifier reports held and stale states with retryAfterSeconds", ()
|
||||
assert.equal(stale.retryAfterSeconds, 0);
|
||||
});
|
||||
|
||||
test("dev-cd status and dry-run are read-only compact JSON", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--dry-run",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({ commandLog }),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const status = JSON.parse(output);
|
||||
assert.equal(code, 0);
|
||||
assert.equal(status.mode, "dry-run");
|
||||
assert.equal(status.mutationAttempted, false);
|
||||
assert.equal(status.prodTouched, false);
|
||||
assert.equal(status.lock.status, "absent");
|
||||
assert.equal(status.deployJson.manifest, undefined);
|
||||
assert.equal(status.live.summary.checked, 2);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("create")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("replace")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
|
||||
});
|
||||
|
||||
test("process candidate discovery only returns dev-cd apply processes", () => {
|
||||
const psOutput = [
|
||||
"101 node scripts/dev-cd-apply.mjs --apply --confirm-dev",
|
||||
"202 node scripts/dev-cd-apply.mjs --status",
|
||||
"303 node scripts/dev-deploy-apply.mjs --apply",
|
||||
"404 node scripts/dev-cd-apply.mjs --apply --confirm-dev"
|
||||
].join("\n");
|
||||
const candidates = findDevCdApplyProcessCandidates(psOutput, 404);
|
||||
assert.deepEqual(candidates.map((candidate) => candidate.pid), [101]);
|
||||
});
|
||||
|
||||
test("deploy-lock-held failure is structured before publish/apply", () => {
|
||||
const failure = deployLockHeldFailure({
|
||||
ownerTaskId: "task-a",
|
||||
@@ -303,7 +353,8 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a
|
||||
"task-b",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig",
|
||||
"--write-report"
|
||||
"--write-report",
|
||||
"--full-output"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
@@ -355,6 +406,88 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a
|
||||
assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2);
|
||||
});
|
||||
|
||||
test("apply stdout is concise by default while write-report keeps the full report", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-c",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig",
|
||||
"--write-report"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand(),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const summary = JSON.parse(output);
|
||||
assert.equal(code, 0);
|
||||
assert.equal(summary.status, "pass");
|
||||
assert.equal(summary.devCdApply, undefined);
|
||||
assert.equal(summary.fullOutputHint.includes("--full-output"), true);
|
||||
assert.equal(summary.lock.acquired, true);
|
||||
assert.equal(summary.steps.some((step) => step.id === "dev-deploy-apply"), true);
|
||||
|
||||
const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
|
||||
assert.equal(writtenReport.devCdApply.deployJson.before.manifest.environment, "dev");
|
||||
});
|
||||
|
||||
test("force-interrupt-lock terminates local apply candidate and takes a held lock", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const killLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--force-interrupt-lock",
|
||||
"--owner-task-id",
|
||||
"task-force",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig",
|
||||
"--full-output"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
killLog,
|
||||
psOutput: "555 node scripts/dev-cd-apply.mjs --apply --confirm-dev\n666 node scripts/dev-cd-apply.mjs --status\n",
|
||||
heldLock: {
|
||||
promotionCommit: "abc1234",
|
||||
deployJsonHash: "sha256:a",
|
||||
ownerTaskId: "task-a",
|
||||
transactionId: "tx-a",
|
||||
phase: "applying",
|
||||
startedAt: iso(0),
|
||||
updatedAt: iso(0),
|
||||
ttlSeconds: 300,
|
||||
liveBefore: { status: "pass" },
|
||||
targetNamespace: "hwlab-dev",
|
||||
targetRef: "origin/main",
|
||||
lockBackend: "Lease"
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const report = JSON.parse(output);
|
||||
assert.equal(code, 0);
|
||||
assert.deepEqual(killLog, [{ signal: "-TERM", pid: "555" }]);
|
||||
assert.equal(report.devCdApply.lock.forceInterrupt.status, "attempted");
|
||||
assert.equal(report.devCdApply.lock.forceInterrupt.processCandidates[0].pid, 555);
|
||||
assert.equal(report.devCdApply.lock.current.forceInterruptStatus, "attempted");
|
||||
assert.equal(report.devCdApply.lock.current.forceInterruptedBy, "task-force");
|
||||
});
|
||||
|
||||
test("live verify summarizes 16666 and 16667 health identity", async () => {
|
||||
const result = await verifyDevLive({
|
||||
now: () => new Date(iso()),
|
||||
|
||||
Reference in New Issue
Block a user