feat: improve dev cd apply observability

Add compact DEV CD status/dry-run output, concise apply summaries, and explicit DEV-only force-interrupt lock takeover.
This commit is contained in:
Lyon
2026-05-23 18:18:04 +08:00
committed by GitHub
parent 2af9748903
commit 97601c9b9d
3 changed files with 579 additions and 27 deletions
+17
View File
@@ -57,6 +57,23 @@ not silently overwritten; taking over a stale lock requires both
`--break-stale-lock` and `--confirm-dev`, and the previous holder is retained
in audit annotations/report fields.
`scripts/dev-cd-apply.mjs` also owns the read-only status surface. Running it
without arguments, with `--status`, or with `--dry-run` must return compact JSON
for the target ref, `deploy/deploy.json`, current Lease, and public
`16666/16667` live health. These modes must not acquire the Lease, publish
artifacts, apply Kubernetes workloads, or write reports. Normal `--apply`
stdout is also compact by default; the complete transaction record is preserved
through `--write-report`, or printed only when `--full-output` is explicit.
When the current DEV apply must be stopped, the operator may use
`--force-interrupt-lock` together with `--apply --confirm-dev
--confirmed-non-production`. This is a DEV-only takeover path: it first tries
to send `TERM` to local `scripts/dev-cd-apply.mjs --apply` processes, then
replaces the Lease holder and records the interrupted holder and interrupt
attempt in Lease annotations and the transaction report. Do not use this flag
for ordinary contention; prefer waiting for the holder or breaking only a
confirmed stale lock.
The old side-effect commands are now internal transaction steps. Direct manual
use of `scripts/dev-artifact-publish.mjs --publish` or
`scripts/dev-deploy-apply.mjs --apply` without `HWLAB_CD_TRANSACTION_ID` is
+427 -25
View File
@@ -38,7 +38,11 @@ const lockAnnotationFields = {
releasedAt: `${lockAnnotationPrefix}/releasedAt`,
releaseStatus: `${lockAnnotationPrefix}/releaseStatus`,
staleLockBrokenAt: `${lockAnnotationPrefix}/staleLockBrokenAt`,
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`,
forceInterruptedAt: `${lockAnnotationPrefix}/forceInterruptedAt`,
forceInterruptedBy: `${lockAnnotationPrefix}/forceInterruptedBy`,
forceInterruptStatus: `${lockAnnotationPrefix}/forceInterruptStatus`,
forceInterruptedPrevious: `${lockAnnotationPrefix}/forceInterruptedPrevious`
};
class DevCdApplyError extends Error {
@@ -64,7 +68,11 @@ export function parseArgs(argv) {
ttlSeconds: defaultTtlSeconds,
ownerTaskId: null,
breakStaleLock: false,
forceInterruptLock: false,
skipLiveVerify: false,
status: false,
dryRun: false,
fullOutput: false,
help: false,
flags: new Set()
};
@@ -86,6 +94,16 @@ export function parseArgs(argv) {
} else if (arg === "--break-stale-lock") {
args.breakStaleLock = true;
args.flags.add(arg);
} else if (arg === "--force-interrupt-lock") {
args.forceInterruptLock = true;
args.flags.add(arg);
} else if (arg === "--dry-run" || arg === "--status") {
args.status = true;
args.dryRun = arg === "--dry-run";
args.flags.add(arg);
} else if (arg === "--full-output") {
args.fullOutput = true;
args.flags.add(arg);
} else if (arg === "--skip-live-verify") {
args.skipLiveVerify = true;
args.flags.add(arg);
@@ -134,20 +152,34 @@ function readOption(argv, index, name) {
}
export function usage() {
return [
"usage: node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production [--write-report]",
"",
"Single DEV CD transaction for latest-main publish, desired-state refresh, apply, and live verify.",
"",
"options:",
" --target-ref REF git ref to publish/apply; default: origin/main",
" --kubeconfig PATH DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG",
" --ttl-seconds SECONDS lock TTL; default: 3600",
" --owner-task-id ID lock holder; default: Code Queue/env/user derived",
" --break-stale-lock may take over an expired lock only with --confirm-dev",
" --report PATH default: reports/dev-gate/dev-cd-apply.json",
" --skip-live-verify test-only escape hatch; do not use for DEV acceptance"
].join("\n");
return {
command: "node scripts/dev-cd-apply.mjs",
summary: "Single DEV CD transaction for latest-main publish, desired-state refresh, apply, and live verify.",
defaultMode: "status",
examples: [
"node scripts/dev-cd-apply.mjs --status",
"node scripts/dev-cd-apply.mjs --dry-run",
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report",
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --force-interrupt-lock --write-report"
],
options: {
"--status": "Read-only status: target ref, deploy.json, current Lease, and public live health.",
"--dry-run": "Read-only alias for --status; does not acquire the Lease or run side effects.",
"--apply": "Run the DEV CD transaction side effects.",
"--confirm-dev": "Required for DEV mutation and stale/force lock takeover.",
"--confirmed-non-production": "Required for DEV mutation.",
"--target-ref REF": "git ref to publish/apply; default: origin/main",
"--kubeconfig PATH": "DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG",
"--ttl-seconds SECONDS": "lock TTL; default: 3600",
"--owner-task-id ID": "lock holder; default: Code Queue/env/user derived",
"--break-stale-lock": "May take over an expired lock only with --confirm-dev.",
"--force-interrupt-lock": "DEV-only explicit takeover: try to interrupt the current local dev-cd apply process, then acquire the Lease.",
"--report PATH": `default: ${defaultReportPath}`,
"--write-report": "Write the full report to --report.",
"--full-output": "Print the full report to stdout instead of the concise summary.",
"--skip-live-verify": "test-only escape hatch; do not use for DEV acceptance"
}
};
}
function nowIso() {
@@ -286,12 +318,12 @@ async function resolveTargetRef(ctx, targetRef) {
function validateArgs(args) {
const blockers = [];
if (!args.apply) {
if (!args.apply && !args.status) {
blockers.push({
type: "safety_blocker",
scope: "apply-mode",
status: "open",
summary: "DEV CD transaction side effects require --apply."
summary: "DEV CD side effects require --apply; use --status or --dry-run for read-only observability."
});
}
if (args.apply && (!args.confirmDev || !args.confirmedNonProduction)) {
@@ -310,6 +342,22 @@ function validateArgs(args) {
summary: "Breaking a stale DEV CD lock requires --break-stale-lock and --confirm-dev."
});
}
if (args.forceInterruptLock && !args.confirmDev) {
blockers.push({
type: "safety_blocker",
scope: "force-interrupt-confirmation",
status: "open",
summary: "Force-interrupting a DEV CD lock requires --force-interrupt-lock and --confirm-dev."
});
}
if (args.forceInterruptLock && !args.apply) {
blockers.push({
type: "safety_blocker",
scope: "force-interrupt-mode",
status: "open",
summary: "--force-interrupt-lock is only valid with --apply."
});
}
if (!Number.isInteger(args.ttlSeconds) || args.ttlSeconds < 60 || args.ttlSeconds > 24 * 60 * 60) {
blockers.push({
type: "safety_blocker",
@@ -394,7 +442,11 @@ export function parseDeployLock(lease) {
releasedAt: readAnnotation(annotations, "releasedAt"),
releaseStatus: readAnnotation(annotations, "releaseStatus"),
staleLockBrokenAt: readAnnotation(annotations, "staleLockBrokenAt"),
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious")
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious"),
forceInterruptedAt: readAnnotation(annotations, "forceInterruptedAt"),
forceInterruptedBy: readAnnotation(annotations, "forceInterruptedBy"),
forceInterruptStatus: readAnnotation(annotations, "forceInterruptStatus"),
forceInterruptedPrevious: readJsonAnnotation(annotations, "forceInterruptedPrevious")
};
}
@@ -456,7 +508,53 @@ function isConflict(result) {
return /conflict|object has been modified|operation cannot be fulfilled/i.test(`${result.stderr}\n${result.stdout}`);
}
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null }) {
function compactInterruptedLock(lock) {
if (!lock) return null;
return {
lockName: lock.lockName ?? defaultLockName,
lockBackend: lock.lockBackend ?? "Lease",
ownerTaskId: lock.ownerTaskId ?? null,
transactionId: lock.transactionId ?? null,
holderIdentity: lock.holderIdentity ?? null,
phase: lock.phase ?? null,
promotionCommit: lock.promotionCommit ?? null,
deployJsonHash: lock.deployJsonHash ?? null,
updatedAt: lock.updatedAt ?? null,
ttlSeconds: lock.ttlSeconds ?? null,
targetNamespace: lock.targetNamespace ?? defaultNamespace,
targetRef: lock.targetRef ?? null
};
}
function compactForceInterruptRecord(record) {
if (!record) return null;
return {
status: record.status,
warning: record.warning ?? null,
processCandidates: (record.processCandidates ?? []).map((candidate) => ({
pid: candidate.pid,
command: candidate.command
})),
attempts: (record.attempts ?? []).map((attempt) => ({
pid: attempt.pid,
signal: attempt.signal,
status: attempt.status
}))
};
}
function attachForceInterruptAnnotations(lock, forceInterrupt, timestamp, ownerTaskId) {
if (!forceInterrupt) return;
lock.forceInterruptedAt = timestamp;
lock.forceInterruptedBy = ownerTaskId;
lock.forceInterruptStatus = forceInterrupt.status;
lock.forceInterruptedPrevious = {
previousLock: compactInterruptedLock(forceInterrupt.previousLock),
interrupt: compactForceInterruptRecord(forceInterrupt)
};
}
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null, forceInterrupt = null }) {
const lock = {
promotionCommit: target.shortCommitId,
deployJsonHash: deployBefore.hash,
@@ -475,6 +573,7 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa
lock.staleLockBrokenAt = startedAt;
lock.staleLockPrevious = staleLockPrevious;
}
attachForceInterruptAnnotations(lock, forceInterrupt, startedAt, ownerTaskId);
return {
apiVersion: "coordination.k8s.io/v1",
kind: "Lease",
@@ -497,7 +596,7 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa
};
}
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null }) {
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null, forceInterrupt = null }) {
const lock = {
promotionCommit: target.shortCommitId,
deployJsonHash: deployBefore.hash,
@@ -516,6 +615,7 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha
lock.staleLockBrokenAt = updatedAt;
lock.staleLockPrevious = staleLockPrevious;
}
attachForceInterruptAnnotations(lock, forceInterrupt, updatedAt, ownerTaskId);
return {
metadata: {
labels: {
@@ -534,6 +634,66 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha
};
}
export function findDevCdApplyProcessCandidates(psOutput, currentPid = process.pid) {
return String(psOutput ?? "")
.split(/\r?\n/u)
.map((line) => line.trim())
.filter(Boolean)
.map((line) => {
const match = line.match(/^(\d+)\s+(.+)$/u);
if (!match) return null;
return {
pid: Number.parseInt(match[1], 10),
command: oneLine(redactSensitiveText(match[2])).slice(0, 500)
};
})
.filter((entry) => {
if (!entry || !Number.isInteger(entry.pid) || entry.pid === currentPid) return false;
return entry.command.includes("scripts/dev-cd-apply.mjs") && entry.command.includes("--apply");
});
}
async function forceInterruptCurrentApplyProcess(ctx, existingLock) {
const ps = await ctx.runCommand("ps", ["-eo", "pid=,args="], {
cwd: ctx.repoRoot,
timeoutMs: 5000
});
if (ps.code !== 0) {
return {
status: "ps-failed",
previousLock: compactInterruptedLock(existingLock),
processCandidates: [],
attempts: [],
warning: oneLine(redactSensitiveText(ps.stderr || ps.stdout || "ps failed"))
};
}
const processCandidates = findDevCdApplyProcessCandidates(ps.stdout, process.pid);
const attempts = [];
for (const candidate of processCandidates) {
const result = await ctx.runCommand("kill", ["-TERM", String(candidate.pid)], {
cwd: ctx.repoRoot,
timeoutMs: 5000
});
attempts.push({
pid: candidate.pid,
signal: "TERM",
status: result.code === 0 ? "sent" : "failed",
stderrTail: redactSensitiveText(result.stderr || result.stdout || "").trim().slice(-500)
});
}
return {
status: processCandidates.length > 0 ? "attempted" : "no-local-process-found",
previousLock: compactInterruptedLock(existingLock),
processCandidates,
attempts,
warning: processCandidates.length > 0
? null
: "No local scripts/dev-cd-apply.mjs --apply process was found; the Lease will still be taken over so the previous holder cannot continue lock-owned updates."
};
}
async function kubectl(ctx, kubectlContext, args, options = {}) {
const result = await ctx.runCommand(kubectlContext.executor, args, {
cwd: ctx.repoRoot,
@@ -596,13 +756,24 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
const lease = JSON.parse(get.stdout);
const existing = parseDeployLock(lease);
const classification = classifyDeployLock(existing, now);
if (classification.held || (classification.stale && !args.breakStaleLock)) {
if (classification.held && !args.forceInterruptLock) {
const failure = deployLockHeldFailure(existing, now);
throw new DevCdApplyError(failure.summary, {
code: "deploy-lock-held",
lockFailure: failure
});
}
if (classification.stale && !args.breakStaleLock && !args.forceInterruptLock) {
const failure = deployLockHeldFailure(existing, now);
throw new DevCdApplyError(failure.summary, {
code: "deploy-lock-held",
lockFailure: failure
});
}
const forceInterrupt = args.forceInterruptLock
? await forceInterruptCurrentApplyProcess(ctx, existing)
: null;
const patch = lockPatch({
args,
@@ -614,7 +785,8 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
startedAt,
updatedAt: startedAt,
liveBefore,
staleLockPrevious: classification.stale ? existing : null
staleLockPrevious: classification.stale ? existing : null,
forceInterrupt
});
const replaced = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch, { resetLockAnnotations: true }));
if (isConflict(replaced)) {
@@ -630,6 +802,7 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
acquired: true,
lock: parseDeployLock(JSON.parse(replaced.stdout)),
staleBreak: classification.stale ? existing : null,
forceInterrupt,
command: replaced.command
};
}
@@ -660,6 +833,7 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
acquired: true,
lock: parseDeployLock(JSON.parse(created.stdout)),
staleBreak: null,
forceInterrupt: null,
command: created.command
};
}
@@ -1006,6 +1180,226 @@ export async function verifyDevLive(ctx, { expectedCommit = null } = {}) {
};
}
function deployServiceCount(manifest) {
const services = manifest?.services;
if (Array.isArray(services)) return services.length;
if (services && typeof services === "object") return Object.keys(services).length;
return null;
}
function summarizeDeployJson(deploy) {
return {
path: deploy.path,
hash: deploy.hash,
commitId: deploy.commitId,
namespace: deploy.namespace,
environment: deploy.environment,
endpoint: deploy.endpoint,
serviceCount: deployServiceCount(deploy.manifest)
};
}
function compactLiveEndpoint(endpoint) {
return {
id: endpoint.id,
url: endpoint.url,
status: endpoint.status,
httpStatus: endpoint.httpStatus ?? null,
reachable: endpoint.reachable,
identityMatches: endpoint.identityMatches,
serviceId: endpoint.serviceId,
environment: endpoint.environment,
observedCommit: endpoint.observedCommit,
commitMatches: endpoint.commitMatches,
blockerCodes: endpoint.blockerCodes ?? [],
error: endpoint.error ?? null
};
}
function compactLiveReport(live) {
return {
status: live.status,
expectedCommit: live.expectedCommit ?? null,
summary: live.summary ?? { checked: 0 },
endpoints: (live.endpoints ?? []).map(compactLiveEndpoint)
};
}
async function readDeployLockStatus({ ctx, args, kubectlContext }) {
const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], {
timeoutMs: 15000
});
if (get.code === 0) {
return {
status: "read",
command: get.command,
lock: parseDeployLock(JSON.parse(get.stdout))
};
}
if (isNotFound(get)) {
return {
status: "absent",
command: get.command,
lockName: args.lockName,
lock: null
};
}
return {
status: "unavailable",
command: get.command,
lockName: args.lockName,
lock: null,
error: redactSensitiveText(get.stderr || get.stdout)
};
}
function summarizeLockStatus(lockRead, now) {
if (lockRead.status === "absent") {
return {
status: "absent",
lockName: lockRead.lockName ?? defaultLockName,
current: null,
retryAfterSeconds: 0,
expiresAt: null
};
}
if (lockRead.status === "unavailable") {
return {
status: "unavailable",
lockName: lockRead.lockName ?? defaultLockName,
current: null,
retryAfterSeconds: null,
expiresAt: null,
error: oneLine(lockRead.error)
};
}
const classification = classifyDeployLock(lockRead.lock, now);
const status = lockRead.lock?.phase === "released"
? "released"
: classification.held
? "held"
: classification.stale
? "stale"
: "unknown";
return {
status,
current: compactInterruptedLock(lockRead.lock),
retryAfterSeconds: classification.retryAfterSeconds,
expiresAt: classification.expiresAt,
takeover: {
stale: classification.stale ? "--apply --confirm-dev --confirmed-non-production --break-stale-lock" : null,
forceInterrupt: status === "held" ? "--apply --confirm-dev --confirmed-non-production --force-interrupt-lock" : null
}
};
}
function buildStatusNextActions({ target, deployBefore, lock, liveVerify }) {
const actions = [];
if (!target.headMatchesTarget) {
actions.push(`Checkout or fast-forward to ${target.ref} ${target.shortCommitId} before apply.`);
}
if (deployBefore.commitId !== target.shortCommitId && deployBefore.commitId !== target.commitId) {
actions.push(`deploy/deploy.json commitId is ${deployBefore.commitId}; target ref is ${target.shortCommitId}.`);
}
if (lock.status === "held") {
actions.push(`Wait ${lock.retryAfterSeconds}s or use explicit --force-interrupt-lock if the current apply must be stopped.`);
} else if (lock.status === "stale") {
actions.push("Use --break-stale-lock only after confirming the holder is inactive.");
} else if (lock.status === "unavailable") {
actions.push("Fix kubectl/KUBECONFIG access before applying.");
}
if (liveVerify.status !== "pass") {
actions.push("Inspect live endpoint identity/health before treating DEV as converged.");
}
if (actions.length === 0) {
actions.push("Run --apply --confirm-dev --confirmed-non-production --write-report when ready to publish/apply DEV.");
}
return actions;
}
async function runDevCdStatus(args, ctx, stdout) {
const generatedAt = ctx.now().toISOString();
const target = await resolveTargetRef(ctx, args.targetRef);
const deployBefore = await readDeployJson(ctx.repoRoot);
const kubectlContext = await resolveKubectlContext(args, ctx.env, ctx.runCommand);
const lockRead = await readDeployLockStatus({ ctx, args, kubectlContext });
const lock = summarizeLockStatus(lockRead, ctx.now());
const liveVerify = args.skipLiveVerify
? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } }
: await verifyDevLive(ctx, { expectedCommit: null });
const status = lock.status === "unavailable" || liveVerify.status !== "pass" ? "degraded" : "pass";
const report = {
ok: true,
status,
mode: args.dryRun ? "dry-run" : "status",
command: "dev-cd-apply",
generatedAt,
mutationAttempted: false,
prodTouched: false,
target: {
ref: target.ref,
promotionCommit: target.commitId,
shortCommitId: target.shortCommitId,
headCommitId: target.headCommitId,
headMatchesTarget: target.headMatchesTarget,
namespace: args.targetNamespace
},
deployJson: summarizeDeployJson(deployBefore),
kubectl: {
kubeconfigSource: kubectlContext.kubeconfigSource,
kubeconfig: kubectlContext.kubeconfig ? "<set>" : "<unset>",
lockReadCommand: lockRead.command
},
lock,
live: compactLiveReport(liveVerify),
nextActions: buildStatusNextActions({ target, deployBefore, lock, liveVerify }),
fullOutputHint: "Use --apply --confirm-dev --confirmed-non-production --write-report for the full transaction report; use --full-output only when stdout needs the full JSON."
};
stdout.write(`${JSON.stringify(report, null, 2)}\n`);
return 0;
}
function summarizeDevCdApplyReport(report) {
const apply = report.devCdApply ?? {};
return {
ok: report.status === "pass",
status: report.status,
mode: apply.mode ?? "apply",
command: "dev-cd-apply",
commitId: report.commitId,
generatedAt: report.generatedAt,
mutationAttempted: apply.safety?.mutationAttempted ?? false,
prodTouched: false,
transaction: {
transactionId: report.transaction?.transactionId ?? null,
ownerTaskId: report.transaction?.ownerTaskId ?? null,
targetNamespace: report.transaction?.targetNamespace ?? defaultNamespace,
lockName: report.transaction?.lockName ?? defaultLockName,
phases: report.transaction?.phases ?? [],
release: report.transaction?.release ?? null
},
target: apply.target ?? null,
lock: {
acquired: Boolean(apply.lock?.acquired),
current: compactInterruptedLock(apply.lock?.current),
staleBreak: compactInterruptedLock(apply.lock?.staleBreak),
forceInterrupt: compactForceInterruptRecord(apply.lock?.forceInterrupt)
},
steps: (apply.steps ?? []).map((step) => ({
id: step.id,
phase: step.phase,
status: step.status,
code: step.code,
reportPath: step.reportPath
})),
liveBefore: apply.liveBefore ? compactLiveReport(apply.liveBefore) : null,
liveVerify: apply.liveVerify ? compactLiveReport(apply.liveVerify) : null,
blockers: report.blockers ?? [],
reportPaths: apply.reportPaths ?? {},
fullOutputHint: "Use --full-output to print the full report to stdout. Use --write-report to persist the full report."
};
}
function buildReport({
args,
transaction,
@@ -1120,7 +1514,8 @@ function buildReport({
lock: {
acquired: Boolean(lockState?.acquired),
current: lockState?.lock ?? null,
staleBreak: lockState?.staleBreak ?? null
staleBreak: lockState?.staleBreak ?? null,
forceInterrupt: lockState?.forceInterrupt ?? null
},
steps,
liveBefore,
@@ -1190,10 +1585,16 @@ export async function runDevCdApply(argv, io = {}) {
try {
args = parseArgs(argv);
if (args.help) {
stdout.write(`${usage()}\n`);
stdout.write(`${JSON.stringify({ ok: true, status: "pass", help: usage() }, null, 2)}\n`);
return 0;
}
if (!args.apply && !args.status) {
args.status = true;
}
validateArgs(args);
if (args.status) {
return await runDevCdStatus(args, ctx, stdout);
}
} catch (error) {
const failure = formatDevCdApplyFailure(error);
stdout.write(`${JSON.stringify(failure, null, 2)}\n`);
@@ -1437,6 +1838,7 @@ export async function runDevCdApply(argv, io = {}) {
await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`);
}
stdout.write(`${JSON.stringify(report, null, 2)}\n`);
const stdoutPayload = args.fullOutput ? report : summarizeDevCdApplyReport(report);
stdout.write(`${JSON.stringify(stdoutPayload, null, 2)}\n`);
return status === "pass" ? 0 : 2;
}
+135 -2
View File
@@ -10,6 +10,7 @@ import {
buildLockAnnotations,
classifyDeployLock,
deployLockHeldFailure,
findDevCdApplyProcessCandidates,
parseDeployLock,
runDevCdApply,
verifyDevLive
@@ -98,7 +99,7 @@ function makeHttpGetJson() {
};
}
function makeRunCommand({ heldLock = null, commandLog = [] } = {}) {
function makeRunCommand({ heldLock = null, commandLog = [], psOutput = "", killLog = [] } = {}) {
let lease = heldLock ? leaseFromLock(heldLock) : null;
const assertLeaseMicroTime = (value) => {
assert.match(value, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/u);
@@ -114,6 +115,13 @@ function makeRunCommand({ heldLock = null, commandLog = [] } = {}) {
if (command === "which" && args[0] === "kubectl") {
return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" };
}
if (command === "ps") {
return { code: 0, stdout: psOutput, stderr: "" };
}
if (command === "kill") {
killLog.push({ signal: args[0], pid: args[1] });
return { code: 0, stdout: "", stderr: "" };
}
if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) {
if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" };
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
@@ -179,6 +187,48 @@ test("lock classifier reports held and stale states with retryAfterSeconds", ()
assert.equal(stale.retryAfterSeconds, 0);
});
test("dev-cd status and dry-run are read-only compact JSON", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const code = await runDevCdApply([
"--dry-run",
"--kubeconfig",
"/tmp/kubeconfig"
], {
repoRoot,
env: {},
runCommand: makeRunCommand({ commandLog }),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const status = JSON.parse(output);
assert.equal(code, 0);
assert.equal(status.mode, "dry-run");
assert.equal(status.mutationAttempted, false);
assert.equal(status.prodTouched, false);
assert.equal(status.lock.status, "absent");
assert.equal(status.deployJson.manifest, undefined);
assert.equal(status.live.summary.checked, 2);
assert.equal(commandLog.some((entry) => entry.args.includes("create")), false);
assert.equal(commandLog.some((entry) => entry.args.includes("replace")), false);
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
});
test("process candidate discovery only returns dev-cd apply processes", () => {
const psOutput = [
"101 node scripts/dev-cd-apply.mjs --apply --confirm-dev",
"202 node scripts/dev-cd-apply.mjs --status",
"303 node scripts/dev-deploy-apply.mjs --apply",
"404 node scripts/dev-cd-apply.mjs --apply --confirm-dev"
].join("\n");
const candidates = findDevCdApplyProcessCandidates(psOutput, 404);
assert.deepEqual(candidates.map((candidate) => candidate.pid), [101]);
});
test("deploy-lock-held failure is structured before publish/apply", () => {
const failure = deployLockHeldFailure({
ownerTaskId: "task-a",
@@ -303,7 +353,8 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a
"task-b",
"--kubeconfig",
"/tmp/kubeconfig",
"--write-report"
"--write-report",
"--full-output"
], {
repoRoot,
env: {},
@@ -355,6 +406,88 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a
assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2);
});
test("apply stdout is concise by default while write-report keeps the full report", async () => {
const repoRoot = await makeRepo();
let output = "";
const code = await runDevCdApply([
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--owner-task-id",
"task-c",
"--kubeconfig",
"/tmp/kubeconfig",
"--write-report"
], {
repoRoot,
env: {},
runCommand: makeRunCommand(),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const summary = JSON.parse(output);
assert.equal(code, 0);
assert.equal(summary.status, "pass");
assert.equal(summary.devCdApply, undefined);
assert.equal(summary.fullOutputHint.includes("--full-output"), true);
assert.equal(summary.lock.acquired, true);
assert.equal(summary.steps.some((step) => step.id === "dev-deploy-apply"), true);
const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
assert.equal(writtenReport.devCdApply.deployJson.before.manifest.environment, "dev");
});
test("force-interrupt-lock terminates local apply candidate and takes a held lock", async () => {
const repoRoot = await makeRepo();
const killLog = [];
let output = "";
const code = await runDevCdApply([
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--force-interrupt-lock",
"--owner-task-id",
"task-force",
"--kubeconfig",
"/tmp/kubeconfig",
"--full-output"
], {
repoRoot,
env: {},
runCommand: makeRunCommand({
killLog,
psOutput: "555 node scripts/dev-cd-apply.mjs --apply --confirm-dev\n666 node scripts/dev-cd-apply.mjs --status\n",
heldLock: {
promotionCommit: "abc1234",
deployJsonHash: "sha256:a",
ownerTaskId: "task-a",
transactionId: "tx-a",
phase: "applying",
startedAt: iso(0),
updatedAt: iso(0),
ttlSeconds: 300,
liveBefore: { status: "pass" },
targetNamespace: "hwlab-dev",
targetRef: "origin/main",
lockBackend: "Lease"
}
}),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const report = JSON.parse(output);
assert.equal(code, 0);
assert.deepEqual(killLog, [{ signal: "-TERM", pid: "555" }]);
assert.equal(report.devCdApply.lock.forceInterrupt.status, "attempted");
assert.equal(report.devCdApply.lock.forceInterrupt.processCandidates[0].pid, 555);
assert.equal(report.devCdApply.lock.current.forceInterruptStatus, "attempted");
assert.equal(report.devCdApply.lock.current.forceInterruptedBy, "task-force");
});
test("live verify summarizes 16666 and 16667 health identity", async () => {
const result = await verifyDevLive({
now: () => new Date(iso()),