From 97601c9b9d210b3c88926d86ee07230f70e243d7 Mon Sep 17 00:00:00 2001 From: Lyon <88232613+pikasTech@users.noreply.github.com> Date: Sat, 23 May 2026 18:18:04 +0800 Subject: [PATCH] feat: improve dev cd apply observability Add compact DEV CD status/dry-run output, concise apply summaries, and explicit DEV-only force-interrupt lock takeover. --- docs/reference/deployment-publish.md | 17 + scripts/src/dev-cd-apply.mjs | 452 +++++++++++++++++++++++++-- scripts/src/dev-cd-apply.test.mjs | 137 +++++++- 3 files changed, 579 insertions(+), 27 deletions(-) diff --git a/docs/reference/deployment-publish.md b/docs/reference/deployment-publish.md index 44a52ac7..818333f0 100644 --- a/docs/reference/deployment-publish.md +++ b/docs/reference/deployment-publish.md @@ -57,6 +57,23 @@ not silently overwritten; taking over a stale lock requires both `--break-stale-lock` and `--confirm-dev`, and the previous holder is retained in audit annotations/report fields. +`scripts/dev-cd-apply.mjs` also owns the read-only status surface. Running it +without arguments, with `--status`, or with `--dry-run` must return compact JSON +for the target ref, `deploy/deploy.json`, current Lease, and public +`16666/16667` live health. These modes must not acquire the Lease, publish +artifacts, apply Kubernetes workloads, or write reports. Normal `--apply` +stdout is also compact by default; the complete transaction record is preserved +through `--write-report`, or printed only when `--full-output` is explicit. + +When the current DEV apply must be stopped, the operator may use +`--force-interrupt-lock` together with `--apply --confirm-dev +--confirmed-non-production`. This is a DEV-only takeover path: it first tries +to send `TERM` to local `scripts/dev-cd-apply.mjs --apply` processes, then +replaces the Lease holder and records the interrupted holder and interrupt +attempt in Lease annotations and the transaction report. Do not use this flag +for ordinary contention; prefer waiting for the holder or breaking only a +confirmed stale lock. + The old side-effect commands are now internal transaction steps. Direct manual use of `scripts/dev-artifact-publish.mjs --publish` or `scripts/dev-deploy-apply.mjs --apply` without `HWLAB_CD_TRANSACTION_ID` is diff --git a/scripts/src/dev-cd-apply.mjs b/scripts/src/dev-cd-apply.mjs index 9d28cad3..c6e9e5ef 100644 --- a/scripts/src/dev-cd-apply.mjs +++ b/scripts/src/dev-cd-apply.mjs @@ -38,7 +38,11 @@ const lockAnnotationFields = { releasedAt: `${lockAnnotationPrefix}/releasedAt`, releaseStatus: `${lockAnnotationPrefix}/releaseStatus`, staleLockBrokenAt: `${lockAnnotationPrefix}/staleLockBrokenAt`, - staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious` + staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`, + forceInterruptedAt: `${lockAnnotationPrefix}/forceInterruptedAt`, + forceInterruptedBy: `${lockAnnotationPrefix}/forceInterruptedBy`, + forceInterruptStatus: `${lockAnnotationPrefix}/forceInterruptStatus`, + forceInterruptedPrevious: `${lockAnnotationPrefix}/forceInterruptedPrevious` }; class DevCdApplyError extends Error { @@ -64,7 +68,11 @@ export function parseArgs(argv) { ttlSeconds: defaultTtlSeconds, ownerTaskId: null, breakStaleLock: false, + forceInterruptLock: false, skipLiveVerify: false, + status: false, + dryRun: false, + fullOutput: false, help: false, flags: new Set() }; @@ -86,6 +94,16 @@ export function parseArgs(argv) { } else if (arg === "--break-stale-lock") { args.breakStaleLock = true; args.flags.add(arg); + } else if (arg === "--force-interrupt-lock") { + args.forceInterruptLock = true; + args.flags.add(arg); + } else if (arg === "--dry-run" || arg === "--status") { + args.status = true; + args.dryRun = arg === "--dry-run"; + args.flags.add(arg); + } else if (arg === "--full-output") { + args.fullOutput = true; + args.flags.add(arg); } else if (arg === "--skip-live-verify") { args.skipLiveVerify = true; args.flags.add(arg); @@ -134,20 +152,34 @@ function readOption(argv, index, name) { } export function usage() { - return [ - "usage: node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production [--write-report]", - "", - "Single DEV CD transaction for latest-main publish, desired-state refresh, apply, and live verify.", - "", - "options:", - " --target-ref REF git ref to publish/apply; default: origin/main", - " --kubeconfig PATH DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG", - " --ttl-seconds SECONDS lock TTL; default: 3600", - " --owner-task-id ID lock holder; default: Code Queue/env/user derived", - " --break-stale-lock may take over an expired lock only with --confirm-dev", - " --report PATH default: reports/dev-gate/dev-cd-apply.json", - " --skip-live-verify test-only escape hatch; do not use for DEV acceptance" - ].join("\n"); + return { + command: "node scripts/dev-cd-apply.mjs", + summary: "Single DEV CD transaction for latest-main publish, desired-state refresh, apply, and live verify.", + defaultMode: "status", + examples: [ + "node scripts/dev-cd-apply.mjs --status", + "node scripts/dev-cd-apply.mjs --dry-run", + "node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report", + "node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --force-interrupt-lock --write-report" + ], + options: { + "--status": "Read-only status: target ref, deploy.json, current Lease, and public live health.", + "--dry-run": "Read-only alias for --status; does not acquire the Lease or run side effects.", + "--apply": "Run the DEV CD transaction side effects.", + "--confirm-dev": "Required for DEV mutation and stale/force lock takeover.", + "--confirmed-non-production": "Required for DEV mutation.", + "--target-ref REF": "git ref to publish/apply; default: origin/main", + "--kubeconfig PATH": "DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG", + "--ttl-seconds SECONDS": "lock TTL; default: 3600", + "--owner-task-id ID": "lock holder; default: Code Queue/env/user derived", + "--break-stale-lock": "May take over an expired lock only with --confirm-dev.", + "--force-interrupt-lock": "DEV-only explicit takeover: try to interrupt the current local dev-cd apply process, then acquire the Lease.", + "--report PATH": `default: ${defaultReportPath}`, + "--write-report": "Write the full report to --report.", + "--full-output": "Print the full report to stdout instead of the concise summary.", + "--skip-live-verify": "test-only escape hatch; do not use for DEV acceptance" + } + }; } function nowIso() { @@ -286,12 +318,12 @@ async function resolveTargetRef(ctx, targetRef) { function validateArgs(args) { const blockers = []; - if (!args.apply) { + if (!args.apply && !args.status) { blockers.push({ type: "safety_blocker", scope: "apply-mode", status: "open", - summary: "DEV CD transaction side effects require --apply." + summary: "DEV CD side effects require --apply; use --status or --dry-run for read-only observability." }); } if (args.apply && (!args.confirmDev || !args.confirmedNonProduction)) { @@ -310,6 +342,22 @@ function validateArgs(args) { summary: "Breaking a stale DEV CD lock requires --break-stale-lock and --confirm-dev." }); } + if (args.forceInterruptLock && !args.confirmDev) { + blockers.push({ + type: "safety_blocker", + scope: "force-interrupt-confirmation", + status: "open", + summary: "Force-interrupting a DEV CD lock requires --force-interrupt-lock and --confirm-dev." + }); + } + if (args.forceInterruptLock && !args.apply) { + blockers.push({ + type: "safety_blocker", + scope: "force-interrupt-mode", + status: "open", + summary: "--force-interrupt-lock is only valid with --apply." + }); + } if (!Number.isInteger(args.ttlSeconds) || args.ttlSeconds < 60 || args.ttlSeconds > 24 * 60 * 60) { blockers.push({ type: "safety_blocker", @@ -394,7 +442,11 @@ export function parseDeployLock(lease) { releasedAt: readAnnotation(annotations, "releasedAt"), releaseStatus: readAnnotation(annotations, "releaseStatus"), staleLockBrokenAt: readAnnotation(annotations, "staleLockBrokenAt"), - staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious") + staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious"), + forceInterruptedAt: readAnnotation(annotations, "forceInterruptedAt"), + forceInterruptedBy: readAnnotation(annotations, "forceInterruptedBy"), + forceInterruptStatus: readAnnotation(annotations, "forceInterruptStatus"), + forceInterruptedPrevious: readJsonAnnotation(annotations, "forceInterruptedPrevious") }; } @@ -456,7 +508,53 @@ function isConflict(result) { return /conflict|object has been modified|operation cannot be fulfilled/i.test(`${result.stderr}\n${result.stdout}`); } -function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null }) { +function compactInterruptedLock(lock) { + if (!lock) return null; + return { + lockName: lock.lockName ?? defaultLockName, + lockBackend: lock.lockBackend ?? "Lease", + ownerTaskId: lock.ownerTaskId ?? null, + transactionId: lock.transactionId ?? null, + holderIdentity: lock.holderIdentity ?? null, + phase: lock.phase ?? null, + promotionCommit: lock.promotionCommit ?? null, + deployJsonHash: lock.deployJsonHash ?? null, + updatedAt: lock.updatedAt ?? null, + ttlSeconds: lock.ttlSeconds ?? null, + targetNamespace: lock.targetNamespace ?? defaultNamespace, + targetRef: lock.targetRef ?? null + }; +} + +function compactForceInterruptRecord(record) { + if (!record) return null; + return { + status: record.status, + warning: record.warning ?? null, + processCandidates: (record.processCandidates ?? []).map((candidate) => ({ + pid: candidate.pid, + command: candidate.command + })), + attempts: (record.attempts ?? []).map((attempt) => ({ + pid: attempt.pid, + signal: attempt.signal, + status: attempt.status + })) + }; +} + +function attachForceInterruptAnnotations(lock, forceInterrupt, timestamp, ownerTaskId) { + if (!forceInterrupt) return; + lock.forceInterruptedAt = timestamp; + lock.forceInterruptedBy = ownerTaskId; + lock.forceInterruptStatus = forceInterrupt.status; + lock.forceInterruptedPrevious = { + previousLock: compactInterruptedLock(forceInterrupt.previousLock), + interrupt: compactForceInterruptRecord(forceInterrupt) + }; +} + +function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null, forceInterrupt = null }) { const lock = { promotionCommit: target.shortCommitId, deployJsonHash: deployBefore.hash, @@ -475,6 +573,7 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa lock.staleLockBrokenAt = startedAt; lock.staleLockPrevious = staleLockPrevious; } + attachForceInterruptAnnotations(lock, forceInterrupt, startedAt, ownerTaskId); return { apiVersion: "coordination.k8s.io/v1", kind: "Lease", @@ -497,7 +596,7 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa }; } -function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null }) { +function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null, forceInterrupt = null }) { const lock = { promotionCommit: target.shortCommitId, deployJsonHash: deployBefore.hash, @@ -516,6 +615,7 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha lock.staleLockBrokenAt = updatedAt; lock.staleLockPrevious = staleLockPrevious; } + attachForceInterruptAnnotations(lock, forceInterrupt, updatedAt, ownerTaskId); return { metadata: { labels: { @@ -534,6 +634,66 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha }; } +export function findDevCdApplyProcessCandidates(psOutput, currentPid = process.pid) { + return String(psOutput ?? "") + .split(/\r?\n/u) + .map((line) => line.trim()) + .filter(Boolean) + .map((line) => { + const match = line.match(/^(\d+)\s+(.+)$/u); + if (!match) return null; + return { + pid: Number.parseInt(match[1], 10), + command: oneLine(redactSensitiveText(match[2])).slice(0, 500) + }; + }) + .filter((entry) => { + if (!entry || !Number.isInteger(entry.pid) || entry.pid === currentPid) return false; + return entry.command.includes("scripts/dev-cd-apply.mjs") && entry.command.includes("--apply"); + }); +} + +async function forceInterruptCurrentApplyProcess(ctx, existingLock) { + const ps = await ctx.runCommand("ps", ["-eo", "pid=,args="], { + cwd: ctx.repoRoot, + timeoutMs: 5000 + }); + if (ps.code !== 0) { + return { + status: "ps-failed", + previousLock: compactInterruptedLock(existingLock), + processCandidates: [], + attempts: [], + warning: oneLine(redactSensitiveText(ps.stderr || ps.stdout || "ps failed")) + }; + } + + const processCandidates = findDevCdApplyProcessCandidates(ps.stdout, process.pid); + const attempts = []; + for (const candidate of processCandidates) { + const result = await ctx.runCommand("kill", ["-TERM", String(candidate.pid)], { + cwd: ctx.repoRoot, + timeoutMs: 5000 + }); + attempts.push({ + pid: candidate.pid, + signal: "TERM", + status: result.code === 0 ? "sent" : "failed", + stderrTail: redactSensitiveText(result.stderr || result.stdout || "").trim().slice(-500) + }); + } + + return { + status: processCandidates.length > 0 ? "attempted" : "no-local-process-found", + previousLock: compactInterruptedLock(existingLock), + processCandidates, + attempts, + warning: processCandidates.length > 0 + ? null + : "No local scripts/dev-cd-apply.mjs --apply process was found; the Lease will still be taken over so the previous holder cannot continue lock-owned updates." + }; +} + async function kubectl(ctx, kubectlContext, args, options = {}) { const result = await ctx.runCommand(kubectlContext.executor, args, { cwd: ctx.repoRoot, @@ -596,13 +756,24 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo const lease = JSON.parse(get.stdout); const existing = parseDeployLock(lease); const classification = classifyDeployLock(existing, now); - if (classification.held || (classification.stale && !args.breakStaleLock)) { + if (classification.held && !args.forceInterruptLock) { const failure = deployLockHeldFailure(existing, now); throw new DevCdApplyError(failure.summary, { code: "deploy-lock-held", lockFailure: failure }); } + if (classification.stale && !args.breakStaleLock && !args.forceInterruptLock) { + const failure = deployLockHeldFailure(existing, now); + throw new DevCdApplyError(failure.summary, { + code: "deploy-lock-held", + lockFailure: failure + }); + } + + const forceInterrupt = args.forceInterruptLock + ? await forceInterruptCurrentApplyProcess(ctx, existing) + : null; const patch = lockPatch({ args, @@ -614,7 +785,8 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo startedAt, updatedAt: startedAt, liveBefore, - staleLockPrevious: classification.stale ? existing : null + staleLockPrevious: classification.stale ? existing : null, + forceInterrupt }); const replaced = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch, { resetLockAnnotations: true })); if (isConflict(replaced)) { @@ -630,6 +802,7 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo acquired: true, lock: parseDeployLock(JSON.parse(replaced.stdout)), staleBreak: classification.stale ? existing : null, + forceInterrupt, command: replaced.command }; } @@ -660,6 +833,7 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo acquired: true, lock: parseDeployLock(JSON.parse(created.stdout)), staleBreak: null, + forceInterrupt: null, command: created.command }; } @@ -1006,6 +1180,226 @@ export async function verifyDevLive(ctx, { expectedCommit = null } = {}) { }; } +function deployServiceCount(manifest) { + const services = manifest?.services; + if (Array.isArray(services)) return services.length; + if (services && typeof services === "object") return Object.keys(services).length; + return null; +} + +function summarizeDeployJson(deploy) { + return { + path: deploy.path, + hash: deploy.hash, + commitId: deploy.commitId, + namespace: deploy.namespace, + environment: deploy.environment, + endpoint: deploy.endpoint, + serviceCount: deployServiceCount(deploy.manifest) + }; +} + +function compactLiveEndpoint(endpoint) { + return { + id: endpoint.id, + url: endpoint.url, + status: endpoint.status, + httpStatus: endpoint.httpStatus ?? null, + reachable: endpoint.reachable, + identityMatches: endpoint.identityMatches, + serviceId: endpoint.serviceId, + environment: endpoint.environment, + observedCommit: endpoint.observedCommit, + commitMatches: endpoint.commitMatches, + blockerCodes: endpoint.blockerCodes ?? [], + error: endpoint.error ?? null + }; +} + +function compactLiveReport(live) { + return { + status: live.status, + expectedCommit: live.expectedCommit ?? null, + summary: live.summary ?? { checked: 0 }, + endpoints: (live.endpoints ?? []).map(compactLiveEndpoint) + }; +} + +async function readDeployLockStatus({ ctx, args, kubectlContext }) { + const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], { + timeoutMs: 15000 + }); + if (get.code === 0) { + return { + status: "read", + command: get.command, + lock: parseDeployLock(JSON.parse(get.stdout)) + }; + } + if (isNotFound(get)) { + return { + status: "absent", + command: get.command, + lockName: args.lockName, + lock: null + }; + } + return { + status: "unavailable", + command: get.command, + lockName: args.lockName, + lock: null, + error: redactSensitiveText(get.stderr || get.stdout) + }; +} + +function summarizeLockStatus(lockRead, now) { + if (lockRead.status === "absent") { + return { + status: "absent", + lockName: lockRead.lockName ?? defaultLockName, + current: null, + retryAfterSeconds: 0, + expiresAt: null + }; + } + if (lockRead.status === "unavailable") { + return { + status: "unavailable", + lockName: lockRead.lockName ?? defaultLockName, + current: null, + retryAfterSeconds: null, + expiresAt: null, + error: oneLine(lockRead.error) + }; + } + const classification = classifyDeployLock(lockRead.lock, now); + const status = lockRead.lock?.phase === "released" + ? "released" + : classification.held + ? "held" + : classification.stale + ? "stale" + : "unknown"; + return { + status, + current: compactInterruptedLock(lockRead.lock), + retryAfterSeconds: classification.retryAfterSeconds, + expiresAt: classification.expiresAt, + takeover: { + stale: classification.stale ? "--apply --confirm-dev --confirmed-non-production --break-stale-lock" : null, + forceInterrupt: status === "held" ? "--apply --confirm-dev --confirmed-non-production --force-interrupt-lock" : null + } + }; +} + +function buildStatusNextActions({ target, deployBefore, lock, liveVerify }) { + const actions = []; + if (!target.headMatchesTarget) { + actions.push(`Checkout or fast-forward to ${target.ref} ${target.shortCommitId} before apply.`); + } + if (deployBefore.commitId !== target.shortCommitId && deployBefore.commitId !== target.commitId) { + actions.push(`deploy/deploy.json commitId is ${deployBefore.commitId}; target ref is ${target.shortCommitId}.`); + } + if (lock.status === "held") { + actions.push(`Wait ${lock.retryAfterSeconds}s or use explicit --force-interrupt-lock if the current apply must be stopped.`); + } else if (lock.status === "stale") { + actions.push("Use --break-stale-lock only after confirming the holder is inactive."); + } else if (lock.status === "unavailable") { + actions.push("Fix kubectl/KUBECONFIG access before applying."); + } + if (liveVerify.status !== "pass") { + actions.push("Inspect live endpoint identity/health before treating DEV as converged."); + } + if (actions.length === 0) { + actions.push("Run --apply --confirm-dev --confirmed-non-production --write-report when ready to publish/apply DEV."); + } + return actions; +} + +async function runDevCdStatus(args, ctx, stdout) { + const generatedAt = ctx.now().toISOString(); + const target = await resolveTargetRef(ctx, args.targetRef); + const deployBefore = await readDeployJson(ctx.repoRoot); + const kubectlContext = await resolveKubectlContext(args, ctx.env, ctx.runCommand); + const lockRead = await readDeployLockStatus({ ctx, args, kubectlContext }); + const lock = summarizeLockStatus(lockRead, ctx.now()); + const liveVerify = args.skipLiveVerify + ? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } } + : await verifyDevLive(ctx, { expectedCommit: null }); + const status = lock.status === "unavailable" || liveVerify.status !== "pass" ? "degraded" : "pass"; + const report = { + ok: true, + status, + mode: args.dryRun ? "dry-run" : "status", + command: "dev-cd-apply", + generatedAt, + mutationAttempted: false, + prodTouched: false, + target: { + ref: target.ref, + promotionCommit: target.commitId, + shortCommitId: target.shortCommitId, + headCommitId: target.headCommitId, + headMatchesTarget: target.headMatchesTarget, + namespace: args.targetNamespace + }, + deployJson: summarizeDeployJson(deployBefore), + kubectl: { + kubeconfigSource: kubectlContext.kubeconfigSource, + kubeconfig: kubectlContext.kubeconfig ? "" : "", + lockReadCommand: lockRead.command + }, + lock, + live: compactLiveReport(liveVerify), + nextActions: buildStatusNextActions({ target, deployBefore, lock, liveVerify }), + fullOutputHint: "Use --apply --confirm-dev --confirmed-non-production --write-report for the full transaction report; use --full-output only when stdout needs the full JSON." + }; + stdout.write(`${JSON.stringify(report, null, 2)}\n`); + return 0; +} + +function summarizeDevCdApplyReport(report) { + const apply = report.devCdApply ?? {}; + return { + ok: report.status === "pass", + status: report.status, + mode: apply.mode ?? "apply", + command: "dev-cd-apply", + commitId: report.commitId, + generatedAt: report.generatedAt, + mutationAttempted: apply.safety?.mutationAttempted ?? false, + prodTouched: false, + transaction: { + transactionId: report.transaction?.transactionId ?? null, + ownerTaskId: report.transaction?.ownerTaskId ?? null, + targetNamespace: report.transaction?.targetNamespace ?? defaultNamespace, + lockName: report.transaction?.lockName ?? defaultLockName, + phases: report.transaction?.phases ?? [], + release: report.transaction?.release ?? null + }, + target: apply.target ?? null, + lock: { + acquired: Boolean(apply.lock?.acquired), + current: compactInterruptedLock(apply.lock?.current), + staleBreak: compactInterruptedLock(apply.lock?.staleBreak), + forceInterrupt: compactForceInterruptRecord(apply.lock?.forceInterrupt) + }, + steps: (apply.steps ?? []).map((step) => ({ + id: step.id, + phase: step.phase, + status: step.status, + code: step.code, + reportPath: step.reportPath + })), + liveBefore: apply.liveBefore ? compactLiveReport(apply.liveBefore) : null, + liveVerify: apply.liveVerify ? compactLiveReport(apply.liveVerify) : null, + blockers: report.blockers ?? [], + reportPaths: apply.reportPaths ?? {}, + fullOutputHint: "Use --full-output to print the full report to stdout. Use --write-report to persist the full report." + }; +} + function buildReport({ args, transaction, @@ -1120,7 +1514,8 @@ function buildReport({ lock: { acquired: Boolean(lockState?.acquired), current: lockState?.lock ?? null, - staleBreak: lockState?.staleBreak ?? null + staleBreak: lockState?.staleBreak ?? null, + forceInterrupt: lockState?.forceInterrupt ?? null }, steps, liveBefore, @@ -1190,10 +1585,16 @@ export async function runDevCdApply(argv, io = {}) { try { args = parseArgs(argv); if (args.help) { - stdout.write(`${usage()}\n`); + stdout.write(`${JSON.stringify({ ok: true, status: "pass", help: usage() }, null, 2)}\n`); return 0; } + if (!args.apply && !args.status) { + args.status = true; + } validateArgs(args); + if (args.status) { + return await runDevCdStatus(args, ctx, stdout); + } } catch (error) { const failure = formatDevCdApplyFailure(error); stdout.write(`${JSON.stringify(failure, null, 2)}\n`); @@ -1437,6 +1838,7 @@ export async function runDevCdApply(argv, io = {}) { await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`); } - stdout.write(`${JSON.stringify(report, null, 2)}\n`); + const stdoutPayload = args.fullOutput ? report : summarizeDevCdApplyReport(report); + stdout.write(`${JSON.stringify(stdoutPayload, null, 2)}\n`); return status === "pass" ? 0 : 2; } diff --git a/scripts/src/dev-cd-apply.test.mjs b/scripts/src/dev-cd-apply.test.mjs index 6ce11bed..04d36ce4 100644 --- a/scripts/src/dev-cd-apply.test.mjs +++ b/scripts/src/dev-cd-apply.test.mjs @@ -10,6 +10,7 @@ import { buildLockAnnotations, classifyDeployLock, deployLockHeldFailure, + findDevCdApplyProcessCandidates, parseDeployLock, runDevCdApply, verifyDevLive @@ -98,7 +99,7 @@ function makeHttpGetJson() { }; } -function makeRunCommand({ heldLock = null, commandLog = [] } = {}) { +function makeRunCommand({ heldLock = null, commandLog = [], psOutput = "", killLog = [] } = {}) { let lease = heldLock ? leaseFromLock(heldLock) : null; const assertLeaseMicroTime = (value) => { assert.match(value, /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\.\d{6}Z$/u); @@ -114,6 +115,13 @@ function makeRunCommand({ heldLock = null, commandLog = [] } = {}) { if (command === "which" && args[0] === "kubectl") { return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" }; } + if (command === "ps") { + return { code: 0, stdout: psOutput, stderr: "" }; + } + if (command === "kill") { + killLog.push({ signal: args[0], pid: args[1] }); + return { code: 0, stdout: "", stderr: "" }; + } if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) { if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" }; return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" }; @@ -179,6 +187,48 @@ test("lock classifier reports held and stale states with retryAfterSeconds", () assert.equal(stale.retryAfterSeconds, 0); }); +test("dev-cd status and dry-run are read-only compact JSON", async () => { + const repoRoot = await makeRepo(); + const commandLog = []; + let output = ""; + const code = await runDevCdApply([ + "--dry-run", + "--kubeconfig", + "/tmp/kubeconfig" + ], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ commandLog }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const status = JSON.parse(output); + assert.equal(code, 0); + assert.equal(status.mode, "dry-run"); + assert.equal(status.mutationAttempted, false); + assert.equal(status.prodTouched, false); + assert.equal(status.lock.status, "absent"); + assert.equal(status.deployJson.manifest, undefined); + assert.equal(status.live.summary.checked, 2); + assert.equal(commandLog.some((entry) => entry.args.includes("create")), false); + assert.equal(commandLog.some((entry) => entry.args.includes("replace")), false); + assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false); + assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false); +}); + +test("process candidate discovery only returns dev-cd apply processes", () => { + const psOutput = [ + "101 node scripts/dev-cd-apply.mjs --apply --confirm-dev", + "202 node scripts/dev-cd-apply.mjs --status", + "303 node scripts/dev-deploy-apply.mjs --apply", + "404 node scripts/dev-cd-apply.mjs --apply --confirm-dev" + ].join("\n"); + const candidates = findDevCdApplyProcessCandidates(psOutput, 404); + assert.deepEqual(candidates.map((candidate) => candidate.pid), [101]); +}); + test("deploy-lock-held failure is structured before publish/apply", () => { const failure = deployLockHeldFailure({ ownerTaskId: "task-a", @@ -303,7 +353,8 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a "task-b", "--kubeconfig", "/tmp/kubeconfig", - "--write-report" + "--write-report", + "--full-output" ], { repoRoot, env: {}, @@ -355,6 +406,88 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a assert.equal(writtenReport.devCdApply.liveVerify.summary.checked, 2); }); +test("apply stdout is concise by default while write-report keeps the full report", async () => { + const repoRoot = await makeRepo(); + let output = ""; + const code = await runDevCdApply([ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--owner-task-id", + "task-c", + "--kubeconfig", + "/tmp/kubeconfig", + "--write-report" + ], { + repoRoot, + env: {}, + runCommand: makeRunCommand(), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const summary = JSON.parse(output); + assert.equal(code, 0); + assert.equal(summary.status, "pass"); + assert.equal(summary.devCdApply, undefined); + assert.equal(summary.fullOutputHint.includes("--full-output"), true); + assert.equal(summary.lock.acquired, true); + assert.equal(summary.steps.some((step) => step.id === "dev-deploy-apply"), true); + + const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8")); + assert.equal(writtenReport.devCdApply.deployJson.before.manifest.environment, "dev"); +}); + +test("force-interrupt-lock terminates local apply candidate and takes a held lock", async () => { + const repoRoot = await makeRepo(); + const killLog = []; + let output = ""; + const code = await runDevCdApply([ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--force-interrupt-lock", + "--owner-task-id", + "task-force", + "--kubeconfig", + "/tmp/kubeconfig", + "--full-output" + ], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ + killLog, + psOutput: "555 node scripts/dev-cd-apply.mjs --apply --confirm-dev\n666 node scripts/dev-cd-apply.mjs --status\n", + heldLock: { + promotionCommit: "abc1234", + deployJsonHash: "sha256:a", + ownerTaskId: "task-a", + transactionId: "tx-a", + phase: "applying", + startedAt: iso(0), + updatedAt: iso(0), + ttlSeconds: 300, + liveBefore: { status: "pass" }, + targetNamespace: "hwlab-dev", + targetRef: "origin/main", + lockBackend: "Lease" + } + }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const report = JSON.parse(output); + assert.equal(code, 0); + assert.deepEqual(killLog, [{ signal: "-TERM", pid: "555" }]); + assert.equal(report.devCdApply.lock.forceInterrupt.status, "attempted"); + assert.equal(report.devCdApply.lock.forceInterrupt.processCandidates[0].pid, 555); + assert.equal(report.devCdApply.lock.current.forceInterruptStatus, "attempted"); + assert.equal(report.devCdApply.lock.current.forceInterruptedBy, "task-force"); +}); + test("live verify summarizes 16666 and 16667 health identity", async () => { const result = await verifyDevLive({ now: () => new Date(iso()),