mirrors/pikasTech-HWLAB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add read-only dev cd status surface

Browse Source
This commit is contained in:
Code Queue Review
2026-05-23 10:43:52 +00:00
parent 3de0b62eec
commit 4c12985068
3 changed files with 281 additions and 267 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/src/dev-cd-apply.mjs
+59 -156
View File
@@ -42,11 +42,7 @@ const lockAnnotationFields = {
releasedAt: `${lockAnnotationPrefix}/releasedAt`,
releaseStatus: `${lockAnnotationPrefix}/releaseStatus`,
staleLockBrokenAt: `${lockAnnotationPrefix}/staleLockBrokenAt`,
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`,
forceInterruptedAt: `${lockAnnotationPrefix}/forceInterruptedAt`,
forceInterruptedBy: `${lockAnnotationPrefix}/forceInterruptedBy`,
forceInterruptStatus: `${lockAnnotationPrefix}/forceInterruptStatus`,
forceInterruptedPrevious: `${lockAnnotationPrefix}/forceInterruptedPrevious`
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`
};
class DevCdApplyError extends Error {
@@ -72,7 +68,6 @@ export function parseArgs(argv) {
ttlSeconds: defaultTtlSeconds,
ownerTaskId: null,
breakStaleLock: false,
forceInterruptLock: false,
skipLiveVerify: false,
status: false,
dryRun: false,
@@ -98,9 +93,6 @@ export function parseArgs(argv) {
} else if (arg === "--break-stale-lock") {
args.breakStaleLock = true;
args.flags.add(arg);
} else if (arg === "--force-interrupt-lock") {
args.forceInterruptLock = true;
args.flags.add(arg);
} else if (arg === "--dry-run" || arg === "--status") {
args.status = true;
args.dryRun = arg === "--dry-run";
@@ -140,7 +132,15 @@ export function parseArgs(argv) {
} else if (arg === "--help" || arg === "-h") {
args.help = true;
} else {
throw new Error(`unknown argument ${arg}`);
throw new DevCdApplyError(`unknown argument ${arg}`, {
code: "unknown-argument",
blockers: [{
type: "safety_blocker",
scope: "unknown-argument",
status: "open",
summary: `${arg} is not a supported DEV CD option. Use --status/--dry-run for read-only observability or --apply with explicit DEV confirmations.`
}]
});
}
}
@@ -163,21 +163,19 @@ export function usage() {
examples: [
"node scripts/dev-cd-apply.mjs --status",
"node scripts/dev-cd-apply.mjs --dry-run",
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report",
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --force-interrupt-lock --write-report"
"node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --write-report"
],
options: {
"--status": "Read-only status: target ref, deploy.json, current Lease, and public live health.",
"--dry-run": "Read-only alias for --status; does not acquire the Lease or run side effects.",
"--apply": "Run the DEV CD transaction side effects.",
"--confirm-dev": "Required for DEV mutation and stale/force lock takeover.",
"--confirm-dev": "Required for DEV mutation and explicit stale-lock recovery.",
"--confirmed-non-production": "Required for DEV mutation.",
"--target-ref REF": "git ref to publish/apply; default: origin/main",
"--kubeconfig PATH": "DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG",
"--ttl-seconds SECONDS": "lock TTL; default: 3600",
"--owner-task-id ID": "lock holder; default: Code Queue/env/user derived",
"--break-stale-lock": "May take over an expired lock only with --confirm-dev.",
"--force-interrupt-lock": "DEV-only explicit takeover: try to interrupt the current local dev-cd apply process, then acquire the Lease.",
"--report PATH": `default: ${defaultReportPath}`,
"--write-report": "Write the full report to --report.",
"--full-output": "Print the full report to stdout instead of the concise summary.",
@@ -241,15 +239,17 @@ function defaultOwnerTaskId(env = process.env) {
function defaultRunCommand(command, args, options = {}) {
return new Promise((resolve) => {
const abortController = options.timeoutMs ? new AbortController() : null;
const child = spawn(command, args, {
cwd: options.cwd ?? defaultRepoRoot,
env: options.env ?? process.env,
stdio: ["pipe", "pipe", "pipe"]
stdio: ["pipe", "pipe", "pipe"],
signal: abortController?.signal
});
let stdout = "";
let stderr = "";
const timeout = options.timeoutMs
? setTimeout(() => child.kill("SIGTERM"), options.timeoutMs)
? setTimeout(() => abortController.abort(), options.timeoutMs)
: null;
child.stdin.on("error", () => {});
@@ -262,7 +262,12 @@ function defaultRunCommand(command, args, options = {}) {
});
child.on("error", (error) => {
if (timeout) clearTimeout(timeout);
resolve({ code: 127, signal: null, stdout, stderr: error.message });
resolve({
code: error.name === "AbortError" ? 124 : 127,
signal: null,
stdout,
stderr: error.message
});
});
child.on("close", (code, signal) => {
if (timeout) clearTimeout(timeout);
@@ -346,22 +351,6 @@ function validateArgs(args) {
summary: "Breaking a stale DEV CD lock requires --break-stale-lock and --confirm-dev."
});
}
if (args.forceInterruptLock && !args.confirmDev) {
blockers.push({
type: "safety_blocker",
scope: "force-interrupt-confirmation",
status: "open",
summary: "Force-interrupting a DEV CD lock requires --force-interrupt-lock and --confirm-dev."
});
}
if (args.forceInterruptLock && !args.apply) {
blockers.push({
type: "safety_blocker",
scope: "force-interrupt-mode",
status: "open",
summary: "--force-interrupt-lock is only valid with --apply."
});
}
if (!Number.isInteger(args.ttlSeconds) || args.ttlSeconds < 60 || args.ttlSeconds > 24 * 60 * 60) {
blockers.push({
type: "safety_blocker",
@@ -446,11 +435,7 @@ export function parseDeployLock(lease) {
releasedAt: readAnnotation(annotations, "releasedAt"),
releaseStatus: readAnnotation(annotations, "releaseStatus"),
staleLockBrokenAt: readAnnotation(annotations, "staleLockBrokenAt"),
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious"),
forceInterruptedAt: readAnnotation(annotations, "forceInterruptedAt"),
forceInterruptedBy: readAnnotation(annotations, "forceInterruptedBy"),
forceInterruptStatus: readAnnotation(annotations, "forceInterruptStatus"),
forceInterruptedPrevious: readJsonAnnotation(annotations, "forceInterruptedPrevious")
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious")
};
}
@@ -512,7 +497,7 @@ function isConflict(result) {
return /conflict|object has been modified|operation cannot be fulfilled/i.test(`${result.stderr}\n${result.stdout}`);
}
function compactInterruptedLock(lock) {
function compactLockSummary(lock) {
if (!lock) return null;
return {
lockName: lock.lockName ?? defaultLockName,
@@ -526,39 +511,14 @@ function compactInterruptedLock(lock) {
updatedAt: lock.updatedAt ?? null,
ttlSeconds: lock.ttlSeconds ?? null,
targetNamespace: lock.targetNamespace ?? defaultNamespace,
targetRef: lock.targetRef ?? null
targetRef: lock.targetRef ?? null,
releasedAt: lock.releasedAt ?? null,
releaseStatus: lock.releaseStatus ?? null,
staleLockBrokenAt: lock.staleLockBrokenAt ?? null
};
}
function compactForceInterruptRecord(record) {
if (!record) return null;
return {
status: record.status,
warning: record.warning ?? null,
processCandidates: (record.processCandidates ?? []).map((candidate) => ({
pid: candidate.pid,
command: candidate.command
})),
attempts: (record.attempts ?? []).map((attempt) => ({
pid: attempt.pid,
signal: attempt.signal,
status: attempt.status
}))
};
}
function attachForceInterruptAnnotations(lock, forceInterrupt, timestamp, ownerTaskId) {
if (!forceInterrupt) return;
lock.forceInterruptedAt = timestamp;
lock.forceInterruptedBy = ownerTaskId;
lock.forceInterruptStatus = forceInterrupt.status;
lock.forceInterruptedPrevious = {
previousLock: compactInterruptedLock(forceInterrupt.previousLock),
interrupt: compactForceInterruptRecord(forceInterrupt)
};
}
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null, forceInterrupt = null }) {
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null }) {
const lock = {
promotionCommit: target.shortCommitId,
deployJsonHash: deployBefore.hash,
@@ -577,7 +537,6 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa
lock.staleLockBrokenAt = startedAt;
lock.staleLockPrevious = staleLockPrevious;
}
attachForceInterruptAnnotations(lock, forceInterrupt, startedAt, ownerTaskId);
return {
apiVersion: "coordination.k8s.io/v1",
kind: "Lease",
@@ -600,7 +559,7 @@ function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTa
};
}
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null, forceInterrupt = null }) {
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null }) {
const lock = {
promotionCommit: target.shortCommitId,
deployJsonHash: deployBefore.hash,
@@ -619,7 +578,6 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha
lock.staleLockBrokenAt = updatedAt;
lock.staleLockPrevious = staleLockPrevious;
}
attachForceInterruptAnnotations(lock, forceInterrupt, updatedAt, ownerTaskId);
return {
metadata: {
labels: {
@@ -638,66 +596,6 @@ function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, pha
};
}
export function findDevCdApplyProcessCandidates(psOutput, currentPid = process.pid) {
return String(psOutput ?? "")
.split(/\r?\n/u)
.map((line) => line.trim())
.filter(Boolean)
.map((line) => {
const match = line.match(/^(\d+)\s+(.+)$/u);
if (!match) return null;
return {
pid: Number.parseInt(match[1], 10),
command: oneLine(redactSensitiveText(match[2])).slice(0, 500)
};
})
.filter((entry) => {
if (!entry || !Number.isInteger(entry.pid) || entry.pid === currentPid) return false;
return entry.command.includes("scripts/dev-cd-apply.mjs") && entry.command.includes("--apply");
});
}
async function forceInterruptCurrentApplyProcess(ctx, existingLock) {
const ps = await ctx.runCommand("ps", ["-eo", "pid=,args="], {
cwd: ctx.repoRoot,
timeoutMs: 5000
});
if (ps.code !== 0) {
return {
status: "ps-failed",
previousLock: compactInterruptedLock(existingLock),
processCandidates: [],
attempts: [],
warning: oneLine(redactSensitiveText(ps.stderr || ps.stdout || "ps failed"))
};
}
const processCandidates = findDevCdApplyProcessCandidates(ps.stdout, process.pid);
const attempts = [];
for (const candidate of processCandidates) {
const result = await ctx.runCommand("kill", ["-TERM", String(candidate.pid)], {
cwd: ctx.repoRoot,
timeoutMs: 5000
});
attempts.push({
pid: candidate.pid,
signal: "TERM",
status: result.code === 0 ? "sent" : "failed",
stderrTail: redactSensitiveText(result.stderr || result.stdout || "").trim().slice(-500)
});
}
return {
status: processCandidates.length > 0 ? "attempted" : "no-local-process-found",
previousLock: compactInterruptedLock(existingLock),
processCandidates,
attempts,
warning: processCandidates.length > 0
? null
: "No local scripts/dev-cd-apply.mjs --apply process was found; the Lease will still be taken over so the previous holder cannot continue lock-owned updates."
};
}
async function kubectl(ctx, kubectlContext, args, options = {}) {
const result = await ctx.runCommand(kubectlContext.executor, args, {
cwd: ctx.repoRoot,
@@ -728,7 +626,7 @@ async function replaceLease(ctx, kubectlContext, args, lease, timeoutMs = 30000)
function mergeLease(lease, patch, { resetLockAnnotations = false } = {}) {
const baseAnnotations = { ...(lease.metadata?.annotations ?? {}) };
if (resetLockAnnotations) {
for (const key of Object.values(lockAnnotationFields)) {
for (const key of Object.keys(baseAnnotations).filter((name) => name.startsWith(`${lockAnnotationPrefix}/`))) {
delete baseAnnotations[key];
}
}
@@ -760,14 +658,14 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
const lease = JSON.parse(get.stdout);
const existing = parseDeployLock(lease);
const classification = classifyDeployLock(existing, now);
if (classification.held && !args.forceInterruptLock) {
if (classification.held) {
const failure = deployLockHeldFailure(existing, now);
throw new DevCdApplyError(failure.summary, {
code: "deploy-lock-held",
lockFailure: failure
});
}
if (classification.stale && !args.breakStaleLock && !args.forceInterruptLock) {
if (classification.stale && !args.breakStaleLock) {
const failure = deployLockHeldFailure(existing, now);
throw new DevCdApplyError(failure.summary, {
code: "deploy-lock-held",
@@ -775,10 +673,6 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
});
}
const forceInterrupt = args.forceInterruptLock
? await forceInterruptCurrentApplyProcess(ctx, existing)
: null;
const patch = lockPatch({
args,
target,
@@ -789,8 +683,7 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
startedAt,
updatedAt: startedAt,
liveBefore,
staleLockPrevious: classification.stale ? existing : null,
forceInterrupt
staleLockPrevious: classification.stale ? existing : null
});
const replaced = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch, { resetLockAnnotations: true }));
if (isConflict(replaced)) {
@@ -806,7 +699,6 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
acquired: true,
lock: parseDeployLock(JSON.parse(replaced.stdout)),
staleBreak: classification.stale ? existing : null,
forceInterrupt,
command: replaced.command
};
}
@@ -837,7 +729,6 @@ async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefo
acquired: true,
lock: parseDeployLock(JSON.parse(created.stdout)),
staleBreak: null,
forceInterrupt: null,
command: created.command
};
}
@@ -1410,6 +1301,10 @@ function summarizeDeployJson(deploy) {
};
}
function deployJsonMatchesTarget(deploy, target) {
return deploy.commitId === target.shortCommitId || deploy.commitId === target.commitId;
}
function compactLiveEndpoint(endpoint) {
return {
id: endpoint.id,
@@ -1481,7 +1376,8 @@ function summarizeLockStatus(lockRead, now) {
current: null,
retryAfterSeconds: null,
expiresAt: null,
error: oneLine(lockRead.error)
reason: "kubectl or DEV kubeconfig unavailable; command output is redacted",
redactedReason: oneLine(lockRead.error).slice(0, 500)
};
}
const classification = classifyDeployLock(lockRead.lock, now);
@@ -1494,12 +1390,12 @@ function summarizeLockStatus(lockRead, now) {
: "unknown";
return {
status,
current: compactInterruptedLock(lockRead.lock),
lockName: lockRead.lock?.lockName ?? lockRead.lockName ?? defaultLockName,
current: compactLockSummary(lockRead.lock),
retryAfterSeconds: classification.retryAfterSeconds,
expiresAt: classification.expiresAt,
takeover: {
stale: classification.stale ? "--apply --confirm-dev --confirmed-non-production --break-stale-lock" : null,
forceInterrupt: status === "held" ? "--apply --confirm-dev --confirmed-non-production --force-interrupt-lock" : null
allowedLockAction: {
breakStale: classification.stale ? "--apply --confirm-dev --confirmed-non-production --break-stale-lock" : null
}
};
}
@@ -1513,7 +1409,7 @@ function buildStatusNextActions({ target, deployBefore, lock, liveVerify }) {
actions.push(`deploy/deploy.json commitId is ${deployBefore.commitId}; target ref is ${target.shortCommitId}.`);
}
if (lock.status === "held") {
actions.push(`Wait ${lock.retryAfterSeconds}s or use explicit --force-interrupt-lock if the current apply must be stopped.`);
actions.push(`Wait for the current DEV CD holder to release the Lease or for the ${lock.retryAfterSeconds}s TTL to expire before retrying apply.`);
} else if (lock.status === "stale") {
actions.push("Use --break-stale-lock only after confirming the holder is inactive.");
} else if (lock.status === "unavailable") {
@@ -1538,7 +1434,13 @@ async function runDevCdStatus(args, ctx, stdout) {
const liveVerify = args.skipLiveVerify
? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } }
: await verifyDevLive(ctx, { expectedCommit: null });
const status = lock.status === "unavailable" || liveVerify.status !== "pass" ? "degraded" : "pass";
const deployMatchesTarget = deployJsonMatchesTarget(deployBefore, target);
const status = (
lock.status === "unavailable" ||
liveVerify.status !== "pass" ||
!target.headMatchesTarget ||
!deployMatchesTarget
) ? "degraded" : "pass";
const report = {
ok: true,
status,
@@ -1555,7 +1457,10 @@ async function runDevCdStatus(args, ctx, stdout) {
headMatchesTarget: target.headMatchesTarget,
namespace: args.targetNamespace
},
deployJson: summarizeDeployJson(deployBefore),
deployJson: {
...summarizeDeployJson(deployBefore),
matchesTarget: deployMatchesTarget
},
kubectl: {
kubeconfigSource: kubectlContext.kubeconfigSource,
kubeconfig: kubectlContext.kubeconfig ? "<set>" : "<unset>",
@@ -1592,9 +1497,8 @@ function summarizeDevCdApplyReport(report) {
target: apply.target ?? null,
lock: {
acquired: Boolean(apply.lock?.acquired),
current: compactInterruptedLock(apply.lock?.current),
staleBreak: compactInterruptedLock(apply.lock?.staleBreak),
forceInterrupt: compactForceInterruptRecord(apply.lock?.forceInterrupt)
current: compactLockSummary(apply.lock?.current),
staleBreak: compactLockSummary(apply.lock?.staleBreak)
},
steps: (apply.steps ?? []).map((step) => ({
id: step.id,
@@ -1726,8 +1630,7 @@ function buildReport({
lock: {
acquired: Boolean(lockState?.acquired),
current: lockState?.lock ?? null,
staleBreak: lockState?.staleBreak ?? null,
forceInterrupt: lockState?.forceInterrupt ?? null
staleBreak: lockState?.staleBreak ?? null
},
steps,
liveBefore,