Code Queue Review
1443 lines
46 KiB
JavaScript
1443 lines
46 KiB
JavaScript
import { spawn } from "node:child_process";
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import { request as httpRequest } from "node:http";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs";
|
|
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
|
import {
|
|
buildKubectlCommandPrefix,
|
|
redactSensitiveText,
|
|
resolveDevKubeconfigSelection
|
|
} from "./dev-deploy-apply.mjs";
|
|
|
|
const defaultRepoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
const defaultNamespace = "hwlab-dev";
|
|
const defaultLockName = "hwlab-dev-cd-lock";
|
|
const defaultTtlSeconds = 3600;
|
|
const defaultReportPath = "reports/dev-gate/dev-cd-apply.json";
|
|
const artifactReportPath = "reports/dev-gate/dev-artifacts.json";
|
|
const browserLiveUrl = "http://74.48.78.17:16666/health/live";
|
|
const apiLiveUrl = `${DEV_ENDPOINT}/health/live`;
|
|
const lockAnnotationPrefix = "hwlab.pikastech.local";
|
|
const lockAnnotationFields = {
|
|
promotionCommit: `${lockAnnotationPrefix}/promotionCommit`,
|
|
deployJsonHash: `${lockAnnotationPrefix}/deployJsonHash`,
|
|
ownerTaskId: `${lockAnnotationPrefix}/ownerTaskId`,
|
|
transactionId: `${lockAnnotationPrefix}/transactionId`,
|
|
phase: `${lockAnnotationPrefix}/phase`,
|
|
startedAt: `${lockAnnotationPrefix}/startedAt`,
|
|
updatedAt: `${lockAnnotationPrefix}/updatedAt`,
|
|
ttlSeconds: `${lockAnnotationPrefix}/ttlSeconds`,
|
|
liveBefore: `${lockAnnotationPrefix}/liveBefore`,
|
|
targetNamespace: `${lockAnnotationPrefix}/targetNamespace`,
|
|
targetRef: `${lockAnnotationPrefix}/targetRef`,
|
|
lockBackend: `${lockAnnotationPrefix}/lockBackend`,
|
|
releasedAt: `${lockAnnotationPrefix}/releasedAt`,
|
|
releaseStatus: `${lockAnnotationPrefix}/releaseStatus`,
|
|
staleLockBrokenAt: `${lockAnnotationPrefix}/staleLockBrokenAt`,
|
|
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`
|
|
};
|
|
|
|
class DevCdApplyError extends Error {
|
|
constructor(message, details = {}) {
|
|
super(message);
|
|
this.name = "DevCdApplyError";
|
|
Object.assign(this, details);
|
|
}
|
|
}
|
|
|
|
export function parseArgs(argv) {
|
|
const args = {
|
|
apply: false,
|
|
confirmDev: false,
|
|
confirmedNonProduction: false,
|
|
writeReport: false,
|
|
reportPath: defaultReportPath,
|
|
targetRef: "origin/main",
|
|
kubeconfig: null,
|
|
kubeconfigSpecified: false,
|
|
lockName: defaultLockName,
|
|
targetNamespace: defaultNamespace,
|
|
ttlSeconds: defaultTtlSeconds,
|
|
ownerTaskId: null,
|
|
breakStaleLock: false,
|
|
skipLiveVerify: false,
|
|
help: false,
|
|
flags: new Set()
|
|
};
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const arg = argv[index];
|
|
if (arg === "--apply") {
|
|
args.apply = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--confirm-dev") {
|
|
args.confirmDev = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--confirmed-non-production") {
|
|
args.confirmedNonProduction = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--write-report") {
|
|
args.writeReport = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--break-stale-lock") {
|
|
args.breakStaleLock = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--skip-live-verify") {
|
|
args.skipLiveVerify = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--report") {
|
|
args.reportPath = readOption(argv, ++index, arg);
|
|
args.flags.add(arg);
|
|
} else if (arg === "--target-ref") {
|
|
args.targetRef = readOption(argv, ++index, arg);
|
|
args.flags.add(arg);
|
|
} else if (arg === "--kubeconfig") {
|
|
args.kubeconfig = readOption(argv, ++index, arg);
|
|
args.kubeconfigSpecified = true;
|
|
args.flags.add(arg);
|
|
} else if (arg.startsWith("--kubeconfig=")) {
|
|
args.kubeconfig = arg.slice("--kubeconfig=".length);
|
|
args.kubeconfigSpecified = true;
|
|
args.flags.add("--kubeconfig");
|
|
} else if (arg === "--lock-name") {
|
|
args.lockName = readOption(argv, ++index, arg);
|
|
args.flags.add(arg);
|
|
} else if (arg === "--target-namespace") {
|
|
args.targetNamespace = readOption(argv, ++index, arg);
|
|
args.flags.add(arg);
|
|
} else if (arg === "--ttl-seconds") {
|
|
args.ttlSeconds = Number.parseInt(readOption(argv, ++index, arg), 10);
|
|
args.flags.add(arg);
|
|
} else if (arg === "--owner-task-id") {
|
|
args.ownerTaskId = readOption(argv, ++index, arg);
|
|
args.flags.add(arg);
|
|
} else if (arg === "--help" || arg === "-h") {
|
|
args.help = true;
|
|
} else {
|
|
throw new Error(`unknown argument ${arg}`);
|
|
}
|
|
}
|
|
|
|
return args;
|
|
}
|
|
|
|
function readOption(argv, index, name) {
|
|
const value = argv[index];
|
|
if (!value || value.startsWith("--")) {
|
|
throw new Error(`${name} requires a value`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
export function usage() {
|
|
return [
|
|
"usage: node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production [--write-report]",
|
|
"",
|
|
"Single DEV CD transaction for latest-main publish, desired-state refresh, apply, and live verify.",
|
|
"",
|
|
"options:",
|
|
" --target-ref REF git ref to publish/apply; default: origin/main",
|
|
" --kubeconfig PATH DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG",
|
|
" --ttl-seconds SECONDS lock TTL; default: 3600",
|
|
" --owner-task-id ID lock holder; default: Code Queue/env/user derived",
|
|
" --break-stale-lock may take over an expired lock only with --confirm-dev",
|
|
" --report PATH default: reports/dev-gate/dev-cd-apply.json",
|
|
" --skip-live-verify test-only escape hatch; do not use for DEV acceptance"
|
|
].join("\n");
|
|
}
|
|
|
|
function nowIso() {
|
|
return new Date().toISOString();
|
|
}
|
|
|
|
function kubernetesMicroTime(value) {
|
|
const date = value instanceof Date ? value : new Date(value);
|
|
if (!Number.isFinite(date.getTime())) {
|
|
throw new DevCdApplyError("invalid Kubernetes Lease timestamp", {
|
|
code: "invalid-lease-timestamp"
|
|
});
|
|
}
|
|
return date.toISOString().replace(/\.(\d{3})Z$/u, ".$1000Z");
|
|
}
|
|
|
|
function parseJsonMaybe(value) {
|
|
try {
|
|
return JSON.parse(value);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function oneLine(value) {
|
|
return String(value ?? "").replace(/\s+/g, " ").trim();
|
|
}
|
|
|
|
function sha256(value) {
|
|
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
}
|
|
|
|
function shortCommit(value) {
|
|
return typeof value === "string" && value.length >= 7 ? value.slice(0, 7) : value;
|
|
}
|
|
|
|
function commitMatches(actual, expected) {
|
|
if (typeof actual !== "string" || typeof expected !== "string") return false;
|
|
return actual === expected || actual.startsWith(expected) || expected.startsWith(actual);
|
|
}
|
|
|
|
function shellCommand(command, args) {
|
|
return [command, ...args].map((part) => (/\s/u.test(part) ? JSON.stringify(part) : part)).join(" ");
|
|
}
|
|
|
|
function defaultOwnerTaskId(env = process.env) {
|
|
return (
|
|
env.CODE_QUEUE_TASK_ID ||
|
|
env.HWLAB_OWNER_TASK_ID ||
|
|
env.GITHUB_RUN_ID ||
|
|
env.USER ||
|
|
"unknown-dev-cd-runner"
|
|
);
|
|
}
|
|
|
|
function defaultRunCommand(command, args, options = {}) {
|
|
return new Promise((resolve) => {
|
|
const child = spawn(command, args, {
|
|
cwd: options.cwd ?? defaultRepoRoot,
|
|
env: options.env ?? process.env,
|
|
stdio: ["pipe", "pipe", "pipe"]
|
|
});
|
|
let stdout = "";
|
|
let stderr = "";
|
|
const timeout = options.timeoutMs
|
|
? setTimeout(() => child.kill("SIGTERM"), options.timeoutMs)
|
|
: null;
|
|
|
|
child.stdin.on("error", () => {});
|
|
child.stdin.end(options.input ?? "");
|
|
child.stdout.on("data", (chunk) => {
|
|
stdout += chunk;
|
|
});
|
|
child.stderr.on("data", (chunk) => {
|
|
stderr += chunk;
|
|
});
|
|
child.on("error", (error) => {
|
|
if (timeout) clearTimeout(timeout);
|
|
resolve({ code: 127, signal: null, stdout, stderr: error.message });
|
|
});
|
|
child.on("close", (code, signal) => {
|
|
if (timeout) clearTimeout(timeout);
|
|
resolve({ code: code ?? 1, signal, stdout, stderr });
|
|
});
|
|
});
|
|
}
|
|
|
|
async function readDeployJson(repoRoot) {
|
|
const relativePath = "deploy/deploy.json";
|
|
const raw = await readFile(path.join(repoRoot, relativePath), "utf8");
|
|
const manifest = JSON.parse(raw);
|
|
return {
|
|
path: relativePath,
|
|
hash: sha256(raw),
|
|
commitId: manifest.commitId ?? "unknown",
|
|
namespace: manifest.namespace ?? defaultNamespace,
|
|
environment: manifest.environment ?? "unknown",
|
|
endpoint: manifest.endpoint ?? null,
|
|
manifest
|
|
};
|
|
}
|
|
|
|
async function resolveTargetRef(ctx, targetRef) {
|
|
const target = await ctx.runCommand("git", ["rev-parse", "--verify", `${targetRef}^{commit}`], {
|
|
cwd: ctx.repoRoot,
|
|
timeoutMs: 10000
|
|
});
|
|
if (target.code !== 0) {
|
|
throw new DevCdApplyError(`target ref ${targetRef} could not be resolved`, {
|
|
code: "target-ref-unresolved",
|
|
targetRef,
|
|
stderr: redactSensitiveText(target.stderr)
|
|
});
|
|
}
|
|
const head = await ctx.runCommand("git", ["rev-parse", "--verify", "HEAD^{commit}"], {
|
|
cwd: ctx.repoRoot,
|
|
timeoutMs: 10000
|
|
});
|
|
if (head.code !== 0) {
|
|
throw new DevCdApplyError("HEAD could not be resolved", {
|
|
code: "head-unresolved",
|
|
stderr: redactSensitiveText(head.stderr)
|
|
});
|
|
}
|
|
const commitId = target.stdout.trim();
|
|
const headCommitId = head.stdout.trim();
|
|
return {
|
|
ref: targetRef,
|
|
commitId,
|
|
shortCommitId: shortCommit(commitId),
|
|
headCommitId,
|
|
headShortCommitId: shortCommit(headCommitId),
|
|
headMatchesTarget: commitId === headCommitId
|
|
};
|
|
}
|
|
|
|
function validateArgs(args) {
|
|
const blockers = [];
|
|
if (!args.apply) {
|
|
blockers.push({
|
|
type: "safety_blocker",
|
|
scope: "apply-mode",
|
|
status: "open",
|
|
summary: "DEV CD transaction side effects require --apply."
|
|
});
|
|
}
|
|
if (args.apply && (!args.confirmDev || !args.confirmedNonProduction)) {
|
|
blockers.push({
|
|
type: "safety_blocker",
|
|
scope: "dev-confirmation",
|
|
status: "open",
|
|
summary: "DEV CD apply requires --confirm-dev and --confirmed-non-production."
|
|
});
|
|
}
|
|
if (args.breakStaleLock && !args.confirmDev) {
|
|
blockers.push({
|
|
type: "safety_blocker",
|
|
scope: "break-stale-lock-confirmation",
|
|
status: "open",
|
|
summary: "Breaking a stale DEV CD lock requires --break-stale-lock and --confirm-dev."
|
|
});
|
|
}
|
|
if (!Number.isInteger(args.ttlSeconds) || args.ttlSeconds < 60 || args.ttlSeconds > 24 * 60 * 60) {
|
|
blockers.push({
|
|
type: "safety_blocker",
|
|
scope: "lock-ttl",
|
|
status: "open",
|
|
summary: "--ttl-seconds must be an integer between 60 and 86400."
|
|
});
|
|
}
|
|
for (const forbidden of ["--prod", "--production", "--read-secret", "--force-push"]) {
|
|
if (args.flags.has(forbidden)) {
|
|
blockers.push({
|
|
type: "safety_blocker",
|
|
scope: forbidden.slice(2),
|
|
status: "open",
|
|
summary: `${forbidden} is forbidden for DEV CD apply.`
|
|
});
|
|
}
|
|
}
|
|
if (blockers.length > 0) {
|
|
throw new DevCdApplyError("invalid DEV CD apply arguments", {
|
|
code: "invalid-arguments",
|
|
blockers
|
|
});
|
|
}
|
|
}
|
|
|
|
function lockAnnotationValue(value) {
|
|
if (value === undefined || value === null) return "";
|
|
if (typeof value === "string") return value;
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
export function buildLockAnnotations(lock) {
|
|
const annotations = {};
|
|
for (const [field, key] of Object.entries(lockAnnotationFields)) {
|
|
if (Object.hasOwn(lock, field)) {
|
|
annotations[key] = lockAnnotationValue(lock[field]);
|
|
}
|
|
}
|
|
return annotations;
|
|
}
|
|
|
|
function readAnnotation(annotations, field) {
|
|
return annotations?.[lockAnnotationFields[field]] ?? null;
|
|
}
|
|
|
|
function readJsonAnnotation(annotations, field) {
|
|
const value = readAnnotation(annotations, field);
|
|
return value ? parseJsonMaybe(value) : null;
|
|
}
|
|
|
|
export function parseDeployLock(lease) {
|
|
if (!lease) return null;
|
|
const annotations = lease.metadata?.annotations ?? {};
|
|
const ttlSeconds = Number.parseInt(
|
|
readAnnotation(annotations, "ttlSeconds") || lease.spec?.leaseDurationSeconds || `${defaultTtlSeconds}`,
|
|
10
|
|
);
|
|
const updatedAt =
|
|
readAnnotation(annotations, "updatedAt") ||
|
|
lease.spec?.renewTime ||
|
|
lease.metadata?.creationTimestamp ||
|
|
null;
|
|
const transactionId = readAnnotation(annotations, "transactionId") || lease.spec?.holderIdentity || null;
|
|
const ownerTaskId = readAnnotation(annotations, "ownerTaskId") || lease.spec?.holderIdentity || null;
|
|
return {
|
|
lockBackend: readAnnotation(annotations, "lockBackend") || "Lease",
|
|
lockName: lease.metadata?.name ?? defaultLockName,
|
|
resourceVersion: lease.metadata?.resourceVersion ?? null,
|
|
holderIdentity: lease.spec?.holderIdentity ?? null,
|
|
promotionCommit: readAnnotation(annotations, "promotionCommit"),
|
|
deployJsonHash: readAnnotation(annotations, "deployJsonHash"),
|
|
ownerTaskId,
|
|
transactionId,
|
|
phase: readAnnotation(annotations, "phase") || "unknown",
|
|
startedAt: readAnnotation(annotations, "startedAt") || lease.spec?.acquireTime || lease.metadata?.creationTimestamp || null,
|
|
updatedAt,
|
|
ttlSeconds: Number.isInteger(ttlSeconds) ? ttlSeconds : defaultTtlSeconds,
|
|
liveBefore: readJsonAnnotation(annotations, "liveBefore"),
|
|
targetNamespace: readAnnotation(annotations, "targetNamespace") || lease.metadata?.namespace || defaultNamespace,
|
|
targetRef: readAnnotation(annotations, "targetRef"),
|
|
releasedAt: readAnnotation(annotations, "releasedAt"),
|
|
releaseStatus: readAnnotation(annotations, "releaseStatus"),
|
|
staleLockBrokenAt: readAnnotation(annotations, "staleLockBrokenAt"),
|
|
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious")
|
|
};
|
|
}
|
|
|
|
export function classifyDeployLock(lock, now = new Date()) {
|
|
if (!lock || lock.phase === "released") {
|
|
return {
|
|
held: false,
|
|
stale: false,
|
|
retryAfterSeconds: 0,
|
|
expiresAt: null
|
|
};
|
|
}
|
|
const updatedAtMs = Number.isFinite(Date.parse(lock.updatedAt)) ? Date.parse(lock.updatedAt) : 0;
|
|
const expiresAtMs = updatedAtMs + lock.ttlSeconds * 1000;
|
|
const retryAfterSeconds = Math.max(0, Math.ceil((expiresAtMs - now.getTime()) / 1000));
|
|
return {
|
|
held: retryAfterSeconds > 0,
|
|
stale: retryAfterSeconds <= 0,
|
|
retryAfterSeconds,
|
|
expiresAt: Number.isFinite(expiresAtMs) ? new Date(expiresAtMs).toISOString() : null
|
|
};
|
|
}
|
|
|
|
export function deployLockHeldFailure(lock, now = new Date()) {
|
|
const classification = classifyDeployLock(lock, now);
|
|
return {
|
|
ok: false,
|
|
status: "blocked",
|
|
error: "deploy-lock-held",
|
|
code: "deploy-lock-held",
|
|
holder: lock.ownerTaskId || lock.transactionId || lock.holderIdentity || "unknown",
|
|
promotionCommit: lock.promotionCommit ?? null,
|
|
deployJsonHash: lock.deployJsonHash ?? null,
|
|
transactionId: lock.transactionId ?? null,
|
|
phase: lock.phase ?? "unknown",
|
|
targetNamespace: lock.targetNamespace ?? defaultNamespace,
|
|
retryAfterSeconds: classification.retryAfterSeconds,
|
|
stale: classification.stale,
|
|
requiresBreakStaleLock: classification.stale,
|
|
lockName: lock.lockName ?? defaultLockName,
|
|
lockBackend: lock.lockBackend ?? "Lease",
|
|
summary: classification.stale
|
|
? "A stale DEV CD transaction lock exists; rerun with --break-stale-lock --confirm-dev only after confirming the holder is inactive."
|
|
: "Another DEV CD transaction is already running; retry after the current holder releases the lock.",
|
|
mutationAttempted: false,
|
|
prodTouched: false
|
|
};
|
|
}
|
|
|
|
function isNotFound(result) {
|
|
return /not\s*found|notfound/i.test(`${result.stderr}\n${result.stdout}`);
|
|
}
|
|
|
|
function isAlreadyExists(result) {
|
|
return /already\s*exists|alreadyexists/i.test(`${result.stderr}\n${result.stdout}`);
|
|
}
|
|
|
|
function isConflict(result) {
|
|
return /conflict|object has been modified|operation cannot be fulfilled/i.test(`${result.stderr}\n${result.stdout}`);
|
|
}
|
|
|
|
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null }) {
|
|
const lock = {
|
|
promotionCommit: target.shortCommitId,
|
|
deployJsonHash: deployBefore.hash,
|
|
ownerTaskId,
|
|
transactionId,
|
|
phase,
|
|
startedAt,
|
|
updatedAt: startedAt,
|
|
ttlSeconds: args.ttlSeconds,
|
|
liveBefore,
|
|
targetNamespace: args.targetNamespace,
|
|
targetRef: target.ref,
|
|
lockBackend: "Lease"
|
|
};
|
|
if (staleLockPrevious) {
|
|
lock.staleLockBrokenAt = startedAt;
|
|
lock.staleLockPrevious = staleLockPrevious;
|
|
}
|
|
return {
|
|
apiVersion: "coordination.k8s.io/v1",
|
|
kind: "Lease",
|
|
metadata: {
|
|
name: args.lockName,
|
|
namespace: args.targetNamespace,
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab",
|
|
"hwlab.pikastech.local/profile": "dev",
|
|
"hwlab.pikastech.local/cd-lock": "true"
|
|
},
|
|
annotations: buildLockAnnotations(lock)
|
|
},
|
|
spec: {
|
|
holderIdentity: `${ownerTaskId}/${transactionId}`,
|
|
leaseDurationSeconds: args.ttlSeconds,
|
|
acquireTime: startedAt,
|
|
renewTime: startedAt
|
|
}
|
|
};
|
|
}
|
|
|
|
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null }) {
|
|
const lock = {
|
|
promotionCommit: target.shortCommitId,
|
|
deployJsonHash: deployBefore.hash,
|
|
ownerTaskId,
|
|
transactionId,
|
|
phase,
|
|
startedAt,
|
|
updatedAt,
|
|
ttlSeconds: args.ttlSeconds,
|
|
liveBefore,
|
|
targetNamespace: args.targetNamespace,
|
|
targetRef: target.ref,
|
|
lockBackend: "Lease"
|
|
};
|
|
if (staleLockPrevious) {
|
|
lock.staleLockBrokenAt = updatedAt;
|
|
lock.staleLockPrevious = staleLockPrevious;
|
|
}
|
|
return {
|
|
metadata: {
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab",
|
|
"hwlab.pikastech.local/profile": "dev",
|
|
"hwlab.pikastech.local/cd-lock": "true"
|
|
},
|
|
annotations: buildLockAnnotations(lock)
|
|
},
|
|
spec: {
|
|
holderIdentity: `${ownerTaskId}/${transactionId}`,
|
|
leaseDurationSeconds: args.ttlSeconds,
|
|
acquireTime: startedAt,
|
|
renewTime: updatedAt
|
|
}
|
|
};
|
|
}
|
|
|
|
async function kubectl(ctx, kubectlContext, args, options = {}) {
|
|
const result = await ctx.runCommand(kubectlContext.executor, args, {
|
|
cwd: ctx.repoRoot,
|
|
env: kubectlContext.env,
|
|
input: options.input,
|
|
timeoutMs: options.timeoutMs ?? 30000
|
|
});
|
|
return {
|
|
...result,
|
|
command: shellCommand("kubectl", args),
|
|
stdout: result.stdout ?? "",
|
|
stderr: result.stderr ?? ""
|
|
};
|
|
}
|
|
|
|
async function replaceLease(ctx, kubectlContext, args, lease, timeoutMs = 30000) {
|
|
return kubectl(
|
|
ctx,
|
|
kubectlContext,
|
|
["-n", args.targetNamespace, "replace", "-f", "-", "-o", "json"],
|
|
{
|
|
input: JSON.stringify(lease),
|
|
timeoutMs
|
|
}
|
|
);
|
|
}
|
|
|
|
function mergeLease(lease, patch, { resetLockAnnotations = false } = {}) {
|
|
const baseAnnotations = { ...(lease.metadata?.annotations ?? {}) };
|
|
if (resetLockAnnotations) {
|
|
for (const key of Object.values(lockAnnotationFields)) {
|
|
delete baseAnnotations[key];
|
|
}
|
|
}
|
|
return {
|
|
...lease,
|
|
metadata: {
|
|
...lease.metadata,
|
|
...(patch.metadata ?? {}),
|
|
labels: {
|
|
...(lease.metadata?.labels ?? {}),
|
|
...(patch.metadata?.labels ?? {})
|
|
},
|
|
annotations: {
|
|
...baseAnnotations,
|
|
...(patch.metadata?.annotations ?? {})
|
|
}
|
|
},
|
|
spec: {
|
|
...(lease.spec ?? {}),
|
|
...(patch.spec ?? {})
|
|
}
|
|
};
|
|
}
|
|
|
|
async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now }) {
|
|
const startedAt = kubernetesMicroTime(now);
|
|
const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"]);
|
|
if (get.code === 0) {
|
|
const lease = JSON.parse(get.stdout);
|
|
const existing = parseDeployLock(lease);
|
|
const classification = classifyDeployLock(existing, now);
|
|
if (classification.held || (classification.stale && !args.breakStaleLock)) {
|
|
const failure = deployLockHeldFailure(existing, now);
|
|
throw new DevCdApplyError(failure.summary, {
|
|
code: "deploy-lock-held",
|
|
lockFailure: failure
|
|
});
|
|
}
|
|
|
|
const patch = lockPatch({
|
|
args,
|
|
target,
|
|
deployBefore,
|
|
transactionId,
|
|
ownerTaskId,
|
|
phase: "publishing",
|
|
startedAt,
|
|
updatedAt: startedAt,
|
|
liveBefore,
|
|
staleLockPrevious: classification.stale ? existing : null
|
|
});
|
|
const replaced = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch, { resetLockAnnotations: true }));
|
|
if (isConflict(replaced)) {
|
|
return acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now });
|
|
}
|
|
if (replaced.code !== 0) {
|
|
throw new DevCdApplyError("failed to acquire DEV CD Lease lock", {
|
|
code: "lock-acquire-failed",
|
|
stderr: redactSensitiveText(replaced.stderr || replaced.stdout)
|
|
});
|
|
}
|
|
return {
|
|
acquired: true,
|
|
lock: parseDeployLock(JSON.parse(replaced.stdout)),
|
|
staleBreak: classification.stale ? existing : null,
|
|
command: replaced.command
|
|
};
|
|
}
|
|
|
|
if (!isNotFound(get)) {
|
|
throw new DevCdApplyError("failed to read DEV CD Lease lock", {
|
|
code: "lock-read-failed",
|
|
stderr: redactSensitiveText(get.stderr || get.stdout)
|
|
});
|
|
}
|
|
|
|
const manifest = buildLeaseManifest({
|
|
args,
|
|
target,
|
|
deployBefore,
|
|
transactionId,
|
|
ownerTaskId,
|
|
startedAt,
|
|
phase: "publishing",
|
|
liveBefore
|
|
});
|
|
const created = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "create", "-f", "-", "-o", "json"], {
|
|
input: JSON.stringify(manifest),
|
|
timeoutMs: 30000
|
|
});
|
|
if (created.code === 0) {
|
|
return {
|
|
acquired: true,
|
|
lock: parseDeployLock(JSON.parse(created.stdout)),
|
|
staleBreak: null,
|
|
command: created.command
|
|
};
|
|
}
|
|
if (isAlreadyExists(created)) {
|
|
return acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now });
|
|
}
|
|
throw new DevCdApplyError("failed to create DEV CD Lease lock", {
|
|
code: "lock-create-failed",
|
|
stderr: redactSensitiveText(created.stderr || created.stdout)
|
|
});
|
|
}
|
|
|
|
async function readOwnedLease({ ctx, args, kubectlContext, transactionId, timeoutMs = 15000 }) {
|
|
const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], {
|
|
timeoutMs
|
|
});
|
|
if (get.code !== 0) {
|
|
throw new DevCdApplyError("failed to read DEV CD lock ownership", {
|
|
code: "lock-owner-read-failed",
|
|
stderr: redactSensitiveText(get.stderr || get.stdout)
|
|
});
|
|
}
|
|
const lease = JSON.parse(get.stdout);
|
|
const lock = parseDeployLock(lease);
|
|
if (lock.transactionId !== transactionId) {
|
|
throw new DevCdApplyError("DEV CD lock ownership changed before transaction update", {
|
|
code: "lock-owner-mismatch",
|
|
holder: lock.ownerTaskId || lock.transactionId || "unknown",
|
|
phase: lock.phase
|
|
});
|
|
}
|
|
return { lease, lock };
|
|
}
|
|
|
|
async function patchOwnedLease({ ctx, args, kubectlContext, lockState, transactionId, patch, errorCode, errorMessage }) {
|
|
const { lease } = await readOwnedLease({ ctx, args, kubectlContext, transactionId });
|
|
const result = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch));
|
|
if (result.code !== 0) {
|
|
throw new DevCdApplyError(errorMessage, {
|
|
code: errorCode,
|
|
stderr: redactSensitiveText(result.stderr || result.stdout)
|
|
});
|
|
}
|
|
lockState.lock = parseDeployLock(JSON.parse(result.stdout));
|
|
return lockState.lock;
|
|
}
|
|
|
|
async function updateDeployLockPhase({ ctx, args, kubectlContext, transactionId, lockState, phase, status = "running" }) {
|
|
const updatedAt = kubernetesMicroTime(ctx.now());
|
|
const patch = {
|
|
metadata: {
|
|
annotations: {
|
|
[lockAnnotationFields.phase]: phase,
|
|
[lockAnnotationFields.updatedAt]: updatedAt,
|
|
[lockAnnotationFields.releaseStatus]: status
|
|
}
|
|
},
|
|
spec: {
|
|
renewTime: updatedAt
|
|
}
|
|
};
|
|
return await patchOwnedLease({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId,
|
|
lockState,
|
|
patch,
|
|
errorCode: "lock-phase-update-failed",
|
|
errorMessage: `failed to update DEV CD lock phase to ${phase}`
|
|
});
|
|
}
|
|
|
|
async function updateDeployLockLiveBefore({ ctx, args, kubectlContext, transactionId, lockState, liveBefore }) {
|
|
const updatedAt = kubernetesMicroTime(ctx.now());
|
|
const patch = {
|
|
metadata: {
|
|
annotations: {
|
|
[lockAnnotationFields.liveBefore]: JSON.stringify(liveBefore),
|
|
[lockAnnotationFields.updatedAt]: updatedAt
|
|
}
|
|
},
|
|
spec: {
|
|
renewTime: updatedAt
|
|
}
|
|
};
|
|
return await patchOwnedLease({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId,
|
|
lockState,
|
|
patch,
|
|
errorCode: "lock-live-before-update-failed",
|
|
errorMessage: "failed to update DEV CD lock liveBefore evidence"
|
|
});
|
|
}
|
|
|
|
async function releaseDeployLock({ ctx, args, kubectlContext, transactionId, status }) {
|
|
const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], {
|
|
timeoutMs: 15000
|
|
});
|
|
if (get.code !== 0) {
|
|
return {
|
|
status: "release_read_failed",
|
|
command: get.command,
|
|
error: redactSensitiveText(get.stderr || get.stdout)
|
|
};
|
|
}
|
|
const current = parseDeployLock(JSON.parse(get.stdout));
|
|
if (current.transactionId !== transactionId) {
|
|
return {
|
|
status: "not_owner",
|
|
holder: current.ownerTaskId || current.transactionId || "unknown",
|
|
phase: current.phase
|
|
};
|
|
}
|
|
const releasedAt = kubernetesMicroTime(ctx.now());
|
|
const patch = {
|
|
metadata: {
|
|
annotations: {
|
|
[lockAnnotationFields.phase]: "released",
|
|
[lockAnnotationFields.updatedAt]: releasedAt,
|
|
[lockAnnotationFields.releasedAt]: releasedAt,
|
|
[lockAnnotationFields.releaseStatus]: status
|
|
}
|
|
},
|
|
spec: {
|
|
holderIdentity: "",
|
|
leaseDurationSeconds: 1,
|
|
renewTime: releasedAt
|
|
}
|
|
};
|
|
const result = await replaceLease(ctx, kubectlContext, args, mergeLease(JSON.parse(get.stdout), patch), 15000);
|
|
if (isConflict(result)) {
|
|
return {
|
|
status: "release_conflict",
|
|
command: result.command,
|
|
error: "Lease changed before release; lock was not overwritten."
|
|
};
|
|
}
|
|
if (result.code !== 0) {
|
|
return {
|
|
status: "release_failed",
|
|
command: result.command,
|
|
error: redactSensitiveText(result.stderr || result.stdout)
|
|
};
|
|
}
|
|
return {
|
|
status: "released",
|
|
command: result.command,
|
|
lock: parseDeployLock(JSON.parse(result.stdout))
|
|
};
|
|
}
|
|
|
|
async function resolveKubectlContext(args, env, runCommand) {
|
|
const selection = resolveDevKubeconfigSelection({ flagValue: args.kubeconfig, env });
|
|
const which = await runCommand("which", ["kubectl"], { timeoutMs: 5000 });
|
|
const executor = which.code === 0 ? which.stdout.trim() : "kubectl";
|
|
return {
|
|
executor,
|
|
kubeconfig: selection.kubeconfig,
|
|
kubeconfigSource: selection.source,
|
|
commandPrefix: buildKubectlCommandPrefix(selection.kubeconfig),
|
|
env: {
|
|
...env,
|
|
KUBECONFIG: selection.kubeconfig
|
|
}
|
|
};
|
|
}
|
|
|
|
function commandEnv(ctx, transaction) {
|
|
return {
|
|
...ctx.env,
|
|
HWLAB_CD_TRANSACTION_ID: transaction.transactionId,
|
|
HWLAB_CD_TRANSACTION_OWNER: transaction.ownerTaskId,
|
|
HWLAB_CD_LOCK_NAME: transaction.lockName,
|
|
HWLAB_CD_LOCK_NAMESPACE: transaction.targetNamespace
|
|
};
|
|
}
|
|
|
|
async function runStep(ctx, transaction, step) {
|
|
const startedAt = ctx.now().toISOString();
|
|
const command = shellCommand(step.command, step.args);
|
|
const result = await ctx.runCommand(step.command, step.args, {
|
|
cwd: ctx.repoRoot,
|
|
env: commandEnv(ctx, transaction),
|
|
timeoutMs: step.timeoutMs ?? 300000
|
|
});
|
|
const finishedAt = ctx.now().toISOString();
|
|
const stdout = redactSensitiveText(result.stdout ?? "");
|
|
const stderr = redactSensitiveText(result.stderr ?? "");
|
|
const parsedStdout = parseJsonMaybe(stdout);
|
|
return {
|
|
id: step.id,
|
|
phase: step.phase,
|
|
status: result.code === 0 ? "pass" : "blocked",
|
|
command,
|
|
code: result.code,
|
|
startedAt,
|
|
finishedAt,
|
|
stdoutJson: parsedStdout,
|
|
stdoutTail: parsedStdout ? null : stdout.trim().slice(-2000),
|
|
stderrTail: stderr.trim().slice(-2000),
|
|
reportPath: step.reportPath ?? null
|
|
};
|
|
}
|
|
|
|
function stepBlocker(stepResult) {
|
|
return {
|
|
type: "runtime_blocker",
|
|
scope: stepResult.id,
|
|
status: "open",
|
|
summary: `${stepResult.command} exited ${stepResult.code}.`
|
|
};
|
|
}
|
|
|
|
function healthCommit(json) {
|
|
return (
|
|
json?.commit?.id ||
|
|
json?.commitId ||
|
|
json?.revision ||
|
|
json?.image?.tag ||
|
|
null
|
|
);
|
|
}
|
|
|
|
function healthImage(json) {
|
|
if (typeof json?.image === "string") return json.image;
|
|
return json?.image?.reference ?? null;
|
|
}
|
|
|
|
function summarizeHealthJson(json, expectedServiceId, expectedCommit) {
|
|
const observedCommit = healthCommit(json);
|
|
return {
|
|
serviceId: json?.serviceId ?? json?.service?.id ?? null,
|
|
environment: json?.environment ?? null,
|
|
applicationStatus: json?.status ?? null,
|
|
ready: Object.hasOwn(json ?? {}, "ready") ? json.ready : null,
|
|
observedCommit,
|
|
image: healthImage(json),
|
|
imageTag: json?.image?.tag ?? null,
|
|
observedAt: json?.observedAt ?? null,
|
|
serviceMatches: (json?.serviceId ?? json?.service?.id) === expectedServiceId,
|
|
environmentMatches: json?.environment === ENVIRONMENT_DEV,
|
|
commitMatches: expectedCommit ? commitMatches(observedCommit, expectedCommit) : null,
|
|
blockerCodes: Array.isArray(json?.blockerCodes) ? json.blockerCodes : []
|
|
};
|
|
}
|
|
|
|
async function defaultHttpGetJson(url, timeoutMs = 10000) {
|
|
return new Promise((resolve, reject) => {
|
|
const request = httpRequest(url, { method: "GET", timeout: timeoutMs }, (response) => {
|
|
response.setEncoding("utf8");
|
|
let body = "";
|
|
response.on("data", (chunk) => {
|
|
body += chunk;
|
|
});
|
|
response.on("end", () => {
|
|
let json = null;
|
|
try {
|
|
json = JSON.parse(body);
|
|
} catch {}
|
|
resolve({
|
|
statusCode: response.statusCode ?? 0,
|
|
body,
|
|
json
|
|
});
|
|
});
|
|
});
|
|
request.on("timeout", () => {
|
|
request.destroy(new Error(`timeout after ${timeoutMs}ms`));
|
|
});
|
|
request.on("error", reject);
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
async function probeLiveEndpoint(ctx, { id, url, expectedServiceId, expectedCommit }) {
|
|
const startedAt = ctx.now().toISOString();
|
|
try {
|
|
const response = await ctx.httpGetJson(url, 10000);
|
|
const summary = summarizeHealthJson(response.json, expectedServiceId, expectedCommit);
|
|
const reachable = response.statusCode >= 200 && response.statusCode < 300 && Boolean(response.json);
|
|
const identityMatches = reachable && summary.serviceMatches && summary.environmentMatches && (expectedCommit ? summary.commitMatches : true);
|
|
return {
|
|
id,
|
|
url,
|
|
status: identityMatches ? "pass" : "blocked",
|
|
httpStatus: response.statusCode,
|
|
reachable,
|
|
identityMatches,
|
|
startedAt,
|
|
finishedAt: ctx.now().toISOString(),
|
|
expectedServiceId,
|
|
expectedCommit: expectedCommit ?? null,
|
|
...summary
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
id,
|
|
url,
|
|
status: "blocked",
|
|
reachable: false,
|
|
identityMatches: false,
|
|
startedAt,
|
|
finishedAt: ctx.now().toISOString(),
|
|
expectedServiceId,
|
|
expectedCommit: expectedCommit ?? null,
|
|
error: oneLine(error.message)
|
|
};
|
|
}
|
|
}
|
|
|
|
export async function verifyDevLive(ctx, { expectedCommit = null } = {}) {
|
|
const [cloudWeb, cloudApi] = await Promise.all([
|
|
probeLiveEndpoint(ctx, {
|
|
id: "cloud-web-16666",
|
|
url: browserLiveUrl,
|
|
expectedServiceId: "hwlab-cloud-web",
|
|
expectedCommit
|
|
}),
|
|
probeLiveEndpoint(ctx, {
|
|
id: "cloud-api-16667",
|
|
url: apiLiveUrl,
|
|
expectedServiceId: "hwlab-cloud-api",
|
|
expectedCommit
|
|
})
|
|
]);
|
|
const endpoints = [cloudWeb, cloudApi];
|
|
return {
|
|
status: endpoints.every((endpoint) => endpoint.status === "pass") ? "pass" : "blocked",
|
|
expectedCommit: expectedCommit ?? null,
|
|
endpoints,
|
|
summary: {
|
|
checked: endpoints.length,
|
|
reachable: endpoints.filter((endpoint) => endpoint.reachable).length,
|
|
identityMatches: endpoints.filter((endpoint) => endpoint.identityMatches).length,
|
|
commitMatches: expectedCommit
|
|
? endpoints.filter((endpoint) => endpoint.commitMatches === true).length
|
|
: null,
|
|
ports: [16666, 16667]
|
|
}
|
|
};
|
|
}
|
|
|
|
function buildReport({
|
|
args,
|
|
transaction,
|
|
target,
|
|
deployBefore,
|
|
deployAfter,
|
|
lockState,
|
|
steps,
|
|
blockers,
|
|
status,
|
|
startedAt,
|
|
finishedAt,
|
|
liveBefore,
|
|
liveVerify,
|
|
release
|
|
}) {
|
|
return {
|
|
$schema: "https://hwlab.pikastech.local/schemas/dev-cd-apply.schema.json",
|
|
$id: "https://hwlab.pikastech.local/reports/dev-gate/dev-cd-apply.json",
|
|
reportVersion: "v1",
|
|
issue: "pikasTech/HWLAB#274",
|
|
taskId: "dev-cd-apply",
|
|
commitId: target.shortCommitId,
|
|
acceptanceLevel: "dev_cd_apply",
|
|
devOnly: true,
|
|
prodDisabled: true,
|
|
status,
|
|
generatedAt: finishedAt,
|
|
reportLifecycle: activeReportLifecycle("Current DEV CD transaction report; the only DEV side-effect entrypoint is scripts/dev-cd-apply.mjs."),
|
|
supports: [
|
|
"pikasTech/HWLAB#274",
|
|
"pikasTech/HWLAB#116",
|
|
"pikasTech/HWLAB#235",
|
|
"pikasTech/HWLAB#50",
|
|
"pikasTech/HWLAB#57",
|
|
"pikasTech/HWLAB#67",
|
|
"pikasTech/HWLAB#73"
|
|
],
|
|
sourceContract: {
|
|
status: blockers.length ? "blocked" : "pass",
|
|
documents: [
|
|
"docs/reference/deployment-publish.md",
|
|
"docs/reference/dev-runtime-boundary.md",
|
|
"docs/dev-artifact-publish.md",
|
|
"docs/dev-deploy-apply.md",
|
|
"docs/artifact-catalog.md"
|
|
],
|
|
summary: "DEV CD publish/apply/verify side effects are serialized by one Kubernetes Lease and use deploy/deploy.json as the apply desired-state source."
|
|
},
|
|
validationCommands: [
|
|
"node --check scripts/dev-cd-apply.mjs",
|
|
"node --check scripts/src/dev-cd-apply.mjs",
|
|
"node --test scripts/src/dev-cd-apply.test.mjs",
|
|
"node --check scripts/dev-artifact-publish.mjs",
|
|
"node --check scripts/src/dev-deploy-apply.mjs",
|
|
"node --test scripts/src/dev-deploy-apply.test.mjs",
|
|
"node scripts/deploy-desired-state-plan.mjs --check",
|
|
"node scripts/validate-artifact-catalog.mjs",
|
|
"git diff --check"
|
|
],
|
|
localSmoke: {
|
|
status: "not_run",
|
|
commands: ["npm run check"],
|
|
evidence: ["Local checks are listed as validation commands and are not implied by a DEV CD transaction."],
|
|
summary: "The transaction report records DEV publish/apply/verify evidence; local smoke is a separate gate."
|
|
},
|
|
dryRun: {
|
|
status: "not_run",
|
|
commands: ["node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked --write-report"],
|
|
evidence: ["The transaction path owns live side effects; dry-run remains a preflight support mode."],
|
|
summary: "DEV CD mutation requires --apply and the transaction Lease lock."
|
|
},
|
|
devPreconditions: {
|
|
status: blockers.length ? "blocked" : "pass",
|
|
requirements: [
|
|
"HEAD must match the requested target ref before publish.",
|
|
"Lease/hwlab-dev-cd-lock must be acquired before publish/apply.",
|
|
"Legacy publish/apply side-effect scripts must run with HWLAB_CD_TRANSACTION_ID.",
|
|
"deploy/deploy.json, artifact catalog, and workloads must converge before apply.",
|
|
"Public 16666 and 16667 live health must be recorded in this report."
|
|
],
|
|
summary: blockers.length ? "One or more DEV CD transaction preconditions failed." : "DEV CD transaction preconditions passed."
|
|
},
|
|
transaction: {
|
|
transactionId: transaction.transactionId,
|
|
ownerTaskId: transaction.ownerTaskId,
|
|
targetNamespace: transaction.targetNamespace,
|
|
lockName: transaction.lockName,
|
|
lockBackend: "Lease",
|
|
startedAt,
|
|
finishedAt,
|
|
ttlSeconds: args.ttlSeconds,
|
|
phases: transaction.phases,
|
|
release
|
|
},
|
|
devCdApply: {
|
|
mode: args.apply ? "apply" : "dry-run",
|
|
target: {
|
|
ref: target.ref,
|
|
promotionCommit: target.commitId,
|
|
shortCommitId: target.shortCommitId,
|
|
headCommitId: target.headCommitId,
|
|
headMatchesTarget: target.headMatchesTarget,
|
|
namespace: args.targetNamespace,
|
|
apiEndpoint: DEV_ENDPOINT,
|
|
browserEndpoint: "http://74.48.78.17:16666/"
|
|
},
|
|
deployJson: {
|
|
before: deployBefore,
|
|
after: deployAfter
|
|
},
|
|
lock: {
|
|
acquired: Boolean(lockState?.acquired),
|
|
current: lockState?.lock ?? null,
|
|
staleBreak: lockState?.staleBreak ?? null
|
|
},
|
|
steps,
|
|
liveBefore,
|
|
liveVerify,
|
|
reportPaths: {
|
|
transaction: args.writeReport ? args.reportPath : null,
|
|
artifacts: artifactReportPath,
|
|
deployApply: "reports/dev-gate/dev-deploy-report.json"
|
|
},
|
|
safety: {
|
|
prodTouched: false,
|
|
secretValuesRead: false,
|
|
secretValuesPrinted: false,
|
|
manualDbWrite: false,
|
|
oldPublicPortsUsed: false,
|
|
mutationAttempted: steps.some((step) => ["artifact-publish", "dev-deploy-apply"].includes(step.id)),
|
|
singleLock: "Lease/hwlab-dev/hwlab-dev-cd-lock"
|
|
}
|
|
},
|
|
blockers,
|
|
notes: "DEV-only CD transaction evidence. This does not claim M3 PASS or runtime durability."
|
|
};
|
|
}
|
|
|
|
function buildArgumentFailure(error) {
|
|
const blockers = error.blockers ?? [{
|
|
type: "safety_blocker",
|
|
scope: error.code ?? "dev-cd-apply",
|
|
status: "open",
|
|
summary: oneLine(error.message)
|
|
}];
|
|
return {
|
|
ok: false,
|
|
status: "blocked",
|
|
error: error.code ?? "dev-cd-apply-failed",
|
|
blockers,
|
|
mutationAttempted: false,
|
|
prodTouched: false
|
|
};
|
|
}
|
|
|
|
export function formatDevCdApplyFailure(error) {
|
|
if (error?.lockFailure) return error.lockFailure;
|
|
if (error instanceof DevCdApplyError) return buildArgumentFailure(error);
|
|
return {
|
|
ok: false,
|
|
status: "failed",
|
|
error: "dev-cd-apply-failed",
|
|
summary: error instanceof Error ? error.message : String(error),
|
|
mutationAttempted: false,
|
|
prodTouched: false
|
|
};
|
|
}
|
|
|
|
export async function runDevCdApply(argv, io = {}) {
|
|
const stdout = io.stdout ?? process.stdout;
|
|
const env = io.env ?? process.env;
|
|
const ctx = {
|
|
repoRoot: io.repoRoot ?? defaultRepoRoot,
|
|
env,
|
|
runCommand: io.runCommand ?? defaultRunCommand,
|
|
httpGetJson: io.httpGetJson ?? defaultHttpGetJson,
|
|
now: io.now ?? (() => new Date())
|
|
};
|
|
|
|
let args;
|
|
try {
|
|
args = parseArgs(argv);
|
|
if (args.help) {
|
|
stdout.write(`${usage()}\n`);
|
|
return 0;
|
|
}
|
|
validateArgs(args);
|
|
} catch (error) {
|
|
const failure = formatDevCdApplyFailure(error);
|
|
stdout.write(`${JSON.stringify(failure, null, 2)}\n`);
|
|
return 2;
|
|
}
|
|
|
|
const startedAt = ctx.now().toISOString();
|
|
const transaction = {
|
|
transactionId: randomUUID(),
|
|
ownerTaskId: args.ownerTaskId ?? defaultOwnerTaskId(env),
|
|
targetNamespace: args.targetNamespace,
|
|
lockName: args.lockName,
|
|
phases: []
|
|
};
|
|
const steps = [];
|
|
const blockers = [];
|
|
let lockState = null;
|
|
let release = null;
|
|
let target = null;
|
|
let deployBefore = null;
|
|
let deployAfter = null;
|
|
let liveBefore = null;
|
|
let liveVerify = null;
|
|
let status = "blocked";
|
|
|
|
try {
|
|
target = await resolveTargetRef(ctx, args.targetRef);
|
|
deployBefore = await readDeployJson(ctx.repoRoot);
|
|
const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand);
|
|
liveBefore = {
|
|
status: "pending",
|
|
reason: "liveBefore is captured immediately after Lease acquisition so publish/apply cannot race before the transaction lock."
|
|
};
|
|
lockState = await acquireDeployLock({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
target,
|
|
deployBefore,
|
|
transactionId: transaction.transactionId,
|
|
ownerTaskId: transaction.ownerTaskId,
|
|
liveBefore,
|
|
now: ctx.now()
|
|
});
|
|
liveBefore = await verifyDevLive(ctx, { expectedCommit: null });
|
|
await updateDeployLockLiveBefore({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId: transaction.transactionId,
|
|
lockState,
|
|
liveBefore
|
|
});
|
|
transaction.phases.push({ phase: "publishing", status: "entered", at: ctx.now().toISOString() });
|
|
|
|
if (!target.headMatchesTarget) {
|
|
blockers.push({
|
|
type: "contract_blocker",
|
|
scope: "target-ref-head",
|
|
status: "open",
|
|
summary: `HEAD ${target.headShortCommitId} must match target ref ${target.ref} ${target.shortCommitId} before DEV CD publish.`
|
|
});
|
|
}
|
|
|
|
const plannedSteps = [
|
|
{
|
|
id: "desired-state-check-before-publish",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: ["scripts/deploy-desired-state-plan.mjs", "--check", "--pretty"],
|
|
timeoutMs: 120000
|
|
},
|
|
{
|
|
id: "artifact-publish",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: ["scripts/dev-artifact-publish.mjs", "--publish"],
|
|
timeoutMs: 60 * 60 * 1000,
|
|
reportPath: artifactReportPath
|
|
},
|
|
{
|
|
id: "artifact-catalog-refresh",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: [
|
|
"scripts/refresh-artifact-catalog.mjs",
|
|
"--target-ref",
|
|
target.ref,
|
|
"--publish-report",
|
|
artifactReportPath
|
|
],
|
|
timeoutMs: 120000
|
|
},
|
|
{
|
|
id: "artifact-catalog-validate",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: ["scripts/validate-artifact-catalog.mjs"],
|
|
timeoutMs: 120000
|
|
},
|
|
{
|
|
id: "desired-state-check-after-refresh",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: [
|
|
"scripts/deploy-desired-state-plan.mjs",
|
|
"--promotion-commit",
|
|
target.commitId,
|
|
"--check",
|
|
"--pretty"
|
|
],
|
|
timeoutMs: 120000
|
|
},
|
|
{
|
|
id: "dev-deploy-apply",
|
|
phase: "applying",
|
|
command: process.execPath,
|
|
args: [
|
|
"scripts/dev-deploy-apply.mjs",
|
|
"--apply",
|
|
"--confirm-dev",
|
|
"--confirmed-non-production",
|
|
"--write-report",
|
|
...(args.kubeconfig ? ["--kubeconfig", args.kubeconfig] : [])
|
|
],
|
|
timeoutMs: 20 * 60 * 1000,
|
|
reportPath: "reports/dev-gate/dev-deploy-report.json"
|
|
}
|
|
];
|
|
|
|
if (blockers.length === 0) {
|
|
for (const step of plannedSteps) {
|
|
if (step.phase === "applying" && lockState.lock?.phase !== "applying") {
|
|
await updateDeployLockPhase({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId: transaction.transactionId,
|
|
lockState,
|
|
phase: "applying"
|
|
});
|
|
transaction.phases.push({ phase: "applying", status: "entered", at: ctx.now().toISOString() });
|
|
}
|
|
const result = await runStep(ctx, transaction, step);
|
|
steps.push(result);
|
|
if (result.status !== "pass") {
|
|
blockers.push(stepBlocker(result));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (blockers.length === 0) {
|
|
await updateDeployLockPhase({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId: transaction.transactionId,
|
|
lockState,
|
|
phase: "verifying"
|
|
});
|
|
transaction.phases.push({ phase: "verifying", status: "entered", at: ctx.now().toISOString() });
|
|
deployAfter = await readDeployJson(ctx.repoRoot);
|
|
liveVerify = args.skipLiveVerify
|
|
? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } }
|
|
: await verifyDevLive(ctx, { expectedCommit: target.shortCommitId });
|
|
if (liveVerify.status !== "pass") {
|
|
blockers.push({
|
|
type: "observability_blocker",
|
|
scope: "live-verify",
|
|
status: "open",
|
|
summary: "16666/16667 live health did not match the DEV CD promotion commit."
|
|
});
|
|
}
|
|
} else {
|
|
deployAfter = deployBefore;
|
|
liveVerify = { status: "not_run", reason: "transaction blocked before live verify", endpoints: [], summary: { checked: 0 } };
|
|
}
|
|
|
|
status = blockers.length === 0 ? "pass" : "blocked";
|
|
release = await releaseDeployLock({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId: transaction.transactionId,
|
|
status
|
|
});
|
|
transaction.phases.push({ phase: "released", status: release.status, at: ctx.now().toISOString() });
|
|
} catch (error) {
|
|
if (error?.lockFailure) {
|
|
stdout.write(`${JSON.stringify(error.lockFailure, null, 2)}\n`);
|
|
return 2;
|
|
}
|
|
blockers.push({
|
|
type: "runtime_blocker",
|
|
scope: error.code ?? "dev-cd-apply",
|
|
status: "open",
|
|
summary: oneLine(error.message)
|
|
});
|
|
status = "blocked";
|
|
if (lockState?.acquired) {
|
|
const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand);
|
|
release = await releaseDeployLock({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId: transaction.transactionId,
|
|
status
|
|
});
|
|
transaction.phases.push({ phase: "released", status: release.status, at: ctx.now().toISOString() });
|
|
}
|
|
}
|
|
|
|
const finishedAt = ctx.now().toISOString();
|
|
const report = buildReport({
|
|
args,
|
|
transaction,
|
|
target: target ?? {
|
|
ref: args.targetRef,
|
|
commitId: "unknown",
|
|
shortCommitId: "unknown",
|
|
headCommitId: "unknown",
|
|
headMatchesTarget: false
|
|
},
|
|
deployBefore: deployBefore ?? { path: "deploy/deploy.json", hash: "unknown", commitId: "unknown" },
|
|
deployAfter: deployAfter ?? deployBefore ?? { path: "deploy/deploy.json", hash: "unknown", commitId: "unknown" },
|
|
lockState,
|
|
steps,
|
|
blockers,
|
|
status,
|
|
startedAt,
|
|
finishedAt,
|
|
liveBefore: liveBefore ?? { status: "not_run" },
|
|
liveVerify: liveVerify ?? { status: "not_run", endpoints: [], summary: { checked: 0 } },
|
|
release
|
|
});
|
|
|
|
if (args.writeReport) {
|
|
const absoluteReportPath = path.resolve(ctx.repoRoot, args.reportPath);
|
|
await mkdir(path.dirname(absoluteReportPath), { recursive: true });
|
|
await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
}
|
|
|
|
stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
return status === "pass" ? 0 : 2;
|
|
}
|