import { spawn } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { request as httpRequest } from "node:http"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs"; import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; import { buildKubectlCommandPrefix, redactSensitiveText, resolveDevKubeconfigSelection } from "./dev-deploy-apply.mjs"; const defaultRepoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const defaultNamespace = "hwlab-dev"; const defaultLockName = "hwlab-dev-cd-lock"; const defaultTtlSeconds = 3600; const defaultReportPath = "reports/dev-gate/dev-cd-apply.json"; const artifactReportPath = "reports/dev-gate/dev-artifacts.json"; const browserLiveUrl = "http://74.48.78.17:16666/health/live"; const apiLiveUrl = `${DEV_ENDPOINT}/health/live`; const lockAnnotationPrefix = "hwlab.pikastech.local"; const lockAnnotationFields = { promotionCommit: `${lockAnnotationPrefix}/promotionCommit`, deployJsonHash: `${lockAnnotationPrefix}/deployJsonHash`, ownerTaskId: `${lockAnnotationPrefix}/ownerTaskId`, transactionId: `${lockAnnotationPrefix}/transactionId`, phase: `${lockAnnotationPrefix}/phase`, startedAt: `${lockAnnotationPrefix}/startedAt`, updatedAt: `${lockAnnotationPrefix}/updatedAt`, ttlSeconds: `${lockAnnotationPrefix}/ttlSeconds`, liveBefore: `${lockAnnotationPrefix}/liveBefore`, targetNamespace: `${lockAnnotationPrefix}/targetNamespace`, targetRef: `${lockAnnotationPrefix}/targetRef`, lockBackend: `${lockAnnotationPrefix}/lockBackend`, releasedAt: `${lockAnnotationPrefix}/releasedAt`, releaseStatus: `${lockAnnotationPrefix}/releaseStatus`, staleLockBrokenAt: `${lockAnnotationPrefix}/staleLockBrokenAt`, staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious` }; class DevCdApplyError extends Error { constructor(message, details = {}) { super(message); this.name = "DevCdApplyError"; Object.assign(this, details); } } export function parseArgs(argv) { const args = { apply: false, confirmDev: false, confirmedNonProduction: false, writeReport: false, reportPath: defaultReportPath, targetRef: "origin/main", kubeconfig: null, kubeconfigSpecified: false, lockName: defaultLockName, targetNamespace: defaultNamespace, ttlSeconds: defaultTtlSeconds, ownerTaskId: null, breakStaleLock: false, skipLiveVerify: false, help: false, flags: new Set() }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === "--apply") { args.apply = true; args.flags.add(arg); } else if (arg === "--confirm-dev") { args.confirmDev = true; args.flags.add(arg); } else if (arg === "--confirmed-non-production") { args.confirmedNonProduction = true; args.flags.add(arg); } else if (arg === "--write-report") { args.writeReport = true; args.flags.add(arg); } else if (arg === "--break-stale-lock") { args.breakStaleLock = true; args.flags.add(arg); } else if (arg === "--skip-live-verify") { args.skipLiveVerify = true; args.flags.add(arg); } else if (arg === "--report") { args.reportPath = readOption(argv, ++index, arg); args.flags.add(arg); } else if (arg === "--target-ref") { args.targetRef = readOption(argv, ++index, arg); args.flags.add(arg); } else if (arg === "--kubeconfig") { args.kubeconfig = readOption(argv, ++index, arg); args.kubeconfigSpecified = true; args.flags.add(arg); } else if (arg.startsWith("--kubeconfig=")) { args.kubeconfig = arg.slice("--kubeconfig=".length); args.kubeconfigSpecified = true; args.flags.add("--kubeconfig"); } else if (arg === "--lock-name") { args.lockName = readOption(argv, ++index, arg); args.flags.add(arg); } else if (arg === "--target-namespace") { args.targetNamespace = readOption(argv, ++index, arg); args.flags.add(arg); } else if (arg === "--ttl-seconds") { args.ttlSeconds = Number.parseInt(readOption(argv, ++index, arg), 10); args.flags.add(arg); } else if (arg === "--owner-task-id") { args.ownerTaskId = readOption(argv, ++index, arg); args.flags.add(arg); } else if (arg === "--help" || arg === "-h") { args.help = true; } else { throw new Error(`unknown argument ${arg}`); } } return args; } function readOption(argv, index, name) { const value = argv[index]; if (!value || value.startsWith("--")) { throw new Error(`${name} requires a value`); } return value; } export function usage() { return [ "usage: node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production [--write-report]", "", "Single DEV CD transaction for latest-main publish, desired-state refresh, apply, and live verify.", "", "options:", " --target-ref REF git ref to publish/apply; default: origin/main", " --kubeconfig PATH DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG", " --ttl-seconds SECONDS lock TTL; default: 3600", " --owner-task-id ID lock holder; default: Code Queue/env/user derived", " --break-stale-lock may take over an expired lock only with --confirm-dev", " --report PATH default: reports/dev-gate/dev-cd-apply.json", " --skip-live-verify test-only escape hatch; do not use for DEV acceptance" ].join("\n"); } function nowIso() { return new Date().toISOString(); } function kubernetesMicroTime(value) { const date = value instanceof Date ? value : new Date(value); if (!Number.isFinite(date.getTime())) { throw new DevCdApplyError("invalid Kubernetes Lease timestamp", { code: "invalid-lease-timestamp" }); } return date.toISOString().replace(/\.(\d{3})Z$/u, ".$1000Z"); } function parseJsonMaybe(value) { try { return JSON.parse(value); } catch { return null; } } function oneLine(value) { return String(value ?? "").replace(/\s+/g, " ").trim(); } function sha256(value) { return `sha256:${createHash("sha256").update(value).digest("hex")}`; } function shortCommit(value) { return typeof value === "string" && value.length >= 7 ? value.slice(0, 7) : value; } function commitMatches(actual, expected) { if (typeof actual !== "string" || typeof expected !== "string") return false; return actual === expected || actual.startsWith(expected) || expected.startsWith(actual); } function shellCommand(command, args) { return [command, ...args].map((part) => (/\s/u.test(part) ? JSON.stringify(part) : part)).join(" "); } function defaultOwnerTaskId(env = process.env) { return ( env.CODE_QUEUE_TASK_ID || env.HWLAB_OWNER_TASK_ID || env.GITHUB_RUN_ID || env.USER || "unknown-dev-cd-runner" ); } function defaultRunCommand(command, args, options = {}) { return new Promise((resolve) => { const child = spawn(command, args, { cwd: options.cwd ?? defaultRepoRoot, env: options.env ?? process.env, stdio: ["pipe", "pipe", "pipe"] }); let stdout = ""; let stderr = ""; const timeout = options.timeoutMs ? setTimeout(() => child.kill("SIGTERM"), options.timeoutMs) : null; child.stdin.on("error", () => {}); child.stdin.end(options.input ?? ""); child.stdout.on("data", (chunk) => { stdout += chunk; }); child.stderr.on("data", (chunk) => { stderr += chunk; }); child.on("error", (error) => { if (timeout) clearTimeout(timeout); resolve({ code: 127, signal: null, stdout, stderr: error.message }); }); child.on("close", (code, signal) => { if (timeout) clearTimeout(timeout); resolve({ code: code ?? 1, signal, stdout, stderr }); }); }); } async function readDeployJson(repoRoot) { const relativePath = "deploy/deploy.json"; const raw = await readFile(path.join(repoRoot, relativePath), "utf8"); const manifest = JSON.parse(raw); return { path: relativePath, hash: sha256(raw), commitId: manifest.commitId ?? "unknown", namespace: manifest.namespace ?? defaultNamespace, environment: manifest.environment ?? "unknown", endpoint: manifest.endpoint ?? null, manifest }; } async function resolveTargetRef(ctx, targetRef) { const target = await ctx.runCommand("git", ["rev-parse", "--verify", `${targetRef}^{commit}`], { cwd: ctx.repoRoot, timeoutMs: 10000 }); if (target.code !== 0) { throw new DevCdApplyError(`target ref ${targetRef} could not be resolved`, { code: "target-ref-unresolved", targetRef, stderr: redactSensitiveText(target.stderr) }); } const head = await ctx.runCommand("git", ["rev-parse", "--verify", "HEAD^{commit}"], { cwd: ctx.repoRoot, timeoutMs: 10000 }); if (head.code !== 0) { throw new DevCdApplyError("HEAD could not be resolved", { code: "head-unresolved", stderr: redactSensitiveText(head.stderr) }); } const commitId = target.stdout.trim(); const headCommitId = head.stdout.trim(); return { ref: targetRef, commitId, shortCommitId: shortCommit(commitId), headCommitId, headShortCommitId: shortCommit(headCommitId), headMatchesTarget: commitId === headCommitId }; } function validateArgs(args) { const blockers = []; if (!args.apply) { blockers.push({ type: "safety_blocker", scope: "apply-mode", status: "open", summary: "DEV CD transaction side effects require --apply." }); } if (args.apply && (!args.confirmDev || !args.confirmedNonProduction)) { blockers.push({ type: "safety_blocker", scope: "dev-confirmation", status: "open", summary: "DEV CD apply requires --confirm-dev and --confirmed-non-production." }); } if (args.breakStaleLock && !args.confirmDev) { blockers.push({ type: "safety_blocker", scope: "break-stale-lock-confirmation", status: "open", summary: "Breaking a stale DEV CD lock requires --break-stale-lock and --confirm-dev." }); } if (!Number.isInteger(args.ttlSeconds) || args.ttlSeconds < 60 || args.ttlSeconds > 24 * 60 * 60) { blockers.push({ type: "safety_blocker", scope: "lock-ttl", status: "open", summary: "--ttl-seconds must be an integer between 60 and 86400." }); } for (const forbidden of ["--prod", "--production", "--read-secret", "--force-push"]) { if (args.flags.has(forbidden)) { blockers.push({ type: "safety_blocker", scope: forbidden.slice(2), status: "open", summary: `${forbidden} is forbidden for DEV CD apply.` }); } } if (blockers.length > 0) { throw new DevCdApplyError("invalid DEV CD apply arguments", { code: "invalid-arguments", blockers }); } } function lockAnnotationValue(value) { if (value === undefined || value === null) return ""; if (typeof value === "string") return value; return JSON.stringify(value); } export function buildLockAnnotations(lock) { const annotations = {}; for (const [field, key] of Object.entries(lockAnnotationFields)) { if (Object.hasOwn(lock, field)) { annotations[key] = lockAnnotationValue(lock[field]); } } return annotations; } function readAnnotation(annotations, field) { return annotations?.[lockAnnotationFields[field]] ?? null; } function readJsonAnnotation(annotations, field) { const value = readAnnotation(annotations, field); return value ? parseJsonMaybe(value) : null; } export function parseDeployLock(lease) { if (!lease) return null; const annotations = lease.metadata?.annotations ?? {}; const ttlSeconds = Number.parseInt( readAnnotation(annotations, "ttlSeconds") || lease.spec?.leaseDurationSeconds || `${defaultTtlSeconds}`, 10 ); const updatedAt = readAnnotation(annotations, "updatedAt") || lease.spec?.renewTime || lease.metadata?.creationTimestamp || null; const transactionId = readAnnotation(annotations, "transactionId") || lease.spec?.holderIdentity || null; const ownerTaskId = readAnnotation(annotations, "ownerTaskId") || lease.spec?.holderIdentity || null; return { lockBackend: readAnnotation(annotations, "lockBackend") || "Lease", lockName: lease.metadata?.name ?? defaultLockName, resourceVersion: lease.metadata?.resourceVersion ?? null, holderIdentity: lease.spec?.holderIdentity ?? null, promotionCommit: readAnnotation(annotations, "promotionCommit"), deployJsonHash: readAnnotation(annotations, "deployJsonHash"), ownerTaskId, transactionId, phase: readAnnotation(annotations, "phase") || "unknown", startedAt: readAnnotation(annotations, "startedAt") || lease.spec?.acquireTime || lease.metadata?.creationTimestamp || null, updatedAt, ttlSeconds: Number.isInteger(ttlSeconds) ? ttlSeconds : defaultTtlSeconds, liveBefore: readJsonAnnotation(annotations, "liveBefore"), targetNamespace: readAnnotation(annotations, "targetNamespace") || lease.metadata?.namespace || defaultNamespace, targetRef: readAnnotation(annotations, "targetRef"), releasedAt: readAnnotation(annotations, "releasedAt"), releaseStatus: readAnnotation(annotations, "releaseStatus"), staleLockBrokenAt: readAnnotation(annotations, "staleLockBrokenAt"), staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious") }; } export function classifyDeployLock(lock, now = new Date()) { if (!lock || lock.phase === "released") { return { held: false, stale: false, retryAfterSeconds: 0, expiresAt: null }; } const updatedAtMs = Number.isFinite(Date.parse(lock.updatedAt)) ? Date.parse(lock.updatedAt) : 0; const expiresAtMs = updatedAtMs + lock.ttlSeconds * 1000; const retryAfterSeconds = Math.max(0, Math.ceil((expiresAtMs - now.getTime()) / 1000)); return { held: retryAfterSeconds > 0, stale: retryAfterSeconds <= 0, retryAfterSeconds, expiresAt: Number.isFinite(expiresAtMs) ? new Date(expiresAtMs).toISOString() : null }; } export function deployLockHeldFailure(lock, now = new Date()) { const classification = classifyDeployLock(lock, now); return { ok: false, status: "blocked", error: "deploy-lock-held", code: "deploy-lock-held", holder: lock.ownerTaskId || lock.transactionId || lock.holderIdentity || "unknown", promotionCommit: lock.promotionCommit ?? null, deployJsonHash: lock.deployJsonHash ?? null, transactionId: lock.transactionId ?? null, phase: lock.phase ?? "unknown", targetNamespace: lock.targetNamespace ?? defaultNamespace, retryAfterSeconds: classification.retryAfterSeconds, stale: classification.stale, requiresBreakStaleLock: classification.stale, lockName: lock.lockName ?? defaultLockName, lockBackend: lock.lockBackend ?? "Lease", summary: classification.stale ? "A stale DEV CD transaction lock exists; rerun with --break-stale-lock --confirm-dev only after confirming the holder is inactive." : "Another DEV CD transaction is already running; retry after the current holder releases the lock.", mutationAttempted: false, prodTouched: false }; } function isNotFound(result) { return /not\s*found|notfound/i.test(`${result.stderr}\n${result.stdout}`); } function isAlreadyExists(result) { return /already\s*exists|alreadyexists/i.test(`${result.stderr}\n${result.stdout}`); } function isConflict(result) { return /conflict|object has been modified|operation cannot be fulfilled/i.test(`${result.stderr}\n${result.stdout}`); } function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null }) { const lock = { promotionCommit: target.shortCommitId, deployJsonHash: deployBefore.hash, ownerTaskId, transactionId, phase, startedAt, updatedAt: startedAt, ttlSeconds: args.ttlSeconds, liveBefore, targetNamespace: args.targetNamespace, targetRef: target.ref, lockBackend: "Lease" }; if (staleLockPrevious) { lock.staleLockBrokenAt = startedAt; lock.staleLockPrevious = staleLockPrevious; } return { apiVersion: "coordination.k8s.io/v1", kind: "Lease", metadata: { name: args.lockName, namespace: args.targetNamespace, labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/profile": "dev", "hwlab.pikastech.local/cd-lock": "true" }, annotations: buildLockAnnotations(lock) }, spec: { holderIdentity: `${ownerTaskId}/${transactionId}`, leaseDurationSeconds: args.ttlSeconds, acquireTime: startedAt, renewTime: startedAt } }; } function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null }) { const lock = { promotionCommit: target.shortCommitId, deployJsonHash: deployBefore.hash, ownerTaskId, transactionId, phase, startedAt, updatedAt, ttlSeconds: args.ttlSeconds, liveBefore, targetNamespace: args.targetNamespace, targetRef: target.ref, lockBackend: "Lease" }; if (staleLockPrevious) { lock.staleLockBrokenAt = updatedAt; lock.staleLockPrevious = staleLockPrevious; } return { metadata: { labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/profile": "dev", "hwlab.pikastech.local/cd-lock": "true" }, annotations: buildLockAnnotations(lock) }, spec: { holderIdentity: `${ownerTaskId}/${transactionId}`, leaseDurationSeconds: args.ttlSeconds, acquireTime: startedAt, renewTime: updatedAt } }; } async function kubectl(ctx, kubectlContext, args, options = {}) { const result = await ctx.runCommand(kubectlContext.executor, args, { cwd: ctx.repoRoot, env: kubectlContext.env, input: options.input, timeoutMs: options.timeoutMs ?? 30000 }); return { ...result, command: shellCommand("kubectl", args), stdout: result.stdout ?? "", stderr: result.stderr ?? "" }; } async function replaceLease(ctx, kubectlContext, args, lease, timeoutMs = 30000) { return kubectl( ctx, kubectlContext, ["-n", args.targetNamespace, "replace", "-f", "-", "-o", "json"], { input: JSON.stringify(lease), timeoutMs } ); } function mergeLease(lease, patch, { resetLockAnnotations = false } = {}) { const baseAnnotations = { ...(lease.metadata?.annotations ?? {}) }; if (resetLockAnnotations) { for (const key of Object.values(lockAnnotationFields)) { delete baseAnnotations[key]; } } return { ...lease, metadata: { ...lease.metadata, ...(patch.metadata ?? {}), labels: { ...(lease.metadata?.labels ?? {}), ...(patch.metadata?.labels ?? {}) }, annotations: { ...baseAnnotations, ...(patch.metadata?.annotations ?? {}) } }, spec: { ...(lease.spec ?? {}), ...(patch.spec ?? {}) } }; } async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now }) { const startedAt = kubernetesMicroTime(now); const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"]); if (get.code === 0) { const lease = JSON.parse(get.stdout); const existing = parseDeployLock(lease); const classification = classifyDeployLock(existing, now); if (classification.held || (classification.stale && !args.breakStaleLock)) { const failure = deployLockHeldFailure(existing, now); throw new DevCdApplyError(failure.summary, { code: "deploy-lock-held", lockFailure: failure }); } const patch = lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase: "publishing", startedAt, updatedAt: startedAt, liveBefore, staleLockPrevious: classification.stale ? existing : null }); const replaced = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch, { resetLockAnnotations: true })); if (isConflict(replaced)) { return acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now }); } if (replaced.code !== 0) { throw new DevCdApplyError("failed to acquire DEV CD Lease lock", { code: "lock-acquire-failed", stderr: redactSensitiveText(replaced.stderr || replaced.stdout) }); } return { acquired: true, lock: parseDeployLock(JSON.parse(replaced.stdout)), staleBreak: classification.stale ? existing : null, command: replaced.command }; } if (!isNotFound(get)) { throw new DevCdApplyError("failed to read DEV CD Lease lock", { code: "lock-read-failed", stderr: redactSensitiveText(get.stderr || get.stdout) }); } const manifest = buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase: "publishing", liveBefore }); const created = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "create", "-f", "-", "-o", "json"], { input: JSON.stringify(manifest), timeoutMs: 30000 }); if (created.code === 0) { return { acquired: true, lock: parseDeployLock(JSON.parse(created.stdout)), staleBreak: null, command: created.command }; } if (isAlreadyExists(created)) { return acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now }); } throw new DevCdApplyError("failed to create DEV CD Lease lock", { code: "lock-create-failed", stderr: redactSensitiveText(created.stderr || created.stdout) }); } async function readOwnedLease({ ctx, args, kubectlContext, transactionId, timeoutMs = 15000 }) { const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], { timeoutMs }); if (get.code !== 0) { throw new DevCdApplyError("failed to read DEV CD lock ownership", { code: "lock-owner-read-failed", stderr: redactSensitiveText(get.stderr || get.stdout) }); } const lease = JSON.parse(get.stdout); const lock = parseDeployLock(lease); if (lock.transactionId !== transactionId) { throw new DevCdApplyError("DEV CD lock ownership changed before transaction update", { code: "lock-owner-mismatch", holder: lock.ownerTaskId || lock.transactionId || "unknown", phase: lock.phase }); } return { lease, lock }; } async function patchOwnedLease({ ctx, args, kubectlContext, lockState, transactionId, patch, errorCode, errorMessage }) { const { lease } = await readOwnedLease({ ctx, args, kubectlContext, transactionId }); const result = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch)); if (result.code !== 0) { throw new DevCdApplyError(errorMessage, { code: errorCode, stderr: redactSensitiveText(result.stderr || result.stdout) }); } lockState.lock = parseDeployLock(JSON.parse(result.stdout)); return lockState.lock; } async function updateDeployLockPhase({ ctx, args, kubectlContext, transactionId, lockState, phase, status = "running" }) { const updatedAt = kubernetesMicroTime(ctx.now()); const patch = { metadata: { annotations: { [lockAnnotationFields.phase]: phase, [lockAnnotationFields.updatedAt]: updatedAt, [lockAnnotationFields.releaseStatus]: status } }, spec: { renewTime: updatedAt } }; return await patchOwnedLease({ ctx, args, kubectlContext, transactionId, lockState, patch, errorCode: "lock-phase-update-failed", errorMessage: `failed to update DEV CD lock phase to ${phase}` }); } async function updateDeployLockLiveBefore({ ctx, args, kubectlContext, transactionId, lockState, liveBefore }) { const updatedAt = kubernetesMicroTime(ctx.now()); const patch = { metadata: { annotations: { [lockAnnotationFields.liveBefore]: JSON.stringify(liveBefore), [lockAnnotationFields.updatedAt]: updatedAt } }, spec: { renewTime: updatedAt } }; return await patchOwnedLease({ ctx, args, kubectlContext, transactionId, lockState, patch, errorCode: "lock-live-before-update-failed", errorMessage: "failed to update DEV CD lock liveBefore evidence" }); } async function releaseDeployLock({ ctx, args, kubectlContext, transactionId, status }) { const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], { timeoutMs: 15000 }); if (get.code !== 0) { return { status: "release_read_failed", command: get.command, error: redactSensitiveText(get.stderr || get.stdout) }; } const current = parseDeployLock(JSON.parse(get.stdout)); if (current.transactionId !== transactionId) { return { status: "not_owner", holder: current.ownerTaskId || current.transactionId || "unknown", phase: current.phase }; } const releasedAt = kubernetesMicroTime(ctx.now()); const patch = { metadata: { annotations: { [lockAnnotationFields.phase]: "released", [lockAnnotationFields.updatedAt]: releasedAt, [lockAnnotationFields.releasedAt]: releasedAt, [lockAnnotationFields.releaseStatus]: status } }, spec: { holderIdentity: "", leaseDurationSeconds: 1, renewTime: releasedAt } }; const result = await replaceLease(ctx, kubectlContext, args, mergeLease(JSON.parse(get.stdout), patch), 15000); if (isConflict(result)) { return { status: "release_conflict", command: result.command, error: "Lease changed before release; lock was not overwritten." }; } if (result.code !== 0) { return { status: "release_failed", command: result.command, error: redactSensitiveText(result.stderr || result.stdout) }; } return { status: "released", command: result.command, lock: parseDeployLock(JSON.parse(result.stdout)) }; } async function resolveKubectlContext(args, env, runCommand) { const selection = resolveDevKubeconfigSelection({ flagValue: args.kubeconfig, env }); const which = await runCommand("which", ["kubectl"], { timeoutMs: 5000 }); const executor = which.code === 0 ? which.stdout.trim() : "kubectl"; return { executor, kubeconfig: selection.kubeconfig, kubeconfigSource: selection.source, commandPrefix: buildKubectlCommandPrefix(selection.kubeconfig), env: { ...env, KUBECONFIG: selection.kubeconfig } }; } function commandEnv(ctx, transaction) { return { ...ctx.env, HWLAB_CD_TRANSACTION_ID: transaction.transactionId, HWLAB_CD_TRANSACTION_OWNER: transaction.ownerTaskId, HWLAB_CD_LOCK_NAME: transaction.lockName, HWLAB_CD_LOCK_NAMESPACE: transaction.targetNamespace }; } async function runStep(ctx, transaction, step) { const startedAt = ctx.now().toISOString(); const command = shellCommand(step.command, step.args); const result = await ctx.runCommand(step.command, step.args, { cwd: ctx.repoRoot, env: commandEnv(ctx, transaction), timeoutMs: step.timeoutMs ?? 300000 }); const finishedAt = ctx.now().toISOString(); const stdout = redactSensitiveText(result.stdout ?? ""); const stderr = redactSensitiveText(result.stderr ?? ""); const parsedStdout = parseJsonMaybe(stdout); return { id: step.id, phase: step.phase, status: result.code === 0 ? "pass" : "blocked", command, code: result.code, startedAt, finishedAt, stdoutJson: parsedStdout, stdoutTail: parsedStdout ? null : stdout.trim().slice(-2000), stderrTail: stderr.trim().slice(-2000), reportPath: step.reportPath ?? null }; } function stepBlocker(stepResult) { return { type: "runtime_blocker", scope: stepResult.id, status: "open", summary: `${stepResult.command} exited ${stepResult.code}.` }; } function healthCommit(json) { return ( json?.commit?.id || json?.commitId || json?.revision || json?.image?.tag || null ); } function healthImage(json) { if (typeof json?.image === "string") return json.image; return json?.image?.reference ?? null; } function summarizeHealthJson(json, expectedServiceId, expectedCommit) { const observedCommit = healthCommit(json); return { serviceId: json?.serviceId ?? json?.service?.id ?? null, environment: json?.environment ?? null, applicationStatus: json?.status ?? null, ready: Object.hasOwn(json ?? {}, "ready") ? json.ready : null, observedCommit, image: healthImage(json), imageTag: json?.image?.tag ?? null, observedAt: json?.observedAt ?? null, serviceMatches: (json?.serviceId ?? json?.service?.id) === expectedServiceId, environmentMatches: json?.environment === ENVIRONMENT_DEV, commitMatches: expectedCommit ? commitMatches(observedCommit, expectedCommit) : null, blockerCodes: Array.isArray(json?.blockerCodes) ? json.blockerCodes : [] }; } async function defaultHttpGetJson(url, timeoutMs = 10000) { return new Promise((resolve, reject) => { const request = httpRequest(url, { method: "GET", timeout: timeoutMs }, (response) => { response.setEncoding("utf8"); let body = ""; response.on("data", (chunk) => { body += chunk; }); response.on("end", () => { let json = null; try { json = JSON.parse(body); } catch {} resolve({ statusCode: response.statusCode ?? 0, body, json }); }); }); request.on("timeout", () => { request.destroy(new Error(`timeout after ${timeoutMs}ms`)); }); request.on("error", reject); request.end(); }); } async function probeLiveEndpoint(ctx, { id, url, expectedServiceId, expectedCommit }) { const startedAt = ctx.now().toISOString(); try { const response = await ctx.httpGetJson(url, 10000); const summary = summarizeHealthJson(response.json, expectedServiceId, expectedCommit); const reachable = response.statusCode >= 200 && response.statusCode < 300 && Boolean(response.json); const identityMatches = reachable && summary.serviceMatches && summary.environmentMatches && (expectedCommit ? summary.commitMatches : true); return { id, url, status: identityMatches ? "pass" : "blocked", httpStatus: response.statusCode, reachable, identityMatches, startedAt, finishedAt: ctx.now().toISOString(), expectedServiceId, expectedCommit: expectedCommit ?? null, ...summary }; } catch (error) { return { id, url, status: "blocked", reachable: false, identityMatches: false, startedAt, finishedAt: ctx.now().toISOString(), expectedServiceId, expectedCommit: expectedCommit ?? null, error: oneLine(error.message) }; } } export async function verifyDevLive(ctx, { expectedCommit = null } = {}) { const [cloudWeb, cloudApi] = await Promise.all([ probeLiveEndpoint(ctx, { id: "cloud-web-16666", url: browserLiveUrl, expectedServiceId: "hwlab-cloud-web", expectedCommit }), probeLiveEndpoint(ctx, { id: "cloud-api-16667", url: apiLiveUrl, expectedServiceId: "hwlab-cloud-api", expectedCommit }) ]); const endpoints = [cloudWeb, cloudApi]; return { status: endpoints.every((endpoint) => endpoint.status === "pass") ? "pass" : "blocked", expectedCommit: expectedCommit ?? null, endpoints, summary: { checked: endpoints.length, reachable: endpoints.filter((endpoint) => endpoint.reachable).length, identityMatches: endpoints.filter((endpoint) => endpoint.identityMatches).length, commitMatches: expectedCommit ? endpoints.filter((endpoint) => endpoint.commitMatches === true).length : null, ports: [16666, 16667] } }; } function buildReport({ args, transaction, target, deployBefore, deployAfter, lockState, steps, blockers, status, startedAt, finishedAt, liveBefore, liveVerify, release }) { return { $schema: "https://hwlab.pikastech.local/schemas/dev-cd-apply.schema.json", $id: "https://hwlab.pikastech.local/reports/dev-gate/dev-cd-apply.json", reportVersion: "v1", issue: "pikasTech/HWLAB#274", taskId: "dev-cd-apply", commitId: target.shortCommitId, acceptanceLevel: "dev_cd_apply", devOnly: true, prodDisabled: true, status, generatedAt: finishedAt, reportLifecycle: activeReportLifecycle("Current DEV CD transaction report; the only DEV side-effect entrypoint is scripts/dev-cd-apply.mjs."), supports: [ "pikasTech/HWLAB#274", "pikasTech/HWLAB#116", "pikasTech/HWLAB#235", "pikasTech/HWLAB#50", "pikasTech/HWLAB#57", "pikasTech/HWLAB#67", "pikasTech/HWLAB#73" ], sourceContract: { status: blockers.length ? "blocked" : "pass", documents: [ "docs/reference/deployment-publish.md", "docs/reference/dev-runtime-boundary.md", "docs/dev-artifact-publish.md", "docs/dev-deploy-apply.md", "docs/artifact-catalog.md" ], summary: "DEV CD publish/apply/verify side effects are serialized by one Kubernetes Lease and use deploy/deploy.json as the apply desired-state source." }, validationCommands: [ "node --check scripts/dev-cd-apply.mjs", "node --check scripts/src/dev-cd-apply.mjs", "node --test scripts/src/dev-cd-apply.test.mjs", "node --check scripts/dev-artifact-publish.mjs", "node --check scripts/src/dev-deploy-apply.mjs", "node --test scripts/src/dev-deploy-apply.test.mjs", "node scripts/deploy-desired-state-plan.mjs --check", "node scripts/validate-artifact-catalog.mjs", "git diff --check" ], localSmoke: { status: "not_run", commands: ["npm run check"], evidence: ["Local checks are listed as validation commands and are not implied by a DEV CD transaction."], summary: "The transaction report records DEV publish/apply/verify evidence; local smoke is a separate gate." }, dryRun: { status: "not_run", commands: ["node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked --write-report"], evidence: ["The transaction path owns live side effects; dry-run remains a preflight support mode."], summary: "DEV CD mutation requires --apply and the transaction Lease lock." }, devPreconditions: { status: blockers.length ? "blocked" : "pass", requirements: [ "HEAD must match the requested target ref before publish.", "Lease/hwlab-dev-cd-lock must be acquired before publish/apply.", "Legacy publish/apply side-effect scripts must run with HWLAB_CD_TRANSACTION_ID.", "deploy/deploy.json, artifact catalog, and workloads must converge before apply.", "Public 16666 and 16667 live health must be recorded in this report." ], summary: blockers.length ? "One or more DEV CD transaction preconditions failed." : "DEV CD transaction preconditions passed." }, transaction: { transactionId: transaction.transactionId, ownerTaskId: transaction.ownerTaskId, targetNamespace: transaction.targetNamespace, lockName: transaction.lockName, lockBackend: "Lease", startedAt, finishedAt, ttlSeconds: args.ttlSeconds, phases: transaction.phases, release }, devCdApply: { mode: args.apply ? "apply" : "dry-run", target: { ref: target.ref, promotionCommit: target.commitId, shortCommitId: target.shortCommitId, headCommitId: target.headCommitId, headMatchesTarget: target.headMatchesTarget, namespace: args.targetNamespace, apiEndpoint: DEV_ENDPOINT, browserEndpoint: "http://74.48.78.17:16666/" }, deployJson: { before: deployBefore, after: deployAfter }, lock: { acquired: Boolean(lockState?.acquired), current: lockState?.lock ?? null, staleBreak: lockState?.staleBreak ?? null }, steps, liveBefore, liveVerify, reportPaths: { transaction: args.writeReport ? args.reportPath : null, artifacts: artifactReportPath, deployApply: "reports/dev-gate/dev-deploy-report.json" }, safety: { prodTouched: false, secretValuesRead: false, secretValuesPrinted: false, manualDbWrite: false, oldPublicPortsUsed: false, mutationAttempted: steps.some((step) => ["artifact-publish", "dev-deploy-apply"].includes(step.id)), singleLock: "Lease/hwlab-dev/hwlab-dev-cd-lock" } }, blockers, notes: "DEV-only CD transaction evidence. This does not claim M3 PASS or runtime durability." }; } function buildArgumentFailure(error) { const blockers = error.blockers ?? [{ type: "safety_blocker", scope: error.code ?? "dev-cd-apply", status: "open", summary: oneLine(error.message) }]; return { ok: false, status: "blocked", error: error.code ?? "dev-cd-apply-failed", blockers, mutationAttempted: false, prodTouched: false }; } export function formatDevCdApplyFailure(error) { if (error?.lockFailure) return error.lockFailure; if (error instanceof DevCdApplyError) return buildArgumentFailure(error); return { ok: false, status: "failed", error: "dev-cd-apply-failed", summary: error instanceof Error ? error.message : String(error), mutationAttempted: false, prodTouched: false }; } export async function runDevCdApply(argv, io = {}) { const stdout = io.stdout ?? process.stdout; const env = io.env ?? process.env; const ctx = { repoRoot: io.repoRoot ?? defaultRepoRoot, env, runCommand: io.runCommand ?? defaultRunCommand, httpGetJson: io.httpGetJson ?? defaultHttpGetJson, now: io.now ?? (() => new Date()) }; let args; try { args = parseArgs(argv); if (args.help) { stdout.write(`${usage()}\n`); return 0; } validateArgs(args); } catch (error) { const failure = formatDevCdApplyFailure(error); stdout.write(`${JSON.stringify(failure, null, 2)}\n`); return 2; } const startedAt = ctx.now().toISOString(); const transaction = { transactionId: randomUUID(), ownerTaskId: args.ownerTaskId ?? defaultOwnerTaskId(env), targetNamespace: args.targetNamespace, lockName: args.lockName, phases: [] }; const steps = []; const blockers = []; let lockState = null; let release = null; let target = null; let deployBefore = null; let deployAfter = null; let liveBefore = null; let liveVerify = null; let status = "blocked"; try { target = await resolveTargetRef(ctx, args.targetRef); deployBefore = await readDeployJson(ctx.repoRoot); const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand); liveBefore = { status: "pending", reason: "liveBefore is captured immediately after Lease acquisition so publish/apply cannot race before the transaction lock." }; lockState = await acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId: transaction.transactionId, ownerTaskId: transaction.ownerTaskId, liveBefore, now: ctx.now() }); liveBefore = await verifyDevLive(ctx, { expectedCommit: null }); await updateDeployLockLiveBefore({ ctx, args, kubectlContext, transactionId: transaction.transactionId, lockState, liveBefore }); transaction.phases.push({ phase: "publishing", status: "entered", at: ctx.now().toISOString() }); if (!target.headMatchesTarget) { blockers.push({ type: "contract_blocker", scope: "target-ref-head", status: "open", summary: `HEAD ${target.headShortCommitId} must match target ref ${target.ref} ${target.shortCommitId} before DEV CD publish.` }); } const plannedSteps = [ { id: "desired-state-check-before-publish", phase: "publishing", command: process.execPath, args: ["scripts/deploy-desired-state-plan.mjs", "--check", "--pretty"], timeoutMs: 120000 }, { id: "artifact-publish", phase: "publishing", command: process.execPath, args: ["scripts/dev-artifact-publish.mjs", "--publish"], timeoutMs: 60 * 60 * 1000, reportPath: artifactReportPath }, { id: "artifact-catalog-refresh", phase: "publishing", command: process.execPath, args: [ "scripts/refresh-artifact-catalog.mjs", "--target-ref", target.ref, "--publish-report", artifactReportPath ], timeoutMs: 120000 }, { id: "artifact-catalog-validate", phase: "publishing", command: process.execPath, args: ["scripts/validate-artifact-catalog.mjs"], timeoutMs: 120000 }, { id: "desired-state-check-after-refresh", phase: "publishing", command: process.execPath, args: [ "scripts/deploy-desired-state-plan.mjs", "--promotion-commit", target.commitId, "--check", "--pretty" ], timeoutMs: 120000 }, { id: "dev-deploy-apply", phase: "applying", command: process.execPath, args: [ "scripts/dev-deploy-apply.mjs", "--apply", "--confirm-dev", "--confirmed-non-production", "--write-report", ...(args.kubeconfig ? ["--kubeconfig", args.kubeconfig] : []) ], timeoutMs: 20 * 60 * 1000, reportPath: "reports/dev-gate/dev-deploy-report.json" } ]; if (blockers.length === 0) { for (const step of plannedSteps) { if (step.phase === "applying" && lockState.lock?.phase !== "applying") { await updateDeployLockPhase({ ctx, args, kubectlContext, transactionId: transaction.transactionId, lockState, phase: "applying" }); transaction.phases.push({ phase: "applying", status: "entered", at: ctx.now().toISOString() }); } const result = await runStep(ctx, transaction, step); steps.push(result); if (result.status !== "pass") { blockers.push(stepBlocker(result)); break; } } } if (blockers.length === 0) { await updateDeployLockPhase({ ctx, args, kubectlContext, transactionId: transaction.transactionId, lockState, phase: "verifying" }); transaction.phases.push({ phase: "verifying", status: "entered", at: ctx.now().toISOString() }); deployAfter = await readDeployJson(ctx.repoRoot); liveVerify = args.skipLiveVerify ? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } } : await verifyDevLive(ctx, { expectedCommit: target.shortCommitId }); if (liveVerify.status !== "pass") { blockers.push({ type: "observability_blocker", scope: "live-verify", status: "open", summary: "16666/16667 live health did not match the DEV CD promotion commit." }); } } else { deployAfter = deployBefore; liveVerify = { status: "not_run", reason: "transaction blocked before live verify", endpoints: [], summary: { checked: 0 } }; } status = blockers.length === 0 ? "pass" : "blocked"; release = await releaseDeployLock({ ctx, args, kubectlContext, transactionId: transaction.transactionId, status }); transaction.phases.push({ phase: "released", status: release.status, at: ctx.now().toISOString() }); } catch (error) { if (error?.lockFailure) { stdout.write(`${JSON.stringify(error.lockFailure, null, 2)}\n`); return 2; } blockers.push({ type: "runtime_blocker", scope: error.code ?? "dev-cd-apply", status: "open", summary: oneLine(error.message) }); status = "blocked"; if (lockState?.acquired) { const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand); release = await releaseDeployLock({ ctx, args, kubectlContext, transactionId: transaction.transactionId, status }); transaction.phases.push({ phase: "released", status: release.status, at: ctx.now().toISOString() }); } } const finishedAt = ctx.now().toISOString(); const report = buildReport({ args, transaction, target: target ?? { ref: args.targetRef, commitId: "unknown", shortCommitId: "unknown", headCommitId: "unknown", headMatchesTarget: false }, deployBefore: deployBefore ?? { path: "deploy/deploy.json", hash: "unknown", commitId: "unknown" }, deployAfter: deployAfter ?? deployBefore ?? { path: "deploy/deploy.json", hash: "unknown", commitId: "unknown" }, lockState, steps, blockers, status, startedAt, finishedAt, liveBefore: liveBefore ?? { status: "not_run" }, liveVerify: liveVerify ?? { status: "not_run", endpoints: [], summary: { checked: 0 } }, release }); if (args.writeReport) { const absoluteReportPath = path.resolve(ctx.repoRoot, args.reportPath); await mkdir(path.dirname(absoluteReportPath), { recursive: true }); await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`); } stdout.write(`${JSON.stringify(report, null, 2)}\n`); return status === "pass" ? 0 : 2; }