import { spawn } from "node:child_process"; import { createHash, randomUUID } from "node:crypto"; import { mkdir, readFile, writeFile } from "node:fs/promises"; import { request as httpRequest } from "node:http"; import path from "node:path"; import { fileURLToPath } from "node:url"; import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs"; import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs"; import { buildKubectlCommandPrefix, redactSensitiveText, resolveDevKubeconfigSelection } from "./dev-deploy-apply.mjs"; import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs"; const defaultRepoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../.."); const defaultNamespace = "hwlab-dev"; const defaultLockName = "hwlab-dev-cd-lock"; const defaultTtlSeconds = 3600; const defaultReportPath = tempReportPath("dev-cd-apply.json"); const d601NativeKubeconfigPath = "/etc/rancher/k3s/k3s.yaml"; const artifactCatalogPath = "deploy/artifact-catalog.dev.json"; const artifactReportPath = tempReportPath("dev-artifacts.json"); const artifactDesiredStatePaths = [ "deploy/artifact-catalog.dev.json", "deploy/deploy.json", "deploy/k8s/base/workloads.yaml" ]; const runtimeProvisioningReportPath = tempReportPath("dev-runtime-provisioning-report.json"); const runtimeMigrationReportPath = tempReportPath("dev-runtime-migration-report.json"); const runtimePostflightReportPath = tempReportPath("dev-runtime-postflight-report.json"); const deployApplyReportPath = tempReportPath("dev-deploy-report.json"); const runtimeJobReportPrefix = "/tmp/hwlab-dev-runtime-report"; const browserLiveUrl = "http://74.48.78.17:16666/health/live"; const apiLiveUrl = `${DEV_ENDPOINT}/health/live`; const digestPattern = /^sha256:[a-f0-9]{64}$/u; const forbiddenControlPlanePattern = /docker-desktop|desktop-control-plane|127\.0\.0\.1:11700/iu; const requiredDevSecretRefs = Object.freeze([ Object.freeze({ secretName: "hwlab-cloud-api-dev-db", secretKey: "database-url" }), Object.freeze({ secretName: "hwlab-cloud-api-dev-db-admin", secretKey: "admin-url" }), Object.freeze({ secretName: "hwlab-code-agent-provider", secretKey: "openai-api-key" }), Object.freeze({ secretName: "hwlab-code-agent-codex-auth", secretKey: "auth.json" }) ]); const requiredDevConfigMapRefs = Object.freeze([ Object.freeze({ configMapName: "hwlab-code-agent-codex-config", configMapKey: "config.toml" }) ]); const lockAnnotationPrefix = "hwlab.pikastech.local"; const lockAnnotationFields = { promotionCommit: `${lockAnnotationPrefix}/promotionCommit`, deployJsonHash: `${lockAnnotationPrefix}/deployJsonHash`, ownerTaskId: `${lockAnnotationPrefix}/ownerTaskId`, transactionId: `${lockAnnotationPrefix}/transactionId`, phase: `${lockAnnotationPrefix}/phase`, startedAt: `${lockAnnotationPrefix}/startedAt`, updatedAt: `${lockAnnotationPrefix}/updatedAt`, ttlSeconds: `${lockAnnotationPrefix}/ttlSeconds`, liveBefore: `${lockAnnotationPrefix}/liveBefore`, targetNamespace: `${lockAnnotationPrefix}/targetNamespace`, targetRef: `${lockAnnotationPrefix}/targetRef`, lockBackend: `${lockAnnotationPrefix}/lockBackend`, releasedAt: `${lockAnnotationPrefix}/releasedAt`, releaseStatus: `${lockAnnotationPrefix}/releaseStatus`, staleLockBrokenAt: `${lockAnnotationPrefix}/staleLockBrokenAt`, staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious` }; class DevCdApplyError extends Error { constructor(message, details = {}) { super(message); this.name = "DevCdApplyError"; Object.assign(this, details); } } export function parseArgs(argv) { const args = { apply: false, confirmDev: false, confirmedNonProduction: false, writeReport: false, reportPath: defaultReportPath, targetRef: "origin/main", kubeconfig: null, kubeconfigSpecified: false, registryManifestBaseUrl: null, lockName: defaultLockName, targetNamespace: defaultNamespace, ttlSeconds: defaultTtlSeconds, ownerTaskId: null, breakStaleLock: false, skipLiveVerify: false, skipRuntimePostflight: false, status: false, dryRun: false, fullOutput: false, help: false, flags: new Set() }; for (let index = 0; index < argv.length; index += 1) { const arg = argv[index]; if (arg === "--apply") { args.apply = true; args.flags.add(arg); } else if (arg === "--confirm-dev") { args.confirmDev = true; args.flags.add(arg); } else if (arg === "--confirmed-non-production") { args.confirmedNonProduction = true; args.flags.add(arg); } else if (arg === "--break-stale-lock") { args.breakStaleLock = true; args.flags.add(arg); } else if (arg === "--dry-run" || arg === "--status") { args.status = true; args.dryRun = arg === "--dry-run"; args.flags.add(arg); } else if (arg === "--full-output") { args.fullOutput = true; args.flags.add(arg); } else if (arg === "--skip-live-verify") { args.skipLiveVerify = true; args.flags.add(arg); } else if (arg === "--skip-runtime-postflight") { args.skipRuntimePostflight = true; args.flags.add(arg); } else if (arg === "--report") { args.reportPath = readOption(argv, ++index, arg); args.writeReport = true; args.flags.add(arg); } else if (arg === "--target-ref") { args.targetRef = readOption(argv, ++index, arg); args.flags.add(arg); } else if (arg === "--kubeconfig") { args.kubeconfig = readOption(argv, ++index, arg); args.kubeconfigSpecified = true; args.flags.add(arg); } else if (arg.startsWith("--kubeconfig=")) { args.kubeconfig = arg.slice("--kubeconfig=".length); args.kubeconfigSpecified = true; args.flags.add("--kubeconfig"); } else if (arg === "--registry-manifest-base-url") { args.registryManifestBaseUrl = readOption(argv, ++index, arg); args.flags.add(arg); } else if (arg.startsWith("--registry-manifest-base-url=")) { args.registryManifestBaseUrl = arg.slice("--registry-manifest-base-url=".length); args.flags.add("--registry-manifest-base-url"); } else if (arg === "--lock-name") { args.lockName = readOption(argv, ++index, arg); args.flags.add(arg); } else if (arg === "--target-namespace") { args.targetNamespace = readOption(argv, ++index, arg); args.flags.add(arg); } else if (arg === "--ttl-seconds") { args.ttlSeconds = Number.parseInt(readOption(argv, ++index, arg), 10); args.flags.add(arg); } else if (arg === "--owner-task-id") { args.ownerTaskId = readOption(argv, ++index, arg); args.flags.add(arg); } else if (arg === "--help" || arg === "-h") { args.help = true; } else { throw new DevCdApplyError(`unknown argument ${arg}`, { code: "unknown-argument", blockers: [{ type: "safety_blocker", scope: "unknown-argument", status: "open", summary: `${arg} is not a supported DEV CD option. Use --status/--dry-run for read-only observability or --apply with explicit DEV confirmations.` }] }); } } return args; } function readOption(argv, index, name) { const value = argv[index]; if (!value || value.startsWith("--")) { throw new Error(`${name} requires a value`); } return value; } export function usage() { return { command: "node scripts/dev-cd-apply.mjs", summary: "Single DEV CD transaction for latest-main publish, desired-state refresh, apply, and live verify.", defaultMode: "status", examples: [ "node scripts/dev-cd-apply.mjs --status", "node scripts/dev-cd-apply.mjs --dry-run", `node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report ${defaultReportPath}` ], options: { "--status": "Read-only status: target ref, deploy.json, current Lease, and public live health.", "--dry-run": "Read-only alias for --status; does not acquire the Lease or run side effects.", "--apply": "Run the DEV CD transaction side effects.", "--confirm-dev": "Required for DEV mutation and explicit stale-lock recovery.", "--confirmed-non-production": "Required for DEV mutation.", "--target-ref REF": "git ref to publish/apply; default: origin/main", "--kubeconfig PATH": "DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG", "--ttl-seconds SECONDS": "lock TTL; default: 3600", "--owner-task-id ID": "lock holder; default: Code Queue/env/user derived", "--break-stale-lock": "May take over an expired lock only with --confirm-dev.", "--report PATH": `write the full transaction JSON outside the repo; packaged scripts use ${defaultReportPath}`, "--full-output": "Print the full report to stdout instead of the concise summary.", "--skip-runtime-postflight": "Skip the M3/durable runtime postflight while still applying DEV and verifying 16666/16667 health.", "--skip-live-verify": "test-only escape hatch; do not use for DEV acceptance" } }; } function nowIso() { return new Date().toISOString(); } function kubernetesMicroTime(value) { const date = value instanceof Date ? value : new Date(value); if (!Number.isFinite(date.getTime())) { throw new DevCdApplyError("invalid Kubernetes Lease timestamp", { code: "invalid-lease-timestamp" }); } return date.toISOString().replace(/\.(\d{3})Z$/u, ".$1000Z"); } function parseJsonMaybe(value) { try { return JSON.parse(value); } catch { return null; } } function oneLine(value) { return String(value ?? "").replace(/\s+/g, " ").trim(); } function blocker({ type = "environment_blocker", scope, reason, impact, safeNextAction, retryable }) { return { type, scope, status: "open", reason: oneLine(reason), impact: oneLine(impact), safeNextAction: oneLine(safeNextAction), retryable: Boolean(retryable), summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`) }; } function sha256(value) { return `sha256:${createHash("sha256").update(value).digest("hex")}`; } function shortCommit(value) { return typeof value === "string" && value.length >= 7 ? value.slice(0, 7) : value; } function commitMatches(actual, expected) { if (typeof actual !== "string" || typeof expected !== "string") return false; return actual === expected || actual.startsWith(expected) || expected.startsWith(actual); } function shellCommand(command, args) { return [command, ...args].map((part) => (/\s/u.test(part) ? JSON.stringify(part) : part)).join(" "); } function defaultOwnerTaskId(env = process.env) { return ( env.CODE_QUEUE_TASK_ID || env.HWLAB_OWNER_TASK_ID || env.GITHUB_RUN_ID || env.USER || "unknown-dev-cd-runner" ); } function defaultRunCommand(command, args, options = {}) { return new Promise((resolve) => { const abortController = options.timeoutMs ? new AbortController() : null; const child = spawn(command, args, { cwd: options.cwd ?? defaultRepoRoot, env: options.env ?? process.env, stdio: ["pipe", "pipe", "pipe"], signal: abortController?.signal }); let stdout = ""; let stderr = ""; const timeout = options.timeoutMs ? setTimeout(() => abortController.abort(), options.timeoutMs) : null; child.stdin.on("error", () => {}); child.stdin.end(options.input ?? ""); child.stdout.on("data", (chunk) => { stdout += chunk; }); child.stderr.on("data", (chunk) => { stderr += chunk; }); child.on("error", (error) => { if (timeout) clearTimeout(timeout); resolve({ code: error.name === "AbortError" ? 124 : 127, signal: null, stdout, stderr: error.message }); }); child.on("close", (code, signal) => { if (timeout) clearTimeout(timeout); resolve({ code: code ?? 1, signal, stdout, stderr }); }); }); } async function readDeployJson(repoRoot) { const relativePath = "deploy/deploy.json"; const raw = await readFile(path.join(repoRoot, relativePath), "utf8"); const manifest = JSON.parse(raw); return { path: relativePath, hash: sha256(raw), commitId: manifest.commitId ?? "unknown", namespace: manifest.namespace ?? defaultNamespace, environment: manifest.environment ?? "unknown", endpoint: manifest.endpoint ?? null, manifest }; } async function readJsonFileSummary(repoRoot, relativePath, summarize) { try { const absolutePath = path.isAbsolute(relativePath) ? relativePath : path.join(repoRoot, relativePath); const raw = await readFile(absolutePath, "utf8"); const json = JSON.parse(raw); return { path: relativePath, hash: sha256(raw), exists: true, ...summarize(json) }; } catch (error) { if (error?.code === "ENOENT") { return { path: relativePath, hash: null, exists: false, error: "missing" }; } throw error; } } function servicesArray(value) { if (Array.isArray(value)) return value; if (value && typeof value === "object") return Object.values(value); return []; } function summarizeArtifactCatalog(catalog) { const services = servicesArray(catalog?.services); const digestCounts = { sha256: 0, notPublished: 0, invalid: 0 }; for (const service of services) { if (digestPattern.test(service?.digest ?? "")) digestCounts.sha256 += 1; else if (service?.digest === "not_published") digestCounts.notPublished += 1; else digestCounts.invalid += 1; } return { kind: catalog?.kind ?? null, commitId: catalog?.commitId ?? null, artifactState: catalog?.artifactState ?? null, ciPublished: catalog?.publish?.ciPublished ?? null, registryVerified: catalog?.publish?.registryVerified ?? null, provenance: catalog?.publish?.provenance ?? null, serviceCount: services.length, digestCounts }; } function summarizeArtifactReport(report) { const publish = report?.artifactPublish ?? {}; const services = servicesArray(publish.services); const requiredServices = services.filter((service) => service?.artifactRequired !== false); const publishedRequiredServices = requiredServices.filter((service) => service?.status === "published"); const digestCounts = { sha256: 0, notPublished: 0, invalid: 0 }; for (const service of services) { if (digestPattern.test(service?.digest ?? "")) digestCounts.sha256 += 1; else if (service?.digest === "not_published") digestCounts.notPublished += 1; else digestCounts.invalid += 1; } return { commitId: report?.commitId ?? null, status: report?.status ?? null, sourceCommitId: publish.sourceCommitId ?? null, registryPrefix: publish.registryPrefix ?? null, serviceCount: publish.serviceCount ?? services.length, requiredServiceCount: publish.requiredServiceCount ?? requiredServices.length, publishedCount: publish.publishedCount ?? publishedRequiredServices.length, digestCounts, generatedAt: publish.generatedAt ?? null }; } async function readArtifactEvidence(repoRoot) { const [catalog, report] = await Promise.all([ readJsonFileSummary(repoRoot, artifactCatalogPath, summarizeArtifactCatalog), readJsonFileSummary(repoRoot, artifactReportPath, summarizeArtifactReport) ]); return { catalog, report }; } async function resolveTargetRef(ctx, targetRef) { const target = await ctx.runCommand("git", ["rev-parse", "--verify", `${targetRef}^{commit}`], { cwd: ctx.repoRoot, timeoutMs: 10000 }); if (target.code !== 0) { throw new DevCdApplyError(`target ref ${targetRef} could not be resolved`, { code: "target-ref-unresolved", targetRef, stderr: redactSensitiveText(target.stderr) }); } const head = await ctx.runCommand("git", ["rev-parse", "--verify", "HEAD^{commit}"], { cwd: ctx.repoRoot, timeoutMs: 10000 }); if (head.code !== 0) { throw new DevCdApplyError("HEAD could not be resolved", { code: "head-unresolved", stderr: redactSensitiveText(head.stderr) }); } const commitId = target.stdout.trim(); const headCommitId = head.stdout.trim(); return { ref: targetRef, commitId, shortCommitId: shortCommit(commitId), headCommitId, headShortCommitId: shortCommit(headCommitId), headMatchesTarget: commitId === headCommitId }; } async function gitOutput(ctx, args, { allowFailure = false, timeoutMs = 10000 } = {}) { const result = await ctx.runCommand("git", args, { cwd: ctx.repoRoot, timeoutMs }); if (result.code !== 0) { if (allowFailure) return null; throw new DevCdApplyError("git command failed while resolving DEV CD provenance", { code: "git-provenance-failed", command: shellCommand("git", args), stderr: redactSensitiveText(result.stderr || result.stdout) }); } return result.stdout.trim(); } async function commitParents(ctx, commitId) { const line = await gitOutput(ctx, ["rev-list", "--parents", "-n", "1", commitId], { allowFailure: true }); if (!line) return []; return line.split(/\s+/u).slice(1); } async function shortGitCommit(ctx, commitId) { const value = await gitOutput(ctx, ["rev-parse", "--short=7", commitId], { allowFailure: true }); return value || shortCommit(commitId); } async function filesChangedBetween(ctx, left, right) { const output = await gitOutput(ctx, ["diff-tree", "--no-commit-id", "--name-only", "-r", left, right], { allowFailure: true }); if (!output) return []; return output.split(/\r?\n/u).map((item) => item.trim()).filter(Boolean); } async function latestArtifactMergeCommit(ctx, controlCommit) { const output = await gitOutput(ctx, [ "log", "--first-parent", "--format=%H", "-n", "1", controlCommit, "--", ...artifactDesiredStatePaths ], { allowFailure: true }); return output?.split(/\r?\n/u).find(Boolean) ?? null; } async function resolveArtifactBoundary(ctx, control, promotion, deployBefore, artifactEvidence) { const mergeCommitId = await latestArtifactMergeCommit(ctx, control.commitId); if (!mergeCommitId) { return { status: "unavailable", reason: "No first-parent commit touching artifact desired-state files was found.", desiredStatePaths: artifactDesiredStatePaths }; } const parents = await commitParents(ctx, mergeCommitId); const mergeShortCommitId = await shortGitCommit(ctx, mergeCommitId); const firstParent = parents[0] ?? null; const secondParent = parents[1] ?? null; const promotionParentMatches = firstParent ? commitMatches(firstParent, promotion.commitId) : false; const changedPaths = firstParent ? await filesChangedBetween(ctx, firstParent, mergeCommitId) : []; const touchedDesiredStatePaths = artifactDesiredStatePaths.filter((entry) => changedPaths.includes(entry)); const artifactCommitId = secondParent || mergeCommitId; const artifactShortCommitId = await shortGitCommit(ctx, artifactCommitId); const artifactParents = secondParent ? await commitParents(ctx, secondParent) : []; const artifactParentMatches = artifactParents.some((parent) => commitMatches(parent, promotion.commitId)); const catalogCommitMatches = commitMatches(artifactEvidence.catalog.commitId, promotion.commitId); const reportSource = artifactEvidence.report.sourceCommitId ?? artifactEvidence.report.commitId; const reportCommitMatches = commitMatches(reportSource, promotion.commitId); const deployCommitMatches = commitMatches(deployBefore.commitId, promotion.commitId); const isMergeCommit = parents.length >= 2; const desiredStateStatus = ( deployCommitMatches && catalogCommitMatches && ( promotionParentMatches || artifactParentMatches || commitMatches(mergeCommitId, promotion.commitId) ) ) ? "pass" : "degraded"; const reportStatus = artifactEvidence.report.exists ? (reportCommitMatches ? "matches_promotion" : "stale_or_unrelated") : "missing"; return { status: desiredStateStatus, reportStatus, reportIsReleaseGate: false, reportPolicy: "CI artifact reports are audit snapshots; deploy.json, artifact catalog, workloads, and registry manifests are the CD release gate.", controlCommit: { commitId: control.commitId, shortCommitId: control.shortCommitId }, artifactMergeCommit: { commitId: mergeCommitId, shortCommitId: mergeShortCommitId, isMergeCommit, parents, promotionParent: firstParent, artifactParent: secondParent, promotionParentMatches, changedPaths: touchedDesiredStatePaths }, artifactCommit: { commitId: artifactCommitId, shortCommitId: artifactShortCommitId, parents: artifactParents, promotionParentMatches: artifactParentMatches }, desiredState: { deployCommitId: deployBefore.commitId, deployJsonHash: deployBefore.hash, catalogCommitId: artifactEvidence.catalog.commitId, catalogHash: artifactEvidence.catalog.hash, reportCommitId: artifactEvidence.report.commitId, reportSourceCommitId: artifactEvidence.report.sourceCommitId, reportHash: artifactEvidence.report.hash, deployCommitMatches, catalogCommitMatches, reportCommitMatches, reportExists: artifactEvidence.report.exists, desiredStatePaths: artifactDesiredStatePaths } }; } async function resolveCommitRef(ctx, ref) { const result = await ctx.runCommand("git", ["rev-parse", "--verify", `${ref}^{commit}`], { cwd: ctx.repoRoot, timeoutMs: 10000 }); if (result.code !== 0) { throw new DevCdApplyError(`promotion commit ${ref} could not be resolved`, { code: "promotion-commit-unresolved", targetRef: ref, stderr: redactSensitiveText(result.stderr) }); } const commitId = result.stdout.trim(); return { commitId, shortCommitId: shortCommit(commitId) }; } function compactDesiredStateCheck(result) { const parsed = parseJsonMaybe(result.stdout ?? ""); return { status: result.code === 0 ? "pass" : "blocked", code: result.code, summary: parsed?.summary ?? null, convergence: parsed?.target?.convergence?.state ?? null, diagnostics: Array.isArray(parsed?.diagnostics) ? parsed.diagnostics.slice(0, 5).map((item) => ({ code: item.code ?? null, path: item.path ?? null, message: item.message ?? null })) : [], stderrTail: redactSensitiveText(result.stderr ?? "").trim().slice(-1000) }; } async function checkPromotionDesiredState(ctx, promotionCommit) { const result = await ctx.runCommand(process.execPath, [ "scripts/deploy-desired-state-plan.mjs", "--promotion-commit", promotionCommit, "--check", "--pretty" ], { cwd: ctx.repoRoot, timeoutMs: 120000 }); return compactDesiredStateCheck(result); } async function resolveEffectiveTarget(ctx, args, deployBefore, artifactEvidence = null) { const control = await resolveTargetRef(ctx, args.targetRef); if (deployJsonMatchesTarget(deployBefore, control)) { return { ...control, controlRef: { ref: control.ref, commitId: control.commitId, shortCommitId: control.shortCommitId }, promotionSource: "target-ref", publishRequired: true, desiredStateCheck: null, artifactBoundary: artifactEvidence ? await resolveArtifactBoundary(ctx, control, control, deployBefore, artifactEvidence) : null }; } const promotion = await resolveCommitRef(ctx, deployBefore.commitId); const desiredStateCheck = await checkPromotionDesiredState(ctx, promotion.commitId); if (desiredStateCheck.status !== "pass") { return { ...control, controlRef: { ref: control.ref, commitId: control.commitId, shortCommitId: control.shortCommitId }, promotionSource: "target-ref", publishRequired: true, desiredStateCheck, artifactBoundary: artifactEvidence ? await resolveArtifactBoundary(ctx, control, promotion, deployBefore, artifactEvidence) : null }; } return { ref: control.ref, commitId: promotion.commitId, shortCommitId: promotion.shortCommitId, headCommitId: control.headCommitId, headShortCommitId: control.headShortCommitId, headMatchesTarget: control.headMatchesTarget, controlRef: { ref: control.ref, commitId: control.commitId, shortCommitId: control.shortCommitId }, promotionSource: "deploy-json", publishRequired: false, desiredStateCheck, artifactBoundary: artifactEvidence ? await resolveArtifactBoundary(ctx, control, promotion, deployBefore, artifactEvidence) : null }; } function validateArgs(args) { const blockers = []; if (!args.apply && !args.status) { blockers.push({ type: "safety_blocker", scope: "apply-mode", status: "open", summary: "DEV CD side effects require --apply; use --status or --dry-run for read-only observability." }); } if (args.apply && (!args.confirmDev || !args.confirmedNonProduction)) { blockers.push({ type: "safety_blocker", scope: "dev-confirmation", status: "open", summary: "DEV CD apply requires --confirm-dev and --confirmed-non-production." }); } if (args.breakStaleLock && !args.confirmDev) { blockers.push({ type: "safety_blocker", scope: "break-stale-lock-confirmation", status: "open", summary: "Breaking a stale DEV CD lock requires --break-stale-lock and --confirm-dev." }); } if (!Number.isInteger(args.ttlSeconds) || args.ttlSeconds < 60 || args.ttlSeconds > 24 * 60 * 60) { blockers.push({ type: "safety_blocker", scope: "lock-ttl", status: "open", summary: "--ttl-seconds must be an integer between 60 and 86400." }); } for (const forbidden of ["--prod", "--production", "--read-secret", "--force-push"]) { if (args.flags.has(forbidden)) { blockers.push({ type: "safety_blocker", scope: forbidden.slice(2), status: "open", summary: `${forbidden} is forbidden for DEV CD apply.` }); } } if (blockers.length > 0) { throw new DevCdApplyError("invalid DEV CD apply arguments", { code: "invalid-arguments", blockers }); } } function artifactBoundaryRequiresApplyBlock(target) { if (target.publishRequired !== false) return false; if (target.artifactBoundary?.status !== "degraded") return false; const desiredState = target.artifactBoundary.desiredState ?? {}; const desiredStateCheckPass = target.desiredStateCheck?.status === "pass"; const releaseGatePinsMatch = desiredState.deployCommitMatches === true && desiredState.catalogCommitMatches === true; return !(desiredStateCheckPass && releaseGatePinsMatch); } function lockAnnotationValue(value) { if (value === undefined || value === null) return ""; if (typeof value === "string") return value; return JSON.stringify(value); } export function buildLockAnnotations(lock) { const annotations = {}; for (const [field, key] of Object.entries(lockAnnotationFields)) { if (Object.hasOwn(lock, field)) { annotations[key] = lockAnnotationValue(lock[field]); } } return annotations; } function readAnnotation(annotations, field) { return annotations?.[lockAnnotationFields[field]] ?? null; } function readJsonAnnotation(annotations, field) { const value = readAnnotation(annotations, field); return value ? parseJsonMaybe(value) : null; } export function parseDeployLock(lease) { if (!lease) return null; const annotations = lease.metadata?.annotations ?? {}; const ttlSeconds = Number.parseInt( readAnnotation(annotations, "ttlSeconds") || lease.spec?.leaseDurationSeconds || `${defaultTtlSeconds}`, 10 ); const updatedAt = readAnnotation(annotations, "updatedAt") || lease.spec?.renewTime || lease.metadata?.creationTimestamp || null; const transactionId = readAnnotation(annotations, "transactionId") || lease.spec?.holderIdentity || null; const ownerTaskId = readAnnotation(annotations, "ownerTaskId") || lease.spec?.holderIdentity || null; return { lockBackend: readAnnotation(annotations, "lockBackend") || "Lease", lockName: lease.metadata?.name ?? defaultLockName, resourceVersion: lease.metadata?.resourceVersion ?? null, holderIdentity: lease.spec?.holderIdentity ?? null, promotionCommit: readAnnotation(annotations, "promotionCommit"), deployJsonHash: readAnnotation(annotations, "deployJsonHash"), ownerTaskId, transactionId, phase: readAnnotation(annotations, "phase") || "unknown", startedAt: readAnnotation(annotations, "startedAt") || lease.spec?.acquireTime || lease.metadata?.creationTimestamp || null, updatedAt, ttlSeconds: Number.isInteger(ttlSeconds) ? ttlSeconds : defaultTtlSeconds, liveBefore: readJsonAnnotation(annotations, "liveBefore"), targetNamespace: readAnnotation(annotations, "targetNamespace") || lease.metadata?.namespace || defaultNamespace, targetRef: readAnnotation(annotations, "targetRef"), releasedAt: readAnnotation(annotations, "releasedAt"), releaseStatus: readAnnotation(annotations, "releaseStatus"), staleLockBrokenAt: readAnnotation(annotations, "staleLockBrokenAt"), staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious") }; } export function classifyDeployLock(lock, now = new Date()) { if (!lock || lock.phase === "released") { return { held: false, stale: false, retryAfterSeconds: 0, expiresAt: null }; } const updatedAtMs = Number.isFinite(Date.parse(lock.updatedAt)) ? Date.parse(lock.updatedAt) : 0; const expiresAtMs = updatedAtMs + lock.ttlSeconds * 1000; const retryAfterSeconds = Math.max(0, Math.ceil((expiresAtMs - now.getTime()) / 1000)); return { held: retryAfterSeconds > 0, stale: retryAfterSeconds <= 0, retryAfterSeconds, expiresAt: Number.isFinite(expiresAtMs) ? new Date(expiresAtMs).toISOString() : null }; } export function deployLockHeldFailure(lock, now = new Date()) { const classification = classifyDeployLock(lock, now); return { ok: false, status: "blocked", error: "deploy-lock-held", code: "deploy-lock-held", holder: lock.ownerTaskId || lock.transactionId || lock.holderIdentity || "unknown", promotionCommit: lock.promotionCommit ?? null, deployJsonHash: lock.deployJsonHash ?? null, transactionId: lock.transactionId ?? null, phase: lock.phase ?? "unknown", targetNamespace: lock.targetNamespace ?? defaultNamespace, retryAfterSeconds: classification.retryAfterSeconds, stale: classification.stale, requiresBreakStaleLock: classification.stale, lockName: lock.lockName ?? defaultLockName, lockBackend: lock.lockBackend ?? "Lease", summary: classification.stale ? "A stale DEV CD transaction lock exists; rerun with --break-stale-lock --confirm-dev only after confirming the holder is inactive." : "Another DEV CD transaction is already running; retry after the current holder releases the lock.", mutationAttempted: false, prodTouched: false }; } function isNotFound(result) { return /not\s*found|notfound/i.test(`${result.stderr}\n${result.stdout}`); } function isAlreadyExists(result) { return /already\s*exists|alreadyexists/i.test(`${result.stderr}\n${result.stdout}`); } function isConflict(result) { return /conflict|object has been modified|operation cannot be fulfilled/i.test(`${result.stderr}\n${result.stdout}`); } function compactLockSummary(lock) { if (!lock) return null; return { lockName: lock.lockName ?? defaultLockName, lockBackend: lock.lockBackend ?? "Lease", ownerTaskId: lock.ownerTaskId ?? null, transactionId: lock.transactionId ?? null, holderIdentity: lock.holderIdentity ?? null, phase: lock.phase ?? null, promotionCommit: lock.promotionCommit ?? null, deployJsonHash: lock.deployJsonHash ?? null, updatedAt: lock.updatedAt ?? null, ttlSeconds: lock.ttlSeconds ?? null, targetNamespace: lock.targetNamespace ?? defaultNamespace, targetRef: lock.targetRef ?? null, releasedAt: lock.releasedAt ?? null, releaseStatus: lock.releaseStatus ?? null, staleLockBrokenAt: lock.staleLockBrokenAt ?? null }; } function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null }) { const lock = { promotionCommit: target.shortCommitId, deployJsonHash: deployBefore.hash, ownerTaskId, transactionId, phase, startedAt, updatedAt: startedAt, ttlSeconds: args.ttlSeconds, liveBefore, targetNamespace: args.targetNamespace, targetRef: target.ref, lockBackend: "Lease" }; if (staleLockPrevious) { lock.staleLockBrokenAt = startedAt; lock.staleLockPrevious = staleLockPrevious; } return { apiVersion: "coordination.k8s.io/v1", kind: "Lease", metadata: { name: args.lockName, namespace: args.targetNamespace, labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/profile": "dev", "hwlab.pikastech.local/cd-lock": "true" }, annotations: buildLockAnnotations(lock) }, spec: { holderIdentity: `${ownerTaskId}/${transactionId}`, leaseDurationSeconds: args.ttlSeconds, acquireTime: startedAt, renewTime: startedAt } }; } function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null }) { const lock = { promotionCommit: target.shortCommitId, deployJsonHash: deployBefore.hash, ownerTaskId, transactionId, phase, startedAt, updatedAt, ttlSeconds: args.ttlSeconds, liveBefore, targetNamespace: args.targetNamespace, targetRef: target.ref, lockBackend: "Lease" }; if (staleLockPrevious) { lock.staleLockBrokenAt = updatedAt; lock.staleLockPrevious = staleLockPrevious; } return { metadata: { labels: { "app.kubernetes.io/part-of": "hwlab", "hwlab.pikastech.local/profile": "dev", "hwlab.pikastech.local/cd-lock": "true" }, annotations: buildLockAnnotations(lock) }, spec: { holderIdentity: `${ownerTaskId}/${transactionId}`, leaseDurationSeconds: args.ttlSeconds, acquireTime: startedAt, renewTime: updatedAt } }; } async function kubectl(ctx, kubectlContext, args, options = {}) { const result = await ctx.runCommand(kubectlContext.executor, args, { cwd: ctx.repoRoot, env: kubectlContext.env, input: options.input, timeoutMs: options.timeoutMs ?? 30000 }); return { ...result, command: shellCommand("kubectl", args), stdout: result.stdout ?? "", stderr: result.stderr ?? "" }; } async function replaceLease(ctx, kubectlContext, args, lease, timeoutMs = 30000) { return kubectl( ctx, kubectlContext, ["-n", args.targetNamespace, "replace", "-f", "-", "-o", "json"], { input: JSON.stringify(lease), timeoutMs } ); } function mergeLease(lease, patch, { resetLockAnnotations = false } = {}) { const baseAnnotations = { ...(lease.metadata?.annotations ?? {}) }; if (resetLockAnnotations) { for (const key of Object.keys(baseAnnotations).filter((name) => name.startsWith(`${lockAnnotationPrefix}/`))) { delete baseAnnotations[key]; } } return { ...lease, metadata: { ...lease.metadata, ...(patch.metadata ?? {}), labels: { ...(lease.metadata?.labels ?? {}), ...(patch.metadata?.labels ?? {}) }, annotations: { ...baseAnnotations, ...(patch.metadata?.annotations ?? {}) } }, spec: { ...(lease.spec ?? {}), ...(patch.spec ?? {}) } }; } async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now }) { const startedAt = kubernetesMicroTime(now); const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"]); if (get.code === 0) { const lease = JSON.parse(get.stdout); const existing = parseDeployLock(lease); const classification = classifyDeployLock(existing, now); if (classification.held) { const failure = deployLockHeldFailure(existing, now); throw new DevCdApplyError(failure.summary, { code: "deploy-lock-held", lockFailure: failure }); } if (classification.stale && !args.breakStaleLock) { const failure = deployLockHeldFailure(existing, now); throw new DevCdApplyError(failure.summary, { code: "deploy-lock-held", lockFailure: failure }); } const patch = lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase: "publishing", startedAt, updatedAt: startedAt, liveBefore, staleLockPrevious: classification.stale ? existing : null }); const replaced = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch, { resetLockAnnotations: true })); if (isConflict(replaced)) { return acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now }); } if (replaced.code !== 0) { throw new DevCdApplyError("failed to acquire DEV CD Lease lock", { code: "lock-acquire-failed", stderr: redactSensitiveText(replaced.stderr || replaced.stdout) }); } return { acquired: true, lock: parseDeployLock(JSON.parse(replaced.stdout)), staleBreak: classification.stale ? existing : null, command: replaced.command }; } if (!isNotFound(get)) { throw new DevCdApplyError("failed to read DEV CD Lease lock", { code: "lock-read-failed", stderr: redactSensitiveText(get.stderr || get.stdout) }); } const manifest = buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase: "publishing", liveBefore }); const created = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "create", "-f", "-", "-o", "json"], { input: JSON.stringify(manifest), timeoutMs: 30000 }); if (created.code === 0) { return { acquired: true, lock: parseDeployLock(JSON.parse(created.stdout)), staleBreak: null, command: created.command }; } if (isAlreadyExists(created)) { return acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now }); } throw new DevCdApplyError("failed to create DEV CD Lease lock", { code: "lock-create-failed", stderr: redactSensitiveText(created.stderr || created.stdout) }); } async function readOwnedLease({ ctx, args, kubectlContext, transactionId, timeoutMs = 15000 }) { const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], { timeoutMs }); if (get.code !== 0) { throw new DevCdApplyError("failed to read DEV CD lock ownership", { code: "lock-owner-read-failed", stderr: redactSensitiveText(get.stderr || get.stdout) }); } const lease = JSON.parse(get.stdout); const lock = parseDeployLock(lease); if (lock.transactionId !== transactionId) { throw new DevCdApplyError("DEV CD lock ownership changed before transaction update", { code: "lock-owner-mismatch", holder: lock.ownerTaskId || lock.transactionId || "unknown", phase: lock.phase }); } return { lease, lock }; } async function patchOwnedLease({ ctx, args, kubectlContext, lockState, transactionId, patch, errorCode, errorMessage }) { const { lease } = await readOwnedLease({ ctx, args, kubectlContext, transactionId }); const result = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch)); if (result.code !== 0) { throw new DevCdApplyError(errorMessage, { code: errorCode, stderr: redactSensitiveText(result.stderr || result.stdout) }); } lockState.lock = parseDeployLock(JSON.parse(result.stdout)); return lockState.lock; } async function updateDeployLockPhase({ ctx, args, kubectlContext, transactionId, lockState, phase, status = "running" }) { const updatedAt = kubernetesMicroTime(ctx.now()); const patch = { metadata: { annotations: { [lockAnnotationFields.phase]: phase, [lockAnnotationFields.updatedAt]: updatedAt, [lockAnnotationFields.releaseStatus]: status } }, spec: { renewTime: updatedAt } }; return await patchOwnedLease({ ctx, args, kubectlContext, transactionId, lockState, patch, errorCode: "lock-phase-update-failed", errorMessage: `failed to update DEV CD lock phase to ${phase}` }); } async function updateDeployLockLiveBefore({ ctx, args, kubectlContext, transactionId, lockState, liveBefore }) { const updatedAt = kubernetesMicroTime(ctx.now()); const patch = { metadata: { annotations: { [lockAnnotationFields.liveBefore]: JSON.stringify(liveBefore), [lockAnnotationFields.updatedAt]: updatedAt } }, spec: { renewTime: updatedAt } }; return await patchOwnedLease({ ctx, args, kubectlContext, transactionId, lockState, patch, errorCode: "lock-live-before-update-failed", errorMessage: "failed to update DEV CD lock liveBefore evidence" }); } async function releaseDeployLock({ ctx, args, kubectlContext, transactionId, status }) { const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], { timeoutMs: 15000 }); if (get.code !== 0) { return { status: "release_read_failed", command: get.command, error: redactSensitiveText(get.stderr || get.stdout) }; } const current = parseDeployLock(JSON.parse(get.stdout)); if (current.transactionId !== transactionId) { return { status: "not_owner", holder: current.ownerTaskId || current.transactionId || "unknown", phase: current.phase }; } const releasedAt = kubernetesMicroTime(ctx.now()); const patch = { metadata: { annotations: { [lockAnnotationFields.phase]: "released", [lockAnnotationFields.updatedAt]: releasedAt, [lockAnnotationFields.releasedAt]: releasedAt, [lockAnnotationFields.releaseStatus]: status } }, spec: { holderIdentity: "", leaseDurationSeconds: 1, renewTime: releasedAt } }; const result = await replaceLease(ctx, kubectlContext, args, mergeLease(JSON.parse(get.stdout), patch), 15000); if (isConflict(result)) { return { status: "release_conflict", command: result.command, error: "Lease changed before release; lock was not overwritten." }; } if (result.code !== 0) { return { status: "release_failed", command: result.command, error: redactSensitiveText(result.stderr || result.stdout) }; } return { status: "released", command: result.command, lock: parseDeployLock(JSON.parse(result.stdout)) }; } function commandSummary(result, maxLength = 500) { return oneLine(redactSensitiveText(result?.stderr || result?.stdout || "")).slice(0, maxLength); } function envWithoutKubeconfig(env) { const sanitized = { ...env }; delete sanitized.KUBECONFIG; delete sanitized.HWLAB_DEV_KUBECONFIG; return sanitized; } function parseNodeNames(stdout) { return String(stdout ?? "").trim().split(/\s+/u).filter(Boolean); } function parseSecretDescribeKeys(stdout) { const keys = new Set(); let inData = false; for (const rawLine of String(stdout ?? "").split(/\r?\n/u)) { const line = rawLine.trim(); if (/^Data\b/iu.test(line)) { inData = true; continue; } if (!inData || line === "" || /^=+$/u.test(line)) continue; const match = line.match(/^([A-Za-z0-9_.-]+):\s+\d+\s+bytes\b/iu); if (match) keys.add(match[1]); } return [...keys].sort(); } function parseKeyLines(stdout) { return String(stdout ?? "") .split(/\r?\n/u) .map((line) => line.trim()) .filter(Boolean); } function commandObservation(args, result) { return { command: shellCommand("kubectl", args), code: result.code ?? null, ok: result.code === 0, summary: commandSummary(result) }; } async function preflightKubectl(ctx, kubectlContext, args, options = {}) { const result = await ctx.runCommand(kubectlContext.executor, args, { cwd: ctx.repoRoot, env: options.env ?? kubectlContext.env, timeoutMs: options.timeoutMs ?? 15000 }); return { ...result, stdout: result.stdout ?? "", stderr: result.stderr ?? "", command: shellCommand("kubectl", args) }; } function addControlPlaneBlockers(blockers, observation, { scopePrefix, requiredForApply }) { if (observation.contextName && forbiddenControlPlanePattern.test(observation.contextName)) { blockers.push(blocker({ scope: `${scopePrefix}-context`, reason: `kubectl context resolved to forbidden control plane ${observation.contextName}.`, impact: "DEV CD cannot prove it is targeting D601 native k3s before mutation.", safeNextAction: `Use KUBECONFIG=${d601NativeKubeconfigPath} and remove or repair the stale Docker Desktop/default kube context before retrying.`, retryable: true })); } if (observation.serverUrl && forbiddenControlPlanePattern.test(observation.serverUrl)) { blockers.push(blocker({ scope: `${scopePrefix}-server`, reason: `kubectl server resolved to forbidden endpoint ${observation.serverUrl}.`, impact: "DEV CD could write to the wrong control plane or treat stale Docker Desktop output as DEV evidence.", safeNextAction: `Point kubectl at D601 native k3s with KUBECONFIG=${d601NativeKubeconfigPath} before retrying.`, retryable: true })); } if (observation.nodeNames.includes("desktop-control-plane")) { blockers.push(blocker({ scope: `${scopePrefix}-nodes`, reason: "kubectl observed node desktop-control-plane.", impact: "DEV CD is observing Docker Desktop Kubernetes instead of D601 native k3s.", safeNextAction: `Stop and rerun only after kubectl nodes come from KUBECONFIG=${d601NativeKubeconfigPath} and include d601.`, retryable: true })); } else if (requiredForApply && !observation.nodeNames.includes("d601")) { blockers.push(blocker({ scope: `${scopePrefix}-nodes`, reason: `kubectl nodes did not include d601; observed ${observation.nodeNames.join(",") || "none"}.`, impact: "DEV apply/job/rollout side effects are not allowed without proving the D601 native k3s node.", safeNextAction: `Restore access to KUBECONFIG=${d601NativeKubeconfigPath} and verify kubectl get nodes includes d601.`, retryable: true })); } else if (!requiredForApply && observation.nodeNames.length > 0 && !observation.nodeNames.includes("d601")) { blockers.push(blocker({ scope: `${scopePrefix}-nodes`, reason: `bare kubectl observed non-D601 nodes ${observation.nodeNames.join(",")}.`, impact: "A second or stale control plane may still own hwlab-dev evidence, so DEV CD refuses to mutate.", safeNextAction: "Clean the default kubectl context or ensure bare kubectl cannot resolve stale HWLAB DEV resources, then retry.", retryable: true })); } } async function observeControlPlane(ctx, kubectlContext, { env, label, requiredForApply }) { const [context, server, nodes] = await Promise.all([ preflightKubectl(ctx, kubectlContext, ["config", "current-context"], { env, timeoutMs: 10000 }), preflightKubectl(ctx, kubectlContext, ["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], { env, timeoutMs: 10000 }), preflightKubectl(ctx, kubectlContext, ["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], { env, timeoutMs: 15000 }) ]); return { label, contextCommand: commandObservation(["config", "current-context"], context), serverCommand: commandObservation(["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], server), nodesCommand: commandObservation(["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], nodes), contextName: context.code === 0 ? oneLine(context.stdout) : null, serverUrl: server.code === 0 ? oneLine(server.stdout) : null, nodeNames: nodes.code === 0 ? parseNodeNames(nodes.stdout) : [], status: context.code === 0 && server.code === 0 && nodes.code === 0 ? "observed" : requiredForApply ? "blocked" : "unavailable" }; } async function observeBareKubectl(ctx, kubectlContext, forcedObservation) { const bareEnv = envWithoutKubeconfig(ctx.env); const bare = await observeControlPlane(ctx, kubectlContext, { env: bareEnv, label: "bare-kubectl", requiredForApply: false }); const namespace = await preflightKubectl(ctx, kubectlContext, ["get", "namespace", defaultNamespace, "-o", "name"], { env: bareEnv, timeoutMs: 10000 }); const secondControlPlane = namespace.code === 0 && ( (bare.serverUrl && forcedObservation.serverUrl && bare.serverUrl !== forcedObservation.serverUrl) || (!bare.serverUrl && bare.contextName && bare.contextName !== forcedObservation.contextName) ); return { ...bare, namespace: commandObservation(["get", "namespace", defaultNamespace, "-o", "name"], namespace), hwlabDevNamespaceObserved: namespace.code === 0, secondControlPlaneCandidate: secondControlPlane }; } async function observeRequiredSecretRef(ctx, kubectlContext, args, ref) { const existsArgs = ["-n", args.targetNamespace, "get", "secret", ref.secretName, "-o", "name"]; const describeArgs = ["-n", args.targetNamespace, "describe", "secret", ref.secretName]; const exists = await preflightKubectl(ctx, kubectlContext, existsArgs, { timeoutMs: 10000 }); if (exists.code !== 0) { return { secretName: ref.secretName, secretKey: ref.secretKey, status: "missing-secret", exists: false, keyPresent: false, keysObserved: [], existenceCommand: shellCommand("kubectl", existsArgs), keyCommand: null, error: commandSummary(exists), secretValueRead: false, secretValuePrinted: false }; } const describe = await preflightKubectl(ctx, kubectlContext, describeArgs, { timeoutMs: 10000 }); if (describe.code !== 0) { return { secretName: ref.secretName, secretKey: ref.secretKey, status: "key-observation-blocked", exists: true, keyPresent: false, keysObserved: [], existenceCommand: shellCommand("kubectl", existsArgs), keyCommand: shellCommand("kubectl", describeArgs), error: commandSummary(describe), secretValueRead: false, secretValuePrinted: false }; } const keysObserved = parseSecretDescribeKeys(describe.stdout); const keyPresent = keysObserved.includes(ref.secretKey); return { secretName: ref.secretName, secretKey: ref.secretKey, status: keyPresent ? "present" : "missing-key", exists: true, keyPresent, keysObserved, existenceCommand: shellCommand("kubectl", existsArgs), keyCommand: shellCommand("kubectl", describeArgs), secretValueRead: false, secretValuePrinted: false }; } function secretRefBlocker(ref, observation) { const scope = `secretref:${ref.secretName}/${ref.secretKey}`; if (observation.status === "missing-secret") { return blocker({ type: "runtime_blocker", scope, reason: `Required DEV Secret ${ref.secretName} was not observed in ${defaultNamespace}.`, impact: "Runtime provisioning, migration, deploy apply, or Code Agent provider startup would fail after mutation.", safeNextAction: `Create or restore SecretRef ${ref.secretName}/${ref.secretKey} in ${defaultNamespace} without printing the Secret value, then retry DEV CD.`, retryable: true }); } if (observation.status === "key-observation-blocked") { return blocker({ type: "runtime_blocker", scope, reason: `Required DEV Secret ${ref.secretName} exists but key names could not be verified.`, impact: "DEV CD cannot prove the pod SecretRef contract before creating runtime Jobs or applying workloads.", safeNextAction: `Restore metadata/key-name visibility for ${ref.secretName} without granting or printing Secret values, then retry.`, retryable: true }); } return blocker({ type: "runtime_blocker", scope, reason: `Required DEV Secret key ${ref.secretName}/${ref.secretKey} was not observed.`, impact: "The target workload or runtime Job would reference a missing Secret key after mutation.", safeNextAction: `Add key ${ref.secretKey} to Secret ${ref.secretName} without printing the value, then retry DEV CD.`, retryable: true }); } async function observeRequiredConfigMapRef(ctx, kubectlContext, args, ref) { const existsArgs = ["-n", args.targetNamespace, "get", "configmap", ref.configMapName, "-o", "name"]; const keyArgs = [ "-n", args.targetNamespace, "get", "configmap", ref.configMapName, "-o", "go-template={{range $k,$v := .data}}{{printf \"%s\\n\" $k}}{{end}}" ]; const exists = await preflightKubectl(ctx, kubectlContext, existsArgs, { timeoutMs: 10000 }); if (exists.code !== 0) { return { configMapName: ref.configMapName, configMapKey: ref.configMapKey, status: "missing-configmap", exists: false, keyPresent: false, keysObserved: [], existenceCommand: shellCommand("kubectl", existsArgs), keyCommand: null, error: commandSummary(exists), configMapValueRead: false, configMapValuePrinted: false }; } const keys = await preflightKubectl(ctx, kubectlContext, keyArgs, { timeoutMs: 10000 }); if (keys.code !== 0) { return { configMapName: ref.configMapName, configMapKey: ref.configMapKey, status: "key-observation-blocked", exists: true, keyPresent: false, keysObserved: [], existenceCommand: shellCommand("kubectl", existsArgs), keyCommand: shellCommand("kubectl", keyArgs), error: commandSummary(keys), configMapValueRead: false, configMapValuePrinted: false }; } const keysObserved = parseKeyLines(keys.stdout); const keyPresent = keysObserved.includes(ref.configMapKey); return { configMapName: ref.configMapName, configMapKey: ref.configMapKey, status: keyPresent ? "present" : "missing-key", exists: true, keyPresent, keysObserved, existenceCommand: shellCommand("kubectl", existsArgs), keyCommand: shellCommand("kubectl", keyArgs), configMapValueRead: false, configMapValuePrinted: false }; } function configMapRefBlocker(ref, observation) { const scope = `configmap:${ref.configMapName}/${ref.configMapKey}`; if (observation.status === "missing-configmap") { return blocker({ type: "runtime_blocker", scope, reason: `Required DEV ConfigMap ${ref.configMapName} was not observed in ${defaultNamespace}.`, impact: "The Code Agent Codex stdio provider pod would fail to mount config.toml after mutation.", safeNextAction: `Create or restore ConfigMap ${ref.configMapName}/${ref.configMapKey} in ${defaultNamespace}, then retry DEV CD.`, retryable: true }); } if (observation.status === "key-observation-blocked") { return blocker({ type: "runtime_blocker", scope, reason: `Required DEV ConfigMap ${ref.configMapName} exists but key names could not be verified.`, impact: "DEV CD cannot prove the Code Agent Codex config mount before applying workloads.", safeNextAction: `Restore key-name visibility for ${ref.configMapName}, then retry.`, retryable: true }); } return blocker({ type: "runtime_blocker", scope, reason: `Required DEV ConfigMap key ${ref.configMapName}/${ref.configMapKey} was not observed.`, impact: "The Code Agent Codex stdio provider pod would reference a missing config.toml key after mutation.", safeNextAction: `Add key ${ref.configMapKey} to ConfigMap ${ref.configMapName}, then retry DEV CD.`, retryable: true }); } async function runDevCdPreflight({ ctx, args, kubectlContext }) { const blockers = []; const preflight = { status: "blocked", scope: "pre-lock-pre-side-effect", generatedAt: ctx.now().toISOString(), namespace: args.targetNamespace, kubeconfig: { required: d601NativeKubeconfigPath, selected: kubectlContext.kubeconfig, source: kubectlContext.kubeconfigSource, enforcedEnv: kubectlContext.env.KUBECONFIG === d601NativeKubeconfigPath }, kubectl: { executor: kubectlContext.executor, commandPrefix: kubectlContext.commandPrefix }, controlPlane: null, secretRefs: [], configMaps: [], safety: { writeSideEffectsAttempted: false, prodTouched: false, secretValuesRead: false, secretValuesPrinted: false, configMapValuesRead: false, configMapValuesPrinted: false, secretKeyNamesOnly: true }, blockers }; if (kubectlContext.kubeconfig !== d601NativeKubeconfigPath) { blockers.push(blocker({ scope: "d601-kubeconfig-path", reason: `DEV CD selected kubeconfig ${kubectlContext.kubeconfig || ""} from ${kubectlContext.kubeconfigSource}.`, impact: "Any apply/job/rollout could target a non-D601 or stale control plane.", safeNextAction: `Rerun with KUBECONFIG=${d601NativeKubeconfigPath} or --kubeconfig ${d601NativeKubeconfigPath}.`, retryable: true })); preflight.status = "blocked"; return preflight; } const forced = await observeControlPlane(ctx, kubectlContext, { env: kubectlContext.env, label: "forced-d601-k3s", requiredForApply: true }); const bare = await observeBareKubectl(ctx, kubectlContext, forced); preflight.controlPlane = { forced, bare }; if (forced.status !== "observed") { blockers.push(blocker({ scope: "d601-native-k3s-unobservable", reason: `Forced D601 kubeconfig could not prove context/server/nodes: ${[forced.contextCommand.summary, forced.serverCommand.summary, forced.nodesCommand.summary].filter(Boolean).join("; ") || "no kubectl output"}.`, impact: "DEV CD refuses to acquire the Lease or create runtime Jobs without D601 native k3s proof.", safeNextAction: `Restore readable ${d601NativeKubeconfigPath} access and verify kubectl get nodes includes d601.`, retryable: true })); } addControlPlaneBlockers(blockers, forced, { scopePrefix: "d601-native-k3s", requiredForApply: true }); addControlPlaneBlockers(blockers, bare, { scopePrefix: "bare-kubectl", requiredForApply: false }); if (bare.secondControlPlaneCandidate) { blockers.push(blocker({ scope: "second-hwlab-dev-control-plane", reason: `bare kubectl can observe namespace ${defaultNamespace} outside the forced D601 kubeconfig context.`, impact: "A second hwlab-dev control plane may still carry stale HWLAB resources, so DEV CD cannot safely classify apply evidence.", safeNextAction: "Remove the stale hwlab-dev control plane/default kube context or make bare kubectl unable to resolve it, then rerun preflight.", retryable: true })); } if (blockers.length === 0) { [preflight.secretRefs, preflight.configMaps] = await Promise.all([ Promise.all(requiredDevSecretRefs.map((ref) => observeRequiredSecretRef(ctx, kubectlContext, args, ref))), Promise.all(requiredDevConfigMapRefs.map((ref) => observeRequiredConfigMapRef(ctx, kubectlContext, args, ref))) ]); for (const [index, observation] of preflight.secretRefs.entries()) { if (observation.status !== "present") { blockers.push(secretRefBlocker(requiredDevSecretRefs[index], observation)); } } for (const [index, observation] of preflight.configMaps.entries()) { if (observation.status !== "present") { blockers.push(configMapRefBlocker(requiredDevConfigMapRefs[index], observation)); } } } preflight.status = blockers.length === 0 ? "pass" : "blocked"; return preflight; } async function resolveKubectlContext(args, env, runCommand) { const selection = resolveDevKubeconfigSelection({ flagValue: args.kubeconfig, env }); const which = await runCommand("which", ["kubectl"], { timeoutMs: 5000 }); const executor = which.code === 0 ? which.stdout.trim() : "kubectl"; const kubectlEnv = { ...env, KUBECONFIG: selection.kubeconfig }; return { executor, kubeconfig: selection.kubeconfig, kubeconfigSource: selection.source, commandPrefix: buildKubectlCommandPrefix(selection.kubeconfig), env: kubectlEnv }; } function commandEnv(ctx, transaction) { return { ...ctx.env, HWLAB_CD_TRANSACTION_ID: transaction.transactionId, HWLAB_CD_TRANSACTION_OWNER: transaction.ownerTaskId, HWLAB_CD_LOCK_NAME: transaction.lockName, HWLAB_CD_LOCK_NAMESPACE: transaction.targetNamespace }; } async function runStep(ctx, transaction, step) { const startedAt = ctx.now().toISOString(); const command = shellCommand(step.command, step.args); const result = await ctx.runCommand(step.command, step.args, { cwd: ctx.repoRoot, env: commandEnv(ctx, transaction), timeoutMs: step.timeoutMs ?? 300000 }); const finishedAt = ctx.now().toISOString(); const stdout = redactSensitiveText(result.stdout ?? ""); const stderr = redactSensitiveText(result.stderr ?? ""); const parsedStdout = parseJsonMaybe(stdout); return { id: step.id, phase: step.phase, status: result.code === 0 ? "pass" : "blocked", command, code: result.code, startedAt, finishedAt, stdoutJson: parsedStdout, stdoutTail: parsedStdout ? null : stdout.trim().slice(-2000), stderrTail: stderr.trim().slice(-2000), reportPath: step.reportPath ?? null }; } async function runRuntimeK8sJobStep(ctx, transaction, step) { const startedAt = ctx.now().toISOString(); const result = await runRuntimeK8sJob(ctx, transaction, step); const finishedAt = ctx.now().toISOString(); return { id: step.id, phase: step.phase, status: result.status, command: result.command, code: result.status === "pass" ? 0 : 1, startedAt, finishedAt, stdoutJson: result.report, stdoutTail: null, stderrTail: result.error ?? "", reportPath: step.reportPath ?? null, k8sJob: result.k8sJob }; } async function runRuntimeK8sJob(ctx, transaction, step) { const jobName = `${step.jobNamePrefix}-${transaction.transactionId.slice(0, 8)}`; const reportFile = `${runtimeJobReportPrefix}-${step.id}.json`; const image = step.image ?? await resolveRuntimeJobImage(ctx); const manifest = buildRuntimeK8sJobManifest({ jobName, namespace: transaction.targetNamespace, image, commandArgs: [...step.commandArgs, "--report", reportFile], transaction }); const kubectl = transaction.kubectlContext; const apply = await kubectlCommandResult(ctx, kubectl, ["-n", transaction.targetNamespace, "apply", "-f", "-"], { input: JSON.stringify(manifest), timeoutMs: 30000 }); if (apply.code !== 0) { return runtimeJobFailure(step, "apply", apply, jobName, image); } const wait = await kubectlCommandResult(ctx, kubectl, [ "-n", transaction.targetNamespace, "wait", "--for=condition=complete", `job/${jobName}`, "--timeout=300s" ], { timeoutMs: step.timeoutMs ?? 6 * 60 * 1000 }); if (wait.code !== 0) { return runtimeJobFailure(step, "wait", wait, jobName, image); } const logs = await kubectlCommandResult(ctx, kubectl, ["-n", transaction.targetNamespace, "logs", `job/${jobName}`], { timeoutMs: 30000 }); const logText = redactSensitiveText(`${logs.stdout}\n${logs.stderr}`); const report = parseLastJsonObject(logText); if (logs.code !== 0 || !report) { return { status: "blocked", command: `${shellCommand("kubectl", ["-n", transaction.targetNamespace, "logs", `job/${jobName}`])}`, error: logText.trim().slice(-2000) || "runtime job did not emit a JSON report", report: null, k8sJob: runtimeJobSummary(jobName, step, "logs", image) }; } const reportStatus = report.conclusion?.status ?? report.summary?.status ?? report.status; return { status: ["ready", "pass"].includes(reportStatus) ? "pass" : "blocked", command: shellCommand("kubectl", ["-n", transaction.targetNamespace, "apply/wait/logs", `job/${jobName}`]), error: null, report, k8sJob: runtimeJobSummary(jobName, step, "complete", image) }; } function runtimeJobFailure(step, phase, result, jobName, image) { return { status: "blocked", command: result.command ?? shellCommand("kubectl", [phase, `job/${jobName}`]), error: redactSensitiveText(`${result.stderr}\n${result.stdout}`).trim().slice(-2000), report: null, k8sJob: runtimeJobSummary(jobName, step, phase, image) }; } function runtimeJobSummary(jobName, step, phase, image = step.image ?? "deploy-current:hwlab-cloud-api") { return { jobName, phase, serviceId: "hwlab-cloud-api", image, secretRefsOnly: true, secretValuesRead: false, secretValuesPrinted: false }; } async function resolveRuntimeJobImage(ctx) { const deploy = await readDeployJson(ctx.repoRoot); const image = deploy.manifest?.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.image; if (typeof image === "string" && image.trim().length > 0) { return image; } if (typeof deploy.commitId === "string" && deploy.commitId !== "unknown") { return `127.0.0.1:5000/hwlab/hwlab-cloud-api:${shortCommit(deploy.commitId)}`; } throw new DevCdApplyError("could not resolve hwlab-cloud-api image for runtime maintenance Job", { code: "runtime-job-image-unresolved" }); } function buildRuntimeK8sJobManifest({ jobName, namespace, image, commandArgs, transaction }) { return { apiVersion: "batch/v1", kind: "Job", metadata: { name: jobName, namespace, labels: { "app.kubernetes.io/part-of": "hwlab", "app.kubernetes.io/name": "hwlab-runtime-maintenance", "hwlab.pikastech.local/profile": "dev", "hwlab.pikastech.local/service-id": "hwlab-cloud-api", "hwlab.pikastech.local/cd-transaction": transaction.transactionId } }, spec: { backoffLimit: 0, ttlSecondsAfterFinished: 600, template: { metadata: { labels: { "app.kubernetes.io/name": "hwlab-runtime-maintenance", "hwlab.pikastech.local/service-id": "hwlab-cloud-api", "hwlab.pikastech.local/cd-transaction": transaction.transactionId } }, spec: { restartPolicy: "Never", containers: [ { name: "runtime-maintenance", image, command: ["node"], args: commandArgs, env: [ { name: "HWLAB_ENVIRONMENT", value: ENVIRONMENT_DEV }, { name: "HWLAB_CLOUD_RUNTIME_ADAPTER", value: "postgres" }, { name: "HWLAB_CLOUD_RUNTIME_DURABLE", value: "true" }, { name: "HWLAB_CLOUD_DB_URL", valueFrom: { secretKeyRef: { name: "hwlab-cloud-api-dev-db", key: "database-url", optional: false } } }, { name: "HWLAB_CLOUD_DB_ADMIN_URL", valueFrom: { secretKeyRef: { name: "hwlab-cloud-api-dev-db-admin", key: "admin-url", optional: false } } }, { name: "HWLAB_CLOUD_DB_SSL_MODE", value: "disable" }, { name: "HWLAB_CD_TRANSACTION_ID", value: transaction.transactionId }, { name: "HWLAB_CD_TRANSACTION_OWNER", value: transaction.ownerTaskId }, { name: "HWLAB_CD_LOCK_NAME", value: transaction.lockName } ] } ] } } } }; } async function kubectlCommandResult(ctx, kubectlContext, args, options = {}) { const result = await ctx.runCommand(kubectlContext.executor, args, { cwd: ctx.repoRoot, env: kubectlContext.env, input: options.input, timeoutMs: options.timeoutMs ?? 30000 }); return { ...result, command: shellCommand("kubectl", args), stdout: result.stdout ?? "", stderr: result.stderr ?? "" }; } function parseLastJsonObject(text) { const normalized = String(text ?? "").trim(); const lines = normalized.split(/\r?\n/u).reverse(); for (const line of lines) { const parsed = parseJsonMaybe(line.trim()); if (parsed && typeof parsed === "object") return parsed; } const starts = []; for (let index = 0; index < normalized.length; index += 1) { if (normalized[index] === "{") starts.push(index); } for (const start of starts.reverse()) { const parsed = parseJsonMaybe(normalized.slice(start)); if (parsed && typeof parsed === "object") return parsed; } return parseJsonMaybe(normalized); } function stepBlocker(stepResult) { const stdoutBlockers = Array.isArray(stepResult.stdoutJson?.blockers) ? stepResult.stdoutJson.blockers : []; const primary = stdoutBlockers.find((item) => item?.status === "open") ?? stdoutBlockers[0] ?? null; const reason = primary?.reason ?? primary?.summary ?? `${stepResult.command} exited ${stepResult.code}.`; const impact = primary?.impact ?? stepBlockerImpact(stepResult); const safeNextAction = primary?.safeNextAction ?? stepBlockerSafeNextAction(stepResult); return structuredBlocker({ type: "runtime_blocker", scope: stepResult.id, reason, impact, safeNextAction, retryable: primary?.retryable ?? true, evidence: { command: stepResult.command, code: stepResult.code, reportPath: stepResult.reportPath ?? null, blockerScope: primary?.scope ?? null, k8sJob: stepResult.k8sJob ?? null } }); } function stepBlockerImpact(stepResult) { if (stepResult.id === "runtime-db-provisioning") return "DEV CD cannot prove DB role/database provisioning before workload apply."; if (stepResult.id === "runtime-db-migration") return "DEV CD cannot prove runtime schema migration before workload apply."; if (stepResult.id === "runtime-durable-postflight") return "DEV CD cannot prove /health/live, /v1, and M3 durable evidence persistence after apply."; if (stepResult.id === "dev-deploy-apply") return "DEV desired-state apply did not complete, so live readiness cannot be trusted."; return "DEV CD transaction cannot safely continue."; } function stepBlockerSafeNextAction(stepResult) { if (stepResult.id === "runtime-db-provisioning") return "Repair repo-owned runtime provisioning inputs/SecretRefs without printing Secret values, then rerun DEV CD."; if (stepResult.id === "runtime-db-migration") return "Repair repo-owned runtime migration/schema state, then rerun DEV CD."; if (stepResult.id === "runtime-durable-postflight") return "Inspect the runtime postflight report, repair durable readiness or M3 evidence persistence, then rerun DEV CD after host-controlled rollout if needed."; if (stepResult.id === "dev-deploy-apply") return `Inspect ${deployApplyReportPath} and rerun the repo-owned DEV CD path after the apply blocker is fixed.`; return "Fix the reported step blocker and rerun the repo-owned DEV CD transaction."; } function structuredBlocker({ type = "runtime_blocker", scope, reason, impact, safeNextAction, retryable = true, evidence = null }) { return { type, scope, status: "open", reason: oneLine(reason), impact: oneLine(impact), safeNextAction: oneLine(safeNextAction), retryable: Boolean(retryable), summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`), ...(evidence ? { evidence } : {}) }; } function healthCommit(json) { return ( json?.commit?.id || json?.commitId || json?.revision || json?.image?.tag || null ); } function healthImage(json) { if (typeof json?.image === "string") return json.image; return json?.image?.reference ?? null; } function summarizeHealthJson(json, expectedServiceId, expectedCommit) { const observedCommit = healthCommit(json); return { serviceId: json?.serviceId ?? json?.service?.id ?? null, environment: json?.environment ?? null, applicationStatus: json?.status ?? null, ready: Object.hasOwn(json ?? {}, "ready") ? json.ready : null, observedCommit, image: healthImage(json), imageTag: json?.image?.tag ?? null, observedAt: json?.observedAt ?? null, serviceMatches: (json?.serviceId ?? json?.service?.id) === expectedServiceId, environmentMatches: json?.environment === ENVIRONMENT_DEV, commitMatches: expectedCommit ? commitMatches(observedCommit, expectedCommit) : null, blockerCodes: Array.isArray(json?.blockerCodes) ? json.blockerCodes : [] }; } async function defaultHttpGetJson(url, timeoutMs = 10000) { return new Promise((resolve, reject) => { const request = httpRequest(url, { method: "GET", timeout: timeoutMs }, (response) => { response.setEncoding("utf8"); let body = ""; response.on("data", (chunk) => { body += chunk; }); response.on("end", () => { let json = null; try { json = JSON.parse(body); } catch {} resolve({ statusCode: response.statusCode ?? 0, body, json }); }); }); request.on("timeout", () => { request.destroy(new Error(`timeout after ${timeoutMs}ms`)); }); request.on("error", reject); request.end(); }); } async function probeLiveEndpoint(ctx, { id, url, expectedServiceId, expectedCommit }) { const startedAt = ctx.now().toISOString(); try { const response = await ctx.httpGetJson(url, 10000); const summary = summarizeHealthJson(response.json, expectedServiceId, expectedCommit); const reachable = response.statusCode >= 200 && response.statusCode < 300 && Boolean(response.json); const identityMatches = reachable && summary.serviceMatches && summary.environmentMatches && (expectedCommit ? summary.commitMatches : true); return { id, url, status: identityMatches ? "pass" : "blocked", httpStatus: response.statusCode, reachable, identityMatches, startedAt, finishedAt: ctx.now().toISOString(), expectedServiceId, expectedCommit: expectedCommit ?? null, ...summary }; } catch (error) { return { id, url, status: "blocked", reachable: false, identityMatches: false, startedAt, finishedAt: ctx.now().toISOString(), expectedServiceId, expectedCommit: expectedCommit ?? null, error: oneLine(error.message) }; } } export async function verifyDevLive(ctx, { expectedCommit = null } = {}) { const [cloudWeb, cloudApi] = await Promise.all([ probeLiveEndpoint(ctx, { id: "cloud-web-16666", url: browserLiveUrl, expectedServiceId: "hwlab-cloud-web", expectedCommit }), probeLiveEndpoint(ctx, { id: "cloud-api-16667", url: apiLiveUrl, expectedServiceId: "hwlab-cloud-api", expectedCommit }) ]); const endpoints = [cloudWeb, cloudApi]; return { status: endpoints.every((endpoint) => endpoint.status === "pass") ? "pass" : "blocked", expectedCommit: expectedCommit ?? null, endpoints, summary: { checked: endpoints.length, reachable: endpoints.filter((endpoint) => endpoint.reachable).length, identityMatches: endpoints.filter((endpoint) => endpoint.identityMatches).length, commitMatches: expectedCommit ? endpoints.filter((endpoint) => endpoint.commitMatches === true).length : null, ports: [16666, 16667] } }; } function deployServiceCount(manifest) { const services = manifest?.services; if (Array.isArray(services)) return services.length; if (services && typeof services === "object") return Object.keys(services).length; return null; } function summarizeDeployJson(deploy) { return { path: deploy.path, hash: deploy.hash, commitId: deploy.commitId, namespace: deploy.namespace, environment: deploy.environment, endpoint: deploy.endpoint, serviceCount: deployServiceCount(deploy.manifest) }; } function deployJsonMatchesTarget(deploy, target) { return deploy.commitId === target.shortCommitId || deploy.commitId === target.commitId; } function compactLiveEndpoint(endpoint) { return { id: endpoint.id, url: endpoint.url, status: endpoint.status, httpStatus: endpoint.httpStatus ?? null, reachable: endpoint.reachable, identityMatches: endpoint.identityMatches, serviceId: endpoint.serviceId, environment: endpoint.environment, observedCommit: endpoint.observedCommit, commitMatches: endpoint.commitMatches, blockerCodes: endpoint.blockerCodes ?? [], error: endpoint.error ?? null }; } function compactLiveReport(live) { return { status: live.status, expectedCommit: live.expectedCommit ?? null, summary: live.summary ?? { checked: 0 }, endpoints: (live.endpoints ?? []).map(compactLiveEndpoint) }; } function buildLiveDelta(live, target) { const expectedCommit = target?.shortCommitId ?? target?.commitId ?? null; const endpoints = (live?.endpoints ?? []).map((endpoint) => ({ id: endpoint.id, observedCommit: endpoint.observedCommit ?? null, expectedCommit, matchesPromotion: expectedCommit ? commitMatches(endpoint.observedCommit, expectedCommit) : null, status: endpoint.status })); const mismatched = endpoints.filter((endpoint) => endpoint.matchesPromotion === false); return { status: expectedCommit && mismatched.length === 0 && endpoints.length > 0 ? "pass" : expectedCommit ? "mismatch" : "unknown", expectedCommit, mismatched: mismatched.length, endpoints }; } async function readDeployLockStatus({ ctx, args, kubectlContext }) { const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], { timeoutMs: 15000 }); if (get.code === 0) { return { status: "read", command: get.command, lock: parseDeployLock(JSON.parse(get.stdout)) }; } if (isNotFound(get)) { return { status: "absent", command: get.command, lockName: args.lockName, lock: null }; } return { status: "unavailable", command: get.command, lockName: args.lockName, lock: null, error: redactSensitiveText(get.stderr || get.stdout) }; } function summarizeLockStatus(lockRead, now) { if (lockRead.status === "absent") { return { status: "absent", lockName: lockRead.lockName ?? defaultLockName, current: null, retryAfterSeconds: 0, expiresAt: null }; } if (lockRead.status === "unavailable") { return { status: "unavailable", lockName: lockRead.lockName ?? defaultLockName, current: null, retryAfterSeconds: null, expiresAt: null, reason: "kubectl or DEV kubeconfig unavailable; command output is redacted", redactedReason: oneLine(lockRead.error).slice(0, 500) }; } const classification = classifyDeployLock(lockRead.lock, now); const status = lockRead.lock?.phase === "released" ? "released" : classification.held ? "held" : classification.stale ? "stale" : "unknown"; return { status, lockName: lockRead.lock?.lockName ?? lockRead.lockName ?? defaultLockName, current: compactLockSummary(lockRead.lock), retryAfterSeconds: classification.retryAfterSeconds, expiresAt: classification.expiresAt, allowedLockAction: { breakStale: classification.stale ? "--apply --confirm-dev --confirmed-non-production --break-stale-lock" : null } }; } function buildStatusNextActions({ target, deployBefore, lock, liveVerify }) { const actions = []; if (target.desiredStateCheck?.status === "blocked") { actions.push("deploy/deploy.json does not match the target ref and the deploy-json promotion desired-state check failed; refresh artifacts/catalog or choose an explicit published promotion."); } if (!target.headMatchesTarget) { const controlShortCommit = target.controlRef?.shortCommitId ?? target.shortCommitId; actions.push(`Checkout or fast-forward to ${target.ref} ${controlShortCommit} before apply.`); } if (deployBefore.commitId !== target.shortCommitId && deployBefore.commitId !== target.commitId) { actions.push(`deploy/deploy.json commitId is ${deployBefore.commitId}; target ref is ${target.shortCommitId}.`); } if (lock.status === "held") { actions.push(`Wait for the current DEV CD holder to release the Lease or for the ${lock.retryAfterSeconds}s TTL to expire before retrying apply.`); } else if (lock.status === "stale") { actions.push("Use --break-stale-lock only after confirming the holder is inactive."); } else if (lock.status === "unavailable") { actions.push("Fix kubectl/KUBECONFIG access before applying."); } if (liveVerify.status !== "pass") { actions.push("Inspect live endpoint identity/health before treating DEV as converged."); } const liveDelta = buildLiveDelta(liveVerify, target); if (liveDelta.status === "mismatch") { actions.push(`Live DEV is still serving ${liveDelta.mismatched} endpoint(s) that do not match promotion ${target.shortCommitId}; host commander should run the controlled DEV rollout after the apply report is ready.`); } if (actions.length === 0) { actions.push(target.publishRequired === false ? `Run --apply --confirm-dev --confirmed-non-production --report ${defaultReportPath} when ready to apply the published deploy.json promotion without republishing HEAD.` : `Run --apply --confirm-dev --confirmed-non-production --report ${defaultReportPath} when ready to publish/apply DEV.`); } return actions; } async function runDevCdStatus(args, ctx, stdout) { const generatedAt = ctx.now().toISOString(); const deployBefore = await readDeployJson(ctx.repoRoot); const artifactEvidence = await readArtifactEvidence(ctx.repoRoot); const target = await resolveEffectiveTarget(ctx, args, deployBefore, artifactEvidence); const kubectlContext = await resolveKubectlContext(args, ctx.env, ctx.runCommand); const lockRead = await readDeployLockStatus({ ctx, args, kubectlContext }); const lock = summarizeLockStatus(lockRead, ctx.now()); const liveVerify = args.skipLiveVerify ? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } } : await verifyDevLive(ctx, { expectedCommit: null }); const deployMatchesTarget = deployJsonMatchesTarget(deployBefore, target); const liveDelta = buildLiveDelta(liveVerify, target); const status = ( lock.status === "unavailable" || liveVerify.status !== "pass" || liveDelta.status === "mismatch" || !target.headMatchesTarget || !deployMatchesTarget || target.publishRequired === false && target.artifactBoundary?.status === "degraded" ) ? "degraded" : "pass"; const report = { ok: true, status, mode: args.dryRun ? "dry-run" : "status", command: "dev-cd-apply", generatedAt, mutationAttempted: false, prodTouched: false, target: { ref: target.ref, promotionCommit: target.commitId, shortCommitId: target.shortCommitId, promotionSource: target.promotionSource, publishRequired: target.publishRequired, controlRef: target.controlRef, headCommitId: target.headCommitId, headMatchesTarget: target.headMatchesTarget, desiredStateCheck: target.desiredStateCheck, artifactBoundary: target.artifactBoundary, namespace: args.targetNamespace }, deployJson: { ...summarizeDeployJson(deployBefore), matchesTarget: deployMatchesTarget }, artifactCatalog: artifactEvidence.catalog, artifactReport: artifactEvidence.report, kubectl: { kubeconfigSource: kubectlContext.kubeconfigSource, kubeconfig: kubectlContext.kubeconfig ? "" : "", lockReadCommand: lockRead.command }, lock, live: compactLiveReport(liveVerify), liveDelta, nextActions: buildStatusNextActions({ target, deployBefore, lock, liveVerify }), fullOutputHint: `Use --apply --confirm-dev --confirmed-non-production --report ${defaultReportPath} for the full transaction JSON; use --full-output only when stdout needs the full JSON.` }; stdout.write(`${JSON.stringify(report, null, 2)}\n`); return 0; } function summarizeDevCdApplyReport(report) { const apply = report.devCdApply ?? {}; return { ok: report.status === "pass", status: report.status, mode: apply.mode ?? "apply", command: "dev-cd-apply", commitId: report.commitId, generatedAt: report.generatedAt, mutationAttempted: apply.safety?.mutationAttempted ?? false, prodTouched: false, transaction: { transactionId: report.transaction?.transactionId ?? null, ownerTaskId: report.transaction?.ownerTaskId ?? null, targetNamespace: report.transaction?.targetNamespace ?? defaultNamespace, lockName: report.transaction?.lockName ?? defaultLockName, phases: report.transaction?.phases ?? [], release: report.transaction?.release ?? null }, target: apply.target ?? null, lock: { acquired: Boolean(apply.lock?.acquired), current: compactLockSummary(apply.lock?.current), staleBreak: compactLockSummary(apply.lock?.staleBreak) }, steps: (apply.steps ?? []).map((step) => ({ id: step.id, phase: step.phase, status: step.status, code: step.code, reportPath: step.reportPath })), liveBefore: apply.liveBefore ? compactLiveReport(apply.liveBefore) : null, liveVerify: apply.liveVerify ? compactLiveReport(apply.liveVerify) : null, preflight: apply.preflight ? { status: apply.preflight.status, scope: apply.preflight.scope, kubeconfig: apply.preflight.kubeconfig, blockerCount: apply.preflight.blockers?.length ?? 0, secretRefs: (apply.preflight.secretRefs ?? []).map((ref) => ({ secretName: ref.secretName, secretKey: ref.secretKey, status: ref.status, exists: ref.exists, keyPresent: ref.keyPresent })) } : null, blockers: report.blockers ?? [], reportPaths: apply.reportPaths ?? {}, fullOutputHint: `Use --full-output to print the full report to stdout. Use --report ${defaultReportPath} to persist the full report outside the repo.` }; } function buildReport({ args, transaction, target, deployBefore, deployAfter, lockState, steps, blockers, status, startedAt, finishedAt, preflight, liveBefore, liveVerify, release }) { return { $schema: "https://hwlab.pikastech.local/schemas/dev-cd-apply.schema.json", $id: "https://hwlab.pikastech.local/dev-cd/dev-cd-apply.json", reportVersion: "v1", issue: "pikasTech/HWLAB#274", taskId: "dev-cd-apply", commitId: target.shortCommitId, acceptanceLevel: "dev_cd_apply", devOnly: true, prodDisabled: true, status, generatedAt: finishedAt, reportLifecycle: activeReportLifecycle("Current DEV CD transaction report; the only DEV side-effect entrypoint is scripts/dev-cd-apply.mjs."), supports: [ "pikasTech/HWLAB#274", "pikasTech/HWLAB#116", "pikasTech/HWLAB#235", "pikasTech/HWLAB#50", "pikasTech/HWLAB#57", "pikasTech/HWLAB#67", "pikasTech/HWLAB#73" ], sourceContract: { status: blockers.length ? "blocked" : "pass", documents: [ "docs/reference/deployment-publish.md", "docs/reference/dev-runtime-boundary.md", "docs/dev-artifact-publish.md", "docs/dev-deploy-apply.md", "docs/artifact-catalog.md" ], summary: "DEV CD apply/verify side effects are serialized by one Kubernetes Lease and use deploy/deploy.json as the publish-control desired-state source; CI artifact reports are audit snapshots." }, validationCommands: [ "node --check scripts/dev-cd-apply.mjs", "node --check scripts/src/dev-cd-apply.mjs", "node --test scripts/src/dev-cd-apply.test.mjs", "node --check scripts/dev-artifact-publish.mjs", "node --check scripts/src/dev-deploy-apply.mjs", "node --test scripts/src/dev-deploy-apply.test.mjs", "node scripts/deploy-desired-state-plan.mjs --check", "node scripts/validate-artifact-catalog.mjs", "git diff --check" ], localSmoke: { status: "not_run", commands: ["npm run check"], evidence: ["Local checks are listed as validation commands and are not implied by a DEV CD transaction."], summary: "The transaction report records DEV publish/apply/verify evidence; local smoke is a separate gate." }, dryRun: { status: "not_run", commands: [`node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked --report ${deployApplyReportPath}`], evidence: ["The transaction path owns live side effects; dry-run remains a preflight support mode."], summary: "DEV CD mutation requires --apply and the transaction Lease lock." }, devPreconditions: { status: blockers.length ? "blocked" : "pass", requirements: [ "DEV CD apply must force KUBECONFIG=/etc/rancher/k3s/k3s.yaml and verify native node d601 before any Lease, Job, apply, or rollout side effect.", "DEV CD apply must reject docker-desktop, desktop-control-plane, 127.0.0.1:11700, bare kubectl ambiguity, and second hwlab-dev control-plane signals before mutation.", "DEV CD apply must verify required SecretRef existence and key names without reading or printing Secret values.", "HEAD must match the requested target ref before publish.", "Lease/hwlab-dev-cd-lock must be acquired before DEV apply/verify side effects.", "Artifact publish must run under CI artifact identity, with HWLAB_CD_TRANSACTION_ID accepted only for the legacy transition path.", "DEV DB role/database provisioning and runtime migration must run through repo-owned commands before workload apply.", "deploy/deploy.json, artifact catalog, and workloads must converge before apply.", "When deploy/deploy.json already points to a published promotion commit, HEAD may be a later control commit and the transaction must consume that promotion without republishing HEAD.", "Public 16666 and 16667 live health plus M3 durable postflight must be recorded in this report." ], summary: blockers.length ? "One or more DEV CD transaction preconditions failed." : "DEV CD transaction preconditions passed." }, transaction: { transactionId: transaction.transactionId, ownerTaskId: transaction.ownerTaskId, targetNamespace: transaction.targetNamespace, lockName: transaction.lockName, lockBackend: "Lease", startedAt, finishedAt, ttlSeconds: args.ttlSeconds, phases: transaction.phases, release }, devCdApply: { mode: args.apply ? "apply" : "dry-run", target: { ref: target.ref, promotionCommit: target.commitId, shortCommitId: target.shortCommitId, promotionSource: target.promotionSource, publishRequired: target.publishRequired, controlRef: target.controlRef, headCommitId: target.headCommitId, headMatchesTarget: target.headMatchesTarget, desiredStateCheck: target.desiredStateCheck, artifactBoundary: target.artifactBoundary, namespace: args.targetNamespace, apiEndpoint: DEV_ENDPOINT, browserEndpoint: "http://74.48.78.17:16666/" }, deployJson: { before: deployBefore, after: deployAfter }, lock: { acquired: Boolean(lockState?.acquired), current: lockState?.lock ?? null, staleBreak: lockState?.staleBreak ?? null }, steps, liveBefore, liveVerify, preflight, reportPaths: { transaction: args.writeReport ? args.reportPath : null, artifacts: artifactReportPath, deployApply: deployApplyReportPath, runtimeProvisioning: runtimeProvisioningReportPath, runtimeMigration: runtimeMigrationReportPath, runtimePostflight: runtimePostflightReportPath }, safety: { prodTouched: false, secretValuesRead: false, secretValuesPrinted: false, manualDbWrite: false, oldPublicPortsUsed: false, mutationAttempted: steps.some((step) => ["artifact-publish", "dev-deploy-apply"].includes(step.id)), singleLock: "Lease/hwlab-dev/hwlab-dev-cd-lock" } }, blockers, notes: "DEV-only CD transaction evidence. This does not claim M3 PASS or runtime durability." }; } function buildArgumentFailure(error) { const blockers = error.blockers ?? [{ type: "safety_blocker", scope: error.code ?? "dev-cd-apply", status: "open", summary: oneLine(error.message) }]; return { ok: false, status: "blocked", error: error.code ?? "dev-cd-apply-failed", blockers, mutationAttempted: false, prodTouched: false }; } export function formatDevCdApplyFailure(error) { if (error?.lockFailure) return error.lockFailure; if (error instanceof DevCdApplyError) return buildArgumentFailure(error); return { ok: false, status: "failed", error: "dev-cd-apply-failed", summary: error instanceof Error ? error.message : String(error), mutationAttempted: false, prodTouched: false }; } export async function runDevCdApply(argv, io = {}) { const stdout = io.stdout ?? process.stdout; const env = io.env ?? process.env; const ctx = { repoRoot: io.repoRoot ?? defaultRepoRoot, env, runCommand: io.runCommand ?? defaultRunCommand, httpGetJson: io.httpGetJson ?? defaultHttpGetJson, now: io.now ?? (() => new Date()) }; let args; try { args = parseArgs(argv); if (args.help) { stdout.write(`${JSON.stringify({ ok: true, status: "pass", help: usage() }, null, 2)}\n`); return 0; } if (!args.apply && !args.status) { args.status = true; } validateArgs(args); if (args.status) { return await runDevCdStatus(args, ctx, stdout); } } catch (error) { const failure = formatDevCdApplyFailure(error); stdout.write(`${JSON.stringify(failure, null, 2)}\n`); return 2; } const startedAt = ctx.now().toISOString(); const transaction = { transactionId: randomUUID(), ownerTaskId: args.ownerTaskId ?? defaultOwnerTaskId(env), targetNamespace: args.targetNamespace, lockName: args.lockName, phases: [], kubectlContext: null }; const steps = []; const blockers = []; let lockState = null; let release = null; let target = null; let deployBefore = null; let deployAfter = null; let liveBefore = null; let liveVerify = null; let preflight = null; let status = "blocked"; try { deployBefore = await readDeployJson(ctx.repoRoot); const artifactEvidence = await readArtifactEvidence(ctx.repoRoot); target = await resolveEffectiveTarget(ctx, args, deployBefore, artifactEvidence); if (target.desiredStateCheck?.status === "blocked") { blockers.push({ type: "contract_blocker", scope: "desired-state-promotion", status: "open", summary: "deploy/deploy.json is not aligned with the target ref, and the deploy-json promotion check did not pass." }); throw new DevCdApplyError("deploy desired-state does not match the target ref or a published deploy.json promotion", { code: "desired-state-promotion-check-failed", suppressCatchBlocker: true }); } if (artifactBoundaryRequiresApplyBlock(target)) { blockers.push({ type: "contract_blocker", scope: "artifact-boundary", status: "open", summary: "deploy/deploy.json, deploy/artifact-catalog.dev.json, deploy/k8s/base/workloads.yaml, and registry manifests must prove the same promotion commit before apply." }); throw new DevCdApplyError("artifact desired-state provenance does not match the deploy.json promotion", { code: "artifact-boundary-provenance-mismatch", suppressCatchBlocker: true }); } if (!target.headMatchesTarget) { const controlShortCommit = target.controlRef?.shortCommitId ?? target.shortCommitId; blockers.push({ type: "contract_blocker", scope: "target-ref-head", status: "open", summary: `HEAD ${target.headShortCommitId} must match target ref ${target.ref} ${controlShortCommit} before DEV CD apply.` }); throw new DevCdApplyError("checkout is behind the requested DEV CD control ref", { code: "target-ref-head-mismatch", suppressCatchBlocker: true }); } const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand); transaction.kubectlContext = kubectlContext; preflight = await runDevCdPreflight({ ctx, args, kubectlContext }); if (preflight.status !== "pass") { blockers.push(...preflight.blockers); throw new DevCdApplyError("DEV CD preflight blocked before any write side effect", { code: "dev-cd-preflight-blocked", suppressCatchBlocker: true }); } liveBefore = { status: "pending", reason: "liveBefore is captured immediately after Lease acquisition so publish/apply cannot race before the transaction lock." }; lockState = await acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId: transaction.transactionId, ownerTaskId: transaction.ownerTaskId, liveBefore, now: ctx.now() }); liveBefore = await verifyDevLive(ctx, { expectedCommit: null }); await updateDeployLockLiveBefore({ ctx, args, kubectlContext, transactionId: transaction.transactionId, lockState, liveBefore }); transaction.phases.push({ phase: "publishing", status: "entered", at: ctx.now().toISOString() }); const publishSteps = target.publishRequired === false ? [ { id: "desired-state-check-existing-promotion", phase: "publishing", command: process.execPath, args: [ "scripts/deploy-desired-state-plan.mjs", "--promotion-commit", target.commitId, "--check", "--pretty" ], timeoutMs: 120000 }, { id: "artifact-catalog-validate", phase: "publishing", command: process.execPath, args: ["scripts/validate-artifact-catalog.mjs"], timeoutMs: 120000 } ] : [ { id: "desired-state-check-before-publish", phase: "publishing", command: process.execPath, args: ["scripts/deploy-desired-state-plan.mjs", "--check", "--pretty"], timeoutMs: 120000 }, { id: "artifact-publish", phase: "publishing", command: process.execPath, args: ["scripts/dev-artifact-publish.mjs", "--publish", "--report", artifactReportPath], timeoutMs: 60 * 60 * 1000, reportPath: artifactReportPath }, { id: "artifact-catalog-refresh", phase: "publishing", command: process.execPath, args: [ "scripts/refresh-artifact-catalog.mjs", "--target-ref", target.commitId, "--publish-report", artifactReportPath ], timeoutMs: 120000 }, { id: "artifact-catalog-validate", phase: "publishing", command: process.execPath, args: ["scripts/validate-artifact-catalog.mjs"], timeoutMs: 120000 }, { id: "desired-state-check-after-refresh", phase: "publishing", command: process.execPath, args: [ "scripts/deploy-desired-state-plan.mjs", "--promotion-commit", target.commitId, "--check", "--pretty" ], timeoutMs: 120000 }, ]; const plannedSteps = [ ...publishSteps, { id: "runtime-db-provisioning", phase: "applying", kind: "runtime-k8s-job", jobNamePrefix: "hwlab-runtime-provision", commandArgs: [ "cmd/hwlab-cloud-api/provision.mjs", "--apply", "--confirm-dev", "--confirmed-non-production", "--report", runtimeProvisioningReportPath, "--fail-on-blocked" ], timeoutMs: 5 * 60 * 1000, reportPath: runtimeProvisioningReportPath }, { id: "runtime-db-migration", phase: "applying", kind: "runtime-k8s-job", jobNamePrefix: "hwlab-runtime-migrate", commandArgs: [ "cmd/hwlab-cloud-api/migrate.mjs", "--apply", "--confirm-dev", "--confirmed-non-production", "--report", runtimeMigrationReportPath, "--fail-on-blocked" ], timeoutMs: 5 * 60 * 1000, reportPath: runtimeMigrationReportPath }, { id: "dev-deploy-apply", phase: "applying", command: process.execPath, args: [ "scripts/dev-deploy-apply.mjs", "--apply", "--confirm-dev", "--confirmed-non-production", "--report", deployApplyReportPath, ...(args.kubeconfig ? ["--kubeconfig", args.kubeconfig] : []), ...(args.registryManifestBaseUrl ? ["--registry-manifest-base-url", args.registryManifestBaseUrl] : []) ], timeoutMs: 20 * 60 * 1000, reportPath: deployApplyReportPath }, ...( args.skipRuntimePostflight ? [] : [{ id: "runtime-durable-postflight", phase: "verifying", command: process.execPath, args: [ "scripts/dev-runtime-postflight.mjs", "--live", "--confirm-dev", "--confirmed-non-production", "--target", "api", "--report", runtimePostflightReportPath, "--fail-on-blocked" ], timeoutMs: 2 * 60 * 1000, reportPath: runtimePostflightReportPath }] ) ]; if (blockers.length === 0) { for (const step of plannedSteps) { if (step.phase !== lockState.lock?.phase) { await updateDeployLockPhase({ ctx, args, kubectlContext, transactionId: transaction.transactionId, lockState, phase: step.phase }); transaction.phases.push({ phase: step.phase, status: "entered", at: ctx.now().toISOString() }); } const result = step.kind === "runtime-k8s-job" ? await runRuntimeK8sJobStep(ctx, transaction, step) : await runStep(ctx, transaction, step); steps.push(result); if (result.status !== "pass") { blockers.push(stepBlocker(result)); break; } } } if (blockers.length === 0) { if (lockState.lock?.phase !== "verifying") { await updateDeployLockPhase({ ctx, args, kubectlContext, transactionId: transaction.transactionId, lockState, phase: "verifying" }); transaction.phases.push({ phase: "verifying", status: "entered", at: ctx.now().toISOString() }); } deployAfter = await readDeployJson(ctx.repoRoot); liveVerify = args.skipLiveVerify ? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } } : await verifyDevLive(ctx, { expectedCommit: target.shortCommitId }); if (liveVerify.status !== "pass") { blockers.push({ type: "observability_blocker", scope: "live-verify", status: "open", summary: "16666/16667 live health did not match the DEV CD promotion commit." }); } } else { deployAfter = deployBefore; liveVerify = { status: "not_run", reason: "transaction blocked before live verify", endpoints: [], summary: { checked: 0 } }; } status = blockers.length === 0 ? "pass" : "blocked"; release = await releaseDeployLock({ ctx, args, kubectlContext, transactionId: transaction.transactionId, status }); transaction.phases.push({ phase: "released", status: release.status, at: ctx.now().toISOString() }); } catch (error) { if (error?.lockFailure) { stdout.write(`${JSON.stringify(error.lockFailure, null, 2)}\n`); return 2; } if (!error?.suppressCatchBlocker) { blockers.push({ type: "runtime_blocker", scope: error.code ?? "dev-cd-apply", status: "open", summary: oneLine(error.message) }); } status = "blocked"; if (lockState?.acquired) { const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand); release = await releaseDeployLock({ ctx, args, kubectlContext, transactionId: transaction.transactionId, status }); transaction.phases.push({ phase: "released", status: release.status, at: ctx.now().toISOString() }); } } const finishedAt = ctx.now().toISOString(); const report = buildReport({ args, transaction, target: target ?? { ref: args.targetRef, commitId: "unknown", shortCommitId: "unknown", headCommitId: "unknown", headMatchesTarget: false }, deployBefore: deployBefore ?? { path: "deploy/deploy.json", hash: "unknown", commitId: "unknown" }, deployAfter: deployAfter ?? deployBefore ?? { path: "deploy/deploy.json", hash: "unknown", commitId: "unknown" }, lockState, steps, blockers, status, startedAt, finishedAt, preflight: preflight ?? { status: "not_run", scope: "pre-lock-pre-side-effect", blockers: [], safety: { writeSideEffectsAttempted: false, prodTouched: false, secretValuesRead: false, secretValuesPrinted: false } }, liveBefore: liveBefore ?? { status: "not_run" }, liveVerify: liveVerify ?? { status: "not_run", endpoints: [], summary: { checked: 0 } }, release }); if (args.writeReport) { const absoluteReportPath = ensureNotRepoReportsPath(ctx.repoRoot, args.reportPath, "--report"); await mkdir(path.dirname(absoluteReportPath), { recursive: true }); await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`); } const stdoutPayload = args.fullOutput ? report : summarizeDevCdApplyReport(report); stdout.write(`${JSON.stringify(stdoutPayload, null, 2)}\n`); return status === "pass" ? 0 : 2; }