Codex
3025 lines
106 KiB
JavaScript
3025 lines
106 KiB
JavaScript
import { spawn } from "node:child_process";
|
|
import { createHash, randomUUID } from "node:crypto";
|
|
import { mkdir, readFile, writeFile } from "node:fs/promises";
|
|
import { request as httpRequest } from "node:http";
|
|
import path from "node:path";
|
|
import { fileURLToPath } from "node:url";
|
|
|
|
import { activeReportLifecycle } from "../../internal/dev-report-lifecycle.mjs";
|
|
import { DEV_ENDPOINT, ENVIRONMENT_DEV } from "../../internal/protocol/index.mjs";
|
|
import {
|
|
buildKubectlCommandPrefix,
|
|
redactSensitiveText,
|
|
resolveDevKubeconfigSelection
|
|
} from "./dev-deploy-apply.mjs";
|
|
import { ensureNotRepoReportsPath, tempReportPath } from "./report-paths.mjs";
|
|
|
|
const defaultRepoRoot = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "../..");
|
|
const defaultNamespace = "hwlab-dev";
|
|
const defaultLockName = "hwlab-dev-cd-lock";
|
|
const defaultTtlSeconds = 3600;
|
|
const defaultReportPath = tempReportPath("dev-cd-apply.json");
|
|
const d601NativeKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
|
|
const artifactCatalogPath = "deploy/artifact-catalog.dev.json";
|
|
const artifactReportPath = tempReportPath("dev-artifacts.json");
|
|
const artifactDesiredStatePaths = [
|
|
"deploy/artifact-catalog.dev.json",
|
|
"deploy/deploy.json",
|
|
"deploy/k8s/base/workloads.yaml"
|
|
];
|
|
const runtimeProvisioningReportPath = tempReportPath("dev-runtime-provisioning-report.json");
|
|
const runtimeMigrationReportPath = tempReportPath("dev-runtime-migration-report.json");
|
|
const runtimePostflightReportPath = tempReportPath("dev-runtime-postflight-report.json");
|
|
const deployApplyReportPath = tempReportPath("dev-deploy-report.json");
|
|
const runtimeJobReportPrefix = "/tmp/hwlab-dev-runtime-report";
|
|
const browserLiveUrl = "http://74.48.78.17:16666/health/live";
|
|
const apiLiveUrl = `${DEV_ENDPOINT}/health/live`;
|
|
const digestPattern = /^sha256:[a-f0-9]{64}$/u;
|
|
const forbiddenControlPlanePattern = /docker-desktop|desktop-control-plane|127\.0\.0\.1:11700/iu;
|
|
const requiredDevSecretRefs = Object.freeze([
|
|
Object.freeze({ secretName: "hwlab-cloud-api-dev-db", secretKey: "database-url" }),
|
|
Object.freeze({ secretName: "hwlab-cloud-api-dev-db-admin", secretKey: "admin-url" }),
|
|
Object.freeze({ secretName: "hwlab-code-agent-provider", secretKey: "openai-api-key" }),
|
|
Object.freeze({ secretName: "hwlab-code-agent-codex-auth", secretKey: "auth.json" })
|
|
]);
|
|
const requiredDevConfigMapRefs = Object.freeze([
|
|
Object.freeze({ configMapName: "hwlab-code-agent-codex-config", configMapKey: "config.toml" })
|
|
]);
|
|
const lockAnnotationPrefix = "hwlab.pikastech.local";
|
|
const lockAnnotationFields = {
|
|
promotionCommit: `${lockAnnotationPrefix}/promotionCommit`,
|
|
deployJsonHash: `${lockAnnotationPrefix}/deployJsonHash`,
|
|
ownerTaskId: `${lockAnnotationPrefix}/ownerTaskId`,
|
|
transactionId: `${lockAnnotationPrefix}/transactionId`,
|
|
phase: `${lockAnnotationPrefix}/phase`,
|
|
startedAt: `${lockAnnotationPrefix}/startedAt`,
|
|
updatedAt: `${lockAnnotationPrefix}/updatedAt`,
|
|
ttlSeconds: `${lockAnnotationPrefix}/ttlSeconds`,
|
|
liveBefore: `${lockAnnotationPrefix}/liveBefore`,
|
|
targetNamespace: `${lockAnnotationPrefix}/targetNamespace`,
|
|
targetRef: `${lockAnnotationPrefix}/targetRef`,
|
|
lockBackend: `${lockAnnotationPrefix}/lockBackend`,
|
|
releasedAt: `${lockAnnotationPrefix}/releasedAt`,
|
|
releaseStatus: `${lockAnnotationPrefix}/releaseStatus`,
|
|
staleLockBrokenAt: `${lockAnnotationPrefix}/staleLockBrokenAt`,
|
|
staleLockPrevious: `${lockAnnotationPrefix}/staleLockPrevious`
|
|
};
|
|
|
|
class DevCdApplyError extends Error {
|
|
constructor(message, details = {}) {
|
|
super(message);
|
|
this.name = "DevCdApplyError";
|
|
Object.assign(this, details);
|
|
}
|
|
}
|
|
|
|
export function parseArgs(argv) {
|
|
const args = {
|
|
apply: false,
|
|
confirmDev: false,
|
|
confirmedNonProduction: false,
|
|
writeReport: false,
|
|
reportPath: defaultReportPath,
|
|
targetRef: "origin/main",
|
|
kubeconfig: null,
|
|
kubeconfigSpecified: false,
|
|
registryManifestBaseUrl: null,
|
|
lockName: defaultLockName,
|
|
targetNamespace: defaultNamespace,
|
|
ttlSeconds: defaultTtlSeconds,
|
|
ownerTaskId: null,
|
|
breakStaleLock: false,
|
|
skipLiveVerify: false,
|
|
skipRuntimePostflight: false,
|
|
status: false,
|
|
dryRun: false,
|
|
fullOutput: false,
|
|
help: false,
|
|
flags: new Set()
|
|
};
|
|
|
|
for (let index = 0; index < argv.length; index += 1) {
|
|
const arg = argv[index];
|
|
if (arg === "--apply") {
|
|
args.apply = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--confirm-dev") {
|
|
args.confirmDev = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--confirmed-non-production") {
|
|
args.confirmedNonProduction = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--break-stale-lock") {
|
|
args.breakStaleLock = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--dry-run" || arg === "--status") {
|
|
args.status = true;
|
|
args.dryRun = arg === "--dry-run";
|
|
args.flags.add(arg);
|
|
} else if (arg === "--full-output") {
|
|
args.fullOutput = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--skip-live-verify") {
|
|
args.skipLiveVerify = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--skip-runtime-postflight") {
|
|
args.skipRuntimePostflight = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--report") {
|
|
args.reportPath = readOption(argv, ++index, arg);
|
|
args.writeReport = true;
|
|
args.flags.add(arg);
|
|
} else if (arg === "--target-ref") {
|
|
args.targetRef = readOption(argv, ++index, arg);
|
|
args.flags.add(arg);
|
|
} else if (arg === "--kubeconfig") {
|
|
args.kubeconfig = readOption(argv, ++index, arg);
|
|
args.kubeconfigSpecified = true;
|
|
args.flags.add(arg);
|
|
} else if (arg.startsWith("--kubeconfig=")) {
|
|
args.kubeconfig = arg.slice("--kubeconfig=".length);
|
|
args.kubeconfigSpecified = true;
|
|
args.flags.add("--kubeconfig");
|
|
} else if (arg === "--registry-manifest-base-url") {
|
|
args.registryManifestBaseUrl = readOption(argv, ++index, arg);
|
|
args.flags.add(arg);
|
|
} else if (arg.startsWith("--registry-manifest-base-url=")) {
|
|
args.registryManifestBaseUrl = arg.slice("--registry-manifest-base-url=".length);
|
|
args.flags.add("--registry-manifest-base-url");
|
|
} else if (arg === "--lock-name") {
|
|
args.lockName = readOption(argv, ++index, arg);
|
|
args.flags.add(arg);
|
|
} else if (arg === "--target-namespace") {
|
|
args.targetNamespace = readOption(argv, ++index, arg);
|
|
args.flags.add(arg);
|
|
} else if (arg === "--ttl-seconds") {
|
|
args.ttlSeconds = Number.parseInt(readOption(argv, ++index, arg), 10);
|
|
args.flags.add(arg);
|
|
} else if (arg === "--owner-task-id") {
|
|
args.ownerTaskId = readOption(argv, ++index, arg);
|
|
args.flags.add(arg);
|
|
} else if (arg === "--help" || arg === "-h") {
|
|
args.help = true;
|
|
} else {
|
|
throw new DevCdApplyError(`unknown argument ${arg}`, {
|
|
code: "unknown-argument",
|
|
blockers: [{
|
|
type: "safety_blocker",
|
|
scope: "unknown-argument",
|
|
status: "open",
|
|
summary: `${arg} is not a supported DEV CD option. Use --status/--dry-run for read-only observability or --apply with explicit DEV confirmations.`
|
|
}]
|
|
});
|
|
}
|
|
}
|
|
|
|
return args;
|
|
}
|
|
|
|
function readOption(argv, index, name) {
|
|
const value = argv[index];
|
|
if (!value || value.startsWith("--")) {
|
|
throw new Error(`${name} requires a value`);
|
|
}
|
|
return value;
|
|
}
|
|
|
|
export function usage() {
|
|
return {
|
|
command: "node scripts/dev-cd-apply.mjs",
|
|
summary: "Single DEV CD transaction for latest-main publish, desired-state refresh, apply, and live verify.",
|
|
defaultMode: "status",
|
|
examples: [
|
|
"node scripts/dev-cd-apply.mjs --status",
|
|
"node scripts/dev-cd-apply.mjs --dry-run",
|
|
`node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production --report ${defaultReportPath}`
|
|
],
|
|
options: {
|
|
"--status": "Read-only status: target ref, deploy.json, current Lease, and public live health.",
|
|
"--dry-run": "Read-only alias for --status; does not acquire the Lease or run side effects.",
|
|
"--apply": "Run the DEV CD transaction side effects.",
|
|
"--confirm-dev": "Required for DEV mutation and explicit stale-lock recovery.",
|
|
"--confirmed-non-production": "Required for DEV mutation.",
|
|
"--target-ref REF": "git ref to publish/apply; default: origin/main",
|
|
"--kubeconfig PATH": "DEV kubeconfig path; also honors HWLAB_DEV_KUBECONFIG/KUBECONFIG",
|
|
"--ttl-seconds SECONDS": "lock TTL; default: 3600",
|
|
"--owner-task-id ID": "lock holder; default: Code Queue/env/user derived",
|
|
"--break-stale-lock": "May take over an expired lock only with --confirm-dev.",
|
|
"--report PATH": `write the full transaction JSON outside the repo; packaged scripts use ${defaultReportPath}`,
|
|
"--full-output": "Print the full report to stdout instead of the concise summary.",
|
|
"--skip-runtime-postflight": "Skip the M3/durable runtime postflight while still applying DEV and verifying 16666/16667 health.",
|
|
"--skip-live-verify": "test-only escape hatch; do not use for DEV acceptance"
|
|
}
|
|
};
|
|
}
|
|
|
|
function nowIso() {
|
|
return new Date().toISOString();
|
|
}
|
|
|
|
function kubernetesMicroTime(value) {
|
|
const date = value instanceof Date ? value : new Date(value);
|
|
if (!Number.isFinite(date.getTime())) {
|
|
throw new DevCdApplyError("invalid Kubernetes Lease timestamp", {
|
|
code: "invalid-lease-timestamp"
|
|
});
|
|
}
|
|
return date.toISOString().replace(/\.(\d{3})Z$/u, ".$1000Z");
|
|
}
|
|
|
|
function parseJsonMaybe(value) {
|
|
try {
|
|
return JSON.parse(value);
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
function oneLine(value) {
|
|
return String(value ?? "").replace(/\s+/g, " ").trim();
|
|
}
|
|
|
|
function blocker({ type = "environment_blocker", scope, reason, impact, safeNextAction, retryable }) {
|
|
return {
|
|
type,
|
|
scope,
|
|
status: "open",
|
|
reason: oneLine(reason),
|
|
impact: oneLine(impact),
|
|
safeNextAction: oneLine(safeNextAction),
|
|
retryable: Boolean(retryable),
|
|
summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`)
|
|
};
|
|
}
|
|
|
|
function sha256(value) {
|
|
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
|
}
|
|
|
|
function shortCommit(value) {
|
|
return typeof value === "string" && value.length >= 7 ? value.slice(0, 7) : value;
|
|
}
|
|
|
|
function commitMatches(actual, expected) {
|
|
if (typeof actual !== "string" || typeof expected !== "string") return false;
|
|
return actual === expected || actual.startsWith(expected) || expected.startsWith(actual);
|
|
}
|
|
|
|
function shellCommand(command, args) {
|
|
return [command, ...args].map((part) => (/\s/u.test(part) ? JSON.stringify(part) : part)).join(" ");
|
|
}
|
|
|
|
function defaultOwnerTaskId(env = process.env) {
|
|
return (
|
|
env.CODE_QUEUE_TASK_ID ||
|
|
env.HWLAB_OWNER_TASK_ID ||
|
|
env.GITHUB_RUN_ID ||
|
|
env.USER ||
|
|
"unknown-dev-cd-runner"
|
|
);
|
|
}
|
|
|
|
function defaultRunCommand(command, args, options = {}) {
|
|
return new Promise((resolve) => {
|
|
const abortController = options.timeoutMs ? new AbortController() : null;
|
|
const child = spawn(command, args, {
|
|
cwd: options.cwd ?? defaultRepoRoot,
|
|
env: options.env ?? process.env,
|
|
stdio: ["pipe", "pipe", "pipe"],
|
|
signal: abortController?.signal
|
|
});
|
|
let stdout = "";
|
|
let stderr = "";
|
|
const timeout = options.timeoutMs
|
|
? setTimeout(() => abortController.abort(), options.timeoutMs)
|
|
: null;
|
|
|
|
child.stdin.on("error", () => {});
|
|
child.stdin.end(options.input ?? "");
|
|
child.stdout.on("data", (chunk) => {
|
|
stdout += chunk;
|
|
});
|
|
child.stderr.on("data", (chunk) => {
|
|
stderr += chunk;
|
|
});
|
|
child.on("error", (error) => {
|
|
if (timeout) clearTimeout(timeout);
|
|
resolve({
|
|
code: error.name === "AbortError" ? 124 : 127,
|
|
signal: null,
|
|
stdout,
|
|
stderr: error.message
|
|
});
|
|
});
|
|
child.on("close", (code, signal) => {
|
|
if (timeout) clearTimeout(timeout);
|
|
resolve({ code: code ?? 1, signal, stdout, stderr });
|
|
});
|
|
});
|
|
}
|
|
|
|
async function readDeployJson(repoRoot) {
|
|
const relativePath = "deploy/deploy.json";
|
|
const raw = await readFile(path.join(repoRoot, relativePath), "utf8");
|
|
const manifest = JSON.parse(raw);
|
|
return {
|
|
path: relativePath,
|
|
hash: sha256(raw),
|
|
commitId: manifest.commitId ?? "unknown",
|
|
namespace: manifest.namespace ?? defaultNamespace,
|
|
environment: manifest.environment ?? "unknown",
|
|
endpoint: manifest.endpoint ?? null,
|
|
manifest
|
|
};
|
|
}
|
|
|
|
async function readJsonFileSummary(repoRoot, relativePath, summarize) {
|
|
try {
|
|
const absolutePath = path.isAbsolute(relativePath) ? relativePath : path.join(repoRoot, relativePath);
|
|
const raw = await readFile(absolutePath, "utf8");
|
|
const json = JSON.parse(raw);
|
|
return {
|
|
path: relativePath,
|
|
hash: sha256(raw),
|
|
exists: true,
|
|
...summarize(json)
|
|
};
|
|
} catch (error) {
|
|
if (error?.code === "ENOENT") {
|
|
return {
|
|
path: relativePath,
|
|
hash: null,
|
|
exists: false,
|
|
error: "missing"
|
|
};
|
|
}
|
|
throw error;
|
|
}
|
|
}
|
|
|
|
function servicesArray(value) {
|
|
if (Array.isArray(value)) return value;
|
|
if (value && typeof value === "object") return Object.values(value);
|
|
return [];
|
|
}
|
|
|
|
function summarizeArtifactCatalog(catalog) {
|
|
const services = servicesArray(catalog?.services);
|
|
const digestCounts = { sha256: 0, notPublished: 0, invalid: 0 };
|
|
for (const service of services) {
|
|
if (digestPattern.test(service?.digest ?? "")) digestCounts.sha256 += 1;
|
|
else if (service?.digest === "not_published") digestCounts.notPublished += 1;
|
|
else digestCounts.invalid += 1;
|
|
}
|
|
return {
|
|
kind: catalog?.kind ?? null,
|
|
commitId: catalog?.commitId ?? null,
|
|
artifactState: catalog?.artifactState ?? null,
|
|
ciPublished: catalog?.publish?.ciPublished ?? null,
|
|
registryVerified: catalog?.publish?.registryVerified ?? null,
|
|
provenance: catalog?.publish?.provenance ?? null,
|
|
serviceCount: services.length,
|
|
digestCounts
|
|
};
|
|
}
|
|
|
|
function summarizeArtifactReport(report) {
|
|
const publish = report?.artifactPublish ?? {};
|
|
const services = servicesArray(publish.services);
|
|
const requiredServices = services.filter((service) => service?.artifactRequired !== false);
|
|
const publishedRequiredServices = requiredServices.filter((service) => service?.status === "published");
|
|
const digestCounts = { sha256: 0, notPublished: 0, invalid: 0 };
|
|
for (const service of services) {
|
|
if (digestPattern.test(service?.digest ?? "")) digestCounts.sha256 += 1;
|
|
else if (service?.digest === "not_published") digestCounts.notPublished += 1;
|
|
else digestCounts.invalid += 1;
|
|
}
|
|
return {
|
|
commitId: report?.commitId ?? null,
|
|
status: report?.status ?? null,
|
|
sourceCommitId: publish.sourceCommitId ?? null,
|
|
registryPrefix: publish.registryPrefix ?? null,
|
|
serviceCount: publish.serviceCount ?? services.length,
|
|
requiredServiceCount: publish.requiredServiceCount ?? requiredServices.length,
|
|
publishedCount: publish.publishedCount ?? publishedRequiredServices.length,
|
|
digestCounts,
|
|
generatedAt: publish.generatedAt ?? null
|
|
};
|
|
}
|
|
|
|
async function readArtifactEvidence(repoRoot) {
|
|
const [catalog, report] = await Promise.all([
|
|
readJsonFileSummary(repoRoot, artifactCatalogPath, summarizeArtifactCatalog),
|
|
readJsonFileSummary(repoRoot, artifactReportPath, summarizeArtifactReport)
|
|
]);
|
|
return { catalog, report };
|
|
}
|
|
|
|
async function resolveTargetRef(ctx, targetRef) {
|
|
const target = await ctx.runCommand("git", ["rev-parse", "--verify", `${targetRef}^{commit}`], {
|
|
cwd: ctx.repoRoot,
|
|
timeoutMs: 10000
|
|
});
|
|
if (target.code !== 0) {
|
|
throw new DevCdApplyError(`target ref ${targetRef} could not be resolved`, {
|
|
code: "target-ref-unresolved",
|
|
targetRef,
|
|
stderr: redactSensitiveText(target.stderr)
|
|
});
|
|
}
|
|
const head = await ctx.runCommand("git", ["rev-parse", "--verify", "HEAD^{commit}"], {
|
|
cwd: ctx.repoRoot,
|
|
timeoutMs: 10000
|
|
});
|
|
if (head.code !== 0) {
|
|
throw new DevCdApplyError("HEAD could not be resolved", {
|
|
code: "head-unresolved",
|
|
stderr: redactSensitiveText(head.stderr)
|
|
});
|
|
}
|
|
const commitId = target.stdout.trim();
|
|
const headCommitId = head.stdout.trim();
|
|
return {
|
|
ref: targetRef,
|
|
commitId,
|
|
shortCommitId: shortCommit(commitId),
|
|
headCommitId,
|
|
headShortCommitId: shortCommit(headCommitId),
|
|
headMatchesTarget: commitId === headCommitId
|
|
};
|
|
}
|
|
|
|
async function gitOutput(ctx, args, { allowFailure = false, timeoutMs = 10000 } = {}) {
|
|
const result = await ctx.runCommand("git", args, {
|
|
cwd: ctx.repoRoot,
|
|
timeoutMs
|
|
});
|
|
if (result.code !== 0) {
|
|
if (allowFailure) return null;
|
|
throw new DevCdApplyError("git command failed while resolving DEV CD provenance", {
|
|
code: "git-provenance-failed",
|
|
command: shellCommand("git", args),
|
|
stderr: redactSensitiveText(result.stderr || result.stdout)
|
|
});
|
|
}
|
|
return result.stdout.trim();
|
|
}
|
|
|
|
async function commitParents(ctx, commitId) {
|
|
const line = await gitOutput(ctx, ["rev-list", "--parents", "-n", "1", commitId], { allowFailure: true });
|
|
if (!line) return [];
|
|
return line.split(/\s+/u).slice(1);
|
|
}
|
|
|
|
async function shortGitCommit(ctx, commitId) {
|
|
const value = await gitOutput(ctx, ["rev-parse", "--short=7", commitId], { allowFailure: true });
|
|
return value || shortCommit(commitId);
|
|
}
|
|
|
|
async function filesChangedBetween(ctx, left, right) {
|
|
const output = await gitOutput(ctx, ["diff-tree", "--no-commit-id", "--name-only", "-r", left, right], {
|
|
allowFailure: true
|
|
});
|
|
if (!output) return [];
|
|
return output.split(/\r?\n/u).map((item) => item.trim()).filter(Boolean);
|
|
}
|
|
|
|
async function latestArtifactMergeCommit(ctx, controlCommit) {
|
|
const output = await gitOutput(ctx, [
|
|
"log",
|
|
"--first-parent",
|
|
"--format=%H",
|
|
"-n",
|
|
"1",
|
|
controlCommit,
|
|
"--",
|
|
...artifactDesiredStatePaths
|
|
], { allowFailure: true });
|
|
return output?.split(/\r?\n/u).find(Boolean) ?? null;
|
|
}
|
|
|
|
async function resolveArtifactBoundary(ctx, control, promotion, deployBefore, artifactEvidence) {
|
|
const mergeCommitId = await latestArtifactMergeCommit(ctx, control.commitId);
|
|
if (!mergeCommitId) {
|
|
return {
|
|
status: "unavailable",
|
|
reason: "No first-parent commit touching artifact desired-state files was found.",
|
|
desiredStatePaths: artifactDesiredStatePaths
|
|
};
|
|
}
|
|
|
|
const parents = await commitParents(ctx, mergeCommitId);
|
|
const mergeShortCommitId = await shortGitCommit(ctx, mergeCommitId);
|
|
const firstParent = parents[0] ?? null;
|
|
const secondParent = parents[1] ?? null;
|
|
const promotionParentMatches = firstParent ? commitMatches(firstParent, promotion.commitId) : false;
|
|
const changedPaths = firstParent ? await filesChangedBetween(ctx, firstParent, mergeCommitId) : [];
|
|
const touchedDesiredStatePaths = artifactDesiredStatePaths.filter((entry) => changedPaths.includes(entry));
|
|
const artifactCommitId = secondParent || mergeCommitId;
|
|
const artifactShortCommitId = await shortGitCommit(ctx, artifactCommitId);
|
|
const artifactParents = secondParent ? await commitParents(ctx, secondParent) : [];
|
|
const artifactParentMatches = artifactParents.some((parent) => commitMatches(parent, promotion.commitId));
|
|
const catalogCommitMatches = commitMatches(artifactEvidence.catalog.commitId, promotion.commitId);
|
|
const reportSource = artifactEvidence.report.sourceCommitId ?? artifactEvidence.report.commitId;
|
|
const reportCommitMatches = commitMatches(reportSource, promotion.commitId);
|
|
const deployCommitMatches = commitMatches(deployBefore.commitId, promotion.commitId);
|
|
const isMergeCommit = parents.length >= 2;
|
|
const desiredStateStatus = (
|
|
deployCommitMatches &&
|
|
catalogCommitMatches &&
|
|
(
|
|
promotionParentMatches ||
|
|
artifactParentMatches ||
|
|
commitMatches(mergeCommitId, promotion.commitId)
|
|
)
|
|
) ? "pass" : "degraded";
|
|
const reportStatus = artifactEvidence.report.exists
|
|
? (reportCommitMatches ? "matches_promotion" : "stale_or_unrelated")
|
|
: "missing";
|
|
|
|
return {
|
|
status: desiredStateStatus,
|
|
reportStatus,
|
|
reportIsReleaseGate: false,
|
|
reportPolicy: "CI artifact reports are audit snapshots; deploy.json, artifact catalog, workloads, and registry manifests are the CD release gate.",
|
|
controlCommit: {
|
|
commitId: control.commitId,
|
|
shortCommitId: control.shortCommitId
|
|
},
|
|
artifactMergeCommit: {
|
|
commitId: mergeCommitId,
|
|
shortCommitId: mergeShortCommitId,
|
|
isMergeCommit,
|
|
parents,
|
|
promotionParent: firstParent,
|
|
artifactParent: secondParent,
|
|
promotionParentMatches,
|
|
changedPaths: touchedDesiredStatePaths
|
|
},
|
|
artifactCommit: {
|
|
commitId: artifactCommitId,
|
|
shortCommitId: artifactShortCommitId,
|
|
parents: artifactParents,
|
|
promotionParentMatches: artifactParentMatches
|
|
},
|
|
desiredState: {
|
|
deployCommitId: deployBefore.commitId,
|
|
deployJsonHash: deployBefore.hash,
|
|
catalogCommitId: artifactEvidence.catalog.commitId,
|
|
catalogHash: artifactEvidence.catalog.hash,
|
|
reportCommitId: artifactEvidence.report.commitId,
|
|
reportSourceCommitId: artifactEvidence.report.sourceCommitId,
|
|
reportHash: artifactEvidence.report.hash,
|
|
deployCommitMatches,
|
|
catalogCommitMatches,
|
|
reportCommitMatches,
|
|
reportExists: artifactEvidence.report.exists,
|
|
desiredStatePaths: artifactDesiredStatePaths
|
|
}
|
|
};
|
|
}
|
|
|
|
async function resolveCommitRef(ctx, ref) {
|
|
const result = await ctx.runCommand("git", ["rev-parse", "--verify", `${ref}^{commit}`], {
|
|
cwd: ctx.repoRoot,
|
|
timeoutMs: 10000
|
|
});
|
|
if (result.code !== 0) {
|
|
throw new DevCdApplyError(`promotion commit ${ref} could not be resolved`, {
|
|
code: "promotion-commit-unresolved",
|
|
targetRef: ref,
|
|
stderr: redactSensitiveText(result.stderr)
|
|
});
|
|
}
|
|
const commitId = result.stdout.trim();
|
|
return {
|
|
commitId,
|
|
shortCommitId: shortCommit(commitId)
|
|
};
|
|
}
|
|
|
|
function compactDesiredStateCheck(result) {
|
|
const parsed = parseJsonMaybe(result.stdout ?? "");
|
|
return {
|
|
status: result.code === 0 ? "pass" : "blocked",
|
|
code: result.code,
|
|
summary: parsed?.summary ?? null,
|
|
convergence: parsed?.target?.convergence?.state ?? null,
|
|
diagnostics: Array.isArray(parsed?.diagnostics)
|
|
? parsed.diagnostics.slice(0, 5).map((item) => ({
|
|
code: item.code ?? null,
|
|
path: item.path ?? null,
|
|
message: item.message ?? null
|
|
}))
|
|
: [],
|
|
stderrTail: redactSensitiveText(result.stderr ?? "").trim().slice(-1000)
|
|
};
|
|
}
|
|
|
|
async function checkPromotionDesiredState(ctx, promotionCommit) {
|
|
const result = await ctx.runCommand(process.execPath, [
|
|
"scripts/deploy-desired-state-plan.mjs",
|
|
"--promotion-commit",
|
|
promotionCommit,
|
|
"--check",
|
|
"--pretty"
|
|
], {
|
|
cwd: ctx.repoRoot,
|
|
timeoutMs: 120000
|
|
});
|
|
return compactDesiredStateCheck(result);
|
|
}
|
|
|
|
async function resolveEffectiveTarget(ctx, args, deployBefore, artifactEvidence = null) {
|
|
const control = await resolveTargetRef(ctx, args.targetRef);
|
|
if (deployJsonMatchesTarget(deployBefore, control)) {
|
|
return {
|
|
...control,
|
|
controlRef: {
|
|
ref: control.ref,
|
|
commitId: control.commitId,
|
|
shortCommitId: control.shortCommitId
|
|
},
|
|
promotionSource: "target-ref",
|
|
publishRequired: true,
|
|
desiredStateCheck: null,
|
|
artifactBoundary: artifactEvidence
|
|
? await resolveArtifactBoundary(ctx, control, control, deployBefore, artifactEvidence)
|
|
: null
|
|
};
|
|
}
|
|
|
|
const promotion = await resolveCommitRef(ctx, deployBefore.commitId);
|
|
const desiredStateCheck = await checkPromotionDesiredState(ctx, promotion.commitId);
|
|
if (desiredStateCheck.status !== "pass") {
|
|
return {
|
|
...control,
|
|
controlRef: {
|
|
ref: control.ref,
|
|
commitId: control.commitId,
|
|
shortCommitId: control.shortCommitId
|
|
},
|
|
promotionSource: "target-ref",
|
|
publishRequired: true,
|
|
desiredStateCheck,
|
|
artifactBoundary: artifactEvidence
|
|
? await resolveArtifactBoundary(ctx, control, promotion, deployBefore, artifactEvidence)
|
|
: null
|
|
};
|
|
}
|
|
|
|
return {
|
|
ref: control.ref,
|
|
commitId: promotion.commitId,
|
|
shortCommitId: promotion.shortCommitId,
|
|
headCommitId: control.headCommitId,
|
|
headShortCommitId: control.headShortCommitId,
|
|
headMatchesTarget: control.headMatchesTarget,
|
|
controlRef: {
|
|
ref: control.ref,
|
|
commitId: control.commitId,
|
|
shortCommitId: control.shortCommitId
|
|
},
|
|
promotionSource: "deploy-json",
|
|
publishRequired: false,
|
|
desiredStateCheck,
|
|
artifactBoundary: artifactEvidence
|
|
? await resolveArtifactBoundary(ctx, control, promotion, deployBefore, artifactEvidence)
|
|
: null
|
|
};
|
|
}
|
|
|
|
function validateArgs(args) {
|
|
const blockers = [];
|
|
if (!args.apply && !args.status) {
|
|
blockers.push({
|
|
type: "safety_blocker",
|
|
scope: "apply-mode",
|
|
status: "open",
|
|
summary: "DEV CD side effects require --apply; use --status or --dry-run for read-only observability."
|
|
});
|
|
}
|
|
if (args.apply && (!args.confirmDev || !args.confirmedNonProduction)) {
|
|
blockers.push({
|
|
type: "safety_blocker",
|
|
scope: "dev-confirmation",
|
|
status: "open",
|
|
summary: "DEV CD apply requires --confirm-dev and --confirmed-non-production."
|
|
});
|
|
}
|
|
if (args.breakStaleLock && !args.confirmDev) {
|
|
blockers.push({
|
|
type: "safety_blocker",
|
|
scope: "break-stale-lock-confirmation",
|
|
status: "open",
|
|
summary: "Breaking a stale DEV CD lock requires --break-stale-lock and --confirm-dev."
|
|
});
|
|
}
|
|
if (!Number.isInteger(args.ttlSeconds) || args.ttlSeconds < 60 || args.ttlSeconds > 24 * 60 * 60) {
|
|
blockers.push({
|
|
type: "safety_blocker",
|
|
scope: "lock-ttl",
|
|
status: "open",
|
|
summary: "--ttl-seconds must be an integer between 60 and 86400."
|
|
});
|
|
}
|
|
for (const forbidden of ["--prod", "--production", "--read-secret", "--force-push"]) {
|
|
if (args.flags.has(forbidden)) {
|
|
blockers.push({
|
|
type: "safety_blocker",
|
|
scope: forbidden.slice(2),
|
|
status: "open",
|
|
summary: `${forbidden} is forbidden for DEV CD apply.`
|
|
});
|
|
}
|
|
}
|
|
if (blockers.length > 0) {
|
|
throw new DevCdApplyError("invalid DEV CD apply arguments", {
|
|
code: "invalid-arguments",
|
|
blockers
|
|
});
|
|
}
|
|
}
|
|
|
|
function artifactBoundaryRequiresApplyBlock(target) {
|
|
if (target.publishRequired !== false) return false;
|
|
if (target.artifactBoundary?.status !== "degraded") return false;
|
|
|
|
const desiredState = target.artifactBoundary.desiredState ?? {};
|
|
const desiredStateCheckPass = target.desiredStateCheck?.status === "pass";
|
|
const releaseGatePinsMatch =
|
|
desiredState.deployCommitMatches === true &&
|
|
desiredState.catalogCommitMatches === true;
|
|
|
|
return !(desiredStateCheckPass && releaseGatePinsMatch);
|
|
}
|
|
|
|
function lockAnnotationValue(value) {
|
|
if (value === undefined || value === null) return "";
|
|
if (typeof value === "string") return value;
|
|
return JSON.stringify(value);
|
|
}
|
|
|
|
export function buildLockAnnotations(lock) {
|
|
const annotations = {};
|
|
for (const [field, key] of Object.entries(lockAnnotationFields)) {
|
|
if (Object.hasOwn(lock, field)) {
|
|
annotations[key] = lockAnnotationValue(lock[field]);
|
|
}
|
|
}
|
|
return annotations;
|
|
}
|
|
|
|
function readAnnotation(annotations, field) {
|
|
return annotations?.[lockAnnotationFields[field]] ?? null;
|
|
}
|
|
|
|
function readJsonAnnotation(annotations, field) {
|
|
const value = readAnnotation(annotations, field);
|
|
return value ? parseJsonMaybe(value) : null;
|
|
}
|
|
|
|
export function parseDeployLock(lease) {
|
|
if (!lease) return null;
|
|
const annotations = lease.metadata?.annotations ?? {};
|
|
const ttlSeconds = Number.parseInt(
|
|
readAnnotation(annotations, "ttlSeconds") || lease.spec?.leaseDurationSeconds || `${defaultTtlSeconds}`,
|
|
10
|
|
);
|
|
const updatedAt =
|
|
readAnnotation(annotations, "updatedAt") ||
|
|
lease.spec?.renewTime ||
|
|
lease.metadata?.creationTimestamp ||
|
|
null;
|
|
const transactionId = readAnnotation(annotations, "transactionId") || lease.spec?.holderIdentity || null;
|
|
const ownerTaskId = readAnnotation(annotations, "ownerTaskId") || lease.spec?.holderIdentity || null;
|
|
return {
|
|
lockBackend: readAnnotation(annotations, "lockBackend") || "Lease",
|
|
lockName: lease.metadata?.name ?? defaultLockName,
|
|
resourceVersion: lease.metadata?.resourceVersion ?? null,
|
|
holderIdentity: lease.spec?.holderIdentity ?? null,
|
|
promotionCommit: readAnnotation(annotations, "promotionCommit"),
|
|
deployJsonHash: readAnnotation(annotations, "deployJsonHash"),
|
|
ownerTaskId,
|
|
transactionId,
|
|
phase: readAnnotation(annotations, "phase") || "unknown",
|
|
startedAt: readAnnotation(annotations, "startedAt") || lease.spec?.acquireTime || lease.metadata?.creationTimestamp || null,
|
|
updatedAt,
|
|
ttlSeconds: Number.isInteger(ttlSeconds) ? ttlSeconds : defaultTtlSeconds,
|
|
liveBefore: readJsonAnnotation(annotations, "liveBefore"),
|
|
targetNamespace: readAnnotation(annotations, "targetNamespace") || lease.metadata?.namespace || defaultNamespace,
|
|
targetRef: readAnnotation(annotations, "targetRef"),
|
|
releasedAt: readAnnotation(annotations, "releasedAt"),
|
|
releaseStatus: readAnnotation(annotations, "releaseStatus"),
|
|
staleLockBrokenAt: readAnnotation(annotations, "staleLockBrokenAt"),
|
|
staleLockPrevious: readJsonAnnotation(annotations, "staleLockPrevious")
|
|
};
|
|
}
|
|
|
|
export function classifyDeployLock(lock, now = new Date()) {
|
|
if (!lock || lock.phase === "released") {
|
|
return {
|
|
held: false,
|
|
stale: false,
|
|
retryAfterSeconds: 0,
|
|
expiresAt: null
|
|
};
|
|
}
|
|
const updatedAtMs = Number.isFinite(Date.parse(lock.updatedAt)) ? Date.parse(lock.updatedAt) : 0;
|
|
const expiresAtMs = updatedAtMs + lock.ttlSeconds * 1000;
|
|
const retryAfterSeconds = Math.max(0, Math.ceil((expiresAtMs - now.getTime()) / 1000));
|
|
return {
|
|
held: retryAfterSeconds > 0,
|
|
stale: retryAfterSeconds <= 0,
|
|
retryAfterSeconds,
|
|
expiresAt: Number.isFinite(expiresAtMs) ? new Date(expiresAtMs).toISOString() : null
|
|
};
|
|
}
|
|
|
|
export function deployLockHeldFailure(lock, now = new Date()) {
|
|
const classification = classifyDeployLock(lock, now);
|
|
return {
|
|
ok: false,
|
|
status: "blocked",
|
|
error: "deploy-lock-held",
|
|
code: "deploy-lock-held",
|
|
holder: lock.ownerTaskId || lock.transactionId || lock.holderIdentity || "unknown",
|
|
promotionCommit: lock.promotionCommit ?? null,
|
|
deployJsonHash: lock.deployJsonHash ?? null,
|
|
transactionId: lock.transactionId ?? null,
|
|
phase: lock.phase ?? "unknown",
|
|
targetNamespace: lock.targetNamespace ?? defaultNamespace,
|
|
retryAfterSeconds: classification.retryAfterSeconds,
|
|
stale: classification.stale,
|
|
requiresBreakStaleLock: classification.stale,
|
|
lockName: lock.lockName ?? defaultLockName,
|
|
lockBackend: lock.lockBackend ?? "Lease",
|
|
summary: classification.stale
|
|
? "A stale DEV CD transaction lock exists; rerun with --break-stale-lock --confirm-dev only after confirming the holder is inactive."
|
|
: "Another DEV CD transaction is already running; retry after the current holder releases the lock.",
|
|
mutationAttempted: false,
|
|
prodTouched: false
|
|
};
|
|
}
|
|
|
|
function isNotFound(result) {
|
|
return /not\s*found|notfound/i.test(`${result.stderr}\n${result.stdout}`);
|
|
}
|
|
|
|
function isAlreadyExists(result) {
|
|
return /already\s*exists|alreadyexists/i.test(`${result.stderr}\n${result.stdout}`);
|
|
}
|
|
|
|
function isConflict(result) {
|
|
return /conflict|object has been modified|operation cannot be fulfilled/i.test(`${result.stderr}\n${result.stdout}`);
|
|
}
|
|
|
|
function compactLockSummary(lock) {
|
|
if (!lock) return null;
|
|
return {
|
|
lockName: lock.lockName ?? defaultLockName,
|
|
lockBackend: lock.lockBackend ?? "Lease",
|
|
ownerTaskId: lock.ownerTaskId ?? null,
|
|
transactionId: lock.transactionId ?? null,
|
|
holderIdentity: lock.holderIdentity ?? null,
|
|
phase: lock.phase ?? null,
|
|
promotionCommit: lock.promotionCommit ?? null,
|
|
deployJsonHash: lock.deployJsonHash ?? null,
|
|
updatedAt: lock.updatedAt ?? null,
|
|
ttlSeconds: lock.ttlSeconds ?? null,
|
|
targetNamespace: lock.targetNamespace ?? defaultNamespace,
|
|
targetRef: lock.targetRef ?? null,
|
|
releasedAt: lock.releasedAt ?? null,
|
|
releaseStatus: lock.releaseStatus ?? null,
|
|
staleLockBrokenAt: lock.staleLockBrokenAt ?? null
|
|
};
|
|
}
|
|
|
|
function buildLeaseManifest({ args, target, deployBefore, transactionId, ownerTaskId, startedAt, phase, liveBefore, staleLockPrevious = null }) {
|
|
const lock = {
|
|
promotionCommit: target.shortCommitId,
|
|
deployJsonHash: deployBefore.hash,
|
|
ownerTaskId,
|
|
transactionId,
|
|
phase,
|
|
startedAt,
|
|
updatedAt: startedAt,
|
|
ttlSeconds: args.ttlSeconds,
|
|
liveBefore,
|
|
targetNamespace: args.targetNamespace,
|
|
targetRef: target.ref,
|
|
lockBackend: "Lease"
|
|
};
|
|
if (staleLockPrevious) {
|
|
lock.staleLockBrokenAt = startedAt;
|
|
lock.staleLockPrevious = staleLockPrevious;
|
|
}
|
|
return {
|
|
apiVersion: "coordination.k8s.io/v1",
|
|
kind: "Lease",
|
|
metadata: {
|
|
name: args.lockName,
|
|
namespace: args.targetNamespace,
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab",
|
|
"hwlab.pikastech.local/profile": "dev",
|
|
"hwlab.pikastech.local/cd-lock": "true"
|
|
},
|
|
annotations: buildLockAnnotations(lock)
|
|
},
|
|
spec: {
|
|
holderIdentity: `${ownerTaskId}/${transactionId}`,
|
|
leaseDurationSeconds: args.ttlSeconds,
|
|
acquireTime: startedAt,
|
|
renewTime: startedAt
|
|
}
|
|
};
|
|
}
|
|
|
|
function lockPatch({ args, target, deployBefore, transactionId, ownerTaskId, phase, startedAt, updatedAt, liveBefore, staleLockPrevious = null }) {
|
|
const lock = {
|
|
promotionCommit: target.shortCommitId,
|
|
deployJsonHash: deployBefore.hash,
|
|
ownerTaskId,
|
|
transactionId,
|
|
phase,
|
|
startedAt,
|
|
updatedAt,
|
|
ttlSeconds: args.ttlSeconds,
|
|
liveBefore,
|
|
targetNamespace: args.targetNamespace,
|
|
targetRef: target.ref,
|
|
lockBackend: "Lease"
|
|
};
|
|
if (staleLockPrevious) {
|
|
lock.staleLockBrokenAt = updatedAt;
|
|
lock.staleLockPrevious = staleLockPrevious;
|
|
}
|
|
return {
|
|
metadata: {
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab",
|
|
"hwlab.pikastech.local/profile": "dev",
|
|
"hwlab.pikastech.local/cd-lock": "true"
|
|
},
|
|
annotations: buildLockAnnotations(lock)
|
|
},
|
|
spec: {
|
|
holderIdentity: `${ownerTaskId}/${transactionId}`,
|
|
leaseDurationSeconds: args.ttlSeconds,
|
|
acquireTime: startedAt,
|
|
renewTime: updatedAt
|
|
}
|
|
};
|
|
}
|
|
|
|
async function kubectl(ctx, kubectlContext, args, options = {}) {
|
|
const result = await ctx.runCommand(kubectlContext.executor, args, {
|
|
cwd: ctx.repoRoot,
|
|
env: kubectlContext.env,
|
|
input: options.input,
|
|
timeoutMs: options.timeoutMs ?? 30000
|
|
});
|
|
return {
|
|
...result,
|
|
command: shellCommand("kubectl", args),
|
|
stdout: result.stdout ?? "",
|
|
stderr: result.stderr ?? ""
|
|
};
|
|
}
|
|
|
|
async function replaceLease(ctx, kubectlContext, args, lease, timeoutMs = 30000) {
|
|
return kubectl(
|
|
ctx,
|
|
kubectlContext,
|
|
["-n", args.targetNamespace, "replace", "-f", "-", "-o", "json"],
|
|
{
|
|
input: JSON.stringify(lease),
|
|
timeoutMs
|
|
}
|
|
);
|
|
}
|
|
|
|
function mergeLease(lease, patch, { resetLockAnnotations = false } = {}) {
|
|
const baseAnnotations = { ...(lease.metadata?.annotations ?? {}) };
|
|
if (resetLockAnnotations) {
|
|
for (const key of Object.keys(baseAnnotations).filter((name) => name.startsWith(`${lockAnnotationPrefix}/`))) {
|
|
delete baseAnnotations[key];
|
|
}
|
|
}
|
|
return {
|
|
...lease,
|
|
metadata: {
|
|
...lease.metadata,
|
|
...(patch.metadata ?? {}),
|
|
labels: {
|
|
...(lease.metadata?.labels ?? {}),
|
|
...(patch.metadata?.labels ?? {})
|
|
},
|
|
annotations: {
|
|
...baseAnnotations,
|
|
...(patch.metadata?.annotations ?? {})
|
|
}
|
|
},
|
|
spec: {
|
|
...(lease.spec ?? {}),
|
|
...(patch.spec ?? {})
|
|
}
|
|
};
|
|
}
|
|
|
|
async function acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now }) {
|
|
const startedAt = kubernetesMicroTime(now);
|
|
const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"]);
|
|
if (get.code === 0) {
|
|
const lease = JSON.parse(get.stdout);
|
|
const existing = parseDeployLock(lease);
|
|
const classification = classifyDeployLock(existing, now);
|
|
if (classification.held) {
|
|
const failure = deployLockHeldFailure(existing, now);
|
|
throw new DevCdApplyError(failure.summary, {
|
|
code: "deploy-lock-held",
|
|
lockFailure: failure
|
|
});
|
|
}
|
|
if (classification.stale && !args.breakStaleLock) {
|
|
const failure = deployLockHeldFailure(existing, now);
|
|
throw new DevCdApplyError(failure.summary, {
|
|
code: "deploy-lock-held",
|
|
lockFailure: failure
|
|
});
|
|
}
|
|
|
|
const patch = lockPatch({
|
|
args,
|
|
target,
|
|
deployBefore,
|
|
transactionId,
|
|
ownerTaskId,
|
|
phase: "publishing",
|
|
startedAt,
|
|
updatedAt: startedAt,
|
|
liveBefore,
|
|
staleLockPrevious: classification.stale ? existing : null
|
|
});
|
|
const replaced = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch, { resetLockAnnotations: true }));
|
|
if (isConflict(replaced)) {
|
|
return acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now });
|
|
}
|
|
if (replaced.code !== 0) {
|
|
throw new DevCdApplyError("failed to acquire DEV CD Lease lock", {
|
|
code: "lock-acquire-failed",
|
|
stderr: redactSensitiveText(replaced.stderr || replaced.stdout)
|
|
});
|
|
}
|
|
return {
|
|
acquired: true,
|
|
lock: parseDeployLock(JSON.parse(replaced.stdout)),
|
|
staleBreak: classification.stale ? existing : null,
|
|
command: replaced.command
|
|
};
|
|
}
|
|
|
|
if (!isNotFound(get)) {
|
|
throw new DevCdApplyError("failed to read DEV CD Lease lock", {
|
|
code: "lock-read-failed",
|
|
stderr: redactSensitiveText(get.stderr || get.stdout)
|
|
});
|
|
}
|
|
|
|
const manifest = buildLeaseManifest({
|
|
args,
|
|
target,
|
|
deployBefore,
|
|
transactionId,
|
|
ownerTaskId,
|
|
startedAt,
|
|
phase: "publishing",
|
|
liveBefore
|
|
});
|
|
const created = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "create", "-f", "-", "-o", "json"], {
|
|
input: JSON.stringify(manifest),
|
|
timeoutMs: 30000
|
|
});
|
|
if (created.code === 0) {
|
|
return {
|
|
acquired: true,
|
|
lock: parseDeployLock(JSON.parse(created.stdout)),
|
|
staleBreak: null,
|
|
command: created.command
|
|
};
|
|
}
|
|
if (isAlreadyExists(created)) {
|
|
return acquireDeployLock({ ctx, args, kubectlContext, target, deployBefore, transactionId, ownerTaskId, liveBefore, now });
|
|
}
|
|
throw new DevCdApplyError("failed to create DEV CD Lease lock", {
|
|
code: "lock-create-failed",
|
|
stderr: redactSensitiveText(created.stderr || created.stdout)
|
|
});
|
|
}
|
|
|
|
async function readOwnedLease({ ctx, args, kubectlContext, transactionId, timeoutMs = 15000 }) {
|
|
const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], {
|
|
timeoutMs
|
|
});
|
|
if (get.code !== 0) {
|
|
throw new DevCdApplyError("failed to read DEV CD lock ownership", {
|
|
code: "lock-owner-read-failed",
|
|
stderr: redactSensitiveText(get.stderr || get.stdout)
|
|
});
|
|
}
|
|
const lease = JSON.parse(get.stdout);
|
|
const lock = parseDeployLock(lease);
|
|
if (lock.transactionId !== transactionId) {
|
|
throw new DevCdApplyError("DEV CD lock ownership changed before transaction update", {
|
|
code: "lock-owner-mismatch",
|
|
holder: lock.ownerTaskId || lock.transactionId || "unknown",
|
|
phase: lock.phase
|
|
});
|
|
}
|
|
return { lease, lock };
|
|
}
|
|
|
|
async function patchOwnedLease({ ctx, args, kubectlContext, lockState, transactionId, patch, errorCode, errorMessage }) {
|
|
const { lease } = await readOwnedLease({ ctx, args, kubectlContext, transactionId });
|
|
const result = await replaceLease(ctx, kubectlContext, args, mergeLease(lease, patch));
|
|
if (result.code !== 0) {
|
|
throw new DevCdApplyError(errorMessage, {
|
|
code: errorCode,
|
|
stderr: redactSensitiveText(result.stderr || result.stdout)
|
|
});
|
|
}
|
|
lockState.lock = parseDeployLock(JSON.parse(result.stdout));
|
|
return lockState.lock;
|
|
}
|
|
|
|
async function updateDeployLockPhase({ ctx, args, kubectlContext, transactionId, lockState, phase, status = "running" }) {
|
|
const updatedAt = kubernetesMicroTime(ctx.now());
|
|
const patch = {
|
|
metadata: {
|
|
annotations: {
|
|
[lockAnnotationFields.phase]: phase,
|
|
[lockAnnotationFields.updatedAt]: updatedAt,
|
|
[lockAnnotationFields.releaseStatus]: status
|
|
}
|
|
},
|
|
spec: {
|
|
renewTime: updatedAt
|
|
}
|
|
};
|
|
return await patchOwnedLease({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId,
|
|
lockState,
|
|
patch,
|
|
errorCode: "lock-phase-update-failed",
|
|
errorMessage: `failed to update DEV CD lock phase to ${phase}`
|
|
});
|
|
}
|
|
|
|
async function updateDeployLockLiveBefore({ ctx, args, kubectlContext, transactionId, lockState, liveBefore }) {
|
|
const updatedAt = kubernetesMicroTime(ctx.now());
|
|
const patch = {
|
|
metadata: {
|
|
annotations: {
|
|
[lockAnnotationFields.liveBefore]: JSON.stringify(liveBefore),
|
|
[lockAnnotationFields.updatedAt]: updatedAt
|
|
}
|
|
},
|
|
spec: {
|
|
renewTime: updatedAt
|
|
}
|
|
};
|
|
return await patchOwnedLease({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId,
|
|
lockState,
|
|
patch,
|
|
errorCode: "lock-live-before-update-failed",
|
|
errorMessage: "failed to update DEV CD lock liveBefore evidence"
|
|
});
|
|
}
|
|
|
|
async function releaseDeployLock({ ctx, args, kubectlContext, transactionId, status }) {
|
|
const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], {
|
|
timeoutMs: 15000
|
|
});
|
|
if (get.code !== 0) {
|
|
return {
|
|
status: "release_read_failed",
|
|
command: get.command,
|
|
error: redactSensitiveText(get.stderr || get.stdout)
|
|
};
|
|
}
|
|
const current = parseDeployLock(JSON.parse(get.stdout));
|
|
if (current.transactionId !== transactionId) {
|
|
return {
|
|
status: "not_owner",
|
|
holder: current.ownerTaskId || current.transactionId || "unknown",
|
|
phase: current.phase
|
|
};
|
|
}
|
|
const releasedAt = kubernetesMicroTime(ctx.now());
|
|
const patch = {
|
|
metadata: {
|
|
annotations: {
|
|
[lockAnnotationFields.phase]: "released",
|
|
[lockAnnotationFields.updatedAt]: releasedAt,
|
|
[lockAnnotationFields.releasedAt]: releasedAt,
|
|
[lockAnnotationFields.releaseStatus]: status
|
|
}
|
|
},
|
|
spec: {
|
|
holderIdentity: "",
|
|
leaseDurationSeconds: 1,
|
|
renewTime: releasedAt
|
|
}
|
|
};
|
|
const result = await replaceLease(ctx, kubectlContext, args, mergeLease(JSON.parse(get.stdout), patch), 15000);
|
|
if (isConflict(result)) {
|
|
return {
|
|
status: "release_conflict",
|
|
command: result.command,
|
|
error: "Lease changed before release; lock was not overwritten."
|
|
};
|
|
}
|
|
if (result.code !== 0) {
|
|
return {
|
|
status: "release_failed",
|
|
command: result.command,
|
|
error: redactSensitiveText(result.stderr || result.stdout)
|
|
};
|
|
}
|
|
return {
|
|
status: "released",
|
|
command: result.command,
|
|
lock: parseDeployLock(JSON.parse(result.stdout))
|
|
};
|
|
}
|
|
|
|
function commandSummary(result, maxLength = 500) {
|
|
return oneLine(redactSensitiveText(result?.stderr || result?.stdout || "")).slice(0, maxLength);
|
|
}
|
|
|
|
function envWithoutKubeconfig(env) {
|
|
const sanitized = { ...env };
|
|
delete sanitized.KUBECONFIG;
|
|
delete sanitized.HWLAB_DEV_KUBECONFIG;
|
|
return sanitized;
|
|
}
|
|
|
|
function parseNodeNames(stdout) {
|
|
return String(stdout ?? "").trim().split(/\s+/u).filter(Boolean);
|
|
}
|
|
|
|
function parseSecretDescribeKeys(stdout) {
|
|
const keys = new Set();
|
|
let inData = false;
|
|
for (const rawLine of String(stdout ?? "").split(/\r?\n/u)) {
|
|
const line = rawLine.trim();
|
|
if (/^Data\b/iu.test(line)) {
|
|
inData = true;
|
|
continue;
|
|
}
|
|
if (!inData || line === "" || /^=+$/u.test(line)) continue;
|
|
const match = line.match(/^([A-Za-z0-9_.-]+):\s+\d+\s+bytes\b/iu);
|
|
if (match) keys.add(match[1]);
|
|
}
|
|
return [...keys].sort();
|
|
}
|
|
|
|
function parseKeyLines(stdout) {
|
|
return String(stdout ?? "")
|
|
.split(/\r?\n/u)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean);
|
|
}
|
|
|
|
function commandObservation(args, result) {
|
|
return {
|
|
command: shellCommand("kubectl", args),
|
|
code: result.code ?? null,
|
|
ok: result.code === 0,
|
|
summary: commandSummary(result)
|
|
};
|
|
}
|
|
|
|
async function preflightKubectl(ctx, kubectlContext, args, options = {}) {
|
|
const result = await ctx.runCommand(kubectlContext.executor, args, {
|
|
cwd: ctx.repoRoot,
|
|
env: options.env ?? kubectlContext.env,
|
|
timeoutMs: options.timeoutMs ?? 15000
|
|
});
|
|
return {
|
|
...result,
|
|
stdout: result.stdout ?? "",
|
|
stderr: result.stderr ?? "",
|
|
command: shellCommand("kubectl", args)
|
|
};
|
|
}
|
|
|
|
function addControlPlaneBlockers(blockers, observation, { scopePrefix, requiredForApply }) {
|
|
if (observation.contextName && forbiddenControlPlanePattern.test(observation.contextName)) {
|
|
blockers.push(blocker({
|
|
scope: `${scopePrefix}-context`,
|
|
reason: `kubectl context resolved to forbidden control plane ${observation.contextName}.`,
|
|
impact: "DEV CD cannot prove it is targeting D601 native k3s before mutation.",
|
|
safeNextAction: `Use KUBECONFIG=${d601NativeKubeconfigPath} and remove or repair the stale Docker Desktop/default kube context before retrying.`,
|
|
retryable: true
|
|
}));
|
|
}
|
|
if (observation.serverUrl && forbiddenControlPlanePattern.test(observation.serverUrl)) {
|
|
blockers.push(blocker({
|
|
scope: `${scopePrefix}-server`,
|
|
reason: `kubectl server resolved to forbidden endpoint ${observation.serverUrl}.`,
|
|
impact: "DEV CD could write to the wrong control plane or treat stale Docker Desktop output as DEV evidence.",
|
|
safeNextAction: `Point kubectl at D601 native k3s with KUBECONFIG=${d601NativeKubeconfigPath} before retrying.`,
|
|
retryable: true
|
|
}));
|
|
}
|
|
if (observation.nodeNames.includes("desktop-control-plane")) {
|
|
blockers.push(blocker({
|
|
scope: `${scopePrefix}-nodes`,
|
|
reason: "kubectl observed node desktop-control-plane.",
|
|
impact: "DEV CD is observing Docker Desktop Kubernetes instead of D601 native k3s.",
|
|
safeNextAction: `Stop and rerun only after kubectl nodes come from KUBECONFIG=${d601NativeKubeconfigPath} and include d601.`,
|
|
retryable: true
|
|
}));
|
|
} else if (requiredForApply && !observation.nodeNames.includes("d601")) {
|
|
blockers.push(blocker({
|
|
scope: `${scopePrefix}-nodes`,
|
|
reason: `kubectl nodes did not include d601; observed ${observation.nodeNames.join(",") || "none"}.`,
|
|
impact: "DEV apply/job/rollout side effects are not allowed without proving the D601 native k3s node.",
|
|
safeNextAction: `Restore access to KUBECONFIG=${d601NativeKubeconfigPath} and verify kubectl get nodes includes d601.`,
|
|
retryable: true
|
|
}));
|
|
} else if (!requiredForApply && observation.nodeNames.length > 0 && !observation.nodeNames.includes("d601")) {
|
|
blockers.push(blocker({
|
|
scope: `${scopePrefix}-nodes`,
|
|
reason: `bare kubectl observed non-D601 nodes ${observation.nodeNames.join(",")}.`,
|
|
impact: "A second or stale control plane may still own hwlab-dev evidence, so DEV CD refuses to mutate.",
|
|
safeNextAction: "Clean the default kubectl context or ensure bare kubectl cannot resolve stale HWLAB DEV resources, then retry.",
|
|
retryable: true
|
|
}));
|
|
}
|
|
}
|
|
|
|
async function observeControlPlane(ctx, kubectlContext, { env, label, requiredForApply }) {
|
|
const [context, server, nodes] = await Promise.all([
|
|
preflightKubectl(ctx, kubectlContext, ["config", "current-context"], { env, timeoutMs: 10000 }),
|
|
preflightKubectl(ctx, kubectlContext, ["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], { env, timeoutMs: 10000 }),
|
|
preflightKubectl(ctx, kubectlContext, ["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], { env, timeoutMs: 15000 })
|
|
]);
|
|
return {
|
|
label,
|
|
contextCommand: commandObservation(["config", "current-context"], context),
|
|
serverCommand: commandObservation(["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], server),
|
|
nodesCommand: commandObservation(["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], nodes),
|
|
contextName: context.code === 0 ? oneLine(context.stdout) : null,
|
|
serverUrl: server.code === 0 ? oneLine(server.stdout) : null,
|
|
nodeNames: nodes.code === 0 ? parseNodeNames(nodes.stdout) : [],
|
|
status: context.code === 0 && server.code === 0 && nodes.code === 0 ? "observed" : requiredForApply ? "blocked" : "unavailable"
|
|
};
|
|
}
|
|
|
|
async function observeBareKubectl(ctx, kubectlContext, forcedObservation) {
|
|
const bareEnv = envWithoutKubeconfig(ctx.env);
|
|
const bare = await observeControlPlane(ctx, kubectlContext, {
|
|
env: bareEnv,
|
|
label: "bare-kubectl",
|
|
requiredForApply: false
|
|
});
|
|
const namespace = await preflightKubectl(ctx, kubectlContext, ["get", "namespace", defaultNamespace, "-o", "name"], {
|
|
env: bareEnv,
|
|
timeoutMs: 10000
|
|
});
|
|
const secondControlPlane =
|
|
namespace.code === 0 &&
|
|
(
|
|
(bare.serverUrl && forcedObservation.serverUrl && bare.serverUrl !== forcedObservation.serverUrl) ||
|
|
(!bare.serverUrl && bare.contextName && bare.contextName !== forcedObservation.contextName)
|
|
);
|
|
return {
|
|
...bare,
|
|
namespace: commandObservation(["get", "namespace", defaultNamespace, "-o", "name"], namespace),
|
|
hwlabDevNamespaceObserved: namespace.code === 0,
|
|
secondControlPlaneCandidate: secondControlPlane
|
|
};
|
|
}
|
|
|
|
async function observeRequiredSecretRef(ctx, kubectlContext, args, ref) {
|
|
const existsArgs = ["-n", args.targetNamespace, "get", "secret", ref.secretName, "-o", "name"];
|
|
const describeArgs = ["-n", args.targetNamespace, "describe", "secret", ref.secretName];
|
|
const exists = await preflightKubectl(ctx, kubectlContext, existsArgs, { timeoutMs: 10000 });
|
|
if (exists.code !== 0) {
|
|
return {
|
|
secretName: ref.secretName,
|
|
secretKey: ref.secretKey,
|
|
status: "missing-secret",
|
|
exists: false,
|
|
keyPresent: false,
|
|
keysObserved: [],
|
|
existenceCommand: shellCommand("kubectl", existsArgs),
|
|
keyCommand: null,
|
|
error: commandSummary(exists),
|
|
secretValueRead: false,
|
|
secretValuePrinted: false
|
|
};
|
|
}
|
|
const describe = await preflightKubectl(ctx, kubectlContext, describeArgs, { timeoutMs: 10000 });
|
|
if (describe.code !== 0) {
|
|
return {
|
|
secretName: ref.secretName,
|
|
secretKey: ref.secretKey,
|
|
status: "key-observation-blocked",
|
|
exists: true,
|
|
keyPresent: false,
|
|
keysObserved: [],
|
|
existenceCommand: shellCommand("kubectl", existsArgs),
|
|
keyCommand: shellCommand("kubectl", describeArgs),
|
|
error: commandSummary(describe),
|
|
secretValueRead: false,
|
|
secretValuePrinted: false
|
|
};
|
|
}
|
|
const keysObserved = parseSecretDescribeKeys(describe.stdout);
|
|
const keyPresent = keysObserved.includes(ref.secretKey);
|
|
return {
|
|
secretName: ref.secretName,
|
|
secretKey: ref.secretKey,
|
|
status: keyPresent ? "present" : "missing-key",
|
|
exists: true,
|
|
keyPresent,
|
|
keysObserved,
|
|
existenceCommand: shellCommand("kubectl", existsArgs),
|
|
keyCommand: shellCommand("kubectl", describeArgs),
|
|
secretValueRead: false,
|
|
secretValuePrinted: false
|
|
};
|
|
}
|
|
|
|
function secretRefBlocker(ref, observation) {
|
|
const scope = `secretref:${ref.secretName}/${ref.secretKey}`;
|
|
if (observation.status === "missing-secret") {
|
|
return blocker({
|
|
type: "runtime_blocker",
|
|
scope,
|
|
reason: `Required DEV Secret ${ref.secretName} was not observed in ${defaultNamespace}.`,
|
|
impact: "Runtime provisioning, migration, deploy apply, or Code Agent provider startup would fail after mutation.",
|
|
safeNextAction: `Create or restore SecretRef ${ref.secretName}/${ref.secretKey} in ${defaultNamespace} without printing the Secret value, then retry DEV CD.`,
|
|
retryable: true
|
|
});
|
|
}
|
|
if (observation.status === "key-observation-blocked") {
|
|
return blocker({
|
|
type: "runtime_blocker",
|
|
scope,
|
|
reason: `Required DEV Secret ${ref.secretName} exists but key names could not be verified.`,
|
|
impact: "DEV CD cannot prove the pod SecretRef contract before creating runtime Jobs or applying workloads.",
|
|
safeNextAction: `Restore metadata/key-name visibility for ${ref.secretName} without granting or printing Secret values, then retry.`,
|
|
retryable: true
|
|
});
|
|
}
|
|
return blocker({
|
|
type: "runtime_blocker",
|
|
scope,
|
|
reason: `Required DEV Secret key ${ref.secretName}/${ref.secretKey} was not observed.`,
|
|
impact: "The target workload or runtime Job would reference a missing Secret key after mutation.",
|
|
safeNextAction: `Add key ${ref.secretKey} to Secret ${ref.secretName} without printing the value, then retry DEV CD.`,
|
|
retryable: true
|
|
});
|
|
}
|
|
|
|
async function observeRequiredConfigMapRef(ctx, kubectlContext, args, ref) {
|
|
const existsArgs = ["-n", args.targetNamespace, "get", "configmap", ref.configMapName, "-o", "name"];
|
|
const keyArgs = [
|
|
"-n",
|
|
args.targetNamespace,
|
|
"get",
|
|
"configmap",
|
|
ref.configMapName,
|
|
"-o",
|
|
"go-template={{range $k,$v := .data}}{{printf \"%s\\n\" $k}}{{end}}"
|
|
];
|
|
const exists = await preflightKubectl(ctx, kubectlContext, existsArgs, { timeoutMs: 10000 });
|
|
if (exists.code !== 0) {
|
|
return {
|
|
configMapName: ref.configMapName,
|
|
configMapKey: ref.configMapKey,
|
|
status: "missing-configmap",
|
|
exists: false,
|
|
keyPresent: false,
|
|
keysObserved: [],
|
|
existenceCommand: shellCommand("kubectl", existsArgs),
|
|
keyCommand: null,
|
|
error: commandSummary(exists),
|
|
configMapValueRead: false,
|
|
configMapValuePrinted: false
|
|
};
|
|
}
|
|
const keys = await preflightKubectl(ctx, kubectlContext, keyArgs, { timeoutMs: 10000 });
|
|
if (keys.code !== 0) {
|
|
return {
|
|
configMapName: ref.configMapName,
|
|
configMapKey: ref.configMapKey,
|
|
status: "key-observation-blocked",
|
|
exists: true,
|
|
keyPresent: false,
|
|
keysObserved: [],
|
|
existenceCommand: shellCommand("kubectl", existsArgs),
|
|
keyCommand: shellCommand("kubectl", keyArgs),
|
|
error: commandSummary(keys),
|
|
configMapValueRead: false,
|
|
configMapValuePrinted: false
|
|
};
|
|
}
|
|
const keysObserved = parseKeyLines(keys.stdout);
|
|
const keyPresent = keysObserved.includes(ref.configMapKey);
|
|
return {
|
|
configMapName: ref.configMapName,
|
|
configMapKey: ref.configMapKey,
|
|
status: keyPresent ? "present" : "missing-key",
|
|
exists: true,
|
|
keyPresent,
|
|
keysObserved,
|
|
existenceCommand: shellCommand("kubectl", existsArgs),
|
|
keyCommand: shellCommand("kubectl", keyArgs),
|
|
configMapValueRead: false,
|
|
configMapValuePrinted: false
|
|
};
|
|
}
|
|
|
|
function configMapRefBlocker(ref, observation) {
|
|
const scope = `configmap:${ref.configMapName}/${ref.configMapKey}`;
|
|
if (observation.status === "missing-configmap") {
|
|
return blocker({
|
|
type: "runtime_blocker",
|
|
scope,
|
|
reason: `Required DEV ConfigMap ${ref.configMapName} was not observed in ${defaultNamespace}.`,
|
|
impact: "The Code Agent Codex stdio provider pod would fail to mount config.toml after mutation.",
|
|
safeNextAction: `Create or restore ConfigMap ${ref.configMapName}/${ref.configMapKey} in ${defaultNamespace}, then retry DEV CD.`,
|
|
retryable: true
|
|
});
|
|
}
|
|
if (observation.status === "key-observation-blocked") {
|
|
return blocker({
|
|
type: "runtime_blocker",
|
|
scope,
|
|
reason: `Required DEV ConfigMap ${ref.configMapName} exists but key names could not be verified.`,
|
|
impact: "DEV CD cannot prove the Code Agent Codex config mount before applying workloads.",
|
|
safeNextAction: `Restore key-name visibility for ${ref.configMapName}, then retry.`,
|
|
retryable: true
|
|
});
|
|
}
|
|
return blocker({
|
|
type: "runtime_blocker",
|
|
scope,
|
|
reason: `Required DEV ConfigMap key ${ref.configMapName}/${ref.configMapKey} was not observed.`,
|
|
impact: "The Code Agent Codex stdio provider pod would reference a missing config.toml key after mutation.",
|
|
safeNextAction: `Add key ${ref.configMapKey} to ConfigMap ${ref.configMapName}, then retry DEV CD.`,
|
|
retryable: true
|
|
});
|
|
}
|
|
|
|
async function runDevCdPreflight({ ctx, args, kubectlContext }) {
|
|
const blockers = [];
|
|
const preflight = {
|
|
status: "blocked",
|
|
scope: "pre-lock-pre-side-effect",
|
|
generatedAt: ctx.now().toISOString(),
|
|
namespace: args.targetNamespace,
|
|
kubeconfig: {
|
|
required: d601NativeKubeconfigPath,
|
|
selected: kubectlContext.kubeconfig,
|
|
source: kubectlContext.kubeconfigSource,
|
|
enforcedEnv: kubectlContext.env.KUBECONFIG === d601NativeKubeconfigPath
|
|
},
|
|
kubectl: {
|
|
executor: kubectlContext.executor,
|
|
commandPrefix: kubectlContext.commandPrefix
|
|
},
|
|
controlPlane: null,
|
|
secretRefs: [],
|
|
configMaps: [],
|
|
safety: {
|
|
writeSideEffectsAttempted: false,
|
|
prodTouched: false,
|
|
secretValuesRead: false,
|
|
secretValuesPrinted: false,
|
|
configMapValuesRead: false,
|
|
configMapValuesPrinted: false,
|
|
secretKeyNamesOnly: true
|
|
},
|
|
blockers
|
|
};
|
|
|
|
if (kubectlContext.kubeconfig !== d601NativeKubeconfigPath) {
|
|
blockers.push(blocker({
|
|
scope: "d601-kubeconfig-path",
|
|
reason: `DEV CD selected kubeconfig ${kubectlContext.kubeconfig || "<unset>"} from ${kubectlContext.kubeconfigSource}.`,
|
|
impact: "Any apply/job/rollout could target a non-D601 or stale control plane.",
|
|
safeNextAction: `Rerun with KUBECONFIG=${d601NativeKubeconfigPath} or --kubeconfig ${d601NativeKubeconfigPath}.`,
|
|
retryable: true
|
|
}));
|
|
preflight.status = "blocked";
|
|
return preflight;
|
|
}
|
|
|
|
const forced = await observeControlPlane(ctx, kubectlContext, {
|
|
env: kubectlContext.env,
|
|
label: "forced-d601-k3s",
|
|
requiredForApply: true
|
|
});
|
|
const bare = await observeBareKubectl(ctx, kubectlContext, forced);
|
|
preflight.controlPlane = { forced, bare };
|
|
|
|
if (forced.status !== "observed") {
|
|
blockers.push(blocker({
|
|
scope: "d601-native-k3s-unobservable",
|
|
reason: `Forced D601 kubeconfig could not prove context/server/nodes: ${[forced.contextCommand.summary, forced.serverCommand.summary, forced.nodesCommand.summary].filter(Boolean).join("; ") || "no kubectl output"}.`,
|
|
impact: "DEV CD refuses to acquire the Lease or create runtime Jobs without D601 native k3s proof.",
|
|
safeNextAction: `Restore readable ${d601NativeKubeconfigPath} access and verify kubectl get nodes includes d601.`,
|
|
retryable: true
|
|
}));
|
|
}
|
|
addControlPlaneBlockers(blockers, forced, {
|
|
scopePrefix: "d601-native-k3s",
|
|
requiredForApply: true
|
|
});
|
|
addControlPlaneBlockers(blockers, bare, {
|
|
scopePrefix: "bare-kubectl",
|
|
requiredForApply: false
|
|
});
|
|
if (bare.secondControlPlaneCandidate) {
|
|
blockers.push(blocker({
|
|
scope: "second-hwlab-dev-control-plane",
|
|
reason: `bare kubectl can observe namespace ${defaultNamespace} outside the forced D601 kubeconfig context.`,
|
|
impact: "A second hwlab-dev control plane may still carry stale HWLAB resources, so DEV CD cannot safely classify apply evidence.",
|
|
safeNextAction: "Remove the stale hwlab-dev control plane/default kube context or make bare kubectl unable to resolve it, then rerun preflight.",
|
|
retryable: true
|
|
}));
|
|
}
|
|
|
|
if (blockers.length === 0) {
|
|
[preflight.secretRefs, preflight.configMaps] = await Promise.all([
|
|
Promise.all(requiredDevSecretRefs.map((ref) => observeRequiredSecretRef(ctx, kubectlContext, args, ref))),
|
|
Promise.all(requiredDevConfigMapRefs.map((ref) => observeRequiredConfigMapRef(ctx, kubectlContext, args, ref)))
|
|
]);
|
|
for (const [index, observation] of preflight.secretRefs.entries()) {
|
|
if (observation.status !== "present") {
|
|
blockers.push(secretRefBlocker(requiredDevSecretRefs[index], observation));
|
|
}
|
|
}
|
|
for (const [index, observation] of preflight.configMaps.entries()) {
|
|
if (observation.status !== "present") {
|
|
blockers.push(configMapRefBlocker(requiredDevConfigMapRefs[index], observation));
|
|
}
|
|
}
|
|
}
|
|
|
|
preflight.status = blockers.length === 0 ? "pass" : "blocked";
|
|
return preflight;
|
|
}
|
|
|
|
async function resolveKubectlContext(args, env, runCommand) {
|
|
const selection = resolveDevKubeconfigSelection({ flagValue: args.kubeconfig, env });
|
|
const which = await runCommand("which", ["kubectl"], { timeoutMs: 5000 });
|
|
const executor = which.code === 0 ? which.stdout.trim() : "kubectl";
|
|
const kubectlEnv = {
|
|
...env,
|
|
KUBECONFIG: selection.kubeconfig
|
|
};
|
|
return {
|
|
executor,
|
|
kubeconfig: selection.kubeconfig,
|
|
kubeconfigSource: selection.source,
|
|
commandPrefix: buildKubectlCommandPrefix(selection.kubeconfig),
|
|
env: kubectlEnv
|
|
};
|
|
}
|
|
|
|
function commandEnv(ctx, transaction) {
|
|
return {
|
|
...ctx.env,
|
|
HWLAB_CD_TRANSACTION_ID: transaction.transactionId,
|
|
HWLAB_CD_TRANSACTION_OWNER: transaction.ownerTaskId,
|
|
HWLAB_CD_LOCK_NAME: transaction.lockName,
|
|
HWLAB_CD_LOCK_NAMESPACE: transaction.targetNamespace
|
|
};
|
|
}
|
|
|
|
async function runStep(ctx, transaction, step) {
|
|
const startedAt = ctx.now().toISOString();
|
|
const command = shellCommand(step.command, step.args);
|
|
const result = await ctx.runCommand(step.command, step.args, {
|
|
cwd: ctx.repoRoot,
|
|
env: commandEnv(ctx, transaction),
|
|
timeoutMs: step.timeoutMs ?? 300000
|
|
});
|
|
const finishedAt = ctx.now().toISOString();
|
|
const stdout = redactSensitiveText(result.stdout ?? "");
|
|
const stderr = redactSensitiveText(result.stderr ?? "");
|
|
const parsedStdout = parseJsonMaybe(stdout);
|
|
return {
|
|
id: step.id,
|
|
phase: step.phase,
|
|
status: result.code === 0 ? "pass" : "blocked",
|
|
command,
|
|
code: result.code,
|
|
startedAt,
|
|
finishedAt,
|
|
stdoutJson: parsedStdout,
|
|
stdoutTail: parsedStdout ? null : stdout.trim().slice(-2000),
|
|
stderrTail: stderr.trim().slice(-2000),
|
|
reportPath: step.reportPath ?? null
|
|
};
|
|
}
|
|
|
|
async function runRuntimeK8sJobStep(ctx, transaction, step) {
|
|
const startedAt = ctx.now().toISOString();
|
|
const result = await runRuntimeK8sJob(ctx, transaction, step);
|
|
const finishedAt = ctx.now().toISOString();
|
|
return {
|
|
id: step.id,
|
|
phase: step.phase,
|
|
status: result.status,
|
|
command: result.command,
|
|
code: result.status === "pass" ? 0 : 1,
|
|
startedAt,
|
|
finishedAt,
|
|
stdoutJson: result.report,
|
|
stdoutTail: null,
|
|
stderrTail: result.error ?? "",
|
|
reportPath: step.reportPath ?? null,
|
|
k8sJob: result.k8sJob
|
|
};
|
|
}
|
|
|
|
async function runRuntimeK8sJob(ctx, transaction, step) {
|
|
const jobName = `${step.jobNamePrefix}-${transaction.transactionId.slice(0, 8)}`;
|
|
const reportFile = `${runtimeJobReportPrefix}-${step.id}.json`;
|
|
const image = step.image ?? await resolveRuntimeJobImage(ctx);
|
|
const manifest = buildRuntimeK8sJobManifest({
|
|
jobName,
|
|
namespace: transaction.targetNamespace,
|
|
image,
|
|
commandArgs: [...step.commandArgs, "--report", reportFile],
|
|
transaction
|
|
});
|
|
const kubectl = transaction.kubectlContext;
|
|
const apply = await kubectlCommandResult(ctx, kubectl, ["-n", transaction.targetNamespace, "apply", "-f", "-"], {
|
|
input: JSON.stringify(manifest),
|
|
timeoutMs: 30000
|
|
});
|
|
if (apply.code !== 0) {
|
|
return runtimeJobFailure(step, "apply", apply, jobName, image);
|
|
}
|
|
|
|
const wait = await kubectlCommandResult(ctx, kubectl, [
|
|
"-n",
|
|
transaction.targetNamespace,
|
|
"wait",
|
|
"--for=condition=complete",
|
|
`job/${jobName}`,
|
|
"--timeout=300s"
|
|
], { timeoutMs: step.timeoutMs ?? 6 * 60 * 1000 });
|
|
if (wait.code !== 0) {
|
|
return runtimeJobFailure(step, "wait", wait, jobName, image);
|
|
}
|
|
|
|
const logs = await kubectlCommandResult(ctx, kubectl, ["-n", transaction.targetNamespace, "logs", `job/${jobName}`], {
|
|
timeoutMs: 30000
|
|
});
|
|
const logText = redactSensitiveText(`${logs.stdout}\n${logs.stderr}`);
|
|
const report = parseLastJsonObject(logText);
|
|
if (logs.code !== 0 || !report) {
|
|
return {
|
|
status: "blocked",
|
|
command: `${shellCommand("kubectl", ["-n", transaction.targetNamespace, "logs", `job/${jobName}`])}`,
|
|
error: logText.trim().slice(-2000) || "runtime job did not emit a JSON report",
|
|
report: null,
|
|
k8sJob: runtimeJobSummary(jobName, step, "logs", image)
|
|
};
|
|
}
|
|
const reportStatus = report.conclusion?.status ?? report.summary?.status ?? report.status;
|
|
return {
|
|
status: ["ready", "pass"].includes(reportStatus) ? "pass" : "blocked",
|
|
command: shellCommand("kubectl", ["-n", transaction.targetNamespace, "apply/wait/logs", `job/${jobName}`]),
|
|
error: null,
|
|
report,
|
|
k8sJob: runtimeJobSummary(jobName, step, "complete", image)
|
|
};
|
|
}
|
|
|
|
function runtimeJobFailure(step, phase, result, jobName, image) {
|
|
return {
|
|
status: "blocked",
|
|
command: result.command ?? shellCommand("kubectl", [phase, `job/${jobName}`]),
|
|
error: redactSensitiveText(`${result.stderr}\n${result.stdout}`).trim().slice(-2000),
|
|
report: null,
|
|
k8sJob: runtimeJobSummary(jobName, step, phase, image)
|
|
};
|
|
}
|
|
|
|
function runtimeJobSummary(jobName, step, phase, image = step.image ?? "deploy-current:hwlab-cloud-api") {
|
|
return {
|
|
jobName,
|
|
phase,
|
|
serviceId: "hwlab-cloud-api",
|
|
image,
|
|
secretRefsOnly: true,
|
|
secretValuesRead: false,
|
|
secretValuesPrinted: false
|
|
};
|
|
}
|
|
|
|
async function resolveRuntimeJobImage(ctx) {
|
|
const deploy = await readDeployJson(ctx.repoRoot);
|
|
const image = deploy.manifest?.services?.find((service) => service.serviceId === "hwlab-cloud-api")?.image;
|
|
if (typeof image === "string" && image.trim().length > 0) {
|
|
return image;
|
|
}
|
|
if (typeof deploy.commitId === "string" && deploy.commitId !== "unknown") {
|
|
return `127.0.0.1:5000/hwlab/hwlab-cloud-api:${shortCommit(deploy.commitId)}`;
|
|
}
|
|
throw new DevCdApplyError("could not resolve hwlab-cloud-api image for runtime maintenance Job", {
|
|
code: "runtime-job-image-unresolved"
|
|
});
|
|
}
|
|
|
|
function buildRuntimeK8sJobManifest({ jobName, namespace, image, commandArgs, transaction }) {
|
|
return {
|
|
apiVersion: "batch/v1",
|
|
kind: "Job",
|
|
metadata: {
|
|
name: jobName,
|
|
namespace,
|
|
labels: {
|
|
"app.kubernetes.io/part-of": "hwlab",
|
|
"app.kubernetes.io/name": "hwlab-runtime-maintenance",
|
|
"hwlab.pikastech.local/profile": "dev",
|
|
"hwlab.pikastech.local/service-id": "hwlab-cloud-api",
|
|
"hwlab.pikastech.local/cd-transaction": transaction.transactionId
|
|
}
|
|
},
|
|
spec: {
|
|
backoffLimit: 0,
|
|
ttlSecondsAfterFinished: 600,
|
|
template: {
|
|
metadata: {
|
|
labels: {
|
|
"app.kubernetes.io/name": "hwlab-runtime-maintenance",
|
|
"hwlab.pikastech.local/service-id": "hwlab-cloud-api",
|
|
"hwlab.pikastech.local/cd-transaction": transaction.transactionId
|
|
}
|
|
},
|
|
spec: {
|
|
restartPolicy: "Never",
|
|
containers: [
|
|
{
|
|
name: "runtime-maintenance",
|
|
image,
|
|
command: ["node"],
|
|
args: commandArgs,
|
|
env: [
|
|
{ name: "HWLAB_ENVIRONMENT", value: ENVIRONMENT_DEV },
|
|
{ name: "HWLAB_CLOUD_RUNTIME_ADAPTER", value: "postgres" },
|
|
{ name: "HWLAB_CLOUD_RUNTIME_DURABLE", value: "true" },
|
|
{
|
|
name: "HWLAB_CLOUD_DB_URL",
|
|
valueFrom: {
|
|
secretKeyRef: {
|
|
name: "hwlab-cloud-api-dev-db",
|
|
key: "database-url",
|
|
optional: false
|
|
}
|
|
}
|
|
},
|
|
{
|
|
name: "HWLAB_CLOUD_DB_ADMIN_URL",
|
|
valueFrom: {
|
|
secretKeyRef: {
|
|
name: "hwlab-cloud-api-dev-db-admin",
|
|
key: "admin-url",
|
|
optional: false
|
|
}
|
|
}
|
|
},
|
|
{ name: "HWLAB_CLOUD_DB_SSL_MODE", value: "disable" },
|
|
{ name: "HWLAB_CD_TRANSACTION_ID", value: transaction.transactionId },
|
|
{ name: "HWLAB_CD_TRANSACTION_OWNER", value: transaction.ownerTaskId },
|
|
{ name: "HWLAB_CD_LOCK_NAME", value: transaction.lockName }
|
|
]
|
|
}
|
|
]
|
|
}
|
|
}
|
|
}
|
|
};
|
|
}
|
|
|
|
async function kubectlCommandResult(ctx, kubectlContext, args, options = {}) {
|
|
const result = await ctx.runCommand(kubectlContext.executor, args, {
|
|
cwd: ctx.repoRoot,
|
|
env: kubectlContext.env,
|
|
input: options.input,
|
|
timeoutMs: options.timeoutMs ?? 30000
|
|
});
|
|
return {
|
|
...result,
|
|
command: shellCommand("kubectl", args),
|
|
stdout: result.stdout ?? "",
|
|
stderr: result.stderr ?? ""
|
|
};
|
|
}
|
|
|
|
function parseLastJsonObject(text) {
|
|
const normalized = String(text ?? "").trim();
|
|
const lines = normalized.split(/\r?\n/u).reverse();
|
|
for (const line of lines) {
|
|
const parsed = parseJsonMaybe(line.trim());
|
|
if (parsed && typeof parsed === "object") return parsed;
|
|
}
|
|
const starts = [];
|
|
for (let index = 0; index < normalized.length; index += 1) {
|
|
if (normalized[index] === "{") starts.push(index);
|
|
}
|
|
for (const start of starts.reverse()) {
|
|
const parsed = parseJsonMaybe(normalized.slice(start));
|
|
if (parsed && typeof parsed === "object") return parsed;
|
|
}
|
|
return parseJsonMaybe(normalized);
|
|
}
|
|
|
|
function stepBlocker(stepResult) {
|
|
const stdoutBlockers = Array.isArray(stepResult.stdoutJson?.blockers) ? stepResult.stdoutJson.blockers : [];
|
|
const primary = stdoutBlockers.find((item) => item?.status === "open") ?? stdoutBlockers[0] ?? null;
|
|
const reason = primary?.reason ?? primary?.summary ?? `${stepResult.command} exited ${stepResult.code}.`;
|
|
const impact = primary?.impact ?? stepBlockerImpact(stepResult);
|
|
const safeNextAction = primary?.safeNextAction ?? stepBlockerSafeNextAction(stepResult);
|
|
return structuredBlocker({
|
|
type: "runtime_blocker",
|
|
scope: stepResult.id,
|
|
reason,
|
|
impact,
|
|
safeNextAction,
|
|
retryable: primary?.retryable ?? true,
|
|
evidence: {
|
|
command: stepResult.command,
|
|
code: stepResult.code,
|
|
reportPath: stepResult.reportPath ?? null,
|
|
blockerScope: primary?.scope ?? null,
|
|
k8sJob: stepResult.k8sJob ?? null
|
|
}
|
|
});
|
|
}
|
|
|
|
function stepBlockerImpact(stepResult) {
|
|
if (stepResult.id === "runtime-db-provisioning") return "DEV CD cannot prove DB role/database provisioning before workload apply.";
|
|
if (stepResult.id === "runtime-db-migration") return "DEV CD cannot prove runtime schema migration before workload apply.";
|
|
if (stepResult.id === "runtime-durable-postflight") return "DEV CD cannot prove /health/live, /v1, and M3 durable evidence persistence after apply.";
|
|
if (stepResult.id === "dev-deploy-apply") return "DEV desired-state apply did not complete, so live readiness cannot be trusted.";
|
|
return "DEV CD transaction cannot safely continue.";
|
|
}
|
|
|
|
function stepBlockerSafeNextAction(stepResult) {
|
|
if (stepResult.id === "runtime-db-provisioning") return "Repair repo-owned runtime provisioning inputs/SecretRefs without printing Secret values, then rerun DEV CD.";
|
|
if (stepResult.id === "runtime-db-migration") return "Repair repo-owned runtime migration/schema state, then rerun DEV CD.";
|
|
if (stepResult.id === "runtime-durable-postflight") return "Inspect the runtime postflight report, repair durable readiness or M3 evidence persistence, then rerun DEV CD after host-controlled rollout if needed.";
|
|
if (stepResult.id === "dev-deploy-apply") return `Inspect ${deployApplyReportPath} and rerun the repo-owned DEV CD path after the apply blocker is fixed.`;
|
|
return "Fix the reported step blocker and rerun the repo-owned DEV CD transaction.";
|
|
}
|
|
|
|
function structuredBlocker({ type = "runtime_blocker", scope, reason, impact, safeNextAction, retryable = true, evidence = null }) {
|
|
return {
|
|
type,
|
|
scope,
|
|
status: "open",
|
|
reason: oneLine(reason),
|
|
impact: oneLine(impact),
|
|
safeNextAction: oneLine(safeNextAction),
|
|
retryable: Boolean(retryable),
|
|
summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`),
|
|
...(evidence ? { evidence } : {})
|
|
};
|
|
}
|
|
|
|
function healthCommit(json) {
|
|
return (
|
|
json?.commit?.id ||
|
|
json?.commitId ||
|
|
json?.revision ||
|
|
json?.image?.tag ||
|
|
null
|
|
);
|
|
}
|
|
|
|
function healthImage(json) {
|
|
if (typeof json?.image === "string") return json.image;
|
|
return json?.image?.reference ?? null;
|
|
}
|
|
|
|
function summarizeHealthJson(json, expectedServiceId, expectedCommit) {
|
|
const observedCommit = healthCommit(json);
|
|
return {
|
|
serviceId: json?.serviceId ?? json?.service?.id ?? null,
|
|
environment: json?.environment ?? null,
|
|
applicationStatus: json?.status ?? null,
|
|
ready: Object.hasOwn(json ?? {}, "ready") ? json.ready : null,
|
|
observedCommit,
|
|
image: healthImage(json),
|
|
imageTag: json?.image?.tag ?? null,
|
|
observedAt: json?.observedAt ?? null,
|
|
serviceMatches: (json?.serviceId ?? json?.service?.id) === expectedServiceId,
|
|
environmentMatches: json?.environment === ENVIRONMENT_DEV,
|
|
commitMatches: expectedCommit ? commitMatches(observedCommit, expectedCommit) : null,
|
|
blockerCodes: Array.isArray(json?.blockerCodes) ? json.blockerCodes : []
|
|
};
|
|
}
|
|
|
|
async function defaultHttpGetJson(url, timeoutMs = 10000) {
|
|
return new Promise((resolve, reject) => {
|
|
const request = httpRequest(url, { method: "GET", timeout: timeoutMs }, (response) => {
|
|
response.setEncoding("utf8");
|
|
let body = "";
|
|
response.on("data", (chunk) => {
|
|
body += chunk;
|
|
});
|
|
response.on("end", () => {
|
|
let json = null;
|
|
try {
|
|
json = JSON.parse(body);
|
|
} catch {}
|
|
resolve({
|
|
statusCode: response.statusCode ?? 0,
|
|
body,
|
|
json
|
|
});
|
|
});
|
|
});
|
|
request.on("timeout", () => {
|
|
request.destroy(new Error(`timeout after ${timeoutMs}ms`));
|
|
});
|
|
request.on("error", reject);
|
|
request.end();
|
|
});
|
|
}
|
|
|
|
async function probeLiveEndpoint(ctx, { id, url, expectedServiceId, expectedCommit }) {
|
|
const startedAt = ctx.now().toISOString();
|
|
try {
|
|
const response = await ctx.httpGetJson(url, 10000);
|
|
const summary = summarizeHealthJson(response.json, expectedServiceId, expectedCommit);
|
|
const reachable = response.statusCode >= 200 && response.statusCode < 300 && Boolean(response.json);
|
|
const identityMatches = reachable && summary.serviceMatches && summary.environmentMatches && (expectedCommit ? summary.commitMatches : true);
|
|
return {
|
|
id,
|
|
url,
|
|
status: identityMatches ? "pass" : "blocked",
|
|
httpStatus: response.statusCode,
|
|
reachable,
|
|
identityMatches,
|
|
startedAt,
|
|
finishedAt: ctx.now().toISOString(),
|
|
expectedServiceId,
|
|
expectedCommit: expectedCommit ?? null,
|
|
...summary
|
|
};
|
|
} catch (error) {
|
|
return {
|
|
id,
|
|
url,
|
|
status: "blocked",
|
|
reachable: false,
|
|
identityMatches: false,
|
|
startedAt,
|
|
finishedAt: ctx.now().toISOString(),
|
|
expectedServiceId,
|
|
expectedCommit: expectedCommit ?? null,
|
|
error: oneLine(error.message)
|
|
};
|
|
}
|
|
}
|
|
|
|
export async function verifyDevLive(ctx, { expectedCommit = null } = {}) {
|
|
const [cloudWeb, cloudApi] = await Promise.all([
|
|
probeLiveEndpoint(ctx, {
|
|
id: "cloud-web-16666",
|
|
url: browserLiveUrl,
|
|
expectedServiceId: "hwlab-cloud-web",
|
|
expectedCommit
|
|
}),
|
|
probeLiveEndpoint(ctx, {
|
|
id: "cloud-api-16667",
|
|
url: apiLiveUrl,
|
|
expectedServiceId: "hwlab-cloud-api",
|
|
expectedCommit
|
|
})
|
|
]);
|
|
const endpoints = [cloudWeb, cloudApi];
|
|
return {
|
|
status: endpoints.every((endpoint) => endpoint.status === "pass") ? "pass" : "blocked",
|
|
expectedCommit: expectedCommit ?? null,
|
|
endpoints,
|
|
summary: {
|
|
checked: endpoints.length,
|
|
reachable: endpoints.filter((endpoint) => endpoint.reachable).length,
|
|
identityMatches: endpoints.filter((endpoint) => endpoint.identityMatches).length,
|
|
commitMatches: expectedCommit
|
|
? endpoints.filter((endpoint) => endpoint.commitMatches === true).length
|
|
: null,
|
|
ports: [16666, 16667]
|
|
}
|
|
};
|
|
}
|
|
|
|
function deployServiceCount(manifest) {
|
|
const services = manifest?.services;
|
|
if (Array.isArray(services)) return services.length;
|
|
if (services && typeof services === "object") return Object.keys(services).length;
|
|
return null;
|
|
}
|
|
|
|
function summarizeDeployJson(deploy) {
|
|
return {
|
|
path: deploy.path,
|
|
hash: deploy.hash,
|
|
commitId: deploy.commitId,
|
|
namespace: deploy.namespace,
|
|
environment: deploy.environment,
|
|
endpoint: deploy.endpoint,
|
|
serviceCount: deployServiceCount(deploy.manifest)
|
|
};
|
|
}
|
|
|
|
function deployJsonMatchesTarget(deploy, target) {
|
|
return deploy.commitId === target.shortCommitId || deploy.commitId === target.commitId;
|
|
}
|
|
|
|
function compactLiveEndpoint(endpoint) {
|
|
return {
|
|
id: endpoint.id,
|
|
url: endpoint.url,
|
|
status: endpoint.status,
|
|
httpStatus: endpoint.httpStatus ?? null,
|
|
reachable: endpoint.reachable,
|
|
identityMatches: endpoint.identityMatches,
|
|
serviceId: endpoint.serviceId,
|
|
environment: endpoint.environment,
|
|
observedCommit: endpoint.observedCommit,
|
|
commitMatches: endpoint.commitMatches,
|
|
blockerCodes: endpoint.blockerCodes ?? [],
|
|
error: endpoint.error ?? null
|
|
};
|
|
}
|
|
|
|
function compactLiveReport(live) {
|
|
return {
|
|
status: live.status,
|
|
expectedCommit: live.expectedCommit ?? null,
|
|
summary: live.summary ?? { checked: 0 },
|
|
endpoints: (live.endpoints ?? []).map(compactLiveEndpoint)
|
|
};
|
|
}
|
|
|
|
function buildLiveDelta(live, target) {
|
|
const expectedCommit = target?.shortCommitId ?? target?.commitId ?? null;
|
|
const endpoints = (live?.endpoints ?? []).map((endpoint) => ({
|
|
id: endpoint.id,
|
|
observedCommit: endpoint.observedCommit ?? null,
|
|
expectedCommit,
|
|
matchesPromotion: expectedCommit ? commitMatches(endpoint.observedCommit, expectedCommit) : null,
|
|
status: endpoint.status
|
|
}));
|
|
const mismatched = endpoints.filter((endpoint) => endpoint.matchesPromotion === false);
|
|
return {
|
|
status: expectedCommit && mismatched.length === 0 && endpoints.length > 0 ? "pass" : expectedCommit ? "mismatch" : "unknown",
|
|
expectedCommit,
|
|
mismatched: mismatched.length,
|
|
endpoints
|
|
};
|
|
}
|
|
|
|
async function readDeployLockStatus({ ctx, args, kubectlContext }) {
|
|
const get = await kubectl(ctx, kubectlContext, ["-n", args.targetNamespace, "get", "lease", args.lockName, "-o", "json"], {
|
|
timeoutMs: 15000
|
|
});
|
|
if (get.code === 0) {
|
|
return {
|
|
status: "read",
|
|
command: get.command,
|
|
lock: parseDeployLock(JSON.parse(get.stdout))
|
|
};
|
|
}
|
|
if (isNotFound(get)) {
|
|
return {
|
|
status: "absent",
|
|
command: get.command,
|
|
lockName: args.lockName,
|
|
lock: null
|
|
};
|
|
}
|
|
return {
|
|
status: "unavailable",
|
|
command: get.command,
|
|
lockName: args.lockName,
|
|
lock: null,
|
|
error: redactSensitiveText(get.stderr || get.stdout)
|
|
};
|
|
}
|
|
|
|
function summarizeLockStatus(lockRead, now) {
|
|
if (lockRead.status === "absent") {
|
|
return {
|
|
status: "absent",
|
|
lockName: lockRead.lockName ?? defaultLockName,
|
|
current: null,
|
|
retryAfterSeconds: 0,
|
|
expiresAt: null
|
|
};
|
|
}
|
|
if (lockRead.status === "unavailable") {
|
|
return {
|
|
status: "unavailable",
|
|
lockName: lockRead.lockName ?? defaultLockName,
|
|
current: null,
|
|
retryAfterSeconds: null,
|
|
expiresAt: null,
|
|
reason: "kubectl or DEV kubeconfig unavailable; command output is redacted",
|
|
redactedReason: oneLine(lockRead.error).slice(0, 500)
|
|
};
|
|
}
|
|
const classification = classifyDeployLock(lockRead.lock, now);
|
|
const status = lockRead.lock?.phase === "released"
|
|
? "released"
|
|
: classification.held
|
|
? "held"
|
|
: classification.stale
|
|
? "stale"
|
|
: "unknown";
|
|
return {
|
|
status,
|
|
lockName: lockRead.lock?.lockName ?? lockRead.lockName ?? defaultLockName,
|
|
current: compactLockSummary(lockRead.lock),
|
|
retryAfterSeconds: classification.retryAfterSeconds,
|
|
expiresAt: classification.expiresAt,
|
|
allowedLockAction: {
|
|
breakStale: classification.stale ? "--apply --confirm-dev --confirmed-non-production --break-stale-lock" : null
|
|
}
|
|
};
|
|
}
|
|
|
|
function buildStatusNextActions({ target, deployBefore, lock, liveVerify }) {
|
|
const actions = [];
|
|
if (target.desiredStateCheck?.status === "blocked") {
|
|
actions.push("deploy/deploy.json does not match the target ref and the deploy-json promotion desired-state check failed; refresh artifacts/catalog or choose an explicit published promotion.");
|
|
}
|
|
if (!target.headMatchesTarget) {
|
|
const controlShortCommit = target.controlRef?.shortCommitId ?? target.shortCommitId;
|
|
actions.push(`Checkout or fast-forward to ${target.ref} ${controlShortCommit} before apply.`);
|
|
}
|
|
if (deployBefore.commitId !== target.shortCommitId && deployBefore.commitId !== target.commitId) {
|
|
actions.push(`deploy/deploy.json commitId is ${deployBefore.commitId}; target ref is ${target.shortCommitId}.`);
|
|
}
|
|
if (lock.status === "held") {
|
|
actions.push(`Wait for the current DEV CD holder to release the Lease or for the ${lock.retryAfterSeconds}s TTL to expire before retrying apply.`);
|
|
} else if (lock.status === "stale") {
|
|
actions.push("Use --break-stale-lock only after confirming the holder is inactive.");
|
|
} else if (lock.status === "unavailable") {
|
|
actions.push("Fix kubectl/KUBECONFIG access before applying.");
|
|
}
|
|
if (liveVerify.status !== "pass") {
|
|
actions.push("Inspect live endpoint identity/health before treating DEV as converged.");
|
|
}
|
|
const liveDelta = buildLiveDelta(liveVerify, target);
|
|
if (liveDelta.status === "mismatch") {
|
|
actions.push(`Live DEV is still serving ${liveDelta.mismatched} endpoint(s) that do not match promotion ${target.shortCommitId}; host commander should run the controlled DEV rollout after the apply report is ready.`);
|
|
}
|
|
if (actions.length === 0) {
|
|
actions.push(target.publishRequired === false
|
|
? `Run --apply --confirm-dev --confirmed-non-production --report ${defaultReportPath} when ready to apply the published deploy.json promotion without republishing HEAD.`
|
|
: `Run --apply --confirm-dev --confirmed-non-production --report ${defaultReportPath} when ready to publish/apply DEV.`);
|
|
}
|
|
return actions;
|
|
}
|
|
|
|
async function runDevCdStatus(args, ctx, stdout) {
|
|
const generatedAt = ctx.now().toISOString();
|
|
const deployBefore = await readDeployJson(ctx.repoRoot);
|
|
const artifactEvidence = await readArtifactEvidence(ctx.repoRoot);
|
|
const target = await resolveEffectiveTarget(ctx, args, deployBefore, artifactEvidence);
|
|
const kubectlContext = await resolveKubectlContext(args, ctx.env, ctx.runCommand);
|
|
const lockRead = await readDeployLockStatus({ ctx, args, kubectlContext });
|
|
const lock = summarizeLockStatus(lockRead, ctx.now());
|
|
const liveVerify = args.skipLiveVerify
|
|
? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } }
|
|
: await verifyDevLive(ctx, { expectedCommit: null });
|
|
const deployMatchesTarget = deployJsonMatchesTarget(deployBefore, target);
|
|
const liveDelta = buildLiveDelta(liveVerify, target);
|
|
const status = (
|
|
lock.status === "unavailable" ||
|
|
liveVerify.status !== "pass" ||
|
|
liveDelta.status === "mismatch" ||
|
|
!target.headMatchesTarget ||
|
|
!deployMatchesTarget ||
|
|
target.publishRequired === false &&
|
|
target.artifactBoundary?.status === "degraded"
|
|
) ? "degraded" : "pass";
|
|
const report = {
|
|
ok: true,
|
|
status,
|
|
mode: args.dryRun ? "dry-run" : "status",
|
|
command: "dev-cd-apply",
|
|
generatedAt,
|
|
mutationAttempted: false,
|
|
prodTouched: false,
|
|
target: {
|
|
ref: target.ref,
|
|
promotionCommit: target.commitId,
|
|
shortCommitId: target.shortCommitId,
|
|
promotionSource: target.promotionSource,
|
|
publishRequired: target.publishRequired,
|
|
controlRef: target.controlRef,
|
|
headCommitId: target.headCommitId,
|
|
headMatchesTarget: target.headMatchesTarget,
|
|
desiredStateCheck: target.desiredStateCheck,
|
|
artifactBoundary: target.artifactBoundary,
|
|
namespace: args.targetNamespace
|
|
},
|
|
deployJson: {
|
|
...summarizeDeployJson(deployBefore),
|
|
matchesTarget: deployMatchesTarget
|
|
},
|
|
artifactCatalog: artifactEvidence.catalog,
|
|
artifactReport: artifactEvidence.report,
|
|
kubectl: {
|
|
kubeconfigSource: kubectlContext.kubeconfigSource,
|
|
kubeconfig: kubectlContext.kubeconfig ? "<set>" : "<unset>",
|
|
lockReadCommand: lockRead.command
|
|
},
|
|
lock,
|
|
live: compactLiveReport(liveVerify),
|
|
liveDelta,
|
|
nextActions: buildStatusNextActions({ target, deployBefore, lock, liveVerify }),
|
|
fullOutputHint: `Use --apply --confirm-dev --confirmed-non-production --report ${defaultReportPath} for the full transaction JSON; use --full-output only when stdout needs the full JSON.`
|
|
};
|
|
stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
return 0;
|
|
}
|
|
|
|
function summarizeDevCdApplyReport(report) {
|
|
const apply = report.devCdApply ?? {};
|
|
return {
|
|
ok: report.status === "pass",
|
|
status: report.status,
|
|
mode: apply.mode ?? "apply",
|
|
command: "dev-cd-apply",
|
|
commitId: report.commitId,
|
|
generatedAt: report.generatedAt,
|
|
mutationAttempted: apply.safety?.mutationAttempted ?? false,
|
|
prodTouched: false,
|
|
transaction: {
|
|
transactionId: report.transaction?.transactionId ?? null,
|
|
ownerTaskId: report.transaction?.ownerTaskId ?? null,
|
|
targetNamespace: report.transaction?.targetNamespace ?? defaultNamespace,
|
|
lockName: report.transaction?.lockName ?? defaultLockName,
|
|
phases: report.transaction?.phases ?? [],
|
|
release: report.transaction?.release ?? null
|
|
},
|
|
target: apply.target ?? null,
|
|
lock: {
|
|
acquired: Boolean(apply.lock?.acquired),
|
|
current: compactLockSummary(apply.lock?.current),
|
|
staleBreak: compactLockSummary(apply.lock?.staleBreak)
|
|
},
|
|
steps: (apply.steps ?? []).map((step) => ({
|
|
id: step.id,
|
|
phase: step.phase,
|
|
status: step.status,
|
|
code: step.code,
|
|
reportPath: step.reportPath
|
|
})),
|
|
liveBefore: apply.liveBefore ? compactLiveReport(apply.liveBefore) : null,
|
|
liveVerify: apply.liveVerify ? compactLiveReport(apply.liveVerify) : null,
|
|
preflight: apply.preflight ? {
|
|
status: apply.preflight.status,
|
|
scope: apply.preflight.scope,
|
|
kubeconfig: apply.preflight.kubeconfig,
|
|
blockerCount: apply.preflight.blockers?.length ?? 0,
|
|
secretRefs: (apply.preflight.secretRefs ?? []).map((ref) => ({
|
|
secretName: ref.secretName,
|
|
secretKey: ref.secretKey,
|
|
status: ref.status,
|
|
exists: ref.exists,
|
|
keyPresent: ref.keyPresent
|
|
}))
|
|
} : null,
|
|
blockers: report.blockers ?? [],
|
|
reportPaths: apply.reportPaths ?? {},
|
|
fullOutputHint: `Use --full-output to print the full report to stdout. Use --report ${defaultReportPath} to persist the full report outside the repo.`
|
|
};
|
|
}
|
|
|
|
function buildReport({
|
|
args,
|
|
transaction,
|
|
target,
|
|
deployBefore,
|
|
deployAfter,
|
|
lockState,
|
|
steps,
|
|
blockers,
|
|
status,
|
|
startedAt,
|
|
finishedAt,
|
|
preflight,
|
|
liveBefore,
|
|
liveVerify,
|
|
release
|
|
}) {
|
|
return {
|
|
$schema: "https://hwlab.pikastech.local/schemas/dev-cd-apply.schema.json",
|
|
$id: "https://hwlab.pikastech.local/dev-cd/dev-cd-apply.json",
|
|
reportVersion: "v1",
|
|
issue: "pikasTech/HWLAB#274",
|
|
taskId: "dev-cd-apply",
|
|
commitId: target.shortCommitId,
|
|
acceptanceLevel: "dev_cd_apply",
|
|
devOnly: true,
|
|
prodDisabled: true,
|
|
status,
|
|
generatedAt: finishedAt,
|
|
reportLifecycle: activeReportLifecycle("Current DEV CD transaction report; the only DEV side-effect entrypoint is scripts/dev-cd-apply.mjs."),
|
|
supports: [
|
|
"pikasTech/HWLAB#274",
|
|
"pikasTech/HWLAB#116",
|
|
"pikasTech/HWLAB#235",
|
|
"pikasTech/HWLAB#50",
|
|
"pikasTech/HWLAB#57",
|
|
"pikasTech/HWLAB#67",
|
|
"pikasTech/HWLAB#73"
|
|
],
|
|
sourceContract: {
|
|
status: blockers.length ? "blocked" : "pass",
|
|
documents: [
|
|
"docs/reference/deployment-publish.md",
|
|
"docs/reference/dev-runtime-boundary.md",
|
|
"docs/dev-artifact-publish.md",
|
|
"docs/dev-deploy-apply.md",
|
|
"docs/artifact-catalog.md"
|
|
],
|
|
summary: "DEV CD apply/verify side effects are serialized by one Kubernetes Lease and use deploy/deploy.json as the publish-control desired-state source; CI artifact reports are audit snapshots."
|
|
},
|
|
validationCommands: [
|
|
"node --check scripts/dev-cd-apply.mjs",
|
|
"node --check scripts/src/dev-cd-apply.mjs",
|
|
"node --test scripts/src/dev-cd-apply.test.mjs",
|
|
"node --check scripts/dev-artifact-publish.mjs",
|
|
"node --check scripts/src/dev-deploy-apply.mjs",
|
|
"node --test scripts/src/dev-deploy-apply.test.mjs",
|
|
"node scripts/deploy-desired-state-plan.mjs --check",
|
|
"node scripts/validate-artifact-catalog.mjs",
|
|
"git diff --check"
|
|
],
|
|
localSmoke: {
|
|
status: "not_run",
|
|
commands: ["npm run check"],
|
|
evidence: ["Local checks are listed as validation commands and are not implied by a DEV CD transaction."],
|
|
summary: "The transaction report records DEV publish/apply/verify evidence; local smoke is a separate gate."
|
|
},
|
|
dryRun: {
|
|
status: "not_run",
|
|
commands: [`node scripts/dev-deploy-apply.mjs --dry-run --expect-blocked --report ${deployApplyReportPath}`],
|
|
evidence: ["The transaction path owns live side effects; dry-run remains a preflight support mode."],
|
|
summary: "DEV CD mutation requires --apply and the transaction Lease lock."
|
|
},
|
|
devPreconditions: {
|
|
status: blockers.length ? "blocked" : "pass",
|
|
requirements: [
|
|
"DEV CD apply must force KUBECONFIG=/etc/rancher/k3s/k3s.yaml and verify native node d601 before any Lease, Job, apply, or rollout side effect.",
|
|
"DEV CD apply must reject docker-desktop, desktop-control-plane, 127.0.0.1:11700, bare kubectl ambiguity, and second hwlab-dev control-plane signals before mutation.",
|
|
"DEV CD apply must verify required SecretRef existence and key names without reading or printing Secret values.",
|
|
"HEAD must match the requested target ref before publish.",
|
|
"Lease/hwlab-dev-cd-lock must be acquired before DEV apply/verify side effects.",
|
|
"Artifact publish must run under CI artifact identity, with HWLAB_CD_TRANSACTION_ID accepted only for the legacy transition path.",
|
|
"DEV DB role/database provisioning and runtime migration must run through repo-owned commands before workload apply.",
|
|
"deploy/deploy.json, artifact catalog, and workloads must converge before apply.",
|
|
"When deploy/deploy.json already points to a published promotion commit, HEAD may be a later control commit and the transaction must consume that promotion without republishing HEAD.",
|
|
"Public 16666 and 16667 live health plus M3 durable postflight must be recorded in this report."
|
|
],
|
|
summary: blockers.length ? "One or more DEV CD transaction preconditions failed." : "DEV CD transaction preconditions passed."
|
|
},
|
|
transaction: {
|
|
transactionId: transaction.transactionId,
|
|
ownerTaskId: transaction.ownerTaskId,
|
|
targetNamespace: transaction.targetNamespace,
|
|
lockName: transaction.lockName,
|
|
lockBackend: "Lease",
|
|
startedAt,
|
|
finishedAt,
|
|
ttlSeconds: args.ttlSeconds,
|
|
phases: transaction.phases,
|
|
release
|
|
},
|
|
devCdApply: {
|
|
mode: args.apply ? "apply" : "dry-run",
|
|
target: {
|
|
ref: target.ref,
|
|
promotionCommit: target.commitId,
|
|
shortCommitId: target.shortCommitId,
|
|
promotionSource: target.promotionSource,
|
|
publishRequired: target.publishRequired,
|
|
controlRef: target.controlRef,
|
|
headCommitId: target.headCommitId,
|
|
headMatchesTarget: target.headMatchesTarget,
|
|
desiredStateCheck: target.desiredStateCheck,
|
|
artifactBoundary: target.artifactBoundary,
|
|
namespace: args.targetNamespace,
|
|
apiEndpoint: DEV_ENDPOINT,
|
|
browserEndpoint: "http://74.48.78.17:16666/"
|
|
},
|
|
deployJson: {
|
|
before: deployBefore,
|
|
after: deployAfter
|
|
},
|
|
lock: {
|
|
acquired: Boolean(lockState?.acquired),
|
|
current: lockState?.lock ?? null,
|
|
staleBreak: lockState?.staleBreak ?? null
|
|
},
|
|
steps,
|
|
liveBefore,
|
|
liveVerify,
|
|
preflight,
|
|
reportPaths: {
|
|
transaction: args.writeReport ? args.reportPath : null,
|
|
artifacts: artifactReportPath,
|
|
deployApply: deployApplyReportPath,
|
|
runtimeProvisioning: runtimeProvisioningReportPath,
|
|
runtimeMigration: runtimeMigrationReportPath,
|
|
runtimePostflight: runtimePostflightReportPath
|
|
},
|
|
safety: {
|
|
prodTouched: false,
|
|
secretValuesRead: false,
|
|
secretValuesPrinted: false,
|
|
manualDbWrite: false,
|
|
oldPublicPortsUsed: false,
|
|
mutationAttempted: steps.some((step) => ["artifact-publish", "dev-deploy-apply"].includes(step.id)),
|
|
singleLock: "Lease/hwlab-dev/hwlab-dev-cd-lock"
|
|
}
|
|
},
|
|
blockers,
|
|
notes: "DEV-only CD transaction evidence. This does not claim M3 PASS or runtime durability."
|
|
};
|
|
}
|
|
|
|
function buildArgumentFailure(error) {
|
|
const blockers = error.blockers ?? [{
|
|
type: "safety_blocker",
|
|
scope: error.code ?? "dev-cd-apply",
|
|
status: "open",
|
|
summary: oneLine(error.message)
|
|
}];
|
|
return {
|
|
ok: false,
|
|
status: "blocked",
|
|
error: error.code ?? "dev-cd-apply-failed",
|
|
blockers,
|
|
mutationAttempted: false,
|
|
prodTouched: false
|
|
};
|
|
}
|
|
|
|
export function formatDevCdApplyFailure(error) {
|
|
if (error?.lockFailure) return error.lockFailure;
|
|
if (error instanceof DevCdApplyError) return buildArgumentFailure(error);
|
|
return {
|
|
ok: false,
|
|
status: "failed",
|
|
error: "dev-cd-apply-failed",
|
|
summary: error instanceof Error ? error.message : String(error),
|
|
mutationAttempted: false,
|
|
prodTouched: false
|
|
};
|
|
}
|
|
|
|
export async function runDevCdApply(argv, io = {}) {
|
|
const stdout = io.stdout ?? process.stdout;
|
|
const env = io.env ?? process.env;
|
|
const ctx = {
|
|
repoRoot: io.repoRoot ?? defaultRepoRoot,
|
|
env,
|
|
runCommand: io.runCommand ?? defaultRunCommand,
|
|
httpGetJson: io.httpGetJson ?? defaultHttpGetJson,
|
|
now: io.now ?? (() => new Date())
|
|
};
|
|
|
|
let args;
|
|
try {
|
|
args = parseArgs(argv);
|
|
if (args.help) {
|
|
stdout.write(`${JSON.stringify({ ok: true, status: "pass", help: usage() }, null, 2)}\n`);
|
|
return 0;
|
|
}
|
|
if (!args.apply && !args.status) {
|
|
args.status = true;
|
|
}
|
|
validateArgs(args);
|
|
if (args.status) {
|
|
return await runDevCdStatus(args, ctx, stdout);
|
|
}
|
|
} catch (error) {
|
|
const failure = formatDevCdApplyFailure(error);
|
|
stdout.write(`${JSON.stringify(failure, null, 2)}\n`);
|
|
return 2;
|
|
}
|
|
|
|
const startedAt = ctx.now().toISOString();
|
|
const transaction = {
|
|
transactionId: randomUUID(),
|
|
ownerTaskId: args.ownerTaskId ?? defaultOwnerTaskId(env),
|
|
targetNamespace: args.targetNamespace,
|
|
lockName: args.lockName,
|
|
phases: [],
|
|
kubectlContext: null
|
|
};
|
|
const steps = [];
|
|
const blockers = [];
|
|
let lockState = null;
|
|
let release = null;
|
|
let target = null;
|
|
let deployBefore = null;
|
|
let deployAfter = null;
|
|
let liveBefore = null;
|
|
let liveVerify = null;
|
|
let preflight = null;
|
|
let status = "blocked";
|
|
|
|
try {
|
|
deployBefore = await readDeployJson(ctx.repoRoot);
|
|
const artifactEvidence = await readArtifactEvidence(ctx.repoRoot);
|
|
target = await resolveEffectiveTarget(ctx, args, deployBefore, artifactEvidence);
|
|
if (target.desiredStateCheck?.status === "blocked") {
|
|
blockers.push({
|
|
type: "contract_blocker",
|
|
scope: "desired-state-promotion",
|
|
status: "open",
|
|
summary: "deploy/deploy.json is not aligned with the target ref, and the deploy-json promotion check did not pass."
|
|
});
|
|
throw new DevCdApplyError("deploy desired-state does not match the target ref or a published deploy.json promotion", {
|
|
code: "desired-state-promotion-check-failed",
|
|
suppressCatchBlocker: true
|
|
});
|
|
}
|
|
if (artifactBoundaryRequiresApplyBlock(target)) {
|
|
blockers.push({
|
|
type: "contract_blocker",
|
|
scope: "artifact-boundary",
|
|
status: "open",
|
|
summary: "deploy/deploy.json, deploy/artifact-catalog.dev.json, deploy/k8s/base/workloads.yaml, and registry manifests must prove the same promotion commit before apply."
|
|
});
|
|
throw new DevCdApplyError("artifact desired-state provenance does not match the deploy.json promotion", {
|
|
code: "artifact-boundary-provenance-mismatch",
|
|
suppressCatchBlocker: true
|
|
});
|
|
}
|
|
if (!target.headMatchesTarget) {
|
|
const controlShortCommit = target.controlRef?.shortCommitId ?? target.shortCommitId;
|
|
blockers.push({
|
|
type: "contract_blocker",
|
|
scope: "target-ref-head",
|
|
status: "open",
|
|
summary: `HEAD ${target.headShortCommitId} must match target ref ${target.ref} ${controlShortCommit} before DEV CD apply.`
|
|
});
|
|
throw new DevCdApplyError("checkout is behind the requested DEV CD control ref", {
|
|
code: "target-ref-head-mismatch",
|
|
suppressCatchBlocker: true
|
|
});
|
|
}
|
|
const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand);
|
|
transaction.kubectlContext = kubectlContext;
|
|
preflight = await runDevCdPreflight({ ctx, args, kubectlContext });
|
|
if (preflight.status !== "pass") {
|
|
blockers.push(...preflight.blockers);
|
|
throw new DevCdApplyError("DEV CD preflight blocked before any write side effect", {
|
|
code: "dev-cd-preflight-blocked",
|
|
suppressCatchBlocker: true
|
|
});
|
|
}
|
|
liveBefore = {
|
|
status: "pending",
|
|
reason: "liveBefore is captured immediately after Lease acquisition so publish/apply cannot race before the transaction lock."
|
|
};
|
|
lockState = await acquireDeployLock({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
target,
|
|
deployBefore,
|
|
transactionId: transaction.transactionId,
|
|
ownerTaskId: transaction.ownerTaskId,
|
|
liveBefore,
|
|
now: ctx.now()
|
|
});
|
|
liveBefore = await verifyDevLive(ctx, { expectedCommit: null });
|
|
await updateDeployLockLiveBefore({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId: transaction.transactionId,
|
|
lockState,
|
|
liveBefore
|
|
});
|
|
transaction.phases.push({ phase: "publishing", status: "entered", at: ctx.now().toISOString() });
|
|
|
|
const publishSteps = target.publishRequired === false
|
|
? [
|
|
{
|
|
id: "desired-state-check-existing-promotion",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: [
|
|
"scripts/deploy-desired-state-plan.mjs",
|
|
"--promotion-commit",
|
|
target.commitId,
|
|
"--check",
|
|
"--pretty"
|
|
],
|
|
timeoutMs: 120000
|
|
},
|
|
{
|
|
id: "artifact-catalog-validate",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: ["scripts/validate-artifact-catalog.mjs"],
|
|
timeoutMs: 120000
|
|
}
|
|
]
|
|
: [
|
|
{
|
|
id: "desired-state-check-before-publish",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: ["scripts/deploy-desired-state-plan.mjs", "--check", "--pretty"],
|
|
timeoutMs: 120000
|
|
},
|
|
{
|
|
id: "artifact-publish",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: ["scripts/dev-artifact-publish.mjs", "--publish", "--report", artifactReportPath],
|
|
timeoutMs: 60 * 60 * 1000,
|
|
reportPath: artifactReportPath
|
|
},
|
|
{
|
|
id: "artifact-catalog-refresh",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: [
|
|
"scripts/refresh-artifact-catalog.mjs",
|
|
"--target-ref",
|
|
target.commitId,
|
|
"--publish-report",
|
|
artifactReportPath
|
|
],
|
|
timeoutMs: 120000
|
|
},
|
|
{
|
|
id: "artifact-catalog-validate",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: ["scripts/validate-artifact-catalog.mjs"],
|
|
timeoutMs: 120000
|
|
},
|
|
{
|
|
id: "desired-state-check-after-refresh",
|
|
phase: "publishing",
|
|
command: process.execPath,
|
|
args: [
|
|
"scripts/deploy-desired-state-plan.mjs",
|
|
"--promotion-commit",
|
|
target.commitId,
|
|
"--check",
|
|
"--pretty"
|
|
],
|
|
timeoutMs: 120000
|
|
},
|
|
];
|
|
|
|
const plannedSteps = [
|
|
...publishSteps,
|
|
{
|
|
id: "runtime-db-provisioning",
|
|
phase: "applying",
|
|
kind: "runtime-k8s-job",
|
|
jobNamePrefix: "hwlab-runtime-provision",
|
|
commandArgs: [
|
|
"cmd/hwlab-cloud-api/provision.mjs",
|
|
"--apply",
|
|
"--confirm-dev",
|
|
"--confirmed-non-production",
|
|
"--report",
|
|
runtimeProvisioningReportPath,
|
|
"--fail-on-blocked"
|
|
],
|
|
timeoutMs: 5 * 60 * 1000,
|
|
reportPath: runtimeProvisioningReportPath
|
|
},
|
|
{
|
|
id: "runtime-db-migration",
|
|
phase: "applying",
|
|
kind: "runtime-k8s-job",
|
|
jobNamePrefix: "hwlab-runtime-migrate",
|
|
commandArgs: [
|
|
"cmd/hwlab-cloud-api/migrate.mjs",
|
|
"--apply",
|
|
"--confirm-dev",
|
|
"--confirmed-non-production",
|
|
"--report",
|
|
runtimeMigrationReportPath,
|
|
"--fail-on-blocked"
|
|
],
|
|
timeoutMs: 5 * 60 * 1000,
|
|
reportPath: runtimeMigrationReportPath
|
|
},
|
|
{
|
|
id: "dev-deploy-apply",
|
|
phase: "applying",
|
|
command: process.execPath,
|
|
args: [
|
|
"scripts/dev-deploy-apply.mjs",
|
|
"--apply",
|
|
"--confirm-dev",
|
|
"--confirmed-non-production",
|
|
"--report",
|
|
deployApplyReportPath,
|
|
...(args.kubeconfig ? ["--kubeconfig", args.kubeconfig] : []),
|
|
...(args.registryManifestBaseUrl ? ["--registry-manifest-base-url", args.registryManifestBaseUrl] : [])
|
|
],
|
|
timeoutMs: 20 * 60 * 1000,
|
|
reportPath: deployApplyReportPath
|
|
},
|
|
...(
|
|
args.skipRuntimePostflight
|
|
? []
|
|
: [{
|
|
id: "runtime-durable-postflight",
|
|
phase: "verifying",
|
|
command: process.execPath,
|
|
args: [
|
|
"scripts/dev-runtime-postflight.mjs",
|
|
"--live",
|
|
"--confirm-dev",
|
|
"--confirmed-non-production",
|
|
"--target",
|
|
"api",
|
|
"--report",
|
|
runtimePostflightReportPath,
|
|
"--fail-on-blocked"
|
|
],
|
|
timeoutMs: 2 * 60 * 1000,
|
|
reportPath: runtimePostflightReportPath
|
|
}]
|
|
)
|
|
];
|
|
|
|
if (blockers.length === 0) {
|
|
for (const step of plannedSteps) {
|
|
if (step.phase !== lockState.lock?.phase) {
|
|
await updateDeployLockPhase({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId: transaction.transactionId,
|
|
lockState,
|
|
phase: step.phase
|
|
});
|
|
transaction.phases.push({ phase: step.phase, status: "entered", at: ctx.now().toISOString() });
|
|
}
|
|
const result = step.kind === "runtime-k8s-job"
|
|
? await runRuntimeK8sJobStep(ctx, transaction, step)
|
|
: await runStep(ctx, transaction, step);
|
|
steps.push(result);
|
|
if (result.status !== "pass") {
|
|
blockers.push(stepBlocker(result));
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
if (blockers.length === 0) {
|
|
if (lockState.lock?.phase !== "verifying") {
|
|
await updateDeployLockPhase({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId: transaction.transactionId,
|
|
lockState,
|
|
phase: "verifying"
|
|
});
|
|
transaction.phases.push({ phase: "verifying", status: "entered", at: ctx.now().toISOString() });
|
|
}
|
|
deployAfter = await readDeployJson(ctx.repoRoot);
|
|
liveVerify = args.skipLiveVerify
|
|
? { status: "not_run", reason: "--skip-live-verify", endpoints: [], summary: { checked: 0 } }
|
|
: await verifyDevLive(ctx, { expectedCommit: target.shortCommitId });
|
|
if (liveVerify.status !== "pass") {
|
|
blockers.push({
|
|
type: "observability_blocker",
|
|
scope: "live-verify",
|
|
status: "open",
|
|
summary: "16666/16667 live health did not match the DEV CD promotion commit."
|
|
});
|
|
}
|
|
} else {
|
|
deployAfter = deployBefore;
|
|
liveVerify = { status: "not_run", reason: "transaction blocked before live verify", endpoints: [], summary: { checked: 0 } };
|
|
}
|
|
|
|
status = blockers.length === 0 ? "pass" : "blocked";
|
|
release = await releaseDeployLock({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId: transaction.transactionId,
|
|
status
|
|
});
|
|
transaction.phases.push({ phase: "released", status: release.status, at: ctx.now().toISOString() });
|
|
} catch (error) {
|
|
if (error?.lockFailure) {
|
|
stdout.write(`${JSON.stringify(error.lockFailure, null, 2)}\n`);
|
|
return 2;
|
|
}
|
|
if (!error?.suppressCatchBlocker) {
|
|
blockers.push({
|
|
type: "runtime_blocker",
|
|
scope: error.code ?? "dev-cd-apply",
|
|
status: "open",
|
|
summary: oneLine(error.message)
|
|
});
|
|
}
|
|
status = "blocked";
|
|
if (lockState?.acquired) {
|
|
const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand);
|
|
release = await releaseDeployLock({
|
|
ctx,
|
|
args,
|
|
kubectlContext,
|
|
transactionId: transaction.transactionId,
|
|
status
|
|
});
|
|
transaction.phases.push({ phase: "released", status: release.status, at: ctx.now().toISOString() });
|
|
}
|
|
}
|
|
|
|
const finishedAt = ctx.now().toISOString();
|
|
const report = buildReport({
|
|
args,
|
|
transaction,
|
|
target: target ?? {
|
|
ref: args.targetRef,
|
|
commitId: "unknown",
|
|
shortCommitId: "unknown",
|
|
headCommitId: "unknown",
|
|
headMatchesTarget: false
|
|
},
|
|
deployBefore: deployBefore ?? { path: "deploy/deploy.json", hash: "unknown", commitId: "unknown" },
|
|
deployAfter: deployAfter ?? deployBefore ?? { path: "deploy/deploy.json", hash: "unknown", commitId: "unknown" },
|
|
lockState,
|
|
steps,
|
|
blockers,
|
|
status,
|
|
startedAt,
|
|
finishedAt,
|
|
preflight: preflight ?? {
|
|
status: "not_run",
|
|
scope: "pre-lock-pre-side-effect",
|
|
blockers: [],
|
|
safety: {
|
|
writeSideEffectsAttempted: false,
|
|
prodTouched: false,
|
|
secretValuesRead: false,
|
|
secretValuesPrinted: false
|
|
}
|
|
},
|
|
liveBefore: liveBefore ?? { status: "not_run" },
|
|
liveVerify: liveVerify ?? { status: "not_run", endpoints: [], summary: { checked: 0 } },
|
|
release
|
|
});
|
|
|
|
if (args.writeReport) {
|
|
const absoluteReportPath = ensureNotRepoReportsPath(ctx.repoRoot, args.reportPath, "--report");
|
|
await mkdir(path.dirname(absoluteReportPath), { recursive: true });
|
|
await writeFile(absoluteReportPath, `${JSON.stringify(report, null, 2)}\n`);
|
|
}
|
|
|
|
const stdoutPayload = args.fullOutput ? report : summarizeDevCdApplyReport(report);
|
|
stdout.write(`${JSON.stringify(stdoutPayload, null, 2)}\n`);
|
|
return status === "pass" ? 0 : 2;
|
|
}
|