fix: gate dev cd apply with d601 preflight
This commit is contained in:
+378
-24
@@ -18,6 +18,7 @@ const defaultNamespace = "hwlab-dev";
|
||||
const defaultLockName = "hwlab-dev-cd-lock";
|
||||
const defaultTtlSeconds = 3600;
|
||||
const defaultReportPath = "reports/dev-gate/dev-cd-apply.json";
|
||||
const d601NativeKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
|
||||
const artifactCatalogPath = "deploy/artifact-catalog.dev.json";
|
||||
const artifactReportPath = "reports/dev-gate/dev-artifacts.json";
|
||||
const artifactDesiredStatePaths = [
|
||||
@@ -33,6 +34,12 @@ const runtimeJobReportPrefix = "/tmp/hwlab-dev-runtime-report";
|
||||
const browserLiveUrl = "http://74.48.78.17:16666/health/live";
|
||||
const apiLiveUrl = `${DEV_ENDPOINT}/health/live`;
|
||||
const digestPattern = /^sha256:[a-f0-9]{64}$/u;
|
||||
const forbiddenControlPlanePattern = /docker-desktop|desktop-control-plane|127\.0\.0\.1:11700/iu;
|
||||
const requiredDevSecretRefs = Object.freeze([
|
||||
Object.freeze({ secretName: "hwlab-cloud-api-dev-db", secretKey: "database-url" }),
|
||||
Object.freeze({ secretName: "hwlab-cloud-api-dev-db-admin", secretKey: "admin-url" }),
|
||||
Object.freeze({ secretName: "hwlab-code-agent-provider", secretKey: "openai-api-key" })
|
||||
]);
|
||||
const lockAnnotationPrefix = "hwlab.pikastech.local";
|
||||
const lockAnnotationFields = {
|
||||
promotionCommit: `${lockAnnotationPrefix}/promotionCommit`,
|
||||
@@ -218,6 +225,19 @@ function oneLine(value) {
|
||||
return String(value ?? "").replace(/\s+/g, " ").trim();
|
||||
}
|
||||
|
||||
function blocker({ type = "environment_blocker", scope, reason, impact, safeNextAction, retryable }) {
|
||||
return {
|
||||
type,
|
||||
scope,
|
||||
status: "open",
|
||||
reason: oneLine(reason),
|
||||
impact: oneLine(impact),
|
||||
safeNextAction: oneLine(safeNextAction),
|
||||
retryable: Boolean(retryable),
|
||||
summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`)
|
||||
};
|
||||
}
|
||||
|
||||
function sha256(value) {
|
||||
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
|
||||
}
|
||||
@@ -1207,6 +1227,323 @@ async function releaseDeployLock({ ctx, args, kubectlContext, transactionId, sta
|
||||
};
|
||||
}
|
||||
|
||||
function commandSummary(result, maxLength = 500) {
|
||||
return oneLine(redactSensitiveText(result?.stderr || result?.stdout || "")).slice(0, maxLength);
|
||||
}
|
||||
|
||||
function envWithoutKubeconfig(env) {
|
||||
const sanitized = { ...env };
|
||||
delete sanitized.KUBECONFIG;
|
||||
delete sanitized.HWLAB_DEV_KUBECONFIG;
|
||||
return sanitized;
|
||||
}
|
||||
|
||||
function parseNodeNames(stdout) {
|
||||
return String(stdout ?? "").trim().split(/\s+/u).filter(Boolean);
|
||||
}
|
||||
|
||||
function parseSecretDescribeKeys(stdout) {
|
||||
const keys = new Set();
|
||||
let inData = false;
|
||||
for (const rawLine of String(stdout ?? "").split(/\r?\n/u)) {
|
||||
const line = rawLine.trim();
|
||||
if (/^Data\b/iu.test(line)) {
|
||||
inData = true;
|
||||
continue;
|
||||
}
|
||||
if (!inData || line === "" || /^=+$/u.test(line)) continue;
|
||||
const match = line.match(/^([A-Za-z0-9_.-]+):\s+\d+\s+bytes\b/iu);
|
||||
if (match) keys.add(match[1]);
|
||||
}
|
||||
return [...keys].sort();
|
||||
}
|
||||
|
||||
function commandObservation(args, result) {
|
||||
return {
|
||||
command: shellCommand("kubectl", args),
|
||||
code: result.code ?? null,
|
||||
ok: result.code === 0,
|
||||
summary: commandSummary(result)
|
||||
};
|
||||
}
|
||||
|
||||
async function preflightKubectl(ctx, kubectlContext, args, options = {}) {
|
||||
const result = await ctx.runCommand(kubectlContext.executor, args, {
|
||||
cwd: ctx.repoRoot,
|
||||
env: options.env ?? kubectlContext.env,
|
||||
timeoutMs: options.timeoutMs ?? 15000
|
||||
});
|
||||
return {
|
||||
...result,
|
||||
stdout: result.stdout ?? "",
|
||||
stderr: result.stderr ?? "",
|
||||
command: shellCommand("kubectl", args)
|
||||
};
|
||||
}
|
||||
|
||||
function addControlPlaneBlockers(blockers, observation, { scopePrefix, requiredForApply }) {
|
||||
if (observation.contextName && forbiddenControlPlanePattern.test(observation.contextName)) {
|
||||
blockers.push(blocker({
|
||||
scope: `${scopePrefix}-context`,
|
||||
reason: `kubectl context resolved to forbidden control plane ${observation.contextName}.`,
|
||||
impact: "DEV CD cannot prove it is targeting D601 native k3s before mutation.",
|
||||
safeNextAction: `Use KUBECONFIG=${d601NativeKubeconfigPath} and remove or repair the stale Docker Desktop/default kube context before retrying.`,
|
||||
retryable: true
|
||||
}));
|
||||
}
|
||||
if (observation.serverUrl && forbiddenControlPlanePattern.test(observation.serverUrl)) {
|
||||
blockers.push(blocker({
|
||||
scope: `${scopePrefix}-server`,
|
||||
reason: `kubectl server resolved to forbidden endpoint ${observation.serverUrl}.`,
|
||||
impact: "DEV CD could write to the wrong control plane or treat stale Docker Desktop output as DEV evidence.",
|
||||
safeNextAction: `Point kubectl at D601 native k3s with KUBECONFIG=${d601NativeKubeconfigPath} before retrying.`,
|
||||
retryable: true
|
||||
}));
|
||||
}
|
||||
if (observation.nodeNames.includes("desktop-control-plane")) {
|
||||
blockers.push(blocker({
|
||||
scope: `${scopePrefix}-nodes`,
|
||||
reason: "kubectl observed node desktop-control-plane.",
|
||||
impact: "DEV CD is observing Docker Desktop Kubernetes instead of D601 native k3s.",
|
||||
safeNextAction: `Stop and rerun only after kubectl nodes come from KUBECONFIG=${d601NativeKubeconfigPath} and include d601.`,
|
||||
retryable: true
|
||||
}));
|
||||
} else if (requiredForApply && !observation.nodeNames.includes("d601")) {
|
||||
blockers.push(blocker({
|
||||
scope: `${scopePrefix}-nodes`,
|
||||
reason: `kubectl nodes did not include d601; observed ${observation.nodeNames.join(",") || "none"}.`,
|
||||
impact: "DEV apply/job/rollout side effects are not allowed without proving the D601 native k3s node.",
|
||||
safeNextAction: `Restore access to KUBECONFIG=${d601NativeKubeconfigPath} and verify kubectl get nodes includes d601.`,
|
||||
retryable: true
|
||||
}));
|
||||
} else if (!requiredForApply && observation.nodeNames.length > 0 && !observation.nodeNames.includes("d601")) {
|
||||
blockers.push(blocker({
|
||||
scope: `${scopePrefix}-nodes`,
|
||||
reason: `bare kubectl observed non-D601 nodes ${observation.nodeNames.join(",")}.`,
|
||||
impact: "A second or stale control plane may still own hwlab-dev evidence, so DEV CD refuses to mutate.",
|
||||
safeNextAction: "Clean the default kubectl context or ensure bare kubectl cannot resolve stale HWLAB DEV resources, then retry.",
|
||||
retryable: true
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
async function observeControlPlane(ctx, kubectlContext, { env, label, requiredForApply }) {
|
||||
const [context, server, nodes] = await Promise.all([
|
||||
preflightKubectl(ctx, kubectlContext, ["config", "current-context"], { env, timeoutMs: 10000 }),
|
||||
preflightKubectl(ctx, kubectlContext, ["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], { env, timeoutMs: 10000 }),
|
||||
preflightKubectl(ctx, kubectlContext, ["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], { env, timeoutMs: 15000 })
|
||||
]);
|
||||
return {
|
||||
label,
|
||||
contextCommand: commandObservation(["config", "current-context"], context),
|
||||
serverCommand: commandObservation(["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], server),
|
||||
nodesCommand: commandObservation(["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], nodes),
|
||||
contextName: context.code === 0 ? oneLine(context.stdout) : null,
|
||||
serverUrl: server.code === 0 ? oneLine(server.stdout) : null,
|
||||
nodeNames: nodes.code === 0 ? parseNodeNames(nodes.stdout) : [],
|
||||
status: context.code === 0 && server.code === 0 && nodes.code === 0 ? "observed" : requiredForApply ? "blocked" : "unavailable"
|
||||
};
|
||||
}
|
||||
|
||||
async function observeBareKubectl(ctx, kubectlContext, forcedObservation) {
|
||||
const bareEnv = envWithoutKubeconfig(ctx.env);
|
||||
const bare = await observeControlPlane(ctx, kubectlContext, {
|
||||
env: bareEnv,
|
||||
label: "bare-kubectl",
|
||||
requiredForApply: false
|
||||
});
|
||||
const namespace = await preflightKubectl(ctx, kubectlContext, ["get", "namespace", defaultNamespace, "-o", "name"], {
|
||||
env: bareEnv,
|
||||
timeoutMs: 10000
|
||||
});
|
||||
const secondControlPlane =
|
||||
namespace.code === 0 &&
|
||||
(
|
||||
(bare.serverUrl && forcedObservation.serverUrl && bare.serverUrl !== forcedObservation.serverUrl) ||
|
||||
(!bare.serverUrl && bare.contextName && bare.contextName !== forcedObservation.contextName)
|
||||
);
|
||||
return {
|
||||
...bare,
|
||||
namespace: commandObservation(["get", "namespace", defaultNamespace, "-o", "name"], namespace),
|
||||
hwlabDevNamespaceObserved: namespace.code === 0,
|
||||
secondControlPlaneCandidate: secondControlPlane
|
||||
};
|
||||
}
|
||||
|
||||
async function observeRequiredSecretRef(ctx, kubectlContext, args, ref) {
|
||||
const existsArgs = ["-n", args.targetNamespace, "get", "secret", ref.secretName, "-o", "name"];
|
||||
const describeArgs = ["-n", args.targetNamespace, "describe", "secret", ref.secretName];
|
||||
const exists = await preflightKubectl(ctx, kubectlContext, existsArgs, { timeoutMs: 10000 });
|
||||
if (exists.code !== 0) {
|
||||
return {
|
||||
secretName: ref.secretName,
|
||||
secretKey: ref.secretKey,
|
||||
status: "missing-secret",
|
||||
exists: false,
|
||||
keyPresent: false,
|
||||
keysObserved: [],
|
||||
existenceCommand: shellCommand("kubectl", existsArgs),
|
||||
keyCommand: null,
|
||||
error: commandSummary(exists),
|
||||
secretValueRead: false,
|
||||
secretValuePrinted: false
|
||||
};
|
||||
}
|
||||
const describe = await preflightKubectl(ctx, kubectlContext, describeArgs, { timeoutMs: 10000 });
|
||||
if (describe.code !== 0) {
|
||||
return {
|
||||
secretName: ref.secretName,
|
||||
secretKey: ref.secretKey,
|
||||
status: "key-observation-blocked",
|
||||
exists: true,
|
||||
keyPresent: false,
|
||||
keysObserved: [],
|
||||
existenceCommand: shellCommand("kubectl", existsArgs),
|
||||
keyCommand: shellCommand("kubectl", describeArgs),
|
||||
error: commandSummary(describe),
|
||||
secretValueRead: false,
|
||||
secretValuePrinted: false
|
||||
};
|
||||
}
|
||||
const keysObserved = parseSecretDescribeKeys(describe.stdout);
|
||||
const keyPresent = keysObserved.includes(ref.secretKey);
|
||||
return {
|
||||
secretName: ref.secretName,
|
||||
secretKey: ref.secretKey,
|
||||
status: keyPresent ? "present" : "missing-key",
|
||||
exists: true,
|
||||
keyPresent,
|
||||
keysObserved,
|
||||
existenceCommand: shellCommand("kubectl", existsArgs),
|
||||
keyCommand: shellCommand("kubectl", describeArgs),
|
||||
secretValueRead: false,
|
||||
secretValuePrinted: false
|
||||
};
|
||||
}
|
||||
|
||||
function secretRefBlocker(ref, observation) {
|
||||
const scope = `secretref:${ref.secretName}/${ref.secretKey}`;
|
||||
if (observation.status === "missing-secret") {
|
||||
return blocker({
|
||||
type: "runtime_blocker",
|
||||
scope,
|
||||
reason: `Required DEV Secret ${ref.secretName} was not observed in ${defaultNamespace}.`,
|
||||
impact: "Runtime provisioning, migration, deploy apply, or Code Agent provider startup would fail after mutation.",
|
||||
safeNextAction: `Create or restore SecretRef ${ref.secretName}/${ref.secretKey} in ${defaultNamespace} without printing the Secret value, then retry DEV CD.`,
|
||||
retryable: true
|
||||
});
|
||||
}
|
||||
if (observation.status === "key-observation-blocked") {
|
||||
return blocker({
|
||||
type: "runtime_blocker",
|
||||
scope,
|
||||
reason: `Required DEV Secret ${ref.secretName} exists but key names could not be verified.`,
|
||||
impact: "DEV CD cannot prove the pod SecretRef contract before creating runtime Jobs or applying workloads.",
|
||||
safeNextAction: `Restore metadata/key-name visibility for ${ref.secretName} without granting or printing Secret values, then retry.`,
|
||||
retryable: true
|
||||
});
|
||||
}
|
||||
return blocker({
|
||||
type: "runtime_blocker",
|
||||
scope,
|
||||
reason: `Required DEV Secret key ${ref.secretName}/${ref.secretKey} was not observed.`,
|
||||
impact: "The target workload or runtime Job would reference a missing Secret key after mutation.",
|
||||
safeNextAction: `Add key ${ref.secretKey} to Secret ${ref.secretName} without printing the value, then retry DEV CD.`,
|
||||
retryable: true
|
||||
});
|
||||
}
|
||||
|
||||
async function runDevCdPreflight({ ctx, args, kubectlContext }) {
|
||||
const blockers = [];
|
||||
const preflight = {
|
||||
status: "blocked",
|
||||
scope: "pre-lock-pre-side-effect",
|
||||
generatedAt: ctx.now().toISOString(),
|
||||
namespace: args.targetNamespace,
|
||||
kubeconfig: {
|
||||
required: d601NativeKubeconfigPath,
|
||||
selected: kubectlContext.kubeconfig,
|
||||
source: kubectlContext.kubeconfigSource,
|
||||
enforcedEnv: kubectlContext.env.KUBECONFIG === d601NativeKubeconfigPath
|
||||
},
|
||||
kubectl: {
|
||||
executor: kubectlContext.executor,
|
||||
commandPrefix: kubectlContext.commandPrefix
|
||||
},
|
||||
controlPlane: null,
|
||||
secretRefs: [],
|
||||
safety: {
|
||||
writeSideEffectsAttempted: false,
|
||||
prodTouched: false,
|
||||
secretValuesRead: false,
|
||||
secretValuesPrinted: false,
|
||||
secretKeyNamesOnly: true
|
||||
},
|
||||
blockers
|
||||
};
|
||||
|
||||
if (kubectlContext.kubeconfig !== d601NativeKubeconfigPath) {
|
||||
blockers.push(blocker({
|
||||
scope: "d601-kubeconfig-path",
|
||||
reason: `DEV CD selected kubeconfig ${kubectlContext.kubeconfig || "<unset>"} from ${kubectlContext.kubeconfigSource}.`,
|
||||
impact: "Any apply/job/rollout could target a non-D601 or stale control plane.",
|
||||
safeNextAction: `Rerun with KUBECONFIG=${d601NativeKubeconfigPath} or --kubeconfig ${d601NativeKubeconfigPath}.`,
|
||||
retryable: true
|
||||
}));
|
||||
preflight.status = "blocked";
|
||||
return preflight;
|
||||
}
|
||||
|
||||
const forced = await observeControlPlane(ctx, kubectlContext, {
|
||||
env: kubectlContext.env,
|
||||
label: "forced-d601-k3s",
|
||||
requiredForApply: true
|
||||
});
|
||||
const bare = await observeBareKubectl(ctx, kubectlContext, forced);
|
||||
preflight.controlPlane = { forced, bare };
|
||||
|
||||
if (forced.status !== "observed") {
|
||||
blockers.push(blocker({
|
||||
scope: "d601-native-k3s-unobservable",
|
||||
reason: `Forced D601 kubeconfig could not prove context/server/nodes: ${[forced.contextCommand.summary, forced.serverCommand.summary, forced.nodesCommand.summary].filter(Boolean).join("; ") || "no kubectl output"}.`,
|
||||
impact: "DEV CD refuses to acquire the Lease or create runtime Jobs without D601 native k3s proof.",
|
||||
safeNextAction: `Restore readable ${d601NativeKubeconfigPath} access and verify kubectl get nodes includes d601.`,
|
||||
retryable: true
|
||||
}));
|
||||
}
|
||||
addControlPlaneBlockers(blockers, forced, {
|
||||
scopePrefix: "d601-native-k3s",
|
||||
requiredForApply: true
|
||||
});
|
||||
addControlPlaneBlockers(blockers, bare, {
|
||||
scopePrefix: "bare-kubectl",
|
||||
requiredForApply: false
|
||||
});
|
||||
if (bare.secondControlPlaneCandidate) {
|
||||
blockers.push(blocker({
|
||||
scope: "second-hwlab-dev-control-plane",
|
||||
reason: `bare kubectl can observe namespace ${defaultNamespace} outside the forced D601 kubeconfig context.`,
|
||||
impact: "A second hwlab-dev control plane may still carry stale HWLAB resources, so DEV CD cannot safely classify apply evidence.",
|
||||
safeNextAction: "Remove the stale hwlab-dev control plane/default kube context or make bare kubectl unable to resolve it, then rerun preflight.",
|
||||
retryable: true
|
||||
}));
|
||||
}
|
||||
|
||||
if (blockers.length === 0) {
|
||||
preflight.secretRefs = await Promise.all(
|
||||
requiredDevSecretRefs.map((ref) => observeRequiredSecretRef(ctx, kubectlContext, args, ref))
|
||||
);
|
||||
for (const [index, observation] of preflight.secretRefs.entries()) {
|
||||
if (observation.status !== "present") {
|
||||
blockers.push(secretRefBlocker(requiredDevSecretRefs[index], observation));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
preflight.status = blockers.length === 0 ? "pass" : "blocked";
|
||||
return preflight;
|
||||
}
|
||||
|
||||
async function resolveKubectlContext(args, env, runCommand) {
|
||||
const selection = resolveDevKubeconfigSelection({ flagValue: args.kubeconfig, env });
|
||||
const which = await runCommand("which", ["kubectl"], { timeoutMs: 5000 });
|
||||
@@ -1215,27 +1552,6 @@ async function resolveKubectlContext(args, env, runCommand) {
|
||||
...env,
|
||||
KUBECONFIG: selection.kubeconfig
|
||||
};
|
||||
if (which.code === 0 && args.apply) {
|
||||
const context = await runCommand(executor, ["config", "current-context"], { env: kubectlEnv, timeoutMs: 10000 });
|
||||
const nodes = await runCommand(executor, ["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], { env: kubectlEnv, timeoutMs: 15000 });
|
||||
const contextName = String(context.stdout ?? "").trim();
|
||||
const nodeNames = String(nodes.stdout ?? "").trim().split(/\s+/u).filter(Boolean);
|
||||
const forbiddenContext = context.code === 0 && /docker-desktop|desktop-control-plane/iu.test(contextName);
|
||||
const forbiddenNode = nodes.code === 0 && (
|
||||
nodeNames.includes("desktop-control-plane") || (nodeNames.length > 0 && !nodeNames.includes("d601"))
|
||||
);
|
||||
if (forbiddenContext || forbiddenNode) {
|
||||
throw new DevCdApplyError("DEV CD kubeconfig does not target D601 native k3s", {
|
||||
code: "wrong-kubernetes-control-plane",
|
||||
blockers: [{
|
||||
type: "environment_blocker",
|
||||
scope: "d601-native-k3s",
|
||||
status: "open",
|
||||
summary: "D601 DEV CD must use native k3s via /etc/rancher/k3s/k3s.yaml; default docker-desktop kubeconfig is forbidden."
|
||||
}]
|
||||
});
|
||||
}
|
||||
}
|
||||
return {
|
||||
executor,
|
||||
kubeconfig: selection.kubeconfig,
|
||||
@@ -1904,6 +2220,19 @@ function summarizeDevCdApplyReport(report) {
|
||||
})),
|
||||
liveBefore: apply.liveBefore ? compactLiveReport(apply.liveBefore) : null,
|
||||
liveVerify: apply.liveVerify ? compactLiveReport(apply.liveVerify) : null,
|
||||
preflight: apply.preflight ? {
|
||||
status: apply.preflight.status,
|
||||
scope: apply.preflight.scope,
|
||||
kubeconfig: apply.preflight.kubeconfig,
|
||||
blockerCount: apply.preflight.blockers?.length ?? 0,
|
||||
secretRefs: (apply.preflight.secretRefs ?? []).map((ref) => ({
|
||||
secretName: ref.secretName,
|
||||
secretKey: ref.secretKey,
|
||||
status: ref.status,
|
||||
exists: ref.exists,
|
||||
keyPresent: ref.keyPresent
|
||||
}))
|
||||
} : null,
|
||||
blockers: report.blockers ?? [],
|
||||
reportPaths: apply.reportPaths ?? {},
|
||||
fullOutputHint: "Use --full-output to print the full report to stdout. Use --write-report to persist the full report."
|
||||
@@ -1922,6 +2251,7 @@ function buildReport({
|
||||
status,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
preflight,
|
||||
liveBefore,
|
||||
liveVerify,
|
||||
release
|
||||
@@ -1985,6 +2315,9 @@ function buildReport({
|
||||
devPreconditions: {
|
||||
status: blockers.length ? "blocked" : "pass",
|
||||
requirements: [
|
||||
"DEV CD apply must force KUBECONFIG=/etc/rancher/k3s/k3s.yaml and verify native node d601 before any Lease, Job, apply, or rollout side effect.",
|
||||
"DEV CD apply must reject docker-desktop, desktop-control-plane, 127.0.0.1:11700, bare kubectl ambiguity, and second hwlab-dev control-plane signals before mutation.",
|
||||
"DEV CD apply must verify required SecretRef existence and key names without reading or printing Secret values.",
|
||||
"HEAD must match the requested target ref before publish.",
|
||||
"Lease/hwlab-dev-cd-lock must be acquired before publish/apply.",
|
||||
"Legacy publish/apply side-effect scripts must run with HWLAB_CD_TRANSACTION_ID.",
|
||||
@@ -2034,9 +2367,10 @@ function buildReport({
|
||||
staleBreak: lockState?.staleBreak ?? null
|
||||
},
|
||||
steps,
|
||||
liveBefore,
|
||||
liveVerify,
|
||||
reportPaths: {
|
||||
liveBefore,
|
||||
liveVerify,
|
||||
preflight,
|
||||
reportPaths: {
|
||||
transaction: args.writeReport ? args.reportPath : null,
|
||||
artifacts: artifactReportPath,
|
||||
deployApply: "reports/dev-gate/dev-deploy-report.json",
|
||||
@@ -2138,6 +2472,7 @@ export async function runDevCdApply(argv, io = {}) {
|
||||
let deployAfter = null;
|
||||
let liveBefore = null;
|
||||
let liveVerify = null;
|
||||
let preflight = null;
|
||||
let status = "blocked";
|
||||
|
||||
try {
|
||||
@@ -2182,6 +2517,14 @@ export async function runDevCdApply(argv, io = {}) {
|
||||
}
|
||||
const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand);
|
||||
transaction.kubectlContext = kubectlContext;
|
||||
preflight = await runDevCdPreflight({ ctx, args, kubectlContext });
|
||||
if (preflight.status !== "pass") {
|
||||
blockers.push(...preflight.blockers);
|
||||
throw new DevCdApplyError("DEV CD preflight blocked before any write side effect", {
|
||||
code: "dev-cd-preflight-blocked",
|
||||
suppressCatchBlocker: true
|
||||
});
|
||||
}
|
||||
liveBefore = {
|
||||
status: "pending",
|
||||
reason: "liveBefore is captured immediately after Lease acquisition so publish/apply cannot race before the transaction lock."
|
||||
@@ -2458,6 +2801,17 @@ export async function runDevCdApply(argv, io = {}) {
|
||||
status,
|
||||
startedAt,
|
||||
finishedAt,
|
||||
preflight: preflight ?? {
|
||||
status: "not_run",
|
||||
scope: "pre-lock-pre-side-effect",
|
||||
blockers: [],
|
||||
safety: {
|
||||
writeSideEffectsAttempted: false,
|
||||
prodTouched: false,
|
||||
secretValuesRead: false,
|
||||
secretValuesPrinted: false
|
||||
}
|
||||
},
|
||||
liveBefore: liveBefore ?? { status: "not_run" },
|
||||
liveVerify: liveVerify ?? { status: "not_run", endpoints: [], summary: { checked: 0 } },
|
||||
release
|
||||
|
||||
@@ -214,7 +214,19 @@ function makeRunCommand({
|
||||
"reports/dev-gate/dev-artifacts.json"
|
||||
],
|
||||
desiredStatePromotionStatus = "pass",
|
||||
runtimeJobLogSuffix = ""
|
||||
runtimeJobLogSuffix = "",
|
||||
kubeContext = "d601",
|
||||
kubeServer = "https://d601:6443",
|
||||
kubeNodes = ["d601"],
|
||||
bareKubeContext = null,
|
||||
bareKubeServer = null,
|
||||
bareKubeNodes = null,
|
||||
bareHwlabDevNamespace = false,
|
||||
secretRefs = {
|
||||
"hwlab-cloud-api-dev-db": ["database-url"],
|
||||
"hwlab-cloud-api-dev-db-admin": ["admin-url"],
|
||||
"hwlab-code-agent-provider": ["openai-api-key"]
|
||||
}
|
||||
} = {}) {
|
||||
let lease = heldLock ? leaseFromLock(heldLock) : null;
|
||||
const assertLeaseMicroTime = (value) => {
|
||||
@@ -255,6 +267,42 @@ function makeRunCommand({
|
||||
if (command === "which" && args[0] === "kubectl") {
|
||||
return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args[0] === "config" && args[1] === "current-context") {
|
||||
const hasKubeconfig = Object.hasOwn(options.env ?? {}, "KUBECONFIG");
|
||||
return { code: 0, stdout: `${hasKubeconfig ? kubeContext : (bareKubeContext ?? kubeContext)}\n`, stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args[0] === "config" && args[1] === "view") {
|
||||
const hasKubeconfig = Object.hasOwn(options.env ?? {}, "KUBECONFIG");
|
||||
return { code: 0, stdout: `${hasKubeconfig ? kubeServer : (bareKubeServer ?? kubeServer)}\n`, stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args[0] === "get" && args[1] === "nodes") {
|
||||
const hasKubeconfig = Object.hasOwn(options.env ?? {}, "KUBECONFIG");
|
||||
return { code: 0, stdout: `${(hasKubeconfig ? kubeNodes : (bareKubeNodes ?? kubeNodes)).join(" ")}\n`, stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args[0] === "get" && args[1] === "namespace") {
|
||||
return bareHwlabDevNamespace
|
||||
? { code: 0, stdout: "namespace/hwlab-dev\n", stderr: "" }
|
||||
: { code: 1, stdout: "", stderr: "Error from server (NotFound): namespaces \"hwlab-dev\" not found" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("get") && args.includes("secret")) {
|
||||
const name = args[args.indexOf("secret") + 1];
|
||||
if (!secretRefs[name]) {
|
||||
return { code: 1, stdout: "", stderr: `Error from server (NotFound): secrets "${name}" not found` };
|
||||
}
|
||||
return { code: 0, stdout: `secret/${name}\n`, stderr: "" };
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("describe") && args.includes("secret")) {
|
||||
const name = args[args.indexOf("secret") + 1];
|
||||
const keys = secretRefs[name];
|
||||
if (!keys) {
|
||||
return { code: 1, stdout: "", stderr: `Error from server (NotFound): secrets "${name}" not found` };
|
||||
}
|
||||
return {
|
||||
code: 0,
|
||||
stdout: `Name: ${name}\nNamespace: hwlab-dev\nType: Opaque\n\nData\n====\n${keys.map((key) => `${key}: 16 bytes`).join("\n")}\n`,
|
||||
stderr: ""
|
||||
};
|
||||
}
|
||||
if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) {
|
||||
if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" };
|
||||
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
|
||||
@@ -360,6 +408,16 @@ function assertNoReadOnlySideEffects(commandLog) {
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/refresh-artifact-catalog.mjs")), false);
|
||||
}
|
||||
|
||||
function assertNoWriteSideEffects(commandLog) {
|
||||
const writeKubectlVerbs = new Set(["apply", "create", "replace", "patch", "delete", "rollout", "wait", "logs"]);
|
||||
assert.equal(commandLog.some((entry) =>
|
||||
entry.command.includes("kubectl") && entry.args.some((arg) => writeKubectlVerbs.has(arg))
|
||||
), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
|
||||
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/refresh-artifact-catalog.mjs")), false);
|
||||
}
|
||||
|
||||
test("lock classifier reports held and stale states with retryAfterSeconds", () => {
|
||||
const lock = {
|
||||
ownerTaskId: "task-a",
|
||||
@@ -391,7 +449,7 @@ test("dev-cd status and dry-run are read-only compact JSON", async () => {
|
||||
const code = await runDevCdApply([
|
||||
"--dry-run",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig",
|
||||
"/etc/rancher/k3s/k3s.yaml",
|
||||
"--write-report"
|
||||
], {
|
||||
repoRoot,
|
||||
@@ -447,7 +505,7 @@ test("dev-cd status accepts a published deploy-json promotion under a later cont
|
||||
const repoRoot = await makeRepo({ commitId: "abc1234" });
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], {
|
||||
const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
@@ -504,7 +562,7 @@ test("dev-cd status exposes moving origin/main when checkout is behind a deploy-
|
||||
const repoRoot = await makeRepo({ commitId: "abc1234" });
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], {
|
||||
const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
@@ -547,7 +605,7 @@ test("apply stops before lock acquisition when checkout is behind deploy-json pr
|
||||
"--owner-task-id",
|
||||
"task-moving-main",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig"
|
||||
"/etc/rancher/k3s/k3s.yaml"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
@@ -593,7 +651,7 @@ test("dev-cd status degrades artifact merge provenance when catalog/report do no
|
||||
);
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], {
|
||||
const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
@@ -651,7 +709,7 @@ test("apply stops before lock acquisition when artifact merge provenance mismatc
|
||||
"--owner-task-id",
|
||||
"task-provenance",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig"
|
||||
"/etc/rancher/k3s/k3s.yaml"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
@@ -713,7 +771,7 @@ test("dev-cd status summarizes held, released, and stale locks without unsafe ta
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], {
|
||||
const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
@@ -870,7 +928,7 @@ test("second transaction exits deploy-lock-held without running side effects", a
|
||||
"--owner-task-id",
|
||||
"task-b",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig"
|
||||
"/etc/rancher/k3s/k3s.yaml"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
@@ -915,7 +973,7 @@ test("stale lock requires explicit break-stale-lock confirmation", async () => {
|
||||
"--owner-task-id",
|
||||
"task-b",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig"
|
||||
"/etc/rancher/k3s/k3s.yaml"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
@@ -947,6 +1005,199 @@ test("stale lock requires explicit break-stale-lock confirmation", async () => {
|
||||
assert.equal(failure.requiresBreakStaleLock, true);
|
||||
});
|
||||
|
||||
test("apply preflight passes with D601 native k3s and required SecretRef keys before mutation", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-preflight-pass",
|
||||
"--kubeconfig",
|
||||
"/etc/rancher/k3s/k3s.yaml",
|
||||
"--write-report",
|
||||
"--full-output"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({ commandLog }),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const report = JSON.parse(output);
|
||||
const preflight = report.devCdApply.preflight;
|
||||
assert.equal(code, 0);
|
||||
assert.equal(preflight.status, "pass");
|
||||
assert.equal(preflight.kubeconfig.required, "/etc/rancher/k3s/k3s.yaml");
|
||||
assert.equal(preflight.kubeconfig.enforcedEnv, true);
|
||||
assert.deepEqual(preflight.controlPlane.forced.nodeNames, ["d601"]);
|
||||
assert.deepEqual(
|
||||
preflight.secretRefs.map((ref) => `${ref.secretName}/${ref.secretKey}:${ref.status}`),
|
||||
[
|
||||
"hwlab-cloud-api-dev-db/database-url:present",
|
||||
"hwlab-cloud-api-dev-db-admin/admin-url:present",
|
||||
"hwlab-code-agent-provider/openai-api-key:present"
|
||||
]
|
||||
);
|
||||
assert.equal(preflight.safety.secretValuesRead, false);
|
||||
assert.equal(preflight.safety.secretValuesPrinted, false);
|
||||
|
||||
const firstWriteIndex = commandLog.findIndex((entry) =>
|
||||
entry.command.includes("kubectl") && ["create", "replace", "apply", "patch", "delete", "rollout", "wait", "logs"].some((verb) => entry.args.includes(verb))
|
||||
);
|
||||
const secretDescribeIndexes = commandLog
|
||||
.map((entry, index) => ({ entry, index }))
|
||||
.filter(({ entry }) => entry.command.includes("kubectl") && entry.args.includes("describe") && entry.args.includes("secret"))
|
||||
.map(({ index }) => index);
|
||||
assert.ok(secretDescribeIndexes.length > 0);
|
||||
assert.ok(secretDescribeIndexes.every((index) => index < firstWriteIndex));
|
||||
});
|
||||
|
||||
test("apply preflight blocks missing SecretRef before Lease or side-effect commands", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-missing-secret",
|
||||
"--kubeconfig",
|
||||
"/etc/rancher/k3s/k3s.yaml",
|
||||
"--write-report"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
commandLog,
|
||||
secretRefs: {
|
||||
"hwlab-cloud-api-dev-db": ["database-url"],
|
||||
"hwlab-cloud-api-dev-db-admin": ["admin-url"]
|
||||
}
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const summary = JSON.parse(output);
|
||||
const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
|
||||
assert.equal(code, 2);
|
||||
assert.equal(summary.status, "blocked");
|
||||
assert.equal(summary.preflight.status, "blocked");
|
||||
assert.equal(summary.preflight.blockerCount, 1);
|
||||
assert.equal(summary.blockers.some((blocker) => blocker.scope === "secretref:hwlab-code-agent-provider/openai-api-key"), true);
|
||||
assert.equal(writtenReport.devCdApply.preflight.safety.writeSideEffectsAttempted, false);
|
||||
assert.equal(writtenReport.devCdApply.preflight.safety.secretValuesRead, false);
|
||||
assert.equal(JSON.stringify(writtenReport).includes("SECRET"), false);
|
||||
assertNoWriteSideEffects(commandLog);
|
||||
});
|
||||
|
||||
test("apply preflight blocks non-D601 kubeconfig path before kubectl mutation", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-wrong-path",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({ commandLog }),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const summary = JSON.parse(output);
|
||||
assert.equal(code, 2);
|
||||
assert.equal(summary.status, "blocked");
|
||||
assert.equal(summary.preflight.status, "blocked");
|
||||
assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-kubeconfig-path"), true);
|
||||
assertNoWriteSideEffects(commandLog);
|
||||
});
|
||||
|
||||
test("apply preflight blocks docker-desktop and 127.0.0.1 control-plane signals before mutation", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-docker-desktop",
|
||||
"--kubeconfig",
|
||||
"/etc/rancher/k3s/k3s.yaml"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
commandLog,
|
||||
kubeContext: "docker-desktop",
|
||||
kubeServer: "https://127.0.0.1:11700",
|
||||
kubeNodes: ["desktop-control-plane"]
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const summary = JSON.parse(output);
|
||||
assert.equal(code, 2);
|
||||
assert.equal(summary.status, "blocked");
|
||||
assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-native-k3s-context"), true);
|
||||
assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-native-k3s-server"), true);
|
||||
assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-native-k3s-nodes"), true);
|
||||
assertNoWriteSideEffects(commandLog);
|
||||
});
|
||||
|
||||
test("apply preflight blocks second hwlab-dev control plane visible through bare kubectl", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
let output = "";
|
||||
const code = await runDevCdApply([
|
||||
"--apply",
|
||||
"--confirm-dev",
|
||||
"--confirmed-non-production",
|
||||
"--owner-task-id",
|
||||
"task-second-control-plane",
|
||||
"--kubeconfig",
|
||||
"/etc/rancher/k3s/k3s.yaml"
|
||||
], {
|
||||
repoRoot,
|
||||
env: {},
|
||||
runCommand: makeRunCommand({
|
||||
commandLog,
|
||||
bareKubeContext: "docker-desktop",
|
||||
bareKubeServer: "https://127.0.0.1:11700",
|
||||
bareKubeNodes: ["desktop-control-plane"],
|
||||
bareHwlabDevNamespace: true
|
||||
}),
|
||||
httpGetJson: makeHttpGetJson(),
|
||||
now: () => new Date(iso(100000)),
|
||||
stdout: { write: (chunk) => { output += chunk; } }
|
||||
});
|
||||
|
||||
const summary = JSON.parse(output);
|
||||
assert.equal(code, 2);
|
||||
assert.equal(summary.status, "blocked");
|
||||
assert.equal(summary.blockers.some((blocker) => blocker.scope === "bare-kubectl-context"), true);
|
||||
assert.equal(summary.blockers.some((blocker) => blocker.scope === "bare-kubectl-server"), true);
|
||||
assert.equal(summary.blockers.some((blocker) => blocker.scope === "second-hwlab-dev-control-plane"), true);
|
||||
assertNoWriteSideEffects(commandLog);
|
||||
});
|
||||
|
||||
test("transaction runs phases, allows internal side-effect env, releases lock, and reports live verify", async () => {
|
||||
const repoRoot = await makeRepo();
|
||||
const commandLog = [];
|
||||
@@ -959,7 +1210,7 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a
|
||||
"--owner-task-id",
|
||||
"task-b",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig",
|
||||
"/etc/rancher/k3s/k3s.yaml",
|
||||
"--write-report",
|
||||
"--full-output"
|
||||
], {
|
||||
@@ -1054,7 +1305,7 @@ test("runtime maintenance job reports are parsed after redacting secret-like log
|
||||
"--owner-task-id",
|
||||
"task-redaction",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig",
|
||||
"/etc/rancher/k3s/k3s.yaml",
|
||||
"--write-report",
|
||||
"--full-output"
|
||||
], {
|
||||
@@ -1091,7 +1342,7 @@ test("transaction can apply an already published deploy-json promotion without r
|
||||
"--owner-task-id",
|
||||
"task-promotion",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig",
|
||||
"/etc/rancher/k3s/k3s.yaml",
|
||||
"--write-report",
|
||||
"--full-output"
|
||||
], {
|
||||
@@ -1158,7 +1409,7 @@ test("apply stdout is concise by default while write-report keeps the full repor
|
||||
"--owner-task-id",
|
||||
"task-c",
|
||||
"--kubeconfig",
|
||||
"/tmp/kubeconfig",
|
||||
"/etc/rancher/k3s/k3s.yaml",
|
||||
"--write-report"
|
||||
], {
|
||||
repoRoot,
|
||||
|
||||
Reference in New Issue
Block a user