fix: gate dev cd apply with d601 preflight

This commit is contained in:
Code Queue Review
2026-05-23 13:24:40 +00:00
parent f590524cd5
commit 5227df7ad9
3 changed files with 662 additions and 41 deletions
+378 -24
View File
@@ -18,6 +18,7 @@ const defaultNamespace = "hwlab-dev";
const defaultLockName = "hwlab-dev-cd-lock";
const defaultTtlSeconds = 3600;
const defaultReportPath = "reports/dev-gate/dev-cd-apply.json";
const d601NativeKubeconfigPath = "/etc/rancher/k3s/k3s.yaml";
const artifactCatalogPath = "deploy/artifact-catalog.dev.json";
const artifactReportPath = "reports/dev-gate/dev-artifacts.json";
const artifactDesiredStatePaths = [
@@ -33,6 +34,12 @@ const runtimeJobReportPrefix = "/tmp/hwlab-dev-runtime-report";
const browserLiveUrl = "http://74.48.78.17:16666/health/live";
const apiLiveUrl = `${DEV_ENDPOINT}/health/live`;
const digestPattern = /^sha256:[a-f0-9]{64}$/u;
const forbiddenControlPlanePattern = /docker-desktop|desktop-control-plane|127\.0\.0\.1:11700/iu;
const requiredDevSecretRefs = Object.freeze([
Object.freeze({ secretName: "hwlab-cloud-api-dev-db", secretKey: "database-url" }),
Object.freeze({ secretName: "hwlab-cloud-api-dev-db-admin", secretKey: "admin-url" }),
Object.freeze({ secretName: "hwlab-code-agent-provider", secretKey: "openai-api-key" })
]);
const lockAnnotationPrefix = "hwlab.pikastech.local";
const lockAnnotationFields = {
promotionCommit: `${lockAnnotationPrefix}/promotionCommit`,
@@ -218,6 +225,19 @@ function oneLine(value) {
return String(value ?? "").replace(/\s+/g, " ").trim();
}
function blocker({ type = "environment_blocker", scope, reason, impact, safeNextAction, retryable }) {
return {
type,
scope,
status: "open",
reason: oneLine(reason),
impact: oneLine(impact),
safeNextAction: oneLine(safeNextAction),
retryable: Boolean(retryable),
summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`)
};
}
function sha256(value) {
return `sha256:${createHash("sha256").update(value).digest("hex")}`;
}
@@ -1207,6 +1227,323 @@ async function releaseDeployLock({ ctx, args, kubectlContext, transactionId, sta
};
}
function commandSummary(result, maxLength = 500) {
return oneLine(redactSensitiveText(result?.stderr || result?.stdout || "")).slice(0, maxLength);
}
function envWithoutKubeconfig(env) {
const sanitized = { ...env };
delete sanitized.KUBECONFIG;
delete sanitized.HWLAB_DEV_KUBECONFIG;
return sanitized;
}
function parseNodeNames(stdout) {
return String(stdout ?? "").trim().split(/\s+/u).filter(Boolean);
}
function parseSecretDescribeKeys(stdout) {
const keys = new Set();
let inData = false;
for (const rawLine of String(stdout ?? "").split(/\r?\n/u)) {
const line = rawLine.trim();
if (/^Data\b/iu.test(line)) {
inData = true;
continue;
}
if (!inData || line === "" || /^=+$/u.test(line)) continue;
const match = line.match(/^([A-Za-z0-9_.-]+):\s+\d+\s+bytes\b/iu);
if (match) keys.add(match[1]);
}
return [...keys].sort();
}
function commandObservation(args, result) {
return {
command: shellCommand("kubectl", args),
code: result.code ?? null,
ok: result.code === 0,
summary: commandSummary(result)
};
}
async function preflightKubectl(ctx, kubectlContext, args, options = {}) {
const result = await ctx.runCommand(kubectlContext.executor, args, {
cwd: ctx.repoRoot,
env: options.env ?? kubectlContext.env,
timeoutMs: options.timeoutMs ?? 15000
});
return {
...result,
stdout: result.stdout ?? "",
stderr: result.stderr ?? "",
command: shellCommand("kubectl", args)
};
}
function addControlPlaneBlockers(blockers, observation, { scopePrefix, requiredForApply }) {
if (observation.contextName && forbiddenControlPlanePattern.test(observation.contextName)) {
blockers.push(blocker({
scope: `${scopePrefix}-context`,
reason: `kubectl context resolved to forbidden control plane ${observation.contextName}.`,
impact: "DEV CD cannot prove it is targeting D601 native k3s before mutation.",
safeNextAction: `Use KUBECONFIG=${d601NativeKubeconfigPath} and remove or repair the stale Docker Desktop/default kube context before retrying.`,
retryable: true
}));
}
if (observation.serverUrl && forbiddenControlPlanePattern.test(observation.serverUrl)) {
blockers.push(blocker({
scope: `${scopePrefix}-server`,
reason: `kubectl server resolved to forbidden endpoint ${observation.serverUrl}.`,
impact: "DEV CD could write to the wrong control plane or treat stale Docker Desktop output as DEV evidence.",
safeNextAction: `Point kubectl at D601 native k3s with KUBECONFIG=${d601NativeKubeconfigPath} before retrying.`,
retryable: true
}));
}
if (observation.nodeNames.includes("desktop-control-plane")) {
blockers.push(blocker({
scope: `${scopePrefix}-nodes`,
reason: "kubectl observed node desktop-control-plane.",
impact: "DEV CD is observing Docker Desktop Kubernetes instead of D601 native k3s.",
safeNextAction: `Stop and rerun only after kubectl nodes come from KUBECONFIG=${d601NativeKubeconfigPath} and include d601.`,
retryable: true
}));
} else if (requiredForApply && !observation.nodeNames.includes("d601")) {
blockers.push(blocker({
scope: `${scopePrefix}-nodes`,
reason: `kubectl nodes did not include d601; observed ${observation.nodeNames.join(",") || "none"}.`,
impact: "DEV apply/job/rollout side effects are not allowed without proving the D601 native k3s node.",
safeNextAction: `Restore access to KUBECONFIG=${d601NativeKubeconfigPath} and verify kubectl get nodes includes d601.`,
retryable: true
}));
} else if (!requiredForApply && observation.nodeNames.length > 0 && !observation.nodeNames.includes("d601")) {
blockers.push(blocker({
scope: `${scopePrefix}-nodes`,
reason: `bare kubectl observed non-D601 nodes ${observation.nodeNames.join(",")}.`,
impact: "A second or stale control plane may still own hwlab-dev evidence, so DEV CD refuses to mutate.",
safeNextAction: "Clean the default kubectl context or ensure bare kubectl cannot resolve stale HWLAB DEV resources, then retry.",
retryable: true
}));
}
}
async function observeControlPlane(ctx, kubectlContext, { env, label, requiredForApply }) {
const [context, server, nodes] = await Promise.all([
preflightKubectl(ctx, kubectlContext, ["config", "current-context"], { env, timeoutMs: 10000 }),
preflightKubectl(ctx, kubectlContext, ["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], { env, timeoutMs: 10000 }),
preflightKubectl(ctx, kubectlContext, ["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], { env, timeoutMs: 15000 })
]);
return {
label,
contextCommand: commandObservation(["config", "current-context"], context),
serverCommand: commandObservation(["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], server),
nodesCommand: commandObservation(["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], nodes),
contextName: context.code === 0 ? oneLine(context.stdout) : null,
serverUrl: server.code === 0 ? oneLine(server.stdout) : null,
nodeNames: nodes.code === 0 ? parseNodeNames(nodes.stdout) : [],
status: context.code === 0 && server.code === 0 && nodes.code === 0 ? "observed" : requiredForApply ? "blocked" : "unavailable"
};
}
async function observeBareKubectl(ctx, kubectlContext, forcedObservation) {
const bareEnv = envWithoutKubeconfig(ctx.env);
const bare = await observeControlPlane(ctx, kubectlContext, {
env: bareEnv,
label: "bare-kubectl",
requiredForApply: false
});
const namespace = await preflightKubectl(ctx, kubectlContext, ["get", "namespace", defaultNamespace, "-o", "name"], {
env: bareEnv,
timeoutMs: 10000
});
const secondControlPlane =
namespace.code === 0 &&
(
(bare.serverUrl && forcedObservation.serverUrl && bare.serverUrl !== forcedObservation.serverUrl) ||
(!bare.serverUrl && bare.contextName && bare.contextName !== forcedObservation.contextName)
);
return {
...bare,
namespace: commandObservation(["get", "namespace", defaultNamespace, "-o", "name"], namespace),
hwlabDevNamespaceObserved: namespace.code === 0,
secondControlPlaneCandidate: secondControlPlane
};
}
async function observeRequiredSecretRef(ctx, kubectlContext, args, ref) {
const existsArgs = ["-n", args.targetNamespace, "get", "secret", ref.secretName, "-o", "name"];
const describeArgs = ["-n", args.targetNamespace, "describe", "secret", ref.secretName];
const exists = await preflightKubectl(ctx, kubectlContext, existsArgs, { timeoutMs: 10000 });
if (exists.code !== 0) {
return {
secretName: ref.secretName,
secretKey: ref.secretKey,
status: "missing-secret",
exists: false,
keyPresent: false,
keysObserved: [],
existenceCommand: shellCommand("kubectl", existsArgs),
keyCommand: null,
error: commandSummary(exists),
secretValueRead: false,
secretValuePrinted: false
};
}
const describe = await preflightKubectl(ctx, kubectlContext, describeArgs, { timeoutMs: 10000 });
if (describe.code !== 0) {
return {
secretName: ref.secretName,
secretKey: ref.secretKey,
status: "key-observation-blocked",
exists: true,
keyPresent: false,
keysObserved: [],
existenceCommand: shellCommand("kubectl", existsArgs),
keyCommand: shellCommand("kubectl", describeArgs),
error: commandSummary(describe),
secretValueRead: false,
secretValuePrinted: false
};
}
const keysObserved = parseSecretDescribeKeys(describe.stdout);
const keyPresent = keysObserved.includes(ref.secretKey);
return {
secretName: ref.secretName,
secretKey: ref.secretKey,
status: keyPresent ? "present" : "missing-key",
exists: true,
keyPresent,
keysObserved,
existenceCommand: shellCommand("kubectl", existsArgs),
keyCommand: shellCommand("kubectl", describeArgs),
secretValueRead: false,
secretValuePrinted: false
};
}
function secretRefBlocker(ref, observation) {
const scope = `secretref:${ref.secretName}/${ref.secretKey}`;
if (observation.status === "missing-secret") {
return blocker({
type: "runtime_blocker",
scope,
reason: `Required DEV Secret ${ref.secretName} was not observed in ${defaultNamespace}.`,
impact: "Runtime provisioning, migration, deploy apply, or Code Agent provider startup would fail after mutation.",
safeNextAction: `Create or restore SecretRef ${ref.secretName}/${ref.secretKey} in ${defaultNamespace} without printing the Secret value, then retry DEV CD.`,
retryable: true
});
}
if (observation.status === "key-observation-blocked") {
return blocker({
type: "runtime_blocker",
scope,
reason: `Required DEV Secret ${ref.secretName} exists but key names could not be verified.`,
impact: "DEV CD cannot prove the pod SecretRef contract before creating runtime Jobs or applying workloads.",
safeNextAction: `Restore metadata/key-name visibility for ${ref.secretName} without granting or printing Secret values, then retry.`,
retryable: true
});
}
return blocker({
type: "runtime_blocker",
scope,
reason: `Required DEV Secret key ${ref.secretName}/${ref.secretKey} was not observed.`,
impact: "The target workload or runtime Job would reference a missing Secret key after mutation.",
safeNextAction: `Add key ${ref.secretKey} to Secret ${ref.secretName} without printing the value, then retry DEV CD.`,
retryable: true
});
}
async function runDevCdPreflight({ ctx, args, kubectlContext }) {
const blockers = [];
const preflight = {
status: "blocked",
scope: "pre-lock-pre-side-effect",
generatedAt: ctx.now().toISOString(),
namespace: args.targetNamespace,
kubeconfig: {
required: d601NativeKubeconfigPath,
selected: kubectlContext.kubeconfig,
source: kubectlContext.kubeconfigSource,
enforcedEnv: kubectlContext.env.KUBECONFIG === d601NativeKubeconfigPath
},
kubectl: {
executor: kubectlContext.executor,
commandPrefix: kubectlContext.commandPrefix
},
controlPlane: null,
secretRefs: [],
safety: {
writeSideEffectsAttempted: false,
prodTouched: false,
secretValuesRead: false,
secretValuesPrinted: false,
secretKeyNamesOnly: true
},
blockers
};
if (kubectlContext.kubeconfig !== d601NativeKubeconfigPath) {
blockers.push(blocker({
scope: "d601-kubeconfig-path",
reason: `DEV CD selected kubeconfig ${kubectlContext.kubeconfig || "<unset>"} from ${kubectlContext.kubeconfigSource}.`,
impact: "Any apply/job/rollout could target a non-D601 or stale control plane.",
safeNextAction: `Rerun with KUBECONFIG=${d601NativeKubeconfigPath} or --kubeconfig ${d601NativeKubeconfigPath}.`,
retryable: true
}));
preflight.status = "blocked";
return preflight;
}
const forced = await observeControlPlane(ctx, kubectlContext, {
env: kubectlContext.env,
label: "forced-d601-k3s",
requiredForApply: true
});
const bare = await observeBareKubectl(ctx, kubectlContext, forced);
preflight.controlPlane = { forced, bare };
if (forced.status !== "observed") {
blockers.push(blocker({
scope: "d601-native-k3s-unobservable",
reason: `Forced D601 kubeconfig could not prove context/server/nodes: ${[forced.contextCommand.summary, forced.serverCommand.summary, forced.nodesCommand.summary].filter(Boolean).join("; ") || "no kubectl output"}.`,
impact: "DEV CD refuses to acquire the Lease or create runtime Jobs without D601 native k3s proof.",
safeNextAction: `Restore readable ${d601NativeKubeconfigPath} access and verify kubectl get nodes includes d601.`,
retryable: true
}));
}
addControlPlaneBlockers(blockers, forced, {
scopePrefix: "d601-native-k3s",
requiredForApply: true
});
addControlPlaneBlockers(blockers, bare, {
scopePrefix: "bare-kubectl",
requiredForApply: false
});
if (bare.secondControlPlaneCandidate) {
blockers.push(blocker({
scope: "second-hwlab-dev-control-plane",
reason: `bare kubectl can observe namespace ${defaultNamespace} outside the forced D601 kubeconfig context.`,
impact: "A second hwlab-dev control plane may still carry stale HWLAB resources, so DEV CD cannot safely classify apply evidence.",
safeNextAction: "Remove the stale hwlab-dev control plane/default kube context or make bare kubectl unable to resolve it, then rerun preflight.",
retryable: true
}));
}
if (blockers.length === 0) {
preflight.secretRefs = await Promise.all(
requiredDevSecretRefs.map((ref) => observeRequiredSecretRef(ctx, kubectlContext, args, ref))
);
for (const [index, observation] of preflight.secretRefs.entries()) {
if (observation.status !== "present") {
blockers.push(secretRefBlocker(requiredDevSecretRefs[index], observation));
}
}
}
preflight.status = blockers.length === 0 ? "pass" : "blocked";
return preflight;
}
async function resolveKubectlContext(args, env, runCommand) {
const selection = resolveDevKubeconfigSelection({ flagValue: args.kubeconfig, env });
const which = await runCommand("which", ["kubectl"], { timeoutMs: 5000 });
@@ -1215,27 +1552,6 @@ async function resolveKubectlContext(args, env, runCommand) {
...env,
KUBECONFIG: selection.kubeconfig
};
if (which.code === 0 && args.apply) {
const context = await runCommand(executor, ["config", "current-context"], { env: kubectlEnv, timeoutMs: 10000 });
const nodes = await runCommand(executor, ["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], { env: kubectlEnv, timeoutMs: 15000 });
const contextName = String(context.stdout ?? "").trim();
const nodeNames = String(nodes.stdout ?? "").trim().split(/\s+/u).filter(Boolean);
const forbiddenContext = context.code === 0 && /docker-desktop|desktop-control-plane/iu.test(contextName);
const forbiddenNode = nodes.code === 0 && (
nodeNames.includes("desktop-control-plane") || (nodeNames.length > 0 && !nodeNames.includes("d601"))
);
if (forbiddenContext || forbiddenNode) {
throw new DevCdApplyError("DEV CD kubeconfig does not target D601 native k3s", {
code: "wrong-kubernetes-control-plane",
blockers: [{
type: "environment_blocker",
scope: "d601-native-k3s",
status: "open",
summary: "D601 DEV CD must use native k3s via /etc/rancher/k3s/k3s.yaml; default docker-desktop kubeconfig is forbidden."
}]
});
}
}
return {
executor,
kubeconfig: selection.kubeconfig,
@@ -1904,6 +2220,19 @@ function summarizeDevCdApplyReport(report) {
})),
liveBefore: apply.liveBefore ? compactLiveReport(apply.liveBefore) : null,
liveVerify: apply.liveVerify ? compactLiveReport(apply.liveVerify) : null,
preflight: apply.preflight ? {
status: apply.preflight.status,
scope: apply.preflight.scope,
kubeconfig: apply.preflight.kubeconfig,
blockerCount: apply.preflight.blockers?.length ?? 0,
secretRefs: (apply.preflight.secretRefs ?? []).map((ref) => ({
secretName: ref.secretName,
secretKey: ref.secretKey,
status: ref.status,
exists: ref.exists,
keyPresent: ref.keyPresent
}))
} : null,
blockers: report.blockers ?? [],
reportPaths: apply.reportPaths ?? {},
fullOutputHint: "Use --full-output to print the full report to stdout. Use --write-report to persist the full report."
@@ -1922,6 +2251,7 @@ function buildReport({
status,
startedAt,
finishedAt,
preflight,
liveBefore,
liveVerify,
release
@@ -1985,6 +2315,9 @@ function buildReport({
devPreconditions: {
status: blockers.length ? "blocked" : "pass",
requirements: [
"DEV CD apply must force KUBECONFIG=/etc/rancher/k3s/k3s.yaml and verify native node d601 before any Lease, Job, apply, or rollout side effect.",
"DEV CD apply must reject docker-desktop, desktop-control-plane, 127.0.0.1:11700, bare kubectl ambiguity, and second hwlab-dev control-plane signals before mutation.",
"DEV CD apply must verify required SecretRef existence and key names without reading or printing Secret values.",
"HEAD must match the requested target ref before publish.",
"Lease/hwlab-dev-cd-lock must be acquired before publish/apply.",
"Legacy publish/apply side-effect scripts must run with HWLAB_CD_TRANSACTION_ID.",
@@ -2034,9 +2367,10 @@ function buildReport({
staleBreak: lockState?.staleBreak ?? null
},
steps,
liveBefore,
liveVerify,
reportPaths: {
liveBefore,
liveVerify,
preflight,
reportPaths: {
transaction: args.writeReport ? args.reportPath : null,
artifacts: artifactReportPath,
deployApply: "reports/dev-gate/dev-deploy-report.json",
@@ -2138,6 +2472,7 @@ export async function runDevCdApply(argv, io = {}) {
let deployAfter = null;
let liveBefore = null;
let liveVerify = null;
let preflight = null;
let status = "blocked";
try {
@@ -2182,6 +2517,14 @@ export async function runDevCdApply(argv, io = {}) {
}
const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand);
transaction.kubectlContext = kubectlContext;
preflight = await runDevCdPreflight({ ctx, args, kubectlContext });
if (preflight.status !== "pass") {
blockers.push(...preflight.blockers);
throw new DevCdApplyError("DEV CD preflight blocked before any write side effect", {
code: "dev-cd-preflight-blocked",
suppressCatchBlocker: true
});
}
liveBefore = {
status: "pending",
reason: "liveBefore is captured immediately after Lease acquisition so publish/apply cannot race before the transaction lock."
@@ -2458,6 +2801,17 @@ export async function runDevCdApply(argv, io = {}) {
status,
startedAt,
finishedAt,
preflight: preflight ?? {
status: "not_run",
scope: "pre-lock-pre-side-effect",
blockers: [],
safety: {
writeSideEffectsAttempted: false,
prodTouched: false,
secretValuesRead: false,
secretValuesPrinted: false
}
},
liveBefore: liveBefore ?? { status: "not_run" },
liveVerify: liveVerify ?? { status: "not_run", endpoints: [], summary: { checked: 0 } },
release
+265 -14
View File
@@ -214,7 +214,19 @@ function makeRunCommand({
"reports/dev-gate/dev-artifacts.json"
],
desiredStatePromotionStatus = "pass",
runtimeJobLogSuffix = ""
runtimeJobLogSuffix = "",
kubeContext = "d601",
kubeServer = "https://d601:6443",
kubeNodes = ["d601"],
bareKubeContext = null,
bareKubeServer = null,
bareKubeNodes = null,
bareHwlabDevNamespace = false,
secretRefs = {
"hwlab-cloud-api-dev-db": ["database-url"],
"hwlab-cloud-api-dev-db-admin": ["admin-url"],
"hwlab-code-agent-provider": ["openai-api-key"]
}
} = {}) {
let lease = heldLock ? leaseFromLock(heldLock) : null;
const assertLeaseMicroTime = (value) => {
@@ -255,6 +267,42 @@ function makeRunCommand({
if (command === "which" && args[0] === "kubectl") {
return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" };
}
if (command.includes("kubectl") && args[0] === "config" && args[1] === "current-context") {
const hasKubeconfig = Object.hasOwn(options.env ?? {}, "KUBECONFIG");
return { code: 0, stdout: `${hasKubeconfig ? kubeContext : (bareKubeContext ?? kubeContext)}\n`, stderr: "" };
}
if (command.includes("kubectl") && args[0] === "config" && args[1] === "view") {
const hasKubeconfig = Object.hasOwn(options.env ?? {}, "KUBECONFIG");
return { code: 0, stdout: `${hasKubeconfig ? kubeServer : (bareKubeServer ?? kubeServer)}\n`, stderr: "" };
}
if (command.includes("kubectl") && args[0] === "get" && args[1] === "nodes") {
const hasKubeconfig = Object.hasOwn(options.env ?? {}, "KUBECONFIG");
return { code: 0, stdout: `${(hasKubeconfig ? kubeNodes : (bareKubeNodes ?? kubeNodes)).join(" ")}\n`, stderr: "" };
}
if (command.includes("kubectl") && args[0] === "get" && args[1] === "namespace") {
return bareHwlabDevNamespace
? { code: 0, stdout: "namespace/hwlab-dev\n", stderr: "" }
: { code: 1, stdout: "", stderr: "Error from server (NotFound): namespaces \"hwlab-dev\" not found" };
}
if (command.includes("kubectl") && args.includes("get") && args.includes("secret")) {
const name = args[args.indexOf("secret") + 1];
if (!secretRefs[name]) {
return { code: 1, stdout: "", stderr: `Error from server (NotFound): secrets "${name}" not found` };
}
return { code: 0, stdout: `secret/${name}\n`, stderr: "" };
}
if (command.includes("kubectl") && args.includes("describe") && args.includes("secret")) {
const name = args[args.indexOf("secret") + 1];
const keys = secretRefs[name];
if (!keys) {
return { code: 1, stdout: "", stderr: `Error from server (NotFound): secrets "${name}" not found` };
}
return {
code: 0,
stdout: `Name: ${name}\nNamespace: hwlab-dev\nType: Opaque\n\nData\n====\n${keys.map((key) => `${key}: 16 bytes`).join("\n")}\n`,
stderr: ""
};
}
if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) {
if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" };
return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" };
@@ -360,6 +408,16 @@ function assertNoReadOnlySideEffects(commandLog) {
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/refresh-artifact-catalog.mjs")), false);
}
function assertNoWriteSideEffects(commandLog) {
const writeKubectlVerbs = new Set(["apply", "create", "replace", "patch", "delete", "rollout", "wait", "logs"]);
assert.equal(commandLog.some((entry) =>
entry.command.includes("kubectl") && entry.args.some((arg) => writeKubectlVerbs.has(arg))
), false);
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false);
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false);
assert.equal(commandLog.some((entry) => entry.args.includes("scripts/refresh-artifact-catalog.mjs")), false);
}
test("lock classifier reports held and stale states with retryAfterSeconds", () => {
const lock = {
ownerTaskId: "task-a",
@@ -391,7 +449,7 @@ test("dev-cd status and dry-run are read-only compact JSON", async () => {
const code = await runDevCdApply([
"--dry-run",
"--kubeconfig",
"/tmp/kubeconfig",
"/etc/rancher/k3s/k3s.yaml",
"--write-report"
], {
repoRoot,
@@ -447,7 +505,7 @@ test("dev-cd status accepts a published deploy-json promotion under a later cont
const repoRoot = await makeRepo({ commitId: "abc1234" });
const commandLog = [];
let output = "";
const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], {
const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], {
repoRoot,
env: {},
runCommand: makeRunCommand({
@@ -504,7 +562,7 @@ test("dev-cd status exposes moving origin/main when checkout is behind a deploy-
const repoRoot = await makeRepo({ commitId: "abc1234" });
const commandLog = [];
let output = "";
const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], {
const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], {
repoRoot,
env: {},
runCommand: makeRunCommand({
@@ -547,7 +605,7 @@ test("apply stops before lock acquisition when checkout is behind deploy-json pr
"--owner-task-id",
"task-moving-main",
"--kubeconfig",
"/tmp/kubeconfig"
"/etc/rancher/k3s/k3s.yaml"
], {
repoRoot,
env: {},
@@ -593,7 +651,7 @@ test("dev-cd status degrades artifact merge provenance when catalog/report do no
);
const commandLog = [];
let output = "";
const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], {
const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], {
repoRoot,
env: {},
runCommand: makeRunCommand({
@@ -651,7 +709,7 @@ test("apply stops before lock acquisition when artifact merge provenance mismatc
"--owner-task-id",
"task-provenance",
"--kubeconfig",
"/tmp/kubeconfig"
"/etc/rancher/k3s/k3s.yaml"
], {
repoRoot,
env: {},
@@ -713,7 +771,7 @@ test("dev-cd status summarizes held, released, and stale locks without unsafe ta
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], {
const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], {
repoRoot,
env: {},
runCommand: makeRunCommand({
@@ -870,7 +928,7 @@ test("second transaction exits deploy-lock-held without running side effects", a
"--owner-task-id",
"task-b",
"--kubeconfig",
"/tmp/kubeconfig"
"/etc/rancher/k3s/k3s.yaml"
], {
repoRoot,
env: {},
@@ -915,7 +973,7 @@ test("stale lock requires explicit break-stale-lock confirmation", async () => {
"--owner-task-id",
"task-b",
"--kubeconfig",
"/tmp/kubeconfig"
"/etc/rancher/k3s/k3s.yaml"
], {
repoRoot,
env: {},
@@ -947,6 +1005,199 @@ test("stale lock requires explicit break-stale-lock confirmation", async () => {
assert.equal(failure.requiresBreakStaleLock, true);
});
test("apply preflight passes with D601 native k3s and required SecretRef keys before mutation", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const code = await runDevCdApply([
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--owner-task-id",
"task-preflight-pass",
"--kubeconfig",
"/etc/rancher/k3s/k3s.yaml",
"--write-report",
"--full-output"
], {
repoRoot,
env: {},
runCommand: makeRunCommand({ commandLog }),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const report = JSON.parse(output);
const preflight = report.devCdApply.preflight;
assert.equal(code, 0);
assert.equal(preflight.status, "pass");
assert.equal(preflight.kubeconfig.required, "/etc/rancher/k3s/k3s.yaml");
assert.equal(preflight.kubeconfig.enforcedEnv, true);
assert.deepEqual(preflight.controlPlane.forced.nodeNames, ["d601"]);
assert.deepEqual(
preflight.secretRefs.map((ref) => `${ref.secretName}/${ref.secretKey}:${ref.status}`),
[
"hwlab-cloud-api-dev-db/database-url:present",
"hwlab-cloud-api-dev-db-admin/admin-url:present",
"hwlab-code-agent-provider/openai-api-key:present"
]
);
assert.equal(preflight.safety.secretValuesRead, false);
assert.equal(preflight.safety.secretValuesPrinted, false);
const firstWriteIndex = commandLog.findIndex((entry) =>
entry.command.includes("kubectl") && ["create", "replace", "apply", "patch", "delete", "rollout", "wait", "logs"].some((verb) => entry.args.includes(verb))
);
const secretDescribeIndexes = commandLog
.map((entry, index) => ({ entry, index }))
.filter(({ entry }) => entry.command.includes("kubectl") && entry.args.includes("describe") && entry.args.includes("secret"))
.map(({ index }) => index);
assert.ok(secretDescribeIndexes.length > 0);
assert.ok(secretDescribeIndexes.every((index) => index < firstWriteIndex));
});
test("apply preflight blocks missing SecretRef before Lease or side-effect commands", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const code = await runDevCdApply([
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--owner-task-id",
"task-missing-secret",
"--kubeconfig",
"/etc/rancher/k3s/k3s.yaml",
"--write-report"
], {
repoRoot,
env: {},
runCommand: makeRunCommand({
commandLog,
secretRefs: {
"hwlab-cloud-api-dev-db": ["database-url"],
"hwlab-cloud-api-dev-db-admin": ["admin-url"]
}
}),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const summary = JSON.parse(output);
const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8"));
assert.equal(code, 2);
assert.equal(summary.status, "blocked");
assert.equal(summary.preflight.status, "blocked");
assert.equal(summary.preflight.blockerCount, 1);
assert.equal(summary.blockers.some((blocker) => blocker.scope === "secretref:hwlab-code-agent-provider/openai-api-key"), true);
assert.equal(writtenReport.devCdApply.preflight.safety.writeSideEffectsAttempted, false);
assert.equal(writtenReport.devCdApply.preflight.safety.secretValuesRead, false);
assert.equal(JSON.stringify(writtenReport).includes("SECRET"), false);
assertNoWriteSideEffects(commandLog);
});
test("apply preflight blocks non-D601 kubeconfig path before kubectl mutation", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const code = await runDevCdApply([
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--owner-task-id",
"task-wrong-path",
"--kubeconfig",
"/tmp/kubeconfig"
], {
repoRoot,
env: {},
runCommand: makeRunCommand({ commandLog }),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const summary = JSON.parse(output);
assert.equal(code, 2);
assert.equal(summary.status, "blocked");
assert.equal(summary.preflight.status, "blocked");
assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-kubeconfig-path"), true);
assertNoWriteSideEffects(commandLog);
});
test("apply preflight blocks docker-desktop and 127.0.0.1 control-plane signals before mutation", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const code = await runDevCdApply([
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--owner-task-id",
"task-docker-desktop",
"--kubeconfig",
"/etc/rancher/k3s/k3s.yaml"
], {
repoRoot,
env: {},
runCommand: makeRunCommand({
commandLog,
kubeContext: "docker-desktop",
kubeServer: "https://127.0.0.1:11700",
kubeNodes: ["desktop-control-plane"]
}),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const summary = JSON.parse(output);
assert.equal(code, 2);
assert.equal(summary.status, "blocked");
assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-native-k3s-context"), true);
assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-native-k3s-server"), true);
assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-native-k3s-nodes"), true);
assertNoWriteSideEffects(commandLog);
});
test("apply preflight blocks second hwlab-dev control plane visible through bare kubectl", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
let output = "";
const code = await runDevCdApply([
"--apply",
"--confirm-dev",
"--confirmed-non-production",
"--owner-task-id",
"task-second-control-plane",
"--kubeconfig",
"/etc/rancher/k3s/k3s.yaml"
], {
repoRoot,
env: {},
runCommand: makeRunCommand({
commandLog,
bareKubeContext: "docker-desktop",
bareKubeServer: "https://127.0.0.1:11700",
bareKubeNodes: ["desktop-control-plane"],
bareHwlabDevNamespace: true
}),
httpGetJson: makeHttpGetJson(),
now: () => new Date(iso(100000)),
stdout: { write: (chunk) => { output += chunk; } }
});
const summary = JSON.parse(output);
assert.equal(code, 2);
assert.equal(summary.status, "blocked");
assert.equal(summary.blockers.some((blocker) => blocker.scope === "bare-kubectl-context"), true);
assert.equal(summary.blockers.some((blocker) => blocker.scope === "bare-kubectl-server"), true);
assert.equal(summary.blockers.some((blocker) => blocker.scope === "second-hwlab-dev-control-plane"), true);
assertNoWriteSideEffects(commandLog);
});
test("transaction runs phases, allows internal side-effect env, releases lock, and reports live verify", async () => {
const repoRoot = await makeRepo();
const commandLog = [];
@@ -959,7 +1210,7 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a
"--owner-task-id",
"task-b",
"--kubeconfig",
"/tmp/kubeconfig",
"/etc/rancher/k3s/k3s.yaml",
"--write-report",
"--full-output"
], {
@@ -1054,7 +1305,7 @@ test("runtime maintenance job reports are parsed after redacting secret-like log
"--owner-task-id",
"task-redaction",
"--kubeconfig",
"/tmp/kubeconfig",
"/etc/rancher/k3s/k3s.yaml",
"--write-report",
"--full-output"
], {
@@ -1091,7 +1342,7 @@ test("transaction can apply an already published deploy-json promotion without r
"--owner-task-id",
"task-promotion",
"--kubeconfig",
"/tmp/kubeconfig",
"/etc/rancher/k3s/k3s.yaml",
"--write-report",
"--full-output"
], {
@@ -1158,7 +1409,7 @@ test("apply stdout is concise by default while write-report keeps the full repor
"--owner-task-id",
"task-c",
"--kubeconfig",
"/tmp/kubeconfig",
"/etc/rancher/k3s/k3s.yaml",
"--write-report"
], {
repoRoot,