diff --git a/docs/reference/deployment-publish.md b/docs/reference/deployment-publish.md index c2bbfc2a..94780aa8 100644 --- a/docs/reference/deployment-publish.md +++ b/docs/reference/deployment-publish.md @@ -56,9 +56,25 @@ node scripts/dev-cd-apply.mjs --apply --confirm-dev --confirmed-non-production - 该命令是唯一允许一次性发布 DEV artifact、从发布报告刷新 desired state、apply 到 `hwlab-dev`、并复验公开 `16666/16667` 的正式写路径。 -事务开始时读取 `deploy/deploy.json`，记录 manifest hash 和 target commit， -获取 `Lease/hwlab-dev/hwlab-dev-cd-lock`，串行执行内部步骤，按需写入 -`reports/dev-gate/dev-cd-apply.json`，最后释放 Lease。 +事务开始时读取 `deploy/deploy.json`，记录 manifest hash 和 target commit。 +在获取 `Lease/hwlab-dev/hwlab-dev-cd-lock` 或运行任何 publish、runtime Job、 +apply、rollout 类副作用前，脚本必须先完成 apply-only preflight：强制 +`KUBECONFIG=/etc/rancher/k3s/k3s.yaml`，验证节点包含 `d601`，拒绝 +`docker-desktop`、`desktop-control-plane`、`127.0.0.1:11700`、裸 +`kubectl` 指向第二套 `hwlab-dev` 控制面或其他不可信控制面信号，并且只验证 +必要 SecretRef 的存在性和 key 名。通过后才获取 Lease、串行执行内部步骤、 +按需写入 `reports/dev-gate/dev-cd-apply.json`，最后释放 Lease。 + +apply preflight 检查的 SecretRef 固定为： + +- `hwlab-cloud-api-dev-db/database-url` +- `hwlab-cloud-api-dev-db-admin/admin-url` +- `hwlab-code-agent-provider/openai-api-key` + +preflight 和报告只能记录 Secret 名、key 名、存在性、key-name 观察结果和脱敏 +命令摘要；不得读取或打印 Secret value。阻塞时输出结构化 blocker，包含 +`scope`、`reason`、`impact`、`safeNextAction` 和 `retryable`，默认 stdout 只 +给有界摘要，完整 preflight 证据写入事务报告。 `deploy/deploy.json` 的 `commitId` 是运行时 promotion commit。正常 latest-main 发布时它可以等于当前 target ref；artifact merge commit 或后续 diff --git a/scripts/src/dev-cd-apply.mjs b/scripts/src/dev-cd-apply.mjs index e1257e95..1a39df30 100644 --- a/scripts/src/dev-cd-apply.mjs +++ b/scripts/src/dev-cd-apply.mjs @@ -18,6 +18,7 @@ const defaultNamespace = "hwlab-dev"; const defaultLockName = "hwlab-dev-cd-lock"; const defaultTtlSeconds = 3600; const defaultReportPath = "reports/dev-gate/dev-cd-apply.json"; +const d601NativeKubeconfigPath = "/etc/rancher/k3s/k3s.yaml"; const artifactCatalogPath = "deploy/artifact-catalog.dev.json"; const artifactReportPath = "reports/dev-gate/dev-artifacts.json"; const artifactDesiredStatePaths = [ @@ -33,6 +34,12 @@ const runtimeJobReportPrefix = "/tmp/hwlab-dev-runtime-report"; const browserLiveUrl = "http://74.48.78.17:16666/health/live"; const apiLiveUrl = `${DEV_ENDPOINT}/health/live`; const digestPattern = /^sha256:[a-f0-9]{64}$/u; +const forbiddenControlPlanePattern = /docker-desktop|desktop-control-plane|127\.0\.0\.1:11700/iu; +const requiredDevSecretRefs = Object.freeze([ + Object.freeze({ secretName: "hwlab-cloud-api-dev-db", secretKey: "database-url" }), + Object.freeze({ secretName: "hwlab-cloud-api-dev-db-admin", secretKey: "admin-url" }), + Object.freeze({ secretName: "hwlab-code-agent-provider", secretKey: "openai-api-key" }) +]); const lockAnnotationPrefix = "hwlab.pikastech.local"; const lockAnnotationFields = { promotionCommit: `${lockAnnotationPrefix}/promotionCommit`, @@ -218,6 +225,19 @@ function oneLine(value) { return String(value ?? "").replace(/\s+/g, " ").trim(); } +function blocker({ type = "environment_blocker", scope, reason, impact, safeNextAction, retryable }) { + return { + type, + scope, + status: "open", + reason: oneLine(reason), + impact: oneLine(impact), + safeNextAction: oneLine(safeNextAction), + retryable: Boolean(retryable), + summary: oneLine(`${reason} Impact: ${impact} Safe next action: ${safeNextAction}`) + }; +} + function sha256(value) { return `sha256:${createHash("sha256").update(value).digest("hex")}`; } @@ -1207,6 +1227,323 @@ async function releaseDeployLock({ ctx, args, kubectlContext, transactionId, sta }; } +function commandSummary(result, maxLength = 500) { + return oneLine(redactSensitiveText(result?.stderr || result?.stdout || "")).slice(0, maxLength); +} + +function envWithoutKubeconfig(env) { + const sanitized = { ...env }; + delete sanitized.KUBECONFIG; + delete sanitized.HWLAB_DEV_KUBECONFIG; + return sanitized; +} + +function parseNodeNames(stdout) { + return String(stdout ?? "").trim().split(/\s+/u).filter(Boolean); +} + +function parseSecretDescribeKeys(stdout) { + const keys = new Set(); + let inData = false; + for (const rawLine of String(stdout ?? "").split(/\r?\n/u)) { + const line = rawLine.trim(); + if (/^Data\b/iu.test(line)) { + inData = true; + continue; + } + if (!inData || line === "" || /^=+$/u.test(line)) continue; + const match = line.match(/^([A-Za-z0-9_.-]+):\s+\d+\s+bytes\b/iu); + if (match) keys.add(match[1]); + } + return [...keys].sort(); +} + +function commandObservation(args, result) { + return { + command: shellCommand("kubectl", args), + code: result.code ?? null, + ok: result.code === 0, + summary: commandSummary(result) + }; +} + +async function preflightKubectl(ctx, kubectlContext, args, options = {}) { + const result = await ctx.runCommand(kubectlContext.executor, args, { + cwd: ctx.repoRoot, + env: options.env ?? kubectlContext.env, + timeoutMs: options.timeoutMs ?? 15000 + }); + return { + ...result, + stdout: result.stdout ?? "", + stderr: result.stderr ?? "", + command: shellCommand("kubectl", args) + }; +} + +function addControlPlaneBlockers(blockers, observation, { scopePrefix, requiredForApply }) { + if (observation.contextName && forbiddenControlPlanePattern.test(observation.contextName)) { + blockers.push(blocker({ + scope: `${scopePrefix}-context`, + reason: `kubectl context resolved to forbidden control plane ${observation.contextName}.`, + impact: "DEV CD cannot prove it is targeting D601 native k3s before mutation.", + safeNextAction: `Use KUBECONFIG=${d601NativeKubeconfigPath} and remove or repair the stale Docker Desktop/default kube context before retrying.`, + retryable: true + })); + } + if (observation.serverUrl && forbiddenControlPlanePattern.test(observation.serverUrl)) { + blockers.push(blocker({ + scope: `${scopePrefix}-server`, + reason: `kubectl server resolved to forbidden endpoint ${observation.serverUrl}.`, + impact: "DEV CD could write to the wrong control plane or treat stale Docker Desktop output as DEV evidence.", + safeNextAction: `Point kubectl at D601 native k3s with KUBECONFIG=${d601NativeKubeconfigPath} before retrying.`, + retryable: true + })); + } + if (observation.nodeNames.includes("desktop-control-plane")) { + blockers.push(blocker({ + scope: `${scopePrefix}-nodes`, + reason: "kubectl observed node desktop-control-plane.", + impact: "DEV CD is observing Docker Desktop Kubernetes instead of D601 native k3s.", + safeNextAction: `Stop and rerun only after kubectl nodes come from KUBECONFIG=${d601NativeKubeconfigPath} and include d601.`, + retryable: true + })); + } else if (requiredForApply && !observation.nodeNames.includes("d601")) { + blockers.push(blocker({ + scope: `${scopePrefix}-nodes`, + reason: `kubectl nodes did not include d601; observed ${observation.nodeNames.join(",") || "none"}.`, + impact: "DEV apply/job/rollout side effects are not allowed without proving the D601 native k3s node.", + safeNextAction: `Restore access to KUBECONFIG=${d601NativeKubeconfigPath} and verify kubectl get nodes includes d601.`, + retryable: true + })); + } else if (!requiredForApply && observation.nodeNames.length > 0 && !observation.nodeNames.includes("d601")) { + blockers.push(blocker({ + scope: `${scopePrefix}-nodes`, + reason: `bare kubectl observed non-D601 nodes ${observation.nodeNames.join(",")}.`, + impact: "A second or stale control plane may still own hwlab-dev evidence, so DEV CD refuses to mutate.", + safeNextAction: "Clean the default kubectl context or ensure bare kubectl cannot resolve stale HWLAB DEV resources, then retry.", + retryable: true + })); + } +} + +async function observeControlPlane(ctx, kubectlContext, { env, label, requiredForApply }) { + const [context, server, nodes] = await Promise.all([ + preflightKubectl(ctx, kubectlContext, ["config", "current-context"], { env, timeoutMs: 10000 }), + preflightKubectl(ctx, kubectlContext, ["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], { env, timeoutMs: 10000 }), + preflightKubectl(ctx, kubectlContext, ["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], { env, timeoutMs: 15000 }) + ]); + return { + label, + contextCommand: commandObservation(["config", "current-context"], context), + serverCommand: commandObservation(["config", "view", "--minify", "-o", "jsonpath={.clusters[0].cluster.server}"], server), + nodesCommand: commandObservation(["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], nodes), + contextName: context.code === 0 ? oneLine(context.stdout) : null, + serverUrl: server.code === 0 ? oneLine(server.stdout) : null, + nodeNames: nodes.code === 0 ? parseNodeNames(nodes.stdout) : [], + status: context.code === 0 && server.code === 0 && nodes.code === 0 ? "observed" : requiredForApply ? "blocked" : "unavailable" + }; +} + +async function observeBareKubectl(ctx, kubectlContext, forcedObservation) { + const bareEnv = envWithoutKubeconfig(ctx.env); + const bare = await observeControlPlane(ctx, kubectlContext, { + env: bareEnv, + label: "bare-kubectl", + requiredForApply: false + }); + const namespace = await preflightKubectl(ctx, kubectlContext, ["get", "namespace", defaultNamespace, "-o", "name"], { + env: bareEnv, + timeoutMs: 10000 + }); + const secondControlPlane = + namespace.code === 0 && + ( + (bare.serverUrl && forcedObservation.serverUrl && bare.serverUrl !== forcedObservation.serverUrl) || + (!bare.serverUrl && bare.contextName && bare.contextName !== forcedObservation.contextName) + ); + return { + ...bare, + namespace: commandObservation(["get", "namespace", defaultNamespace, "-o", "name"], namespace), + hwlabDevNamespaceObserved: namespace.code === 0, + secondControlPlaneCandidate: secondControlPlane + }; +} + +async function observeRequiredSecretRef(ctx, kubectlContext, args, ref) { + const existsArgs = ["-n", args.targetNamespace, "get", "secret", ref.secretName, "-o", "name"]; + const describeArgs = ["-n", args.targetNamespace, "describe", "secret", ref.secretName]; + const exists = await preflightKubectl(ctx, kubectlContext, existsArgs, { timeoutMs: 10000 }); + if (exists.code !== 0) { + return { + secretName: ref.secretName, + secretKey: ref.secretKey, + status: "missing-secret", + exists: false, + keyPresent: false, + keysObserved: [], + existenceCommand: shellCommand("kubectl", existsArgs), + keyCommand: null, + error: commandSummary(exists), + secretValueRead: false, + secretValuePrinted: false + }; + } + const describe = await preflightKubectl(ctx, kubectlContext, describeArgs, { timeoutMs: 10000 }); + if (describe.code !== 0) { + return { + secretName: ref.secretName, + secretKey: ref.secretKey, + status: "key-observation-blocked", + exists: true, + keyPresent: false, + keysObserved: [], + existenceCommand: shellCommand("kubectl", existsArgs), + keyCommand: shellCommand("kubectl", describeArgs), + error: commandSummary(describe), + secretValueRead: false, + secretValuePrinted: false + }; + } + const keysObserved = parseSecretDescribeKeys(describe.stdout); + const keyPresent = keysObserved.includes(ref.secretKey); + return { + secretName: ref.secretName, + secretKey: ref.secretKey, + status: keyPresent ? "present" : "missing-key", + exists: true, + keyPresent, + keysObserved, + existenceCommand: shellCommand("kubectl", existsArgs), + keyCommand: shellCommand("kubectl", describeArgs), + secretValueRead: false, + secretValuePrinted: false + }; +} + +function secretRefBlocker(ref, observation) { + const scope = `secretref:${ref.secretName}/${ref.secretKey}`; + if (observation.status === "missing-secret") { + return blocker({ + type: "runtime_blocker", + scope, + reason: `Required DEV Secret ${ref.secretName} was not observed in ${defaultNamespace}.`, + impact: "Runtime provisioning, migration, deploy apply, or Code Agent provider startup would fail after mutation.", + safeNextAction: `Create or restore SecretRef ${ref.secretName}/${ref.secretKey} in ${defaultNamespace} without printing the Secret value, then retry DEV CD.`, + retryable: true + }); + } + if (observation.status === "key-observation-blocked") { + return blocker({ + type: "runtime_blocker", + scope, + reason: `Required DEV Secret ${ref.secretName} exists but key names could not be verified.`, + impact: "DEV CD cannot prove the pod SecretRef contract before creating runtime Jobs or applying workloads.", + safeNextAction: `Restore metadata/key-name visibility for ${ref.secretName} without granting or printing Secret values, then retry.`, + retryable: true + }); + } + return blocker({ + type: "runtime_blocker", + scope, + reason: `Required DEV Secret key ${ref.secretName}/${ref.secretKey} was not observed.`, + impact: "The target workload or runtime Job would reference a missing Secret key after mutation.", + safeNextAction: `Add key ${ref.secretKey} to Secret ${ref.secretName} without printing the value, then retry DEV CD.`, + retryable: true + }); +} + +async function runDevCdPreflight({ ctx, args, kubectlContext }) { + const blockers = []; + const preflight = { + status: "blocked", + scope: "pre-lock-pre-side-effect", + generatedAt: ctx.now().toISOString(), + namespace: args.targetNamespace, + kubeconfig: { + required: d601NativeKubeconfigPath, + selected: kubectlContext.kubeconfig, + source: kubectlContext.kubeconfigSource, + enforcedEnv: kubectlContext.env.KUBECONFIG === d601NativeKubeconfigPath + }, + kubectl: { + executor: kubectlContext.executor, + commandPrefix: kubectlContext.commandPrefix + }, + controlPlane: null, + secretRefs: [], + safety: { + writeSideEffectsAttempted: false, + prodTouched: false, + secretValuesRead: false, + secretValuesPrinted: false, + secretKeyNamesOnly: true + }, + blockers + }; + + if (kubectlContext.kubeconfig !== d601NativeKubeconfigPath) { + blockers.push(blocker({ + scope: "d601-kubeconfig-path", + reason: `DEV CD selected kubeconfig ${kubectlContext.kubeconfig || ""} from ${kubectlContext.kubeconfigSource}.`, + impact: "Any apply/job/rollout could target a non-D601 or stale control plane.", + safeNextAction: `Rerun with KUBECONFIG=${d601NativeKubeconfigPath} or --kubeconfig ${d601NativeKubeconfigPath}.`, + retryable: true + })); + preflight.status = "blocked"; + return preflight; + } + + const forced = await observeControlPlane(ctx, kubectlContext, { + env: kubectlContext.env, + label: "forced-d601-k3s", + requiredForApply: true + }); + const bare = await observeBareKubectl(ctx, kubectlContext, forced); + preflight.controlPlane = { forced, bare }; + + if (forced.status !== "observed") { + blockers.push(blocker({ + scope: "d601-native-k3s-unobservable", + reason: `Forced D601 kubeconfig could not prove context/server/nodes: ${[forced.contextCommand.summary, forced.serverCommand.summary, forced.nodesCommand.summary].filter(Boolean).join("; ") || "no kubectl output"}.`, + impact: "DEV CD refuses to acquire the Lease or create runtime Jobs without D601 native k3s proof.", + safeNextAction: `Restore readable ${d601NativeKubeconfigPath} access and verify kubectl get nodes includes d601.`, + retryable: true + })); + } + addControlPlaneBlockers(blockers, forced, { + scopePrefix: "d601-native-k3s", + requiredForApply: true + }); + addControlPlaneBlockers(blockers, bare, { + scopePrefix: "bare-kubectl", + requiredForApply: false + }); + if (bare.secondControlPlaneCandidate) { + blockers.push(blocker({ + scope: "second-hwlab-dev-control-plane", + reason: `bare kubectl can observe namespace ${defaultNamespace} outside the forced D601 kubeconfig context.`, + impact: "A second hwlab-dev control plane may still carry stale HWLAB resources, so DEV CD cannot safely classify apply evidence.", + safeNextAction: "Remove the stale hwlab-dev control plane/default kube context or make bare kubectl unable to resolve it, then rerun preflight.", + retryable: true + })); + } + + if (blockers.length === 0) { + preflight.secretRefs = await Promise.all( + requiredDevSecretRefs.map((ref) => observeRequiredSecretRef(ctx, kubectlContext, args, ref)) + ); + for (const [index, observation] of preflight.secretRefs.entries()) { + if (observation.status !== "present") { + blockers.push(secretRefBlocker(requiredDevSecretRefs[index], observation)); + } + } + } + + preflight.status = blockers.length === 0 ? "pass" : "blocked"; + return preflight; +} + async function resolveKubectlContext(args, env, runCommand) { const selection = resolveDevKubeconfigSelection({ flagValue: args.kubeconfig, env }); const which = await runCommand("which", ["kubectl"], { timeoutMs: 5000 }); @@ -1215,27 +1552,6 @@ async function resolveKubectlContext(args, env, runCommand) { ...env, KUBECONFIG: selection.kubeconfig }; - if (which.code === 0 && args.apply) { - const context = await runCommand(executor, ["config", "current-context"], { env: kubectlEnv, timeoutMs: 10000 }); - const nodes = await runCommand(executor, ["get", "nodes", "-o", "jsonpath={.items[*].metadata.name}"], { env: kubectlEnv, timeoutMs: 15000 }); - const contextName = String(context.stdout ?? "").trim(); - const nodeNames = String(nodes.stdout ?? "").trim().split(/\s+/u).filter(Boolean); - const forbiddenContext = context.code === 0 && /docker-desktop|desktop-control-plane/iu.test(contextName); - const forbiddenNode = nodes.code === 0 && ( - nodeNames.includes("desktop-control-plane") || (nodeNames.length > 0 && !nodeNames.includes("d601")) - ); - if (forbiddenContext || forbiddenNode) { - throw new DevCdApplyError("DEV CD kubeconfig does not target D601 native k3s", { - code: "wrong-kubernetes-control-plane", - blockers: [{ - type: "environment_blocker", - scope: "d601-native-k3s", - status: "open", - summary: "D601 DEV CD must use native k3s via /etc/rancher/k3s/k3s.yaml; default docker-desktop kubeconfig is forbidden." - }] - }); - } - } return { executor, kubeconfig: selection.kubeconfig, @@ -1904,6 +2220,19 @@ function summarizeDevCdApplyReport(report) { })), liveBefore: apply.liveBefore ? compactLiveReport(apply.liveBefore) : null, liveVerify: apply.liveVerify ? compactLiveReport(apply.liveVerify) : null, + preflight: apply.preflight ? { + status: apply.preflight.status, + scope: apply.preflight.scope, + kubeconfig: apply.preflight.kubeconfig, + blockerCount: apply.preflight.blockers?.length ?? 0, + secretRefs: (apply.preflight.secretRefs ?? []).map((ref) => ({ + secretName: ref.secretName, + secretKey: ref.secretKey, + status: ref.status, + exists: ref.exists, + keyPresent: ref.keyPresent + })) + } : null, blockers: report.blockers ?? [], reportPaths: apply.reportPaths ?? {}, fullOutputHint: "Use --full-output to print the full report to stdout. Use --write-report to persist the full report." @@ -1922,6 +2251,7 @@ function buildReport({ status, startedAt, finishedAt, + preflight, liveBefore, liveVerify, release @@ -1985,6 +2315,9 @@ function buildReport({ devPreconditions: { status: blockers.length ? "blocked" : "pass", requirements: [ + "DEV CD apply must force KUBECONFIG=/etc/rancher/k3s/k3s.yaml and verify native node d601 before any Lease, Job, apply, or rollout side effect.", + "DEV CD apply must reject docker-desktop, desktop-control-plane, 127.0.0.1:11700, bare kubectl ambiguity, and second hwlab-dev control-plane signals before mutation.", + "DEV CD apply must verify required SecretRef existence and key names without reading or printing Secret values.", "HEAD must match the requested target ref before publish.", "Lease/hwlab-dev-cd-lock must be acquired before publish/apply.", "Legacy publish/apply side-effect scripts must run with HWLAB_CD_TRANSACTION_ID.", @@ -2034,9 +2367,10 @@ function buildReport({ staleBreak: lockState?.staleBreak ?? null }, steps, - liveBefore, - liveVerify, - reportPaths: { + liveBefore, + liveVerify, + preflight, + reportPaths: { transaction: args.writeReport ? args.reportPath : null, artifacts: artifactReportPath, deployApply: "reports/dev-gate/dev-deploy-report.json", @@ -2138,6 +2472,7 @@ export async function runDevCdApply(argv, io = {}) { let deployAfter = null; let liveBefore = null; let liveVerify = null; + let preflight = null; let status = "blocked"; try { @@ -2182,6 +2517,14 @@ export async function runDevCdApply(argv, io = {}) { } const kubectlContext = await resolveKubectlContext(args, env, ctx.runCommand); transaction.kubectlContext = kubectlContext; + preflight = await runDevCdPreflight({ ctx, args, kubectlContext }); + if (preflight.status !== "pass") { + blockers.push(...preflight.blockers); + throw new DevCdApplyError("DEV CD preflight blocked before any write side effect", { + code: "dev-cd-preflight-blocked", + suppressCatchBlocker: true + }); + } liveBefore = { status: "pending", reason: "liveBefore is captured immediately after Lease acquisition so publish/apply cannot race before the transaction lock." @@ -2458,6 +2801,17 @@ export async function runDevCdApply(argv, io = {}) { status, startedAt, finishedAt, + preflight: preflight ?? { + status: "not_run", + scope: "pre-lock-pre-side-effect", + blockers: [], + safety: { + writeSideEffectsAttempted: false, + prodTouched: false, + secretValuesRead: false, + secretValuesPrinted: false + } + }, liveBefore: liveBefore ?? { status: "not_run" }, liveVerify: liveVerify ?? { status: "not_run", endpoints: [], summary: { checked: 0 } }, release diff --git a/scripts/src/dev-cd-apply.test.mjs b/scripts/src/dev-cd-apply.test.mjs index 41c19585..58d02b00 100644 --- a/scripts/src/dev-cd-apply.test.mjs +++ b/scripts/src/dev-cd-apply.test.mjs @@ -214,7 +214,19 @@ function makeRunCommand({ "reports/dev-gate/dev-artifacts.json" ], desiredStatePromotionStatus = "pass", - runtimeJobLogSuffix = "" + runtimeJobLogSuffix = "", + kubeContext = "d601", + kubeServer = "https://d601:6443", + kubeNodes = ["d601"], + bareKubeContext = null, + bareKubeServer = null, + bareKubeNodes = null, + bareHwlabDevNamespace = false, + secretRefs = { + "hwlab-cloud-api-dev-db": ["database-url"], + "hwlab-cloud-api-dev-db-admin": ["admin-url"], + "hwlab-code-agent-provider": ["openai-api-key"] + } } = {}) { let lease = heldLock ? leaseFromLock(heldLock) : null; const assertLeaseMicroTime = (value) => { @@ -255,6 +267,42 @@ function makeRunCommand({ if (command === "which" && args[0] === "kubectl") { return { code: 0, stdout: "/usr/local/bin/kubectl\n", stderr: "" }; } + if (command.includes("kubectl") && args[0] === "config" && args[1] === "current-context") { + const hasKubeconfig = Object.hasOwn(options.env ?? {}, "KUBECONFIG"); + return { code: 0, stdout: `${hasKubeconfig ? kubeContext : (bareKubeContext ?? kubeContext)}\n`, stderr: "" }; + } + if (command.includes("kubectl") && args[0] === "config" && args[1] === "view") { + const hasKubeconfig = Object.hasOwn(options.env ?? {}, "KUBECONFIG"); + return { code: 0, stdout: `${hasKubeconfig ? kubeServer : (bareKubeServer ?? kubeServer)}\n`, stderr: "" }; + } + if (command.includes("kubectl") && args[0] === "get" && args[1] === "nodes") { + const hasKubeconfig = Object.hasOwn(options.env ?? {}, "KUBECONFIG"); + return { code: 0, stdout: `${(hasKubeconfig ? kubeNodes : (bareKubeNodes ?? kubeNodes)).join(" ")}\n`, stderr: "" }; + } + if (command.includes("kubectl") && args[0] === "get" && args[1] === "namespace") { + return bareHwlabDevNamespace + ? { code: 0, stdout: "namespace/hwlab-dev\n", stderr: "" } + : { code: 1, stdout: "", stderr: "Error from server (NotFound): namespaces \"hwlab-dev\" not found" }; + } + if (command.includes("kubectl") && args.includes("get") && args.includes("secret")) { + const name = args[args.indexOf("secret") + 1]; + if (!secretRefs[name]) { + return { code: 1, stdout: "", stderr: `Error from server (NotFound): secrets "${name}" not found` }; + } + return { code: 0, stdout: `secret/${name}\n`, stderr: "" }; + } + if (command.includes("kubectl") && args.includes("describe") && args.includes("secret")) { + const name = args[args.indexOf("secret") + 1]; + const keys = secretRefs[name]; + if (!keys) { + return { code: 1, stdout: "", stderr: `Error from server (NotFound): secrets "${name}" not found` }; + } + return { + code: 0, + stdout: `Name: ${name}\nNamespace: hwlab-dev\nType: Opaque\n\nData\n====\n${keys.map((key) => `${key}: 16 bytes`).join("\n")}\n`, + stderr: "" + }; + } if (command.includes("kubectl") && args.includes("get") && args.includes("lease")) { if (!lease) return { code: 1, stdout: "", stderr: "Error from server (NotFound): leases.coordination.k8s.io \"hwlab-dev-cd-lock\" not found" }; return { code: 0, stdout: `${JSON.stringify(lease)}\n`, stderr: "" }; @@ -360,6 +408,16 @@ function assertNoReadOnlySideEffects(commandLog) { assert.equal(commandLog.some((entry) => entry.args.includes("scripts/refresh-artifact-catalog.mjs")), false); } +function assertNoWriteSideEffects(commandLog) { + const writeKubectlVerbs = new Set(["apply", "create", "replace", "patch", "delete", "rollout", "wait", "logs"]); + assert.equal(commandLog.some((entry) => + entry.command.includes("kubectl") && entry.args.some((arg) => writeKubectlVerbs.has(arg)) + ), false); + assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-artifact-publish.mjs")), false); + assert.equal(commandLog.some((entry) => entry.args.includes("scripts/dev-deploy-apply.mjs")), false); + assert.equal(commandLog.some((entry) => entry.args.includes("scripts/refresh-artifact-catalog.mjs")), false); +} + test("lock classifier reports held and stale states with retryAfterSeconds", () => { const lock = { ownerTaskId: "task-a", @@ -391,7 +449,7 @@ test("dev-cd status and dry-run are read-only compact JSON", async () => { const code = await runDevCdApply([ "--dry-run", "--kubeconfig", - "/tmp/kubeconfig", + "/etc/rancher/k3s/k3s.yaml", "--write-report" ], { repoRoot, @@ -447,7 +505,7 @@ test("dev-cd status accepts a published deploy-json promotion under a later cont const repoRoot = await makeRepo({ commitId: "abc1234" }); const commandLog = []; let output = ""; - const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], { + const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], { repoRoot, env: {}, runCommand: makeRunCommand({ @@ -504,7 +562,7 @@ test("dev-cd status exposes moving origin/main when checkout is behind a deploy- const repoRoot = await makeRepo({ commitId: "abc1234" }); const commandLog = []; let output = ""; - const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], { + const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], { repoRoot, env: {}, runCommand: makeRunCommand({ @@ -547,7 +605,7 @@ test("apply stops before lock acquisition when checkout is behind deploy-json pr "--owner-task-id", "task-moving-main", "--kubeconfig", - "/tmp/kubeconfig" + "/etc/rancher/k3s/k3s.yaml" ], { repoRoot, env: {}, @@ -593,7 +651,7 @@ test("dev-cd status degrades artifact merge provenance when catalog/report do no ); const commandLog = []; let output = ""; - const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], { + const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], { repoRoot, env: {}, runCommand: makeRunCommand({ @@ -651,7 +709,7 @@ test("apply stops before lock acquisition when artifact merge provenance mismatc "--owner-task-id", "task-provenance", "--kubeconfig", - "/tmp/kubeconfig" + "/etc/rancher/k3s/k3s.yaml" ], { repoRoot, env: {}, @@ -713,7 +771,7 @@ test("dev-cd status summarizes held, released, and stale locks without unsafe ta const repoRoot = await makeRepo(); const commandLog = []; let output = ""; - const code = await runDevCdApply(["--status", "--kubeconfig", "/tmp/kubeconfig"], { + const code = await runDevCdApply(["--status", "--kubeconfig", "/etc/rancher/k3s/k3s.yaml"], { repoRoot, env: {}, runCommand: makeRunCommand({ @@ -870,7 +928,7 @@ test("second transaction exits deploy-lock-held without running side effects", a "--owner-task-id", "task-b", "--kubeconfig", - "/tmp/kubeconfig" + "/etc/rancher/k3s/k3s.yaml" ], { repoRoot, env: {}, @@ -915,7 +973,7 @@ test("stale lock requires explicit break-stale-lock confirmation", async () => { "--owner-task-id", "task-b", "--kubeconfig", - "/tmp/kubeconfig" + "/etc/rancher/k3s/k3s.yaml" ], { repoRoot, env: {}, @@ -947,6 +1005,199 @@ test("stale lock requires explicit break-stale-lock confirmation", async () => { assert.equal(failure.requiresBreakStaleLock, true); }); +test("apply preflight passes with D601 native k3s and required SecretRef keys before mutation", async () => { + const repoRoot = await makeRepo(); + const commandLog = []; + let output = ""; + const code = await runDevCdApply([ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--owner-task-id", + "task-preflight-pass", + "--kubeconfig", + "/etc/rancher/k3s/k3s.yaml", + "--write-report", + "--full-output" + ], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ commandLog }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const report = JSON.parse(output); + const preflight = report.devCdApply.preflight; + assert.equal(code, 0); + assert.equal(preflight.status, "pass"); + assert.equal(preflight.kubeconfig.required, "/etc/rancher/k3s/k3s.yaml"); + assert.equal(preflight.kubeconfig.enforcedEnv, true); + assert.deepEqual(preflight.controlPlane.forced.nodeNames, ["d601"]); + assert.deepEqual( + preflight.secretRefs.map((ref) => `${ref.secretName}/${ref.secretKey}:${ref.status}`), + [ + "hwlab-cloud-api-dev-db/database-url:present", + "hwlab-cloud-api-dev-db-admin/admin-url:present", + "hwlab-code-agent-provider/openai-api-key:present" + ] + ); + assert.equal(preflight.safety.secretValuesRead, false); + assert.equal(preflight.safety.secretValuesPrinted, false); + + const firstWriteIndex = commandLog.findIndex((entry) => + entry.command.includes("kubectl") && ["create", "replace", "apply", "patch", "delete", "rollout", "wait", "logs"].some((verb) => entry.args.includes(verb)) + ); + const secretDescribeIndexes = commandLog + .map((entry, index) => ({ entry, index })) + .filter(({ entry }) => entry.command.includes("kubectl") && entry.args.includes("describe") && entry.args.includes("secret")) + .map(({ index }) => index); + assert.ok(secretDescribeIndexes.length > 0); + assert.ok(secretDescribeIndexes.every((index) => index < firstWriteIndex)); +}); + +test("apply preflight blocks missing SecretRef before Lease or side-effect commands", async () => { + const repoRoot = await makeRepo(); + const commandLog = []; + let output = ""; + const code = await runDevCdApply([ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--owner-task-id", + "task-missing-secret", + "--kubeconfig", + "/etc/rancher/k3s/k3s.yaml", + "--write-report" + ], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ + commandLog, + secretRefs: { + "hwlab-cloud-api-dev-db": ["database-url"], + "hwlab-cloud-api-dev-db-admin": ["admin-url"] + } + }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const summary = JSON.parse(output); + const writtenReport = JSON.parse(await readFile(path.join(repoRoot, "reports/dev-gate/dev-cd-apply.json"), "utf8")); + assert.equal(code, 2); + assert.equal(summary.status, "blocked"); + assert.equal(summary.preflight.status, "blocked"); + assert.equal(summary.preflight.blockerCount, 1); + assert.equal(summary.blockers.some((blocker) => blocker.scope === "secretref:hwlab-code-agent-provider/openai-api-key"), true); + assert.equal(writtenReport.devCdApply.preflight.safety.writeSideEffectsAttempted, false); + assert.equal(writtenReport.devCdApply.preflight.safety.secretValuesRead, false); + assert.equal(JSON.stringify(writtenReport).includes("SECRET"), false); + assertNoWriteSideEffects(commandLog); +}); + +test("apply preflight blocks non-D601 kubeconfig path before kubectl mutation", async () => { + const repoRoot = await makeRepo(); + const commandLog = []; + let output = ""; + const code = await runDevCdApply([ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--owner-task-id", + "task-wrong-path", + "--kubeconfig", + "/tmp/kubeconfig" + ], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ commandLog }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const summary = JSON.parse(output); + assert.equal(code, 2); + assert.equal(summary.status, "blocked"); + assert.equal(summary.preflight.status, "blocked"); + assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-kubeconfig-path"), true); + assertNoWriteSideEffects(commandLog); +}); + +test("apply preflight blocks docker-desktop and 127.0.0.1 control-plane signals before mutation", async () => { + const repoRoot = await makeRepo(); + const commandLog = []; + let output = ""; + const code = await runDevCdApply([ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--owner-task-id", + "task-docker-desktop", + "--kubeconfig", + "/etc/rancher/k3s/k3s.yaml" + ], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ + commandLog, + kubeContext: "docker-desktop", + kubeServer: "https://127.0.0.1:11700", + kubeNodes: ["desktop-control-plane"] + }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const summary = JSON.parse(output); + assert.equal(code, 2); + assert.equal(summary.status, "blocked"); + assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-native-k3s-context"), true); + assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-native-k3s-server"), true); + assert.equal(summary.blockers.some((blocker) => blocker.scope === "d601-native-k3s-nodes"), true); + assertNoWriteSideEffects(commandLog); +}); + +test("apply preflight blocks second hwlab-dev control plane visible through bare kubectl", async () => { + const repoRoot = await makeRepo(); + const commandLog = []; + let output = ""; + const code = await runDevCdApply([ + "--apply", + "--confirm-dev", + "--confirmed-non-production", + "--owner-task-id", + "task-second-control-plane", + "--kubeconfig", + "/etc/rancher/k3s/k3s.yaml" + ], { + repoRoot, + env: {}, + runCommand: makeRunCommand({ + commandLog, + bareKubeContext: "docker-desktop", + bareKubeServer: "https://127.0.0.1:11700", + bareKubeNodes: ["desktop-control-plane"], + bareHwlabDevNamespace: true + }), + httpGetJson: makeHttpGetJson(), + now: () => new Date(iso(100000)), + stdout: { write: (chunk) => { output += chunk; } } + }); + + const summary = JSON.parse(output); + assert.equal(code, 2); + assert.equal(summary.status, "blocked"); + assert.equal(summary.blockers.some((blocker) => blocker.scope === "bare-kubectl-context"), true); + assert.equal(summary.blockers.some((blocker) => blocker.scope === "bare-kubectl-server"), true); + assert.equal(summary.blockers.some((blocker) => blocker.scope === "second-hwlab-dev-control-plane"), true); + assertNoWriteSideEffects(commandLog); +}); + test("transaction runs phases, allows internal side-effect env, releases lock, and reports live verify", async () => { const repoRoot = await makeRepo(); const commandLog = []; @@ -959,7 +1210,7 @@ test("transaction runs phases, allows internal side-effect env, releases lock, a "--owner-task-id", "task-b", "--kubeconfig", - "/tmp/kubeconfig", + "/etc/rancher/k3s/k3s.yaml", "--write-report", "--full-output" ], { @@ -1054,7 +1305,7 @@ test("runtime maintenance job reports are parsed after redacting secret-like log "--owner-task-id", "task-redaction", "--kubeconfig", - "/tmp/kubeconfig", + "/etc/rancher/k3s/k3s.yaml", "--write-report", "--full-output" ], { @@ -1091,7 +1342,7 @@ test("transaction can apply an already published deploy-json promotion without r "--owner-task-id", "task-promotion", "--kubeconfig", - "/tmp/kubeconfig", + "/etc/rancher/k3s/k3s.yaml", "--write-report", "--full-output" ], { @@ -1158,7 +1409,7 @@ test("apply stdout is concise by default while write-report keeps the full repor "--owner-task-id", "task-c", "--kubeconfig", - "/tmp/kubeconfig", + "/etc/rancher/k3s/k3s.yaml", "--write-report" ], { repoRoot,